feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware
- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline) - Databricks-style Lakehouse Workbench (Trino engine, live exec matrix, materialize to Iceberg/S3); reused & embedded in every source-DB UI - HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver - Data Flow master pulse switch (Run/Pause/Stop) gating animated edges - Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC), pulsing source -> HDFS edges; toggle in Data Flow - LLM now autonomously aware of all latest platform changes (live platform context) and enforces masking policy: never reveals masked PII, still answers helpfully with aggregates/explanations
This commit is contained in:
mo
+430
-24
@@ -48,6 +48,17 @@ TRINO_USER = os.getenv("TRINO_USER", "atc")
|
||||
SOURCE_ENGINES = ("postgres", "mysql", "mongodb", "cassandra", "neo4j")
|
||||
ENGINES = SOURCE_ENGINES + ("trino",)
|
||||
|
||||
from hadoop_sql import (
|
||||
HADOOP_SAMPLES,
|
||||
catalog_hadoop,
|
||||
connection_info as hadoop_connection_info,
|
||||
health_hadoop,
|
||||
sample_hadoop,
|
||||
table_row_count_hadoop,
|
||||
)
|
||||
|
||||
LAKE_ENGINES = ("hadoop",)
|
||||
|
||||
router = APIRouter(prefix="/api/sql", tags=["sql"])
|
||||
|
||||
SAMPLES: dict[str, list[dict[str, str]]] = {
|
||||
@@ -111,6 +122,7 @@ SAMPLES: dict[str, list[dict[str, str]]] = {
|
||||
{"id": "nj9", "label": "Schema visualization", "sql": "CALL db.schema.visualization()"},
|
||||
{"id": "nj10", "label": "Constraint info", "sql": "SHOW CONSTRAINTS"},
|
||||
],
|
||||
"hadoop": HADOOP_SAMPLES,
|
||||
"trino": [
|
||||
{"id": "tq1", "label": "Trino version", "sql": "SELECT version()"},
|
||||
{"id": "tq2", "label": "Catalogs", "sql": "SHOW CATALOGS"},
|
||||
@@ -332,10 +344,17 @@ def _run_trino(sql: str, limit: int = 200) -> dict[str, Any]:
|
||||
|
||||
|
||||
class SqlRequest(BaseModel):
|
||||
engine: str = Field(..., pattern="^(postgres|mysql|mongodb|trino|cassandra|neo4j)$")
|
||||
engine: str = Field(..., pattern="^(postgres|mysql|mongodb|trino|cassandra|neo4j|hadoop)$")
|
||||
sql: str = Field(..., min_length=1, max_length=8000)
|
||||
|
||||
|
||||
class RowUpdateRequest(BaseModel):
|
||||
engine: str = Field(..., pattern="^(postgres|mysql|mongodb|cassandra|neo4j)$")
|
||||
object: str = Field(..., min_length=1, max_length=256)
|
||||
pk: dict[str, Any] = Field(..., min_length=1)
|
||||
changes: dict[str, Any] = Field(..., min_length=1)
|
||||
|
||||
|
||||
def _connection_info(engine: str) -> dict[str, str]:
|
||||
if engine == "postgres":
|
||||
return {"host": DB_HOST, "port": str(PG_PORT), "database": PG_DB, "user": PG_USER}
|
||||
@@ -361,6 +380,8 @@ def _dispatch(engine: str, sql: str, limit: int = 200) -> dict[str, Any]:
|
||||
return _run_cassandra(sql, limit)
|
||||
if engine == "neo4j":
|
||||
return _run_neo4j(sql, limit)
|
||||
if engine == "hadoop":
|
||||
return _run_trino(sql, limit)
|
||||
return _run_trino(sql, limit)
|
||||
|
||||
|
||||
@@ -458,6 +479,15 @@ def _catalog_neo4j() -> dict[str, Any]:
|
||||
driver.close()
|
||||
|
||||
|
||||
|
||||
def _catalog_hadoop() -> dict[str, Any]:
|
||||
return catalog_hadoop(_run_trino)
|
||||
|
||||
|
||||
def _sample_hadoop(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
|
||||
return sample_hadoop(object_name, limit, offset, _run_trino, _tabular)
|
||||
|
||||
|
||||
def _catalog(engine: str) -> dict[str, Any]:
|
||||
if engine == "postgres":
|
||||
return _catalog_postgres()
|
||||
@@ -469,37 +499,74 @@ def _catalog(engine: str) -> dict[str, Any]:
|
||||
return _catalog_cassandra()
|
||||
if engine == "neo4j":
|
||||
return _catalog_neo4j()
|
||||
if engine == "hadoop":
|
||||
return _catalog_hadoop()
|
||||
raise ValueError(f"Catalog not supported for {engine}")
|
||||
|
||||
|
||||
def _sample_postgres(object_name: str, limit: int) -> dict[str, Any]:
|
||||
def _sample_postgres(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
|
||||
if "." in object_name:
|
||||
schema, table = object_name.split(".", 1)
|
||||
sql = f'SELECT * FROM "{schema}"."{table}" LIMIT {limit}'
|
||||
sql = f'SELECT * FROM "{schema}"."{table}" OFFSET {int(offset)} LIMIT {int(limit)}'
|
||||
else:
|
||||
sql = f'SELECT * FROM public."{object_name}" LIMIT {limit}'
|
||||
sql = f'SELECT * FROM public."{object_name}" OFFSET {int(offset)} LIMIT {int(limit)}'
|
||||
return _run_postgres(sql, limit)
|
||||
|
||||
|
||||
def _sample_mysql(object_name: str, limit: int) -> dict[str, Any]:
|
||||
def _sample_mysql(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
|
||||
table = object_name.split(".")[-1]
|
||||
return _run_mysql(f"SELECT * FROM `{table}` LIMIT {limit}", limit)
|
||||
return _run_mysql(f"SELECT * FROM `{table}` LIMIT {int(limit)} OFFSET {int(offset)}", limit)
|
||||
|
||||
|
||||
def _sample_mongodb(object_name: str, limit: int) -> dict[str, Any]:
|
||||
def _sample_mongodb(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
|
||||
if "." in object_name:
|
||||
db_name, coll = object_name.split(".", 1)
|
||||
else:
|
||||
db_name, coll = MONGO_DB, object_name
|
||||
return _run_mongo(f"FIND {db_name}.{coll} LIMIT {limit}", limit)
|
||||
t0 = time.perf_counter()
|
||||
client = _mongo_client()
|
||||
try:
|
||||
docs = list(client[db_name][coll].find({}).skip(int(offset)).limit(int(limit)))
|
||||
elapsed_ms = int((time.perf_counter() - t0) * 1000)
|
||||
if not docs:
|
||||
return _tabular(["result"], [["(empty)"]], elapsed_ms)
|
||||
columns = sorted({k for d in docs for k in d})
|
||||
rows = [[_fmt(d.get(c)) for c in columns] for d in docs]
|
||||
return _tabular(columns, rows, elapsed_ms, object=object_name)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
def _sample_cassandra(object_name: str, limit: int) -> dict[str, Any]:
|
||||
def _sample_cassandra(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
|
||||
if "." in object_name:
|
||||
ks, table = object_name.split(".", 1)
|
||||
else:
|
||||
ks, table = CASS_KS, object_name
|
||||
return _run_cassandra(f"SELECT * FROM {ks}.{table} LIMIT {limit}", limit)
|
||||
t0 = time.perf_counter()
|
||||
cluster = _cass_cluster()
|
||||
session = cluster.connect()
|
||||
try:
|
||||
stmt = f"SELECT * FROM {ks}.{table}"
|
||||
result = session.execute(stmt, timeout=60)
|
||||
skipped = 0
|
||||
picked: list[Any] = []
|
||||
columns: list[str] = []
|
||||
for row in result:
|
||||
if skipped < offset:
|
||||
skipped += 1
|
||||
continue
|
||||
if not columns:
|
||||
columns = list(row._fields)
|
||||
picked.append(row)
|
||||
if len(picked) >= limit:
|
||||
break
|
||||
elapsed_ms = int((time.perf_counter() - t0) * 1000)
|
||||
if not picked:
|
||||
return _tabular(["result"], [["(empty)"]], elapsed_ms)
|
||||
rows = [[_fmt(getattr(r, c)) for c in columns] for r in picked]
|
||||
return _tabular(columns, rows, elapsed_ms, truncated=len(picked) >= limit)
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def _graph_neo4j(edge_limit: int = 60, rel_type: str | None = None) -> dict[str, Any]:
|
||||
@@ -582,9 +649,9 @@ def _graph_neo4j(edge_limit: int = 60, rel_type: str | None = None) -> dict[str,
|
||||
driver.close()
|
||||
|
||||
|
||||
def _sample_neo4j(object_name: str, limit: int) -> dict[str, Any]:
|
||||
def _sample_neo4j(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
|
||||
label = object_name.split(".")[-1]
|
||||
cypher = f"MATCH (n:`{label}`) RETURN n LIMIT {limit}"
|
||||
cypher = f"MATCH (n:`{label}`) RETURN n SKIP {int(offset)} LIMIT {int(limit)}"
|
||||
t0 = time.perf_counter()
|
||||
driver = _neo4j_driver()
|
||||
try:
|
||||
@@ -601,24 +668,319 @@ def _sample_neo4j(object_name: str, limit: int) -> dict[str, Any]:
|
||||
driver.close()
|
||||
|
||||
|
||||
def _sample(engine: str, object_name: str, limit: int) -> dict[str, Any]:
|
||||
def _sample(engine: str, object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
|
||||
if engine == "postgres":
|
||||
return _sample_postgres(object_name, limit)
|
||||
return _sample_postgres(object_name, limit, offset)
|
||||
if engine == "mysql":
|
||||
return _sample_mysql(object_name, limit)
|
||||
return _sample_mysql(object_name, limit, offset)
|
||||
if engine == "mongodb":
|
||||
return _sample_mongodb(object_name, limit)
|
||||
return _sample_mongodb(object_name, limit, offset)
|
||||
if engine == "cassandra":
|
||||
return _sample_cassandra(object_name, limit)
|
||||
return _sample_cassandra(object_name, limit, offset)
|
||||
if engine == "neo4j":
|
||||
return _sample_neo4j(object_name, limit)
|
||||
return _sample_neo4j(object_name, limit, offset)
|
||||
if engine == "hadoop":
|
||||
return _sample_hadoop(object_name, limit, offset)
|
||||
raise ValueError(f"Sample not supported for {engine}")
|
||||
|
||||
|
||||
def _table_row_count(engine: str, object_name: str) -> int | None:
|
||||
try:
|
||||
if engine == "postgres":
|
||||
schema, table = _parse_fqn(engine, object_name)
|
||||
conn = psycopg2.connect(
|
||||
host=DB_HOST, port=PG_PORT, user=PG_USER, password=PG_PASS, dbname=PG_DB, connect_timeout=8,
|
||||
)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(f'SELECT count(*) FROM "{schema}"."{table}"')
|
||||
return int(cur.fetchone()[0])
|
||||
finally:
|
||||
conn.close()
|
||||
if engine == "mysql":
|
||||
schema, table = _parse_fqn(engine, object_name)
|
||||
conn = pymysql.connect(
|
||||
host=DB_HOST, port=MYSQL_PORT, user=MYSQL_USER, password=MYSQL_PASS,
|
||||
database=MYSQL_DB, connect_timeout=8,
|
||||
)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT count(*) FROM `{table}`")
|
||||
return int(cur.fetchone()[0])
|
||||
finally:
|
||||
conn.close()
|
||||
if engine == "mongodb":
|
||||
db_name, coll = _parse_fqn(engine, object_name)
|
||||
client = _mongo_client()
|
||||
try:
|
||||
return int(client[db_name][coll].estimated_document_count())
|
||||
finally:
|
||||
client.close()
|
||||
if engine == "neo4j":
|
||||
label = object_name.split(".")[-1]
|
||||
driver = _neo4j_driver()
|
||||
try:
|
||||
with driver.session() as session:
|
||||
return int(session.run(f"MATCH (n:`{label}`) RETURN count(n) AS c").single()["c"])
|
||||
finally:
|
||||
driver.close()
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _parse_fqn(engine: str, object_name: str) -> tuple[str, str]:
|
||||
if engine == "postgres":
|
||||
if "." in object_name:
|
||||
return object_name.split(".", 1)
|
||||
return "public", object_name
|
||||
if engine == "mysql":
|
||||
if "." in object_name:
|
||||
return object_name.split(".", 1)
|
||||
return MYSQL_DB, object_name
|
||||
if engine == "mongodb":
|
||||
if "." in object_name:
|
||||
return object_name.split(".", 1)
|
||||
return MONGO_DB, object_name
|
||||
if engine == "cassandra":
|
||||
if "." in object_name:
|
||||
return object_name.split(".", 1)
|
||||
return CASS_KS, object_name
|
||||
return "graph", object_name.split(".")[-1]
|
||||
|
||||
|
||||
def _object_primary_keys(engine: str, object_name: str) -> list[str]:
|
||||
try:
|
||||
if engine == "postgres":
|
||||
schema, table = _parse_fqn(engine, object_name)
|
||||
conn = psycopg2.connect(
|
||||
host=DB_HOST, port=PG_PORT, user=PG_USER, password=PG_PASS, dbname=PG_DB, connect_timeout=8,
|
||||
)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT a.attname
|
||||
FROM pg_constraint c
|
||||
JOIN pg_class t ON t.oid = c.conrelid
|
||||
JOIN pg_namespace n ON n.oid = t.relnamespace
|
||||
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey)
|
||||
WHERE c.contype = 'p' AND n.nspname = %s AND t.relname = %s
|
||||
ORDER BY array_position(c.conkey, a.attnum)
|
||||
""",
|
||||
(schema, table),
|
||||
)
|
||||
return [r[0] for r in cur.fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
if engine == "mysql":
|
||||
schema, table = _parse_fqn(engine, object_name)
|
||||
conn = pymysql.connect(
|
||||
host=DB_HOST, port=MYSQL_PORT, user=MYSQL_USER, password=MYSQL_PASS,
|
||||
database=MYSQL_DB, connect_timeout=8,
|
||||
)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT COLUMN_NAME FROM information_schema.KEY_COLUMN_USAGE
|
||||
WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND CONSTRAINT_NAME = 'PRIMARY'
|
||||
ORDER BY ORDINAL_POSITION
|
||||
""",
|
||||
(schema, table),
|
||||
)
|
||||
return [r[0] for r in cur.fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
if engine == "mongodb":
|
||||
return ["_id"]
|
||||
if engine == "cassandra":
|
||||
ks, table = _parse_fqn(engine, object_name)
|
||||
cluster = _cass_cluster()
|
||||
session = cluster.connect()
|
||||
try:
|
||||
rows = session.execute(
|
||||
"SELECT column_name FROM system_schema.columns "
|
||||
"WHERE keyspace_name=%s AND table_name=%s AND kind IN ('partition_key','clustering') "
|
||||
"ORDER BY position",
|
||||
(ks, table),
|
||||
)
|
||||
return [r.column_name for r in rows]
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
if engine == "neo4j":
|
||||
label = object_name.split(".")[-1]
|
||||
if label == "Product":
|
||||
return ["product_id"]
|
||||
if label == "Supplier":
|
||||
return ["supplier_id"]
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def _coerce_value(val: Any) -> Any:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
if isinstance(val, (int, float, bool)):
|
||||
return val
|
||||
s = str(val)
|
||||
if s.lower() == "null":
|
||||
return None
|
||||
if re.fullmatch(r"-?\d+", s):
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
pass
|
||||
if re.fullmatch(r"-?\d+\.\d+", s):
|
||||
try:
|
||||
return float(s)
|
||||
except ValueError:
|
||||
pass
|
||||
return s
|
||||
|
||||
|
||||
def _safe_ident(name: str) -> bool:
|
||||
return bool(re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_]*", name))
|
||||
|
||||
|
||||
def _update_postgres(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str:
|
||||
schema, table = _parse_fqn("postgres", object_name)
|
||||
if not changes or not pk:
|
||||
raise ValueError("Primary key and changes required")
|
||||
for c in list(changes) + list(pk):
|
||||
if not _safe_ident(c):
|
||||
raise ValueError(f"Invalid column name: {c}")
|
||||
set_sql = ", ".join(f'"{c}"=%s' for c in changes)
|
||||
where_sql = " AND ".join(f'"{c}"=%s' for c in pk)
|
||||
sql = f'UPDATE "{schema}"."{table}" SET {set_sql} WHERE {where_sql}'
|
||||
params = list(changes.values()) + list(pk.values())
|
||||
conn = psycopg2.connect(
|
||||
host=DB_HOST, port=PG_PORT, user=PG_USER, password=PG_PASS, dbname=PG_DB, connect_timeout=8,
|
||||
)
|
||||
try:
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, params)
|
||||
if cur.rowcount == 0:
|
||||
raise ValueError("No row matched primary key")
|
||||
return sql
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _update_mysql(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str:
|
||||
_schema, table = _parse_fqn("mysql", object_name)
|
||||
if not changes or not pk:
|
||||
raise ValueError("Primary key and changes required")
|
||||
for c in list(changes) + list(pk):
|
||||
if not _safe_ident(c):
|
||||
raise ValueError(f"Invalid column name: {c}")
|
||||
set_sql = ", ".join(f"`{c}`=%s" for c in changes)
|
||||
where_sql = " AND ".join(f"`{c}`=%s" for c in pk)
|
||||
sql = f"UPDATE `{table}` SET {set_sql} WHERE {where_sql}"
|
||||
params = list(changes.values()) + list(pk.values())
|
||||
conn = pymysql.connect(
|
||||
host=DB_HOST, port=MYSQL_PORT, user=MYSQL_USER, password=MYSQL_PASS,
|
||||
database=MYSQL_DB, connect_timeout=8,
|
||||
)
|
||||
try:
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, params)
|
||||
if cur.rowcount == 0:
|
||||
raise ValueError("No row matched primary key")
|
||||
return sql
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _update_mongodb(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str:
|
||||
db_name, coll = _parse_fqn("mongodb", object_name)
|
||||
if not pk or not changes:
|
||||
raise ValueError("Primary key and changes required")
|
||||
client = _mongo_client()
|
||||
try:
|
||||
from bson import ObjectId
|
||||
filt = dict(pk)
|
||||
if "_id" in filt and isinstance(filt["_id"], str) and len(filt["_id"]) == 24:
|
||||
try:
|
||||
filt["_id"] = ObjectId(filt["_id"])
|
||||
except Exception:
|
||||
pass
|
||||
res = client[db_name][coll].update_one(filt, {"$set": changes})
|
||||
if res.matched_count == 0:
|
||||
raise ValueError("No document matched primary key")
|
||||
return f"UPDATE {db_name}.{coll} {filt}"
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
def _update_cassandra(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str:
|
||||
ks, table = _parse_fqn("cassandra", object_name)
|
||||
if not pk or not changes:
|
||||
raise ValueError("Primary key and changes required")
|
||||
set_cql = ", ".join(f"{c}=%s" for c in changes)
|
||||
where_cql = " AND ".join(f"{c}=%s" for c in pk)
|
||||
cql = f"UPDATE {ks}.{table} SET {set_cql} WHERE {where_cql}"
|
||||
cluster = _cass_cluster()
|
||||
session = cluster.connect()
|
||||
try:
|
||||
session.execute(cql, list(changes.values()) + list(pk.values()))
|
||||
return cql
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def _update_neo4j(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str:
|
||||
label = object_name.split(".")[-1]
|
||||
if not pk or not changes:
|
||||
raise ValueError("Primary key and changes required")
|
||||
pk_col, pk_val = next(iter(pk.items()))
|
||||
if not _safe_ident(pk_col):
|
||||
raise ValueError(f"Invalid property name: {pk_col}")
|
||||
for c in changes:
|
||||
if not _safe_ident(c):
|
||||
raise ValueError(f"Invalid property name: {c}")
|
||||
set_frag = ", ".join(f"n.{c} = ${c}" for c in changes)
|
||||
cypher = f"MATCH (n:`{label}` {{{pk_col}: $pk}}) SET {set_frag} RETURN n"
|
||||
params: dict[str, Any] = {"pk": _coerce_value(pk_val)}
|
||||
params.update({c: _coerce_value(v) for c, v in changes.items()})
|
||||
driver = _neo4j_driver()
|
||||
try:
|
||||
with driver.session() as session:
|
||||
rec = session.run(cypher, **params).single()
|
||||
if not rec:
|
||||
raise ValueError("No node matched primary key")
|
||||
return cypher
|
||||
finally:
|
||||
driver.close()
|
||||
|
||||
|
||||
def _update_row(engine: str, object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> dict[str, Any]:
|
||||
pk = {k: _coerce_value(v) for k, v in pk.items()}
|
||||
changes = {k: _coerce_value(v) for k, v in changes.items()}
|
||||
if engine == "postgres":
|
||||
sql = _update_postgres(object_name, pk, changes)
|
||||
elif engine == "mysql":
|
||||
sql = _update_mysql(object_name, pk, changes)
|
||||
elif engine == "mongodb":
|
||||
sql = _update_mongodb(object_name, pk, changes)
|
||||
elif engine == "cassandra":
|
||||
sql = _update_cassandra(object_name, pk, changes)
|
||||
elif engine == "neo4j":
|
||||
sql = _update_neo4j(object_name, pk, changes)
|
||||
else:
|
||||
raise ValueError(f"Update not supported for {engine}")
|
||||
return {"ok": True, "engine": engine, "object": object_name, "statement": sql, "cdc": engine in ("postgres", "mysql", "mongodb")}
|
||||
|
||||
|
||||
@router.get("/samples/{engine}")
|
||||
async def get_samples(engine: str):
|
||||
if engine not in SAMPLES:
|
||||
if engine not in SAMPLES and engine not in LAKE_ENGINES:
|
||||
return JSONResponse({"error": "unknown engine"}, status_code=404)
|
||||
if engine == "hadoop":
|
||||
return {"engine": engine, "samples": SAMPLES[engine], "connection": hadoop_connection_info()}
|
||||
return {"engine": engine, "samples": SAMPLES[engine], "connection": _connection_info(engine)}
|
||||
|
||||
|
||||
@@ -647,9 +1009,20 @@ async def sql_health():
|
||||
return out
|
||||
|
||||
|
||||
|
||||
|
||||
@router.get("/health/{engine}")
|
||||
async def sql_health_engine(engine: str):
|
||||
if engine == "hadoop":
|
||||
return health_hadoop(_run_trino)
|
||||
if engine not in SOURCE_ENGINES:
|
||||
return JSONResponse({"error": "unknown engine"}, status_code=404)
|
||||
all_h = await sql_health()
|
||||
return all_h.get(engine) or {"ok": False, "error": "not found"}
|
||||
|
||||
@router.get("/catalog/{engine}")
|
||||
async def get_catalog(engine: str):
|
||||
if engine not in SOURCE_ENGINES:
|
||||
if engine not in SOURCE_ENGINES and engine not in LAKE_ENGINES:
|
||||
return JSONResponse({"error": "unknown engine"}, status_code=404)
|
||||
try:
|
||||
return _catalog(engine)
|
||||
@@ -658,14 +1031,31 @@ async def get_catalog(engine: str):
|
||||
|
||||
|
||||
@router.get("/sample/{engine}")
|
||||
async def get_sample(engine: str, object: str = Query(..., min_length=1), limit: int = Query(50, ge=1, le=200)):
|
||||
if engine not in SOURCE_ENGINES:
|
||||
async def get_sample(
|
||||
engine: str,
|
||||
object: str = Query(..., min_length=1),
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
offset: int = Query(0, ge=0),
|
||||
):
|
||||
if engine not in SOURCE_ENGINES and engine not in LAKE_ENGINES:
|
||||
return JSONResponse({"error": "unknown engine"}, status_code=404)
|
||||
try:
|
||||
result = _sample(engine, object, limit)
|
||||
result = _sample(engine, object, limit, offset)
|
||||
if not result.get("ok"):
|
||||
return JSONResponse(result, status_code=422)
|
||||
return {**result, "engine": engine, "object": object}
|
||||
pks = _object_primary_keys(engine, object)
|
||||
total = _table_row_count(engine, object)
|
||||
return {
|
||||
**result,
|
||||
"engine": engine,
|
||||
"object": object,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"total_count": total,
|
||||
"primary_keys": pks,
|
||||
"cdc": engine in ("postgres", "mysql", "mongodb"),
|
||||
"editable": bool(pks),
|
||||
}
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc)[:500]}, status_code=502)
|
||||
|
||||
@@ -698,6 +1088,22 @@ async def execute_sql(body: SqlRequest):
|
||||
return JSONResponse({"ok": False, "error": str(exc)[:500]}, status_code=502)
|
||||
|
||||
|
||||
@router.post("/row/update")
|
||||
async def update_row(body: RowUpdateRequest):
|
||||
allowed_pks = _object_primary_keys(body.engine, body.object)
|
||||
if allowed_pks:
|
||||
for k in body.pk:
|
||||
if k not in allowed_pks:
|
||||
return JSONResponse({"ok": False, "error": f"Invalid primary key column '{k}'"}, status_code=400)
|
||||
for k in body.changes:
|
||||
if k in body.pk:
|
||||
return JSONResponse({"ok": False, "error": "Cannot change primary key columns"}, status_code=400)
|
||||
try:
|
||||
return _update_row(body.engine, body.object, body.pk, body.changes)
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc)[:500]}, status_code=422)
|
||||
|
||||
|
||||
@router.post("/benchmark")
|
||||
async def benchmark():
|
||||
"""Parallel analytics: PostgreSQL OLTP vs Trino distributed engine (5M sin() rows)."""
|
||||
|
||||
Reference in New Issue
Block a user