feat: Data Explorer tab + manual data entry with live CDC

Add a dedicated business-data view (separate from infra/search): - New "Data Explorer" tab: KPIs (customers, employees, products, source row totals, sample revenue) + charts driven by Elasticsearch aggregations (revenue over time, by region/channel/status, top customers & products, HR by department/role/event, supply chain by type, telemetry averages), with region + free-text filters. - Backend /api/search/business endpoint: battery of ES aggregations with numeric/date index template (so amount sums and date histograms work), source totals via cheap planner estimates; biased the orders sample to rows carrying customer names so people are searchable/visible. Manual data entry on source systems: - "Insert row" action in the Data Hub browser opens a column-aware form; POST /api/sql/row/insert writes to Postgres/MySQL/Mongo/Cassandra/Neo4j. - Inserts into CDC sources (PG/MySQL/Mongo) are captured by Debezium and streamed to Kafka in real time; UI flags this and pulses the data flow.
2026-06-28 17:12:29 +00:00
parent 8a8d779328
commit 8d28695868
7 changed files with 799 additions and 7 deletions
@@ -355,6 +355,12 @@ class RowUpdateRequest(BaseModel):
    changes: dict[str, Any] = Field(..., min_length=1)


+class RowInsertRequest(BaseModel):
+    engine: str = Field(..., pattern="^(postgres|mysql|mongodb|cassandra|neo4j)$")
+    object: str = Field(..., min_length=1, max_length=256)
+    values: dict[str, Any] = Field(..., min_length=1)
+
+
 def _connection_info(engine: str) -> dict[str, str]:
    if engine == "postgres":
        return {"host": DB_HOST, "port": str(PG_PORT), "database": PG_DB, "user": PG_USER}
@@ -1001,6 +1007,115 @@ def _update_row(engine: str, object_name: str, pk: dict[str, Any], changes: dict
    return {"ok": True, "engine": engine, "object": object_name, "statement": sql, "cdc": engine in ("postgres", "mysql", "mongodb")}


+def _insert_postgres(object_name: str, values: dict[str, Any]) -> str:
+    schema, table = _parse_fqn("postgres", object_name)
+    cols = [c for c in values if values[c] is not None]
+    if not cols:
+        raise ValueError("No values provided")
+    for c in cols:
+        if not _safe_ident(c):
+            raise ValueError(f"Invalid column name: {c}")
+    col_sql = ", ".join(f'"{c}"' for c in cols)
+    ph = ", ".join(["%s"] * len(cols))
+    sql = f'INSERT INTO "{schema}"."{table}" ({col_sql}) VALUES ({ph})'
+    conn = psycopg2.connect(host=DB_HOST, port=PG_PORT, user=PG_USER, password=PG_PASS, dbname=PG_DB, connect_timeout=8)
+    try:
+        conn.autocommit = True
+        conn.cursor().execute(sql, [values[c] for c in cols])
+        return sql
+    finally:
+        conn.close()
+
+
+def _insert_mysql(object_name: str, values: dict[str, Any]) -> str:
+    _schema, table = _parse_fqn("mysql", object_name)
+    cols = [c for c in values if values[c] is not None]
+    if not cols:
+        raise ValueError("No values provided")
+    for c in cols:
+        if not _safe_ident(c):
+            raise ValueError(f"Invalid column name: {c}")
+    col_sql = ", ".join(f"`{c}`" for c in cols)
+    ph = ", ".join(["%s"] * len(cols))
+    sql = f"INSERT INTO `{table}` ({col_sql}) VALUES ({ph})"
+    conn = pymysql.connect(host=DB_HOST, port=MYSQL_PORT, user=MYSQL_USER, password=MYSQL_PASS, database=MYSQL_DB, connect_timeout=8)
+    try:
+        conn.autocommit = True
+        conn.cursor().execute(sql, [values[c] for c in cols])
+        return sql
+    finally:
+        conn.close()
+
+
+def _insert_mongodb(object_name: str, values: dict[str, Any]) -> str:
+    db_name, coll = _parse_fqn("mongodb", object_name)
+    doc = {k: v for k, v in values.items() if v is not None}
+    if not doc:
+        raise ValueError("No values provided")
+    client = _mongo_client()
+    try:
+        res = client[db_name][coll].insert_one(doc)
+        return f"INSERT {db_name}.{coll} _id={res.inserted_id}"
+    finally:
+        client.close()
+
+
+def _insert_cassandra(object_name: str, values: dict[str, Any]) -> str:
+    ks, table = _parse_fqn("cassandra", object_name)
+    cols = [c for c in values if values[c] is not None]
+    if not cols:
+        raise ValueError("No values provided")
+    for c in cols:
+        if not _safe_ident(c):
+            raise ValueError(f"Invalid column name: {c}")
+    col_sql = ", ".join(cols)
+    ph = ", ".join(["%s"] * len(cols))
+    cql = f"INSERT INTO {ks}.{table} ({col_sql}) VALUES ({ph})"
+    cluster = _cass_cluster()
+    session = cluster.connect()
+    try:
+        session.execute(cql, [values[c] for c in cols])
+        return cql
+    finally:
+        cluster.shutdown()
+
+
+def _insert_neo4j(object_name: str, values: dict[str, Any]) -> str:
+    label = object_name.split(".")[-1]
+    props = {k: _coerce_value(v) for k, v in values.items() if v is not None}
+    if not props:
+        raise ValueError("No values provided")
+    for c in props:
+        if not _safe_ident(c):
+            raise ValueError(f"Invalid property name: {c}")
+    set_frag = ", ".join(f"n.{c} = ${c}" for c in props)
+    cypher = f"CREATE (n:`{label}`) SET {set_frag} RETURN n"
+    driver = _neo4j_driver()
+    try:
+        with driver.session() as session:
+            session.run(cypher, **props).consume()
+        return cypher
+    finally:
+        driver.close()
+
+
+def _insert_row(engine: str, object_name: str, values: dict[str, Any]) -> dict[str, Any]:
+    values = {k: _coerce_value(v) for k, v in values.items()}
+    if engine == "postgres":
+        sql = _insert_postgres(object_name, values)
+    elif engine == "mysql":
+        sql = _insert_mysql(object_name, values)
+    elif engine == "mongodb":
+        sql = _insert_mongodb(object_name, values)
+    elif engine == "cassandra":
+        sql = _insert_cassandra(object_name, values)
+    elif engine == "neo4j":
+        sql = _insert_neo4j(object_name, values)
+    else:
+        raise ValueError(f"Insert not supported for {engine}")
+    return {"ok": True, "engine": engine, "object": object_name, "statement": sql, "cdc": engine in ("postgres", "mysql", "mongodb")}
+
+
@router.get("/samples/{engine}")
 async def get_samples(engine: str):
    if engine not in SAMPLES and engine not in LAKE_ENGINES:
@@ -1130,6 +1245,16 @@ async def update_row(body: RowUpdateRequest):
        return JSONResponse({"ok": False, "error": str(exc)[:500]}, status_code=422)


+@router.post("/row/insert")
+async def insert_row(body: RowInsertRequest):
+    if not body.values:
+        return JSONResponse({"ok": False, "error": "No values provided"}, status_code=400)
+    try:
+        return _insert_row(body.engine, body.object, body.values)
+    except Exception as exc:
+        return JSONResponse({"ok": False, "error": str(exc)[:500]}, status_code=422)
+
+
@router.post("/benchmark")
 async def benchmark():
    """Parallel analytics: PostgreSQL OLTP vs Trino distributed engine (5M sin() rows)."""