ATC/atc-agents
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Authentik login + switchable GPU prod target

Browse Source 
Add OIDC auth for Command Center and runtime GPU endpoint selection
pointed at atc-gpu-prod (10.0.10.106), matching what is currently deployed.
This commit is contained in:
mo
2026-07-21 23:20:24 +00:00
parent f36c8906bc
commit 9008fbd512
31 changed files with 4667 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/agent_ops.py
+50 -8
View File
@@ -364,10 +364,36 @@ _CUST_INTERVAL = float(os.getenv("CUSTODIAN_OFFLOAD_INTERVAL_SECONDS", "120"))
_CUST_BATCH = int(os.getenv("CUSTODIAN_OFFLOAD_BATCH", "200"))
_CUST_TARGETS = [
{"label": "postgres sales_orders", "src": "postgres_sales.public.sales_orders",
"target": "iceberg.hadoop.sales_orders_offload"},
"target": "iceberg.hadoop.sales_orders_offload",
"create_sql": (
"CREATE TABLE iceberg.hadoop.sales_orders_offload AS "
"SELECT * FROM postgres_sales.public.sales_orders WHERE 1=0"
),
"insert_sql": (
"INSERT INTO iceberg.hadoop.sales_orders_offload "
"SELECT * FROM postgres_sales.public.sales_orders LIMIT {batch}"
)},
{"label": "mysql employee_events", "src": "mysql_hr.hr.employee_events",
"target": "iceberg.hadoop.employee_events_offload"},
"target": "iceberg.hadoop.employee_events_offload",
"create_sql": (
"CREATE TABLE iceberg.hadoop.employee_events_offload AS SELECT "
"event_id, employee_id, department, role_name, region, event_type, "
"CAST(salary_change AS double) AS salary_change, "
"CAST(event_ts AS timestamp(6)) AS event_ts, notes, "
"employee_name, employee_email, employee_phone, national_id, home_address, "
"CAST(date_of_birth AS date) AS date_of_birth "
"FROM mysql_hr.hr.employee_events WHERE 1=0"
),
"insert_sql": (
"INSERT INTO iceberg.hadoop.employee_events_offload SELECT "
"event_id, employee_id, department, role_name, region, event_type, "
"CAST(salary_change AS double), CAST(event_ts AS timestamp(6)), notes, "
"employee_name, employee_email, employee_phone, national_id, home_address, "
"CAST(date_of_birth AS date) "
"FROM mysql_hr.hr.employee_events LIMIT {batch}"
)},
]
_custodian_state: dict[str, Any] = {
"enabled": os.getenv("CUSTODIAN_OFFLOAD_ENABLED", "1") not in ("0", "false", "False", ""),
"interval": _CUST_INTERVAL,
@@ -385,12 +411,28 @@ async def _custodian_offload_once(idx: int | None = None) -> dict[str, Any]:
i = _custodian_state["idx"] if idx is None else idx
tgt = _CUST_TARGETS[i % len(_CUST_TARGETS)]
_custodian_state["idx"] = i + 1
ddl = f"CREATE TABLE IF NOT EXISTS {tgt['target']} AS SELECT * FROM {tgt['src']} WHERE 1=0"
dml = f"INSERT INTO {tgt['target']} SELECT * FROM {tgt['src']} LIMIT {_CUST_BATCH}"
await _term(DML_AGENT, f"$ trino --catalog iceberg # Hadoop offload: {tgt['label']} → {tgt['target']}",
level="cmd", phase="offload")
await _term(DML_AGENT, f" {ddl};", level="cmd", phase="offload")
await _trino_collect(ddl, 1)
await _trino_collect("CREATE SCHEMA IF NOT EXISTS iceberg.hadoop", 1)
probe = await _trino_collect(f"SELECT 1 FROM {tgt['target']} WHERE 1=0", 1)
if not probe.get("ok"):
ddl = tgt["create_sql"]
await _term(DML_AGENT, f"$ trino --catalog iceberg # Hadoop offload: {tgt['label']} → {tgt['target']}",
level="cmd", phase="offload")
await _term(DML_AGENT, f" {ddl};", level="cmd", phase="offload")
created = await _trino_collect(ddl, 1)
if not created.get("ok"):
err = created.get("error")
await _term(DML_AGENT, f" ✗ offload failed: {str(err)[:140]}", level="err", phase="offload")
await _emit(f"[custodian-offload] {tgt['label']} failed: {str(err)[:120]}", "err")
_custodian_state["runs_total"] += 1
_custodian_state["last"] = {
"target": tgt["target"], "src": tgt["src"], "ok": False,
"rows": 0, "ts": datetime.now(timezone.utc).isoformat(), "error": err,
}
return _custodian_state["last"]
else:
await _term(DML_AGENT, f"$ trino --catalog iceberg # Hadoop offload: {tgt['label']} → {tgt['target']}",
level="cmd", phase="offload")
dml = tgt["insert_sql"].format(batch=_CUST_BATCH)
await _term(DML_AGENT, f" {dml};", level="cmd", phase="offload")
ins = await _trino_collect(dml, 1)
ok = bool(ins.get("ok"))