This commit is contained in:
Lakehouse Admin
@@ -1,19 +0,0 @@
|
|||||||
# Forgejo (lokaal op atc-docker01) — primaire Git staat op atc-mgt01:3001
|
|
||||||
services:
|
|
||||||
forgejo:
|
|
||||||
image: codeberg.org/forgejo/forgejo:14
|
|
||||||
container_name: forgejo
|
|
||||||
restart: always
|
|
||||||
environment:
|
|
||||||
USER_UID: "1000"
|
|
||||||
USER_GID: "1000"
|
|
||||||
ports:
|
|
||||||
- "4002:3000"
|
|
||||||
- "222:22"
|
|
||||||
volumes:
|
|
||||||
- forgejo_data:/data
|
|
||||||
- /etc/localtime:/etc/localtime:ro
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
forgejo_data:
|
|
||||||
name: forgejo_forgejo
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
# LDAP Account Manager (lokaal op atc-docker01)
|
|
||||||
# Productie LDAP/LAM: http://atc-mgt01.dell-atc.lan/lam/
|
|
||||||
services:
|
|
||||||
lam:
|
|
||||||
image: ghcr.io/ldapaccountmanager/lam:stable
|
|
||||||
container_name: lam-app-1
|
|
||||||
restart: unless-stopped
|
|
||||||
ports:
|
|
||||||
- "4001:80"
|
|
||||||
environment:
|
|
||||||
DEBIAN_FRONTEND: noninteractive
|
|
||||||
volumes:
|
|
||||||
- lam_data:/var/lib/ldap-account-manager
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
lam_data:
|
|
||||||
name: lam_lam
|
|
||||||
@@ -35,6 +35,19 @@ services:
|
|||||||
- "27017:27017"
|
- "27017:27017"
|
||||||
volumes:
|
volumes:
|
||||||
- mongodb_supplychain_data:/data/db
|
- mongodb_supplychain_data:/data/db
|
||||||
|
- ./init/mongo:/docker-entrypoint-initdb.d:ro
|
||||||
|
|
||||||
|
mongo-express:
|
||||||
|
image: mongo-express
|
||||||
|
container_name: mongo_express
|
||||||
|
depends_on:
|
||||||
|
- mongodb-supplychain
|
||||||
|
ports:
|
||||||
|
- "8081:8081"
|
||||||
|
environment:
|
||||||
|
ME_CONFIG_MONGODB_URL: mongodb://mo:Dell2026%21@mongodb_supplychain:27017/?authSource=admin
|
||||||
|
ME_CONFIG_MONGODB_ENABLE_ADMIN: "true"
|
||||||
|
ME_CONFIG_BASICAUTH: "false"
|
||||||
|
|
||||||
cassandra-telemetry:
|
cassandra-telemetry:
|
||||||
image: cassandra:4.1
|
image: cassandra:4.1
|
||||||
|
|||||||
@@ -0,0 +1,15 @@
|
|||||||
|
// Idempotent admin users for lab (mo + bart)
|
||||||
|
db = db.getSiblingDB("admin");
|
||||||
|
var pw = "Dell2026!";
|
||||||
|
["mo", "bart"].forEach(function(name) {
|
||||||
|
if (db.getUser(name) == null) {
|
||||||
|
db.createUser({
|
||||||
|
user: name,
|
||||||
|
pwd: pw,
|
||||||
|
roles: [{ role: "root", db: "admin" }]
|
||||||
|
});
|
||||||
|
print("Created user: " + name);
|
||||||
|
} else {
|
||||||
|
print("User exists: " + name);
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -9,6 +9,10 @@
|
|||||||
- Proxmox:
|
- Proxmox:
|
||||||
- abbr: PVE
|
- abbr: PVE
|
||||||
href: https://10.0.10.65:8006/
|
href: https://10.0.10.65:8006/
|
||||||
|
- Dockhand:
|
||||||
|
- abbr: DH
|
||||||
|
href: http://atc-docker01.dell-atc.lan:8082/
|
||||||
|
|
||||||
|
|
||||||
- RSS Feeds:
|
- RSS Feeds:
|
||||||
- TLDR Data Engineering:
|
- TLDR Data Engineering:
|
||||||
|
|||||||
@@ -124,6 +124,26 @@
|
|||||||
description: atc-mgt01.dell-atc.lan:3001
|
description: atc-mgt01.dell-atc.lan:3001
|
||||||
siteMonitor: http://atc-mgt01.dell-atc.lan:3001/
|
siteMonitor: http://atc-mgt01.dell-atc.lan:3001/
|
||||||
color: "#F05032"
|
color: "#F05032"
|
||||||
|
- GPU Lab:
|
||||||
|
icon: mdi-gpu
|
||||||
|
href: http://10.0.20.106:9000/
|
||||||
|
description: atc-gpu-dev VM303 · 4× V100 · model manager + chat
|
||||||
|
siteMonitor: http://10.0.20.106:9000/
|
||||||
|
color: "#76B900"
|
||||||
|
- Dockhand:
|
||||||
|
icon: mdi-docker
|
||||||
|
href: http://atc-docker01.dell-atc.lan:8082/
|
||||||
|
description: Docker control plane · GPU-Dev + Bart-GPU + Mo-GPU
|
||||||
|
siteMonitor: http://atc-docker01.dell-atc.lan:8082/api/health
|
||||||
|
color: "#2496ED"
|
||||||
|
|
||||||
|
- ATC Command Center:
|
||||||
|
icon: mdi-robot-outline
|
||||||
|
href: http://10.0.21.33/
|
||||||
|
description: VM304 MCP · agent hub & ops floor · 10.0.21.33
|
||||||
|
siteMonitor: http://10.0.21.33/
|
||||||
|
color: "#0099cc"
|
||||||
|
|
||||||
- LDAP Admin:
|
- LDAP Admin:
|
||||||
icon: http://atc-docker01.dell-atc.lan:8080/ldap.png
|
icon: http://atc-docker01.dell-atc.lan:8080/ldap.png
|
||||||
href: http://atc-mgt01.dell-atc.lan/lam/
|
href: http://atc-mgt01.dell-atc.lan/lam/
|
||||||
@@ -142,11 +162,22 @@
|
|||||||
ping: atc-db01.dell-atc.lan
|
ping: atc-db01.dell-atc.lan
|
||||||
description: atc-db01:3306 · mysql://USER@atc-db01.dell-atc.lan:3306/DB · mysql -h atc-db01
|
description: atc-db01:3306 · mysql://USER@atc-db01.dell-atc.lan:3306/DB · mysql -h atc-db01
|
||||||
color: "#4479A1"
|
color: "#4479A1"
|
||||||
|
- Mongo Express · db02:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/mongodb.svg
|
||||||
|
href: http://atc-db02.dell-atc.lan:8081/
|
||||||
|
description: Web UI · supplychain · 3M events · users mo & bart
|
||||||
|
siteMonitor: http://atc-db02.dell-atc.lan:8081/
|
||||||
|
ping: atc-db02.dell-atc.lan
|
||||||
|
color: "#47A248"
|
||||||
- MongoDB · db02:
|
- MongoDB · db02:
|
||||||
icon: http://atc-docker01.dell-atc.lan:8080/mongodb.svg
|
icon: http://atc-docker01.dell-atc.lan:8080/mongodb.svg
|
||||||
ping: atc-db02.dell-atc.lan
|
ping: atc-db02.dell-atc.lan
|
||||||
description: atc-db02:27017 · mongodb://USER@atc-db02.dell-atc.lan:27017/DB · mongosh
|
description: |
|
||||||
color: "#47A248"
|
:27017 · mongodb_supplychain container
|
||||||
|
DB supplychain · collection events (~3M)
|
||||||
|
mongosh: mongodb://mo@atc-db02.dell-atc.lan:27017/supplychain?authSource=admin
|
||||||
|
Trino: mongodb_supplychain.supplychain.events
|
||||||
|
color: "#3d8b40"
|
||||||
- PostgreSQL · db02:
|
- PostgreSQL · db02:
|
||||||
icon: http://atc-docker01.dell-atc.lan:8080/postgresql.svg
|
icon: http://atc-docker01.dell-atc.lan:8080/postgresql.svg
|
||||||
ping: atc-db02.dell-atc.lan
|
ping: atc-db02.dell-atc.lan
|
||||||
|
|||||||
@@ -0,0 +1,635 @@
|
|||||||
|
---
|
||||||
|
# ATC Lakehouse — Dell Technologies FDE Dashboard (Bart & Mo)
|
||||||
|
|
||||||
|
|
||||||
|
- Lakehouse · Architecture:
|
||||||
|
- Environment Map:
|
||||||
|
icon: mdi-sitemap
|
||||||
|
href: http://atc-docker01.dell-atc.lan:8080/docs/architecture.html
|
||||||
|
description: High-level diagram — data flow, hosts, and service map
|
||||||
|
color: "#007DB8"
|
||||||
|
- Git Docs:
|
||||||
|
icon: mdi-book-open-page-variant
|
||||||
|
href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/landscape.md
|
||||||
|
description: Application landscape (Markdown in Forgejo)
|
||||||
|
color: "#E8752A"
|
||||||
|
|
||||||
|
- Data Pipeline:
|
||||||
|
- Kafka UI:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/apachekafka.svg
|
||||||
|
href: http://atc-kafka01.dell-atc.lan:9000/
|
||||||
|
description: atc-kafka01.dell-atc.lan:9000
|
||||||
|
siteMonitor: http://atc-kafka01.dell-atc.lan:9000/
|
||||||
|
color: "#E02014"
|
||||||
|
- Debezium:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/debezium.png
|
||||||
|
href: http://atc-lake01.dell-atc.lan:8083/
|
||||||
|
description: atc-lake01.dell-atc.lan:8083
|
||||||
|
siteMonitor: http://atc-lake01.dell-atc.lan:8083/
|
||||||
|
color: "#4ECDC4"
|
||||||
|
- Spark:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/apachespark.svg
|
||||||
|
href: http://atc-lake01.dell-atc.lan:8080/
|
||||||
|
description: atc-lake01.dell-atc.lan:8080
|
||||||
|
siteMonitor: http://atc-lake01.dell-atc.lan:8080/
|
||||||
|
color: "#E25A1C"
|
||||||
|
- Trino:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/trino.svg
|
||||||
|
href: http://atc-lake01.dell-atc.lan:8089/ui/
|
||||||
|
description: atc-lake01.dell-atc.lan:8089/ui/
|
||||||
|
siteMonitor: http://atc-lake01.dell-atc.lan:8089/ui/
|
||||||
|
color: "#DD00A1"
|
||||||
|
- Airflow:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/apacheairflow.svg
|
||||||
|
href: http://10.0.21.55:8080/
|
||||||
|
description: 10.0.21.55:8080
|
||||||
|
siteMonitor: http://10.0.21.55:8080/health
|
||||||
|
color: "#017CEE"
|
||||||
|
|
||||||
|
- Analytics:
|
||||||
|
- Superset:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/apachesuperset.svg
|
||||||
|
href: http://atc-docker01.dell-atc.lan:8088/
|
||||||
|
description: atc-docker01.dell-atc.lan:8088
|
||||||
|
siteMonitor: http://atc-docker01.dell-atc.lan:8088/health
|
||||||
|
color: "#6c5ce7"
|
||||||
|
- Kibana:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/kibana.svg
|
||||||
|
href: http://atc-elastic01.dell-atc.lan:5601/
|
||||||
|
description: atc-elastic01.dell-atc.lan:5601
|
||||||
|
siteMonitor: http://atc-elastic01.dell-atc.lan:5601/
|
||||||
|
color: "#F04E98"
|
||||||
|
- Elasticsearch:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/elasticsearch.svg
|
||||||
|
ping: atc-elastic01.dell-atc.lan
|
||||||
|
description: |
|
||||||
|
Host: atc-elastic01.dell-atc.lan
|
||||||
|
Port: 9200 (no public HTTP)
|
||||||
|
curl: curl http://atc-elastic01.dell-atc.lan:9200
|
||||||
|
Use Kibana for browser UI
|
||||||
|
color: "#005571"
|
||||||
|
- Grafana:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/grafana.svg
|
||||||
|
href: http://atc-grafana.dell-atc.lan:3000/
|
||||||
|
ping: atc-grafana.dell-atc.lan
|
||||||
|
description: |
|
||||||
|
Host: atc-grafana.dell-atc.lan
|
||||||
|
IP: 10.0.20.103
|
||||||
|
Port: 3000
|
||||||
|
Note: start with systemctl start grafana-server
|
||||||
|
siteMonitor: http://atc-grafana.dell-atc.lan:3000/
|
||||||
|
color: "#F46800"
|
||||||
|
|
||||||
|
- Infrastructure:
|
||||||
|
- ObjectScale UI:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/dell.svg
|
||||||
|
href: https://10.0.20.111/
|
||||||
|
description: ECS admin · https://10.0.20.111 · luna.local
|
||||||
|
siteMonitor: https://10.0.20.111/
|
||||||
|
color: "#007DB8"
|
||||||
|
- ObjectScale S3:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/minio.svg
|
||||||
|
href: http://10.0.20.111:9020/
|
||||||
|
description: S3 API :9020 · bucket data · Trino Iceberg + Spark
|
||||||
|
siteMonitor: http://10.0.20.111:9020/
|
||||||
|
color: "#C72C48"
|
||||||
|
- ObjectScale SSH:
|
||||||
|
icon: mdi-console
|
||||||
|
href: https://10.0.20.111/
|
||||||
|
description: admin@atc-objectscale · appliance SSH · see docs/objectscale.md
|
||||||
|
ping: 10.0.20.111
|
||||||
|
color: "#64748b"
|
||||||
|
- iDRAC:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/dell.svg
|
||||||
|
href: https://10.0.41.102/
|
||||||
|
description: Dell iDRAC · 10.0.41.102
|
||||||
|
siteMonitor: https://10.0.41.102/
|
||||||
|
color: "#007DB8"
|
||||||
|
- Proxmox:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/proxmox.svg
|
||||||
|
href: https://10.0.10.65:8006/
|
||||||
|
description: 10.0.10.65:8006
|
||||||
|
siteMonitor: https://10.0.10.65:8006/
|
||||||
|
color: "#E57000"
|
||||||
|
widget:
|
||||||
|
type: proxmox
|
||||||
|
url: https://10.0.10.65:8006
|
||||||
|
username: "root@pam!homepage"
|
||||||
|
password: "8890185b-0850-42b7-bab2-a690ea4dc3f1"
|
||||||
|
node: pve01
|
||||||
|
fields: ["vms", "lxc", "resources.cpu", "resources.mem"]
|
||||||
|
- Forgejo:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/git.svg
|
||||||
|
href: http://atc-mgt01.dell-atc.lan:3001/
|
||||||
|
description: atc-mgt01.dell-atc.lan:3001
|
||||||
|
siteMonitor: http://atc-mgt01.dell-atc.lan:3001/
|
||||||
|
color: "#F05032"
|
||||||
|
- LDAP Admin:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/ldap.png
|
||||||
|
href: http://atc-mgt01.dell-atc.lan/lam/
|
||||||
|
description: atc-mgt01.dell-atc.lan/lam/
|
||||||
|
siteMonitor: http://atc-mgt01.dell-atc.lan/lam/
|
||||||
|
color: "#0984e3"
|
||||||
|
|
||||||
|
- Databases:
|
||||||
|
- PostgreSQL · db01:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/postgresql.svg
|
||||||
|
ping: atc-db01.dell-atc.lan
|
||||||
|
description: atc-db01:5432 · postgresql://USER@atc-db01.dell-atc.lan:5432/DB · psql -h atc-db01
|
||||||
|
color: "#4169E1"
|
||||||
|
- MySQL · db01:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/mysql.svg
|
||||||
|
ping: atc-db01.dell-atc.lan
|
||||||
|
description: atc-db01:3306 · mysql://USER@atc-db01.dell-atc.lan:3306/DB · mysql -h atc-db01
|
||||||
|
color: "#4479A1"
|
||||||
|
- Mongo Express · db02:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/mongodb.svg
|
||||||
|
href: http://atc-db02.dell-atc.lan:8081/
|
||||||
|
description: Web UI · supplychain · 3M events · users mo & bart
|
||||||
|
siteMonitor: http://atc-db02.dell-atc.lan:8081/
|
||||||
|
ping: atc-db02.dell-atc.lan
|
||||||
|
color: "#47A248"
|
||||||
|
- MongoDB · db02:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/mongodb.svg
|
||||||
|
ping: atc-db02.dell-atc.lan
|
||||||
|
description: |
|
||||||
|
:27017 · mongodb_supplychain container
|
||||||
|
DB supplychain · collection events (~3M)
|
||||||
|
mongosh: mongodb://mo@atc-db02.dell-atc.lan:27017/supplychain?authSource=admin
|
||||||
|
Trino: mongodb_supplychain.supplychain.events
|
||||||
|
color: "#3d8b40"
|
||||||
|
- PostgreSQL · db02:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/postgresql.svg
|
||||||
|
ping: atc-db02.dell-atc.lan
|
||||||
|
description: atc-db02:5432 · postgresql://USER@atc-db02.dell-atc.lan:5432/DB · psql -h atc-db02
|
||||||
|
color: "#336791"
|
||||||
|
- MySQL · db02:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/mysql.svg
|
||||||
|
ping: atc-db02.dell-atc.lan
|
||||||
|
description: atc-db02:3306 · mysql://USER@atc-db02.dell-atc.lan:3306/DB · mysql -h atc-db02
|
||||||
|
color: "#00758F"
|
||||||
|
- Cassandra · db02:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/apachecassandra.svg
|
||||||
|
ping: atc-db02.dell-atc.lan
|
||||||
|
description: atc-db02:9042 · cqlsh atc-db02.dell-atc.lan 9042 · cassandra://atc-db02:9042
|
||||||
|
color: "#1287B1"
|
||||||
|
- Neo4j · db02:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/neo4j.svg
|
||||||
|
href: http://atc-db02.dell-atc.lan:7474/
|
||||||
|
ping: atc-db02.dell-atc.lan
|
||||||
|
description: HTTP :7474 · bolt://atc-db02:7687 · neo4j://atc-db02.dell-atc.lan:7687
|
||||||
|
siteMonitor: http://atc-db02.dell-atc.lan:7474/
|
||||||
|
color: "#008CC1"
|
||||||
|
|
||||||
|
- Intel · Hacker News:
|
||||||
|
- HN Front Page:
|
||||||
|
icon: mdi-newspaper
|
||||||
|
href: https://news.ycombinator.com/
|
||||||
|
description: Hacker News — front page
|
||||||
|
color: "#E8752A"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/hn-front
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- HN Data Engineering:
|
||||||
|
icon: mdi-database-search
|
||||||
|
href: https://hn.algolia.com/?q=data%20engineering
|
||||||
|
description: Hacker News — data engineering
|
||||||
|
color: "#3b82f6"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/hn-de
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- HN Kafka:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/apachekafka.svg
|
||||||
|
href: https://hn.algolia.com/?q=kafka
|
||||||
|
description: Hacker News — Kafka & streaming
|
||||||
|
color: "#E02014"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/hn-kafka
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- HN Spark:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/apachespark.svg
|
||||||
|
href: https://hn.algolia.com/?q=apache+spark
|
||||||
|
description: Hacker News — Apache Spark
|
||||||
|
color: "#E25A1C"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/hn-spark
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- Lobsters:
|
||||||
|
icon: mdi-lobster
|
||||||
|
href: https://lobste.rs/
|
||||||
|
description: Computing & infra — curated links
|
||||||
|
color: "#ef4444"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/lobsters
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
|
||||||
|
- Intel · Data Architecture:
|
||||||
|
- Data Eng Weekly:
|
||||||
|
icon: mdi-calendar-week
|
||||||
|
href: https://www.dataengineeringweekly.com/
|
||||||
|
description: Weekly newsletter — pipelines & platforms
|
||||||
|
color: "#22d3ee"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/de-weekly
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- Pragmatic Engineer:
|
||||||
|
icon: mdi-account-tie
|
||||||
|
href: https://blog.pragmaticengineer.com/
|
||||||
|
description: Big tech engineering & architecture
|
||||||
|
color: "#a855f7"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/pragmatic
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- Martin Fowler:
|
||||||
|
icon: mdi-arch
|
||||||
|
href: https://martinfowler.com/
|
||||||
|
description: Software architecture & design
|
||||||
|
color: "#64748b"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/martinfowler
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- InfoQ:
|
||||||
|
icon: mdi-information-outline
|
||||||
|
href: https://www.infoq.com/
|
||||||
|
description: Architecture, data & dev practices
|
||||||
|
color: "#94a3b8"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/infoq
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- ByteByteGo:
|
||||||
|
icon: mdi-school
|
||||||
|
href: https://blog.bytebytego.com/
|
||||||
|
description: System design — scalable architectures
|
||||||
|
color: "#f59e0b"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/bytebytego
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- TLDR Data Eng:
|
||||||
|
icon: mdi-lightning-bolt
|
||||||
|
href: https://tldr.tech/dataengineering/
|
||||||
|
description: Daily data engineering digest
|
||||||
|
color: "#E8752A"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/tldr-de
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
|
||||||
|
|
||||||
|
- Seattle Data Guy:
|
||||||
|
icon: mdi-chart-bar
|
||||||
|
href: https://www.seattledataguy.com/
|
||||||
|
description: Practical data engineering tutorials
|
||||||
|
color: "#0ea5e9"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/seattle-de
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- RedMonk:
|
||||||
|
icon: mdi-chart-line
|
||||||
|
href: https://redmonk.com/
|
||||||
|
description: Developer-focused industry analysis
|
||||||
|
color: "#dc2626"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/redmonk
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
|
||||||
|
- Intel · Platforms & Blogs:
|
||||||
|
- Confluent Blog:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/apachekafka.svg
|
||||||
|
href: https://www.confluent.io/blog/
|
||||||
|
description: Kafka & event streaming
|
||||||
|
color: "#007DB8"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/confluent
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- Databricks Blog:
|
||||||
|
icon: mdi-layers-triple
|
||||||
|
href: https://www.databricks.com/blog
|
||||||
|
description: Lakehouse & Spark platform
|
||||||
|
color: "#fb923c"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/databricks
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- Debezium CDC:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/debezium.png
|
||||||
|
href: https://debezium.io/blog/
|
||||||
|
description: Change data capture patterns
|
||||||
|
color: "#4ECDC4"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/debezium
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- AWS Big Data:
|
||||||
|
icon: mdi-aws
|
||||||
|
href: https://aws.amazon.com/blogs/big-data/
|
||||||
|
description: AWS analytics & data lakes
|
||||||
|
color: "#FF9900"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/aws-bigdata
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- Google Cloud Blog:
|
||||||
|
icon: mdi-google-cloud
|
||||||
|
href: https://cloud.google.com/blog
|
||||||
|
description: GCP data & analytics
|
||||||
|
color: "#4285F4"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/google-cloud
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- Cloudflare Eng:
|
||||||
|
icon: mdi-cloud
|
||||||
|
href: https://blog.cloudflare.com/
|
||||||
|
description: Edge, networking & scale
|
||||||
|
color: "#F38020"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/cloudflare
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- Meta Engineering:
|
||||||
|
icon: mdi-facebook
|
||||||
|
href: https://engineering.fb.com/
|
||||||
|
description: Large-scale infra & data
|
||||||
|
color: "#1877F2"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/fb-engineering
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
|
||||||
|
- Airflow Blog:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/apacheairflow.svg
|
||||||
|
href: https://airflow.apache.org/blog/
|
||||||
|
description: Workflow orchestration & DAGs
|
||||||
|
color: "#017CEE"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/airflow
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- Elastic Blog:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/elasticsearch.svg
|
||||||
|
href: https://www.elastic.co/blog/
|
||||||
|
description: Search, observability & security
|
||||||
|
color: "#005571"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/elastic
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
- The New Stack:
|
||||||
|
icon: mdi-newspaper-variant-multiple
|
||||||
|
href: https://thenewstack.io/
|
||||||
|
description: Cloud native & platform engineering
|
||||||
|
color: "#22c55e"
|
||||||
|
widget:
|
||||||
|
type: customapi
|
||||||
|
url: http://atc-docker01.dell-atc.lan:8090/feed/thenewstack
|
||||||
|
refreshInterval: 300000
|
||||||
|
display: dynamic-list
|
||||||
|
mappings:
|
||||||
|
items: items
|
||||||
|
name: title
|
||||||
|
href: link
|
||||||
|
target: _blank
|
||||||
|
|
||||||
|
|
||||||
|
- Documentation:
|
||||||
|
- Architecture Diagram:
|
||||||
|
icon: mdi-sitemap
|
||||||
|
href: http://atc-docker01.dell-atc.lan:8080/docs/architecture.html
|
||||||
|
description: High-level environment map (Mermaid)
|
||||||
|
color: "#007DB8"
|
||||||
|
- Application Landscape:
|
||||||
|
icon: mdi-book-open-variant
|
||||||
|
href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/landscape.md
|
||||||
|
description: docs/landscape.md in Forgejo
|
||||||
|
color: "#E8752A"
|
||||||
|
- Network Diagram:
|
||||||
|
icon: mdi-lan
|
||||||
|
href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/network.md
|
||||||
|
description: Subnets, ports, SSH mesh
|
||||||
|
color: "#22d3ee"
|
||||||
|
- Disaster Recovery:
|
||||||
|
icon: mdi-backup-restore
|
||||||
|
href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/disaster-recovery.md
|
||||||
|
description: Restore procedures from git
|
||||||
|
color: "#a855f7"
|
||||||
|
- Docker Inventory:
|
||||||
|
icon: mdi-docker
|
||||||
|
href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/docs/docker-inventory.md
|
||||||
|
description: Containers per host
|
||||||
|
color: "#2496ED"
|
||||||
|
- Proxmox VMs:
|
||||||
|
icon: mdi-server
|
||||||
|
href: http://atc-mgt01.dell-atc.lan:3001/mo/Lakehouse/src/branch/master/inventory/proxmox-vms.json
|
||||||
|
description: VM inventory (JSON)
|
||||||
|
color: "#E57000"
|
||||||
|
|
||||||
|
- News & Resources:
|
||||||
|
- Dell Technologies:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/dell-technologies.svg
|
||||||
|
href: https://www.delltechnologies.com/
|
||||||
|
description: Corporate site & solutions
|
||||||
|
color: "#007DB8"
|
||||||
|
- Dell Blog · Data:
|
||||||
|
icon: mdi-post-outline
|
||||||
|
href: https://www.dell.com/en-us/blog/categories/products-solutions-analytics
|
||||||
|
description: Analytics & data solutions
|
||||||
|
color: "#007DB8"
|
||||||
|
- Apache Spark:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/apachespark.svg
|
||||||
|
href: https://spark.apache.org/docs/latest/
|
||||||
|
description: Spark documentation
|
||||||
|
color: "#E25A1C"
|
||||||
|
- Trino:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/trino.svg
|
||||||
|
href: https://trino.io/docs/current/
|
||||||
|
description: Distributed SQL engine docs
|
||||||
|
color: "#DD00A1"
|
||||||
|
- Apache Iceberg:
|
||||||
|
icon: mdi-snowflake
|
||||||
|
href: https://iceberg.apache.org/docs/latest/
|
||||||
|
description: Open table format for lakes
|
||||||
|
color: "#38bdf8"
|
||||||
|
- Real Python:
|
||||||
|
icon: mdi-language-python
|
||||||
|
href: https://realpython.com/
|
||||||
|
description: Python tutorials & patterns
|
||||||
|
color: "#3776AB"
|
||||||
|
- MinIO Docs:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/minio.svg
|
||||||
|
href: https://min.io/docs/minio/linux/index.html
|
||||||
|
description: S3-compatible object storage
|
||||||
|
color: "#C72C48"
|
||||||
|
- TLDR Tech:
|
||||||
|
icon: mdi-lightning-bolt-outline
|
||||||
|
href: https://tldr.tech/
|
||||||
|
description: Daily tech newsletter (web)
|
||||||
|
color: "#E8752A"
|
||||||
|
|
||||||
|
- Docker · atc-docker01:
|
||||||
|
- Homepage:
|
||||||
|
icon: mdi-view-dashboard
|
||||||
|
href: http://atc-docker01.dell-atc.lan/
|
||||||
|
description: atc-docker01.dell-atc.lan
|
||||||
|
server: my-docker
|
||||||
|
container: homepage
|
||||||
|
color: "#E8752A"
|
||||||
|
- Superset:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/apachesuperset.svg
|
||||||
|
href: http://atc-docker01.dell-atc.lan:8088/
|
||||||
|
description: atc-docker01.dell-atc.lan:8088
|
||||||
|
server: my-docker
|
||||||
|
container: superset
|
||||||
|
color: "#6c5ce7"
|
||||||
|
- Forgejo Local:
|
||||||
|
icon: http://atc-docker01.dell-atc.lan:8080/git.svg
|
||||||
|
href: http://atc-docker01.dell-atc.lan:4002/
|
||||||
|
description: atc-docker01.dell-atc.lan:4002
|
||||||
|
siteMonitor: http://atc-docker01.dell-atc.lan:4002/
|
||||||
|
server: my-docker
|
||||||
|
container: forgejo
|
||||||
|
color: "#F05032"
|
||||||
@@ -25,9 +25,9 @@
|
|||||||
units: metric
|
units: metric
|
||||||
refresh: 3000
|
refresh: 3000
|
||||||
- openmeteo:
|
- openmeteo:
|
||||||
label: Lab
|
label: Amsterdam
|
||||||
latitude: 52.37
|
latitude: 52.374
|
||||||
longitude: 4.89
|
longitude: 4.890
|
||||||
timezone: Europe/Amsterdam
|
timezone: Europe/Amsterdam
|
||||||
units: metric
|
units: metric
|
||||||
cache: 30
|
cache: 15
|
||||||
|
|||||||
@@ -1 +1,10 @@
|
|||||||
{"connector.class":"io.debezium.connector.mongodb.MongoDbConnector","topic.prefix":"mongodb-supplychain","mongodb.history.kafka.bootstrap.servers":"localhost:9092","mongodb.history.kafka.topic":"schema-changes.supplychain","mongodb.connection.string":"mongodb://10.0.21.51:27017","name":"mongodb-connector","mongodb.name":"supplychain","snapshot.mode":"initial"}
|
{
|
||||||
|
"connector.class": "io.debezium.connector.mongodb.MongoDbConnector",
|
||||||
|
"topic.prefix": "mongodb-supplychain",
|
||||||
|
"mongodb.connection.string": "mongodb://mo:Dell2026%21@10.0.21.51:27017/?authSource=admin",
|
||||||
|
"mongodb.history.kafka.bootstrap.servers": "10.0.21.36:9092",
|
||||||
|
"mongodb.history.kafka.topic": "schema-changes.supplychain",
|
||||||
|
"name": "mongodb-connector",
|
||||||
|
"mongodb.name": "supplychain",
|
||||||
|
"snapshot.mode": "initial"
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,264 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Collect Debezium, Kafka CDC, and Spark metrics into postgres monitor schema."""
|
||||||
|
import json
|
||||||
|
import shlex
|
||||||
|
import subprocess
|
||||||
|
import urllib.request
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
import psycopg2
|
||||||
|
|
||||||
|
PG_DSN = "host=10.0.21.51 dbname=postgres user=mo password=Dell2026!"
|
||||||
|
KAFKA = "10.0.21.36:9092"
|
||||||
|
DEBEZIUM = "http://localhost:8083" # Kafka Connect on kafka01; fallback lake01 :8083
|
||||||
|
SPARK_MASTER = "http://10.0.21.50:8080"
|
||||||
|
|
||||||
|
CDC_TOPICS = [
|
||||||
|
("PostgreSQL", "postgres-sales.public.sales_orders"),
|
||||||
|
("MongoDB", "mongodb-supplychain.supplychain.events"),
|
||||||
|
]
|
||||||
|
|
||||||
|
OP_LABELS = {"c": "INSERT", "u": "UPDATE", "d": "DELETE", "r": "SNAPSHOT", "i": "INSERT"}
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_json(url, timeout=10):
|
||||||
|
with urllib.request.urlopen(url, timeout=timeout) as r:
|
||||||
|
return json.loads(r.read().decode())
|
||||||
|
|
||||||
|
|
||||||
|
def collect_debezium(cur):
|
||||||
|
connectors = fetch_json(f"{DEBEZIUM}/connectors")
|
||||||
|
cur.execute("DELETE FROM monitor.debezium_connectors")
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
for name in connectors:
|
||||||
|
try:
|
||||||
|
st = fetch_json(f"{DEBEZIUM}/connectors/{name}/status")
|
||||||
|
except Exception as e:
|
||||||
|
cur.execute(
|
||||||
|
"""INSERT INTO monitor.debezium_connectors
|
||||||
|
(connector_name, state, task_state, worker_id, checked_at)
|
||||||
|
VALUES (%s,%s,%s,%s,%s)""",
|
||||||
|
(name, "ERROR", str(e)[:32], "", now),
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
conn_state = st.get("connector", {}).get("state", "UNKNOWN")
|
||||||
|
tasks = st.get("tasks") or []
|
||||||
|
task_state = tasks[0].get("state", "NONE") if tasks else "NONE"
|
||||||
|
worker = st.get("connector", {}).get("worker_id", "")
|
||||||
|
cur.execute(
|
||||||
|
"""INSERT INTO monitor.debezium_connectors
|
||||||
|
(connector_name, state, task_state, worker_id, checked_at)
|
||||||
|
VALUES (%s,%s,%s,%s,%s)""",
|
||||||
|
(name, conn_state, task_state, worker, now),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
KAFKA_BIN = "/opt/kafka/bin"
|
||||||
|
USE_SSH_KAFKA = False # set True when running off-host
|
||||||
|
|
||||||
|
|
||||||
|
def _kafka_cmd(bin_name, args):
|
||||||
|
parts = [f"{KAFKA_BIN}/{bin_name}"] + list(args)
|
||||||
|
if USE_SSH_KAFKA:
|
||||||
|
remote = " ".join(shlex.quote(p) for p in parts)
|
||||||
|
full = f"ssh -o StrictHostKeyChecking=no root@10.0.21.36 {remote}"
|
||||||
|
return subprocess.check_output(full, shell=True, stderr=subprocess.DEVNULL, timeout=90, text=True)
|
||||||
|
return subprocess.check_output(parts, stderr=subprocess.DEVNULL, timeout=90, text=True)
|
||||||
|
|
||||||
|
|
||||||
|
def kafka_end_offsets(topic):
|
||||||
|
try:
|
||||||
|
out = _kafka_cmd(
|
||||||
|
"kafka-run-class.sh",
|
||||||
|
[
|
||||||
|
"kafka.tools.GetOffsetShell",
|
||||||
|
"--broker-list",
|
||||||
|
"localhost:9092",
|
||||||
|
"--topic",
|
||||||
|
topic,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
rows = []
|
||||||
|
for line in out.strip().splitlines():
|
||||||
|
parts = line.split(":")
|
||||||
|
if len(parts) >= 3:
|
||||||
|
rows.append((int(parts[1]), int(parts[2])))
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def sample_topic_messages(topic, max_msgs=3000, tail=5000):
|
||||||
|
"""Sample recent messages using kafka-console-consumer from tail."""
|
||||||
|
offsets = kafka_end_offsets(topic)
|
||||||
|
if not offsets:
|
||||||
|
return []
|
||||||
|
# Pick partition 0 for sampling
|
||||||
|
part, end = offsets[0]
|
||||||
|
start = max(0, end - tail)
|
||||||
|
try:
|
||||||
|
out = _kafka_cmd(
|
||||||
|
"kafka-console-consumer.sh",
|
||||||
|
[
|
||||||
|
"--bootstrap-server",
|
||||||
|
"localhost:9092",
|
||||||
|
"--topic",
|
||||||
|
topic,
|
||||||
|
"--partition",
|
||||||
|
str(part),
|
||||||
|
"--offset",
|
||||||
|
str(start),
|
||||||
|
"--max-messages",
|
||||||
|
str(min(max_msgs, tail)),
|
||||||
|
"--timeout-ms",
|
||||||
|
"15000",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
return [ln for ln in out.strip().split("\n") if ln.strip()]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_debezium_line(line):
|
||||||
|
try:
|
||||||
|
doc = json.loads(line)
|
||||||
|
payload = doc.get("payload") or doc
|
||||||
|
op = payload.get("op") or payload.get("operationType") or "?"
|
||||||
|
src = payload.get("source") or {}
|
||||||
|
table = src.get("table") or src.get("collection") or ""
|
||||||
|
ts_ms = payload.get("ts_ms") or src.get("ts_ms")
|
||||||
|
after = payload.get("after") or {}
|
||||||
|
before = payload.get("before") or {}
|
||||||
|
row = after if after else before
|
||||||
|
key = str(row.get("order_id") or row.get("event_id") or row.get("_id") or "")[:200]
|
||||||
|
detail = str(row.get("region") or row.get("type") or row.get("department") or "")[:200]
|
||||||
|
event_ts = None
|
||||||
|
if ts_ms:
|
||||||
|
event_ts = datetime.fromtimestamp(int(ts_ms) / 1000, tz=timezone.utc)
|
||||||
|
return op, table, key, detail, event_ts
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def collect_kafka_cdc(cur):
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
cur.execute("DELETE FROM monitor.kafka_topics")
|
||||||
|
cur.execute("DELETE FROM monitor.cdc_operations")
|
||||||
|
cur.execute("DELETE FROM monitor.cdc_recent_events")
|
||||||
|
|
||||||
|
for source, topic in CDC_TOPICS:
|
||||||
|
for part, end in kafka_end_offsets(topic):
|
||||||
|
cur.execute(
|
||||||
|
"""INSERT INTO monitor.kafka_topics (topic, partition_id, end_offset, checked_at)
|
||||||
|
VALUES (%s,%s,%s,%s)""",
|
||||||
|
(topic, part, end, now),
|
||||||
|
)
|
||||||
|
|
||||||
|
lines = sample_topic_messages(topic, max_msgs=2000, tail=3000)
|
||||||
|
ops = Counter()
|
||||||
|
recent = []
|
||||||
|
for line in lines:
|
||||||
|
parsed = parse_debezium_line(line)
|
||||||
|
if not parsed:
|
||||||
|
continue
|
||||||
|
op, table, key, detail, event_ts = parsed
|
||||||
|
ops[op] += 1
|
||||||
|
if len(recent) < 100:
|
||||||
|
recent.append((op, table, key, detail, event_ts))
|
||||||
|
|
||||||
|
for op, cnt in ops.items():
|
||||||
|
cur.execute(
|
||||||
|
"""INSERT INTO monitor.cdc_operations
|
||||||
|
(source_system, topic, operation, operation_label, event_count, checked_at)
|
||||||
|
VALUES (%s,%s,%s,%s,%s,%s)""",
|
||||||
|
(source, topic, op, OP_LABELS.get(op, op), cnt, now),
|
||||||
|
)
|
||||||
|
|
||||||
|
for op, table, key, detail, event_ts in recent[:50]:
|
||||||
|
cur.execute(
|
||||||
|
"""INSERT INTO monitor.cdc_recent_events
|
||||||
|
(source_system, topic, operation, operation_label, table_name,
|
||||||
|
record_key, detail, event_ts, sampled_at)
|
||||||
|
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
|
||||||
|
(
|
||||||
|
source,
|
||||||
|
topic,
|
||||||
|
op,
|
||||||
|
OP_LABELS.get(op, op),
|
||||||
|
table,
|
||||||
|
key,
|
||||||
|
detail,
|
||||||
|
event_ts,
|
||||||
|
now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def collect_spark(cur):
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
cur.execute("DELETE FROM monitor.spark_applications")
|
||||||
|
try:
|
||||||
|
data = fetch_json(f"{SPARK_MASTER}/json/", timeout=5)
|
||||||
|
apps = []
|
||||||
|
if isinstance(data, dict):
|
||||||
|
# Standalone master JSON
|
||||||
|
for a in data.get("activeapps", []) or []:
|
||||||
|
apps.append(a)
|
||||||
|
for a in data.get("completedapps", []) or []:
|
||||||
|
apps.append(a)
|
||||||
|
for a in apps[:20]:
|
||||||
|
cur.execute(
|
||||||
|
"""INSERT INTO monitor.spark_applications
|
||||||
|
(app_id, app_name, state, cores, memory_mb, duration_sec, checked_at)
|
||||||
|
VALUES (%s,%s,%s,%s,%s,%s,%s)
|
||||||
|
ON CONFLICT (app_id) DO UPDATE SET
|
||||||
|
app_name=EXCLUDED.app_name, state=EXCLUDED.state,
|
||||||
|
cores=EXCLUDED.cores, memory_mb=EXCLUDED.memory_mb,
|
||||||
|
duration_sec=EXCLUDED.duration_sec, checked_at=EXCLUDED.checked_at""",
|
||||||
|
(
|
||||||
|
a.get("id", "unknown"),
|
||||||
|
a.get("name", "Spark App"),
|
||||||
|
"RUNNING" if "attempts" not in a else "COMPLETED",
|
||||||
|
int(a.get("cores", 0) or 0),
|
||||||
|
int((a.get("memory", 0) or 0) / 1024 / 1024),
|
||||||
|
int(a.get("duration", 0) / 1000) if a.get("duration") else 0,
|
||||||
|
now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
# Placeholder row so dashboard shows Spark host status
|
||||||
|
cur.execute(
|
||||||
|
"""INSERT INTO monitor.spark_applications
|
||||||
|
(app_id, app_name, state, cores, memory_mb, duration_sec, checked_at)
|
||||||
|
VALUES (%s,%s,%s,%s,%s,%s,%s)
|
||||||
|
ON CONFLICT (app_id) DO UPDATE SET state=EXCLUDED.state, checked_at=EXCLUDED.checked_at""",
|
||||||
|
(
|
||||||
|
"spark-master",
|
||||||
|
f"Spark Master @ {SPARK_MASTER}",
|
||||||
|
"REACHABLE" if "Connection" not in str(e) else "UNREACHABLE",
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
conn = psycopg2.connect(PG_DSN)
|
||||||
|
conn.autocommit = True
|
||||||
|
cur = conn.cursor()
|
||||||
|
print("Collecting Debezium...")
|
||||||
|
collect_debezium(cur)
|
||||||
|
print("Collecting Kafka CDC samples...")
|
||||||
|
collect_kafka_cdc(cur)
|
||||||
|
print("Collecting Spark...")
|
||||||
|
collect_spark(cur)
|
||||||
|
cur.close()
|
||||||
|
conn.close()
|
||||||
|
print("Done.")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -1,3 +1,7 @@
|
|||||||
FROM apache/superset:latest
|
FROM apache/superset:latest
|
||||||
|
USER root
|
||||||
RUN pip install psycopg2-binary
|
RUN pip3 install --no-cache-dir --target=/app/.venv/lib/python3.10/site-packages --no-deps \
|
||||||
|
sqlalchemy-trino==0.5.0 trino==0.337.0 && \
|
||||||
|
pip3 install --no-cache-dir --target=/app/.venv/lib/python3.10/site-packages \
|
||||||
|
requests lz4 orjson python-dateutil pytz tzlocal zstandard charset-normalizer idna urllib3 certifi six greenlet
|
||||||
|
USER superset
|
||||||
|
|||||||
@@ -0,0 +1,453 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Apply Palantir styling and enrich ATC Lakehouse Superset dashboard."""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import requests
|
||||||
|
|
||||||
|
BASE = "http://127.0.0.1:8088"
|
||||||
|
DASH_ID = 1
|
||||||
|
CSS_PATH = "/tmp/palantir_dashboard.css"
|
||||||
|
|
||||||
|
NEW_CHARTS = [
|
||||||
|
(
|
||||||
|
"Trino · PostgreSQL Sales",
|
||||||
|
"public",
|
||||||
|
"sales_orders",
|
||||||
|
"PostgreSQL · Orders by Channel",
|
||||||
|
"pie",
|
||||||
|
{
|
||||||
|
"metric": {"expressionType": "SQL", "sqlExpression": "COUNT(*)", "label": "COUNT(*)"},
|
||||||
|
"groupby": ["sales_channel"],
|
||||||
|
"row_limit": 10,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Trino · PostgreSQL Sales",
|
||||||
|
"public",
|
||||||
|
"sales_orders",
|
||||||
|
"PostgreSQL · Avg Order by Region",
|
||||||
|
"echarts_timeseries_bar",
|
||||||
|
{
|
||||||
|
"metrics": [
|
||||||
|
{"expressionType": "SQL", "sqlExpression": "AVG(amount)", "label": "Avg Amount"}
|
||||||
|
],
|
||||||
|
"groupby": ["region"],
|
||||||
|
"row_limit": 15,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Trino · PostgreSQL Sales",
|
||||||
|
"public",
|
||||||
|
"sales_orders",
|
||||||
|
"PostgreSQL · Status Breakdown",
|
||||||
|
"pie",
|
||||||
|
{
|
||||||
|
"metric": {"expressionType": "SQL", "sqlExpression": "COUNT(*)", "label": "COUNT(*)"},
|
||||||
|
"groupby": ["order_status"],
|
||||||
|
"row_limit": 10,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Trino · MySQL HR",
|
||||||
|
"hr",
|
||||||
|
"employee_events",
|
||||||
|
"MySQL HR · By Event Type",
|
||||||
|
"pie",
|
||||||
|
{
|
||||||
|
"metric": {"expressionType": "SQL", "sqlExpression": "COUNT(*)", "label": "COUNT(*)"},
|
||||||
|
"groupby": ["event_type"],
|
||||||
|
"row_limit": 12,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Trino · MySQL HR",
|
||||||
|
"hr",
|
||||||
|
"employee_events",
|
||||||
|
"MySQL HR · Events per Month",
|
||||||
|
"echarts_timeseries_line",
|
||||||
|
{
|
||||||
|
"metrics": [
|
||||||
|
{"expressionType": "SQL", "sqlExpression": "COUNT(*)", "label": "Events"}
|
||||||
|
],
|
||||||
|
"groupby": [
|
||||||
|
{
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "date_trunc('month', event_ts)",
|
||||||
|
"label": "Month",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"row_limit": 24,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Trino · MongoDB Supply Chain",
|
||||||
|
"supplychain",
|
||||||
|
"events",
|
||||||
|
"MongoDB · Amount by Source",
|
||||||
|
"echarts_timeseries_bar",
|
||||||
|
{
|
||||||
|
"metrics": [
|
||||||
|
{"expressionType": "SQL", "sqlExpression": "SUM(amount)", "label": "Total Amount"}
|
||||||
|
],
|
||||||
|
"groupby": ["source"],
|
||||||
|
"row_limit": 10,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Trino · MongoDB Supply Chain",
|
||||||
|
"supplychain",
|
||||||
|
"events",
|
||||||
|
"MongoDB · Events per Month",
|
||||||
|
"echarts_timeseries_line",
|
||||||
|
{
|
||||||
|
"metrics": [
|
||||||
|
{"expressionType": "SQL", "sqlExpression": "COUNT(*)", "label": "Events"}
|
||||||
|
],
|
||||||
|
"groupby": [
|
||||||
|
{
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "date_trunc('month', ts)",
|
||||||
|
"label": "Month",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"row_limit": 24,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Trino · Cassandra Telemetry",
|
||||||
|
"telemetry",
|
||||||
|
"device_metrics",
|
||||||
|
"Cassandra · Avg Metric Over Time",
|
||||||
|
"echarts_timeseries_line",
|
||||||
|
{
|
||||||
|
"metrics": [
|
||||||
|
{
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "AVG(metric_value)",
|
||||||
|
"label": "Avg Value",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"groupby": [
|
||||||
|
{
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "date_trunc('day', metric_ts)",
|
||||||
|
"label": "Day",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"row_limit": 30,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
SECTIONS = [
|
||||||
|
("HEADER", "ATC Lakehouse · Federated Data Platform", "Trino · PostgreSQL · MySQL · MongoDB · Cassandra · Dell Technologies FDE"),
|
||||||
|
("PostgreSQL Sales", "30M orders · postgres_sales.public.sales_orders"),
|
||||||
|
("MySQL HR", "569K events · mysql_hr.hr.employee_events"),
|
||||||
|
("MongoDB Supply Chain", "3M events · mongodb_supplychain.supplychain.events"),
|
||||||
|
("Cassandra Telemetry", "Device metrics · cassandra_telemetry.telemetry.device_metrics"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def session():
|
||||||
|
s = requests.Session()
|
||||||
|
r = s.post(
|
||||||
|
f"{BASE}/api/v1/security/login",
|
||||||
|
json={"username": "admin", "password": "admin", "provider": "db", "refresh": True},
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
h = {
|
||||||
|
"Authorization": "Bearer " + r.json()["access_token"],
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
}
|
||||||
|
h["X-CSRFToken"] = s.get(f"{BASE}/api/v1/security/csrf_token/", headers=h).json()["result"]
|
||||||
|
h["Referer"] = BASE
|
||||||
|
return s, h
|
||||||
|
|
||||||
|
|
||||||
|
def get_db_map(s, h):
|
||||||
|
r = s.get(f"{BASE}/api/v1/database/", headers=h)
|
||||||
|
r.raise_for_status()
|
||||||
|
return {d["database_name"]: d["id"] for d in r.json().get("result", [])}
|
||||||
|
|
||||||
|
|
||||||
|
def get_or_create_dataset(s, h, db_id, schema, table):
|
||||||
|
r = s.get(f"{BASE}/api/v1/dataset/", headers=h)
|
||||||
|
for d in r.json().get("result", []):
|
||||||
|
if (
|
||||||
|
d.get("table_name") == table
|
||||||
|
and d.get("schema") == schema
|
||||||
|
and d.get("database", {}).get("id") == db_id
|
||||||
|
):
|
||||||
|
return d["id"]
|
||||||
|
r = s.post(
|
||||||
|
f"{BASE}/api/v1/dataset/",
|
||||||
|
headers=h,
|
||||||
|
json={"database": db_id, "schema": schema, "table_name": table},
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()["id"]
|
||||||
|
|
||||||
|
|
||||||
|
def create_chart(s, h, name, ds_id, viz_type, params):
|
||||||
|
r = s.get(f"{BASE}/api/v1/chart/", headers=h)
|
||||||
|
for c in r.json().get("result", []):
|
||||||
|
if c.get("slice_name") == name:
|
||||||
|
return c["id"]
|
||||||
|
full = {"datasource": f"{ds_id}__table", "viz_type": viz_type, "row_limit": 1000, **params}
|
||||||
|
r = s.post(
|
||||||
|
f"{BASE}/api/v1/chart/",
|
||||||
|
headers=h,
|
||||||
|
json={
|
||||||
|
"slice_name": name,
|
||||||
|
"viz_type": viz_type,
|
||||||
|
"datasource_id": ds_id,
|
||||||
|
"datasource_type": "table",
|
||||||
|
"params": json.dumps(full),
|
||||||
|
"owners": [1, 2, 3],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if r.status_code not in (200, 201):
|
||||||
|
raise RuntimeError(f"chart {name}: {r.text[:300]}")
|
||||||
|
return r.json()["id"]
|
||||||
|
|
||||||
|
|
||||||
|
def build_layout(chart_items):
|
||||||
|
"""chart_items: list of (chart_id, name, section) or ('md', title, subtitle)."""
|
||||||
|
layout = {
|
||||||
|
"DASHBOARD_VERSION": "v2",
|
||||||
|
"ROOT_ID": {"type": "ROOT", "id": "ROOT_ID", "children": ["GRID_ID"]},
|
||||||
|
"GRID_ID": {"type": "GRID", "id": "GRID_ID", "children": [], "parents": ["ROOT_ID"]},
|
||||||
|
}
|
||||||
|
row_idx = 0
|
||||||
|
|
||||||
|
def add_row():
|
||||||
|
nonlocal row_idx
|
||||||
|
row_idx += 1
|
||||||
|
rid = f"ROW-{row_idx}"
|
||||||
|
layout["GRID_ID"]["children"].append(rid)
|
||||||
|
layout[rid] = {
|
||||||
|
"type": "ROW",
|
||||||
|
"id": rid,
|
||||||
|
"children": [],
|
||||||
|
"parents": ["ROOT_ID", "GRID_ID"],
|
||||||
|
"meta": {"background": "BACKGROUND_TRANSPARENT"},
|
||||||
|
}
|
||||||
|
return rid
|
||||||
|
|
||||||
|
for item in chart_items:
|
||||||
|
if item[0] == "md":
|
||||||
|
_, title, subtitle = item
|
||||||
|
rid = add_row()
|
||||||
|
mid = f"MARKDOWN-{row_idx}"
|
||||||
|
layout[rid]["children"].append(mid)
|
||||||
|
layout[mid] = {
|
||||||
|
"type": "MARKDOWN",
|
||||||
|
"id": mid,
|
||||||
|
"children": [],
|
||||||
|
"parents": ["ROOT_ID", "GRID_ID", rid],
|
||||||
|
"meta": {
|
||||||
|
"width": 12,
|
||||||
|
"height": 12,
|
||||||
|
"code": f"## {title}\n\n{subtitle}",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
cid, name, _section = item
|
||||||
|
rid = add_row()
|
||||||
|
# up to 3 charts per row
|
||||||
|
existing = [
|
||||||
|
k
|
||||||
|
for k in layout[rid]["children"]
|
||||||
|
if k.startswith("CHART-")
|
||||||
|
]
|
||||||
|
if len(existing) >= 3:
|
||||||
|
rid = add_row()
|
||||||
|
chart_key = f"CHART-explore-{cid}"
|
||||||
|
layout[rid]["children"].append(chart_key)
|
||||||
|
col = len([k for k in layout[rid]["children"] if k.startswith("CHART-")]) - 1
|
||||||
|
layout[chart_key] = {
|
||||||
|
"type": "CHART",
|
||||||
|
"id": chart_key,
|
||||||
|
"children": [],
|
||||||
|
"parents": ["ROOT_ID", "GRID_ID", rid],
|
||||||
|
"meta": {
|
||||||
|
"width": 4,
|
||||||
|
"height": 55 if "Total" in name or "Records" in name else 65,
|
||||||
|
"chartId": cid,
|
||||||
|
"sliceName": name,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
return layout
|
||||||
|
|
||||||
|
|
||||||
|
def save_query_contexts():
|
||||||
|
app = __import__("superset.app", fromlist=["create_app"]).create_app()
|
||||||
|
with app.app_context():
|
||||||
|
from flask import g
|
||||||
|
from superset.extensions import db
|
||||||
|
from superset.models.slice import Slice
|
||||||
|
from superset.charts.schemas import ChartDataQueryContextSchema
|
||||||
|
from superset import security_manager
|
||||||
|
|
||||||
|
g.user = security_manager.find_user(username="admin")
|
||||||
|
for sl in db.session.query(Slice).all():
|
||||||
|
try:
|
||||||
|
fd = sl.form_data
|
||||||
|
metric = fd.get("metric")
|
||||||
|
metrics = fd.get("metrics") or ([metric] if metric else [])
|
||||||
|
if not metrics:
|
||||||
|
metrics = [
|
||||||
|
{
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "COUNT(*)",
|
||||||
|
"label": "COUNT(*)",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
groupby = fd.get("groupby") or []
|
||||||
|
payload = {
|
||||||
|
"datasource": {"id": sl.datasource_id, "type": sl.datasource_type},
|
||||||
|
"force": False,
|
||||||
|
"queries": [
|
||||||
|
{
|
||||||
|
"filters": [],
|
||||||
|
"extras": {"having": "", "where": ""},
|
||||||
|
"applied_time_extras": {},
|
||||||
|
"columns": groupby if isinstance(groupby, list) else [],
|
||||||
|
"metrics": metrics,
|
||||||
|
"orderby": [],
|
||||||
|
"annotation_layers": [],
|
||||||
|
"row_limit": int(fd.get("row_limit") or 1000),
|
||||||
|
"series_limit": 0,
|
||||||
|
"order_desc": True,
|
||||||
|
"url_params": {},
|
||||||
|
"custom_params": {},
|
||||||
|
"custom_form_data": {},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"form_data": fd,
|
||||||
|
"result_format": "json",
|
||||||
|
"result_type": "full",
|
||||||
|
}
|
||||||
|
ChartDataQueryContextSchema().load(payload)
|
||||||
|
sl.query_context = json.dumps(payload)
|
||||||
|
sl.query_context_generation = True
|
||||||
|
db.session.add(sl)
|
||||||
|
except Exception as e:
|
||||||
|
print("qc err", sl.id, e)
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
s, h = session()
|
||||||
|
db_map = get_db_map(s, h)
|
||||||
|
|
||||||
|
# Create new charts
|
||||||
|
new_ids = []
|
||||||
|
for db_name, schema, table, name, viz, params in NEW_CHARTS:
|
||||||
|
db_id = db_map.get(db_name)
|
||||||
|
if not db_id:
|
||||||
|
print("skip, no db:", db_name)
|
||||||
|
continue
|
||||||
|
ds_id = get_or_create_dataset(s, h, db_id, schema, table)
|
||||||
|
cid = create_chart(s, h, name, ds_id, viz, params)
|
||||||
|
new_ids.append((cid, name, db_name.split("·")[-1].strip()))
|
||||||
|
print("new chart", cid, name)
|
||||||
|
|
||||||
|
# All charts for dashboard
|
||||||
|
r = s.get(f"{BASE}/api/v1/chart/?q=(page:0,page_size:200)", headers=h)
|
||||||
|
all_charts = r.json().get("result", [])
|
||||||
|
|
||||||
|
def sort_key(c):
|
||||||
|
n = c.get("slice_name") or ""
|
||||||
|
if "Lakehouse" in n or "Records per Source" in n:
|
||||||
|
return (0, n)
|
||||||
|
if "PostgreSQL" in n:
|
||||||
|
return (1, n)
|
||||||
|
if "MySQL" in n:
|
||||||
|
return (2, n)
|
||||||
|
if "MongoDB" in n:
|
||||||
|
return (3, n)
|
||||||
|
if "Cassandra" in n:
|
||||||
|
return (4, n)
|
||||||
|
return (5, n)
|
||||||
|
|
||||||
|
all_charts.sort(key=sort_key)
|
||||||
|
|
||||||
|
chart_items = [
|
||||||
|
("md", "ATC Lakehouse · Federated Data Platform", "Real-time analytics across all Trino catalogs · Dell Technologies"),
|
||||||
|
]
|
||||||
|
current_section = None
|
||||||
|
for c in all_charts:
|
||||||
|
name = c.get("slice_name") or ""
|
||||||
|
if "PostgreSQL" in name and current_section != "pg":
|
||||||
|
chart_items.append(("md", "PostgreSQL Sales", "30M orders · CDC-enabled · atc-db02"))
|
||||||
|
current_section = "pg"
|
||||||
|
elif "MySQL" in name and current_section != "mysql":
|
||||||
|
chart_items.append(("md", "MySQL HR", "569K employee events · HR domain"))
|
||||||
|
current_section = "mysql"
|
||||||
|
elif "MongoDB" in name and current_section != "mongo":
|
||||||
|
chart_items.append(("md", "MongoDB Supply Chain", "3M supply chain events"))
|
||||||
|
current_section = "mongo"
|
||||||
|
elif "Cassandra" in name and current_section != "cass":
|
||||||
|
chart_items.append(("md", "Cassandra Telemetry", "IoT device metrics"))
|
||||||
|
current_section = "cass"
|
||||||
|
chart_items.append((c["id"], name, current_section))
|
||||||
|
|
||||||
|
chart_ids = [x[0] for x in chart_items if x[0] != "md"]
|
||||||
|
position = build_layout(chart_items)
|
||||||
|
|
||||||
|
css = ""
|
||||||
|
if os.path.exists(CSS_PATH):
|
||||||
|
css = open(CSS_PATH, encoding="utf-8").read()
|
||||||
|
|
||||||
|
chart_configuration = {
|
||||||
|
str(cid): {"id": cid, "crossFilters": {"scope": "global", "chartsInScope": chart_ids}}
|
||||||
|
for cid in chart_ids
|
||||||
|
}
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"dashboard_title": "ATC Lakehouse · Trino Federated",
|
||||||
|
"published": True,
|
||||||
|
"position_json": json.dumps(position),
|
||||||
|
"css": css,
|
||||||
|
"json_metadata": json.dumps(
|
||||||
|
{
|
||||||
|
"color_scheme": "palantir_ops",
|
||||||
|
"label_colors": {},
|
||||||
|
"refresh_frequency": 120,
|
||||||
|
"timed_refresh_immune_slices": [],
|
||||||
|
"expanded_slices": {},
|
||||||
|
"chart_configuration": chart_configuration,
|
||||||
|
"global_chart_configuration": {
|
||||||
|
"scope": {"rootPath": ["ROOT_ID"], "excluded": []},
|
||||||
|
"chartsInScope": chart_ids,
|
||||||
|
},
|
||||||
|
"native_filter_configuration": [],
|
||||||
|
"color_scheme_domain": [],
|
||||||
|
"shared_label_colors": {},
|
||||||
|
}
|
||||||
|
),
|
||||||
|
"owners": [1, 2, 3],
|
||||||
|
}
|
||||||
|
|
||||||
|
r = s.put(f"{BASE}/api/v1/dashboard/{DASH_ID}", headers=h, json=payload)
|
||||||
|
print("dashboard update", r.status_code)
|
||||||
|
if r.status_code >= 400:
|
||||||
|
print(r.text[:500])
|
||||||
|
return
|
||||||
|
|
||||||
|
for cid in chart_ids:
|
||||||
|
s.put(
|
||||||
|
f"{BASE}/api/v1/chart/{cid}",
|
||||||
|
headers=h,
|
||||||
|
json={"dashboards": [DASH_ID], "owners": [1, 2, 3]},
|
||||||
|
)
|
||||||
|
|
||||||
|
print("Saving query contexts...")
|
||||||
|
save_query_contexts()
|
||||||
|
print(f"Done — {len(chart_ids)} charts, Palantir theme applied to dashboard.")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import json, sys, requests
|
||||||
|
BASE = "http://127.0.0.1:8088"
|
||||||
|
HOST = "10.0.21.50:8089"
|
||||||
|
USER = "mo"
|
||||||
|
SOURCES = [
|
||||||
|
("Trino · PostgreSQL Sales", f"trino://{USER}@{HOST}/postgres_sales/public", "public", "sales_orders", [
|
||||||
|
("PostgreSQL · Total Orders", "big_number_total", {"metric":{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"COUNT(*)"}}),
|
||||||
|
("PostgreSQL · Revenue by Region", "pie", {"metric":{"expressionType":"SQL","sqlExpression":"SUM(amount)","label":"Revenue"},"groupby":["region"],"row_limit":20}),
|
||||||
|
("PostgreSQL · Orders per Month", "echarts_timeseries_bar", {"metrics":[{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"Orders"}],"groupby":[{"expressionType":"SQL","sqlExpression":"date_trunc('month', order_ts)","label":"Month"}],"row_limit":24}),
|
||||||
|
]),
|
||||||
|
("Trino · MySQL HR", f"trino://{USER}@{HOST}/mysql_hr/hr", "hr", "employee_events", [
|
||||||
|
("MySQL HR · Total Events", "big_number_total", {"metric":{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"COUNT(*)"}}),
|
||||||
|
("MySQL HR · By Department", "pie", {"metric":{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"COUNT(*)"},"groupby":["department"],"row_limit":15}),
|
||||||
|
("MySQL HR · By Region", "echarts_timeseries_bar", {"metrics":[{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"Events"}],"groupby":["region"],"row_limit":20}),
|
||||||
|
]),
|
||||||
|
("Trino · MongoDB Supply Chain", f"trino://{USER}@{HOST}/mongodb_supplychain/supplychain", "supplychain", "events", [
|
||||||
|
("MongoDB · Total Events", "big_number_total", {"metric":{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"COUNT(*)"}}),
|
||||||
|
("MongoDB · By Type", "pie", {"metric":{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"COUNT(*)"},"groupby":["type"],"row_limit":10}),
|
||||||
|
("MongoDB · By Region", "echarts_timeseries_bar", {"metrics":[{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"Events"}],"groupby":["region"],"row_limit":10}),
|
||||||
|
]),
|
||||||
|
("Trino · Cassandra Telemetry", f"trino://{USER}@{HOST}/cassandra_telemetry/telemetry", "telemetry", "device_metrics", [
|
||||||
|
("Cassandra · Total Metrics", "big_number_total", {"metric":{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"COUNT(*)"}}),
|
||||||
|
("Cassandra · By Metric Type", "pie", {"metric":{"expressionType":"SQL","sqlExpression":"COUNT(*)","label":"COUNT(*)"},"groupby":["metric_type"],"row_limit":20}),
|
||||||
|
("Cassandra · Avg by Device", "echarts_timeseries_bar", {"metrics":[{"expressionType":"SQL","sqlExpression":"AVG(metric_value)","label":"Avg"}],"groupby":["device_id"],"row_limit":20}),
|
||||||
|
]),
|
||||||
|
]
|
||||||
|
OVERVIEW_SQL = """SELECT 'PostgreSQL sales_orders' AS source, COUNT(*) AS records FROM postgres_sales.public.sales_orders
|
||||||
|
UNION ALL SELECT 'MySQL employee_events', COUNT(*) FROM mysql_hr.hr.employee_events
|
||||||
|
UNION ALL SELECT 'MongoDB events', COUNT(*) FROM mongodb_supplychain.supplychain.events
|
||||||
|
UNION ALL SELECT 'Cassandra device_metrics', COUNT(*) FROM cassandra_telemetry.telemetry.device_metrics"""
|
||||||
|
|
||||||
|
def headers(s):
|
||||||
|
r=s.post(f"{BASE}/api/v1/security/login",json={"username":"admin","password":"admin","provider":"db","refresh":True}); r.raise_for_status()
|
||||||
|
h={"Authorization":"Bearer "+r.json()["access_token"],"Content-Type":"application/json"}
|
||||||
|
h["X-CSRFToken"]=s.get(f"{BASE}/api/v1/security/csrf_token/",headers=h).json()["result"]; h["Referer"]=BASE; return h
|
||||||
|
|
||||||
|
def get_db(s,h,name,uri):
|
||||||
|
r=s.get(f"{BASE}/api/v1/database/",headers=h); r.raise_for_status()
|
||||||
|
for d in r.json().get("result",[]):
|
||||||
|
if d["database_name"]==name: return d["id"]
|
||||||
|
r=s.post(f"{BASE}/api/v1/database/",headers=h,json={"database_name":name,"sqlalchemy_uri":uri,"expose_in_sqllab":True,"allow_run_async":True})
|
||||||
|
if r.status_code not in (200,201): raise SystemExit(r.text)
|
||||||
|
print("DB",name,r.json()["id"]); return r.json()["id"]
|
||||||
|
|
||||||
|
def get_ds(s,h,db,schema,table,sql=None):
|
||||||
|
r=s.get(f"{BASE}/api/v1/dataset/",headers=h); r.raise_for_status()
|
||||||
|
for d in r.json().get("result",[]):
|
||||||
|
if d.get("table_name")==table and d.get("schema")==schema and d.get("database",{}).get("id")==db: return d["id"]
|
||||||
|
p={"database":db,"table_name":table,"schema":schema} if not sql else {"database":db,"table_name":table,"sql":sql}
|
||||||
|
r=s.post(f"{BASE}/api/v1/dataset/",headers=h,json=p)
|
||||||
|
if r.status_code not in (200,201): raise SystemExit(r.text)
|
||||||
|
print(" DS",table,r.json()["id"]); return r.json()["id"]
|
||||||
|
|
||||||
|
def mk_chart(s,h,name,ds,viz,params):
|
||||||
|
r=s.get(f"{BASE}/api/v1/chart/",headers=h); r.raise_for_status()
|
||||||
|
for c in r.json().get("result",[]):
|
||||||
|
if c.get("slice_name")==name: return c["id"]
|
||||||
|
p={"datasource":f"{ds}__table","viz_type":viz,"row_limit":1000,**params}
|
||||||
|
r=s.post(f"{BASE}/api/v1/chart/",headers=h,json={"slice_name":name,"viz_type":viz,"datasource_id":ds,"datasource_type":"table","params":json.dumps(p)})
|
||||||
|
if r.status_code not in (200,201): raise SystemExit(f"chart {name}: {r.text[:300]}")
|
||||||
|
print(" chart",name,r.json()["id"]); return r.json()["id"]
|
||||||
|
|
||||||
|
def mk_dash(s,h,title,cids):
|
||||||
|
layout={"DASHBOARD_VERSION":"v2","ROOT_ID":{"type":"ROOT","id":"ROOT_ID","children":["GRID_ID"]},"GRID_ID":{"type":"GRID","id":"GRID_ID","children":[],"parents":["ROOT_ID"]}}
|
||||||
|
row=col=0
|
||||||
|
for cid in cids:
|
||||||
|
k=f"CHART-{cid}"; x=(col%3)*4; y=row*12
|
||||||
|
layout[k]={"type":"CHART","id":k,"children":[],"meta":{"width":4,"height":10,"chartId":cid},"parents":["ROOT_ID","GRID_ID"]}
|
||||||
|
layout["GRID_ID"]["children"].append(k); col+=1
|
||||||
|
if col%3==0: row+=1
|
||||||
|
payload={"dashboard_title":title,"published":True,"position_json":json.dumps(layout),"json_metadata":"{}"}
|
||||||
|
r=s.get(f"{BASE}/api/v1/dashboard/",headers=h); r.raise_for_status()
|
||||||
|
for d in r.json().get("result",[]):
|
||||||
|
if d.get("dashboard_title")==title:
|
||||||
|
did=d["id"]; s.put(f"{BASE}/api/v1/dashboard/{did}",headers=h,json=payload)
|
||||||
|
for cid in cids: s.put(f"{BASE}/api/v1/chart/{cid}",headers=h,json={"dashboards":[did]})
|
||||||
|
print("Dashboard",did); return did
|
||||||
|
r=s.post(f"{BASE}/api/v1/dashboard/",headers=h,json=payload)
|
||||||
|
if r.status_code not in (200,201): raise SystemExit(r.text)
|
||||||
|
did=r.json()["id"]
|
||||||
|
for cid in cids: s.put(f"{BASE}/api/v1/chart/{cid}",headers=h,json={"dashboards":[did]})
|
||||||
|
print("Dashboard",did); return did
|
||||||
|
|
||||||
|
def main():
|
||||||
|
s=requests.Session(); h=headers(s)
|
||||||
|
cids=[]
|
||||||
|
odb=get_db(s,h,"Trino · Lakehouse Overview",f"trino://{USER}@{HOST}/postgres_sales/public")
|
||||||
|
ods=get_ds(s,h,odb,None,"lakehouse_counts",OVERVIEW_SQL)
|
||||||
|
cids.append(mk_chart(s,h,"Lakehouse · Records per Source",ods,"pie",{"metric":{"expressionType":"SQL","sqlExpression":"SUM(records)","label":"Records"},"groupby":["source"],"row_limit":10}))
|
||||||
|
for dbn,uri,sch,tbl,charts in SOURCES:
|
||||||
|
db=get_db(s,h,dbn,uri); ds=get_ds(s,h,db,sch,tbl)
|
||||||
|
for nm,viz,pr in charts: cids.append(mk_chart(s,h,nm,ds,viz,pr))
|
||||||
|
mk_dash(s,h,"ATC Lakehouse · Trino Federated",cids)
|
||||||
|
if __name__=="__main__": main()
|
||||||
@@ -0,0 +1,367 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Create Superset dashboard for Debezium, Kafka CDC changes, and Spark."""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import requests
|
||||||
|
|
||||||
|
BASE = "http://127.0.0.1:8088"
|
||||||
|
DASH_TITLE = "ATC Lakehouse · Pipeline & CDC"
|
||||||
|
CSS_PATH = "/tmp/palantir_dashboard.css"
|
||||||
|
|
||||||
|
MONITOR_URI = "trino://mo@10.0.21.50:8089/postgres_sales/monitor"
|
||||||
|
KAFKA_URI = "trino://mo@10.0.21.50:8089/kafka/default"
|
||||||
|
|
||||||
|
|
||||||
|
def session():
|
||||||
|
s = requests.Session()
|
||||||
|
r = s.post(
|
||||||
|
f"{BASE}/api/v1/security/login",
|
||||||
|
json={"username": "admin", "password": "admin", "provider": "db", "refresh": True},
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
h = {"Authorization": "Bearer " + r.json()["access_token"], "Content-Type": "application/json"}
|
||||||
|
h["X-CSRFToken"] = s.get(f"{BASE}/api/v1/security/csrf_token/", headers=h).json()["result"]
|
||||||
|
h["Referer"] = BASE
|
||||||
|
return s, h
|
||||||
|
|
||||||
|
|
||||||
|
def get_db(s, h, name, uri):
|
||||||
|
r = s.get(f"{BASE}/api/v1/database/", headers=h)
|
||||||
|
for d in r.json().get("result", []):
|
||||||
|
if d["database_name"] == name:
|
||||||
|
return d["id"]
|
||||||
|
r = s.post(
|
||||||
|
f"{BASE}/api/v1/database/",
|
||||||
|
headers=h,
|
||||||
|
json={"database_name": name, "sqlalchemy_uri": uri, "expose_in_sqllab": True},
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()["id"]
|
||||||
|
|
||||||
|
|
||||||
|
def ds_table(s, h, db_id, schema, table):
|
||||||
|
r = s.get(f"{BASE}/api/v1/dataset/", headers=h)
|
||||||
|
for d in r.json().get("result", []):
|
||||||
|
if d.get("table_name") == table and d.get("schema") == schema and d.get("database", {}).get("id") == db_id:
|
||||||
|
return d["id"]
|
||||||
|
r = s.post(
|
||||||
|
f"{BASE}/api/v1/dataset/",
|
||||||
|
headers=h,
|
||||||
|
json={"database": db_id, "schema": schema, "table_name": table},
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()["id"]
|
||||||
|
|
||||||
|
|
||||||
|
def ds_sql(s, h, db_id, name, sql):
|
||||||
|
r = s.get(f"{BASE}/api/v1/dataset/", headers=h)
|
||||||
|
for d in r.json().get("result", []):
|
||||||
|
if d.get("table_name") == name:
|
||||||
|
return d["id"]
|
||||||
|
r = s.post(
|
||||||
|
f"{BASE}/api/v1/dataset/",
|
||||||
|
headers=h,
|
||||||
|
json={"database": db_id, "table_name": name, "sql": sql},
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()["id"]
|
||||||
|
|
||||||
|
|
||||||
|
def chart(s, h, name, ds_id, viz, params):
|
||||||
|
r = s.get(f"{BASE}/api/v1/chart/", headers=h)
|
||||||
|
for c in r.json().get("result", []):
|
||||||
|
if c.get("slice_name") == name:
|
||||||
|
return c["id"]
|
||||||
|
p = {"datasource": f"{ds_id}__table", "viz_type": viz, "row_limit": 1000, **params}
|
||||||
|
r = s.post(
|
||||||
|
f"{BASE}/api/v1/chart/",
|
||||||
|
headers=h,
|
||||||
|
json={
|
||||||
|
"slice_name": name,
|
||||||
|
"viz_type": viz,
|
||||||
|
"datasource_id": ds_id,
|
||||||
|
"datasource_type": "table",
|
||||||
|
"params": json.dumps(p),
|
||||||
|
"owners": [1, 2, 3],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()["id"]
|
||||||
|
|
||||||
|
|
||||||
|
def layout(items):
|
||||||
|
L = {
|
||||||
|
"DASHBOARD_VERSION": "v2",
|
||||||
|
"ROOT_ID": {"type": "ROOT", "id": "ROOT_ID", "children": ["GRID_ID"]},
|
||||||
|
"GRID_ID": {"type": "GRID", "id": "GRID_ID", "children": [], "parents": ["ROOT_ID"]},
|
||||||
|
}
|
||||||
|
ri = 0
|
||||||
|
|
||||||
|
def row():
|
||||||
|
nonlocal ri
|
||||||
|
ri += 1
|
||||||
|
rid = f"ROW-{ri}"
|
||||||
|
L["GRID_ID"]["children"].append(rid)
|
||||||
|
L[rid] = {
|
||||||
|
"type": "ROW",
|
||||||
|
"id": rid,
|
||||||
|
"children": [],
|
||||||
|
"parents": ["ROOT_ID", "GRID_ID"],
|
||||||
|
"meta": {"background": "BACKGROUND_TRANSPARENT"},
|
||||||
|
}
|
||||||
|
return rid
|
||||||
|
|
||||||
|
for item in items:
|
||||||
|
if item[0] == "md":
|
||||||
|
rid = row()
|
||||||
|
mid = f"MD-{ri}"
|
||||||
|
L[rid]["children"].append(mid)
|
||||||
|
L[mid] = {
|
||||||
|
"type": "MARKDOWN",
|
||||||
|
"id": mid,
|
||||||
|
"children": [],
|
||||||
|
"parents": ["ROOT_ID", "GRID_ID", rid],
|
||||||
|
"meta": {"width": 12, "height": 10, "code": f"## {item[1]}\n\n{item[2]}"},
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
cid, name = item
|
||||||
|
rid = row()
|
||||||
|
charts_in_row = [k for k in L[rid]["children"] if k.startswith("CHART-")]
|
||||||
|
if len(charts_in_row) >= 3:
|
||||||
|
rid = row()
|
||||||
|
key = f"CHART-{cid}"
|
||||||
|
L[rid]["children"].append(key)
|
||||||
|
hgt = 70 if "Recent" in name or "table" in name.lower() else 55
|
||||||
|
L[key] = {
|
||||||
|
"type": "CHART",
|
||||||
|
"id": key,
|
||||||
|
"children": [],
|
||||||
|
"parents": ["ROOT_ID", "GRID_ID", rid],
|
||||||
|
"meta": {"width": 4 if "Recent" not in name else 12, "height": hgt, "chartId": cid, "sliceName": name},
|
||||||
|
}
|
||||||
|
return L
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
s, h = session()
|
||||||
|
db_mon = get_db(s, h, "Trino · Pipeline Monitor", MONITOR_URI)
|
||||||
|
db_kfk = get_db(s, h, "Trino · Kafka CDC", KAFKA_URI)
|
||||||
|
|
||||||
|
ds_conn = ds_table(s, h, db_mon, "monitor", "debezium_connectors")
|
||||||
|
ds_topics = ds_table(s, h, db_mon, "monitor", "kafka_topics")
|
||||||
|
ds_ops = ds_table(s, h, db_mon, "monitor", "cdc_operations")
|
||||||
|
ds_recent = ds_table(s, h, db_mon, "monitor", "cdc_recent_events")
|
||||||
|
ds_spark = ds_table(s, h, db_mon, "monitor", "spark_applications")
|
||||||
|
|
||||||
|
LIVE_CDC_SQL = """
|
||||||
|
SELECT
|
||||||
|
'PostgreSQL' AS source_system,
|
||||||
|
CASE json_extract_scalar(_message, '$.payload.op')
|
||||||
|
WHEN 'c' THEN 'INSERT' WHEN 'u' THEN 'UPDATE' WHEN 'd' THEN 'DELETE' WHEN 'r' THEN 'SNAPSHOT' ELSE 'OTHER'
|
||||||
|
END AS change_type,
|
||||||
|
json_extract_scalar(_message, '$.payload.source.table') AS table_name,
|
||||||
|
json_extract_scalar(_message, '$.payload.after.order_id') AS record_key,
|
||||||
|
_timestamp AS event_time
|
||||||
|
FROM kafka.default."postgres-sales.public.sales_orders"
|
||||||
|
WHERE _timestamp > current_timestamp - INTERVAL '7' DAY
|
||||||
|
LIMIT 500
|
||||||
|
"""
|
||||||
|
ds_live = ds_sql(s, h, db_kfk, "live_cdc_postgres_sample", LIVE_CDC_SQL)
|
||||||
|
|
||||||
|
charts = []
|
||||||
|
charts.append(chart(s, h, "Debezium · Connector Status", ds_conn, "table", {}))
|
||||||
|
charts.append(
|
||||||
|
chart(
|
||||||
|
s,
|
||||||
|
h,
|
||||||
|
"Debezium · RUNNING vs State",
|
||||||
|
ds_conn,
|
||||||
|
"pie",
|
||||||
|
{
|
||||||
|
"metric": {"expressionType": "SQL", "sqlExpression": "COUNT(*)", "label": "COUNT(*)"},
|
||||||
|
"groupby": ["state"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
charts.append(
|
||||||
|
chart(
|
||||||
|
s,
|
||||||
|
h,
|
||||||
|
"Kafka · Topic Offsets",
|
||||||
|
ds_topics,
|
||||||
|
"echarts_timeseries_bar",
|
||||||
|
{
|
||||||
|
"metrics": [
|
||||||
|
{"expressionType": "SQL", "sqlExpression": "SUM(end_offset)", "label": "Messages"}
|
||||||
|
],
|
||||||
|
"groupby": ["topic"],
|
||||||
|
"row_limit": 20,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
charts.append(
|
||||||
|
chart(
|
||||||
|
s,
|
||||||
|
h,
|
||||||
|
"Kafka · Partitions per Topic",
|
||||||
|
ds_topics,
|
||||||
|
"echarts_timeseries_bar",
|
||||||
|
{
|
||||||
|
"metrics": [
|
||||||
|
{"expressionType": "SQL", "sqlExpression": "SUM(end_offset)", "label": "Offset"}
|
||||||
|
],
|
||||||
|
"groupby": ["topic", "partition_id"],
|
||||||
|
"row_limit": 30,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
charts.append(
|
||||||
|
chart(
|
||||||
|
s,
|
||||||
|
h,
|
||||||
|
"CDC · Changes by Type",
|
||||||
|
ds_ops,
|
||||||
|
"pie",
|
||||||
|
{
|
||||||
|
"metric": {"expressionType": "SQL", "sqlExpression": "SUM(event_count)", "label": "Events"},
|
||||||
|
"groupby": ["operation_label"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
charts.append(
|
||||||
|
chart(
|
||||||
|
s,
|
||||||
|
h,
|
||||||
|
"CDC · Changes per Source",
|
||||||
|
ds_ops,
|
||||||
|
"echarts_timeseries_bar",
|
||||||
|
{
|
||||||
|
"metrics": [
|
||||||
|
{"expressionType": "SQL", "sqlExpression": "SUM(event_count)", "label": "Events"}
|
||||||
|
],
|
||||||
|
"groupby": ["source_system", "operation_label"],
|
||||||
|
"row_limit": 20,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
charts.append(
|
||||||
|
chart(
|
||||||
|
s,
|
||||||
|
h,
|
||||||
|
"CDC · Recent Changes (sampled)",
|
||||||
|
ds_recent,
|
||||||
|
"table",
|
||||||
|
{
|
||||||
|
"all_columns": [
|
||||||
|
"source_system",
|
||||||
|
"operation_label",
|
||||||
|
"table_name",
|
||||||
|
"record_key",
|
||||||
|
"detail",
|
||||||
|
"event_ts",
|
||||||
|
],
|
||||||
|
"row_limit": 50,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
charts.append(
|
||||||
|
chart(
|
||||||
|
s,
|
||||||
|
h,
|
||||||
|
"CDC · Live Stream Sample (PostgreSQL)",
|
||||||
|
ds_live,
|
||||||
|
"table",
|
||||||
|
{
|
||||||
|
"all_columns": ["source_system", "change_type", "table_name", "record_key", "event_time"],
|
||||||
|
"row_limit": 100,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
charts.append(
|
||||||
|
chart(
|
||||||
|
s,
|
||||||
|
h,
|
||||||
|
"Spark · Applications",
|
||||||
|
ds_spark,
|
||||||
|
"table",
|
||||||
|
{"all_columns": ["app_id", "app_name", "state", "cores", "memory_mb", "duration_sec"]},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
charts.append(
|
||||||
|
chart(
|
||||||
|
s,
|
||||||
|
h,
|
||||||
|
"Pipeline · Total Kafka Messages",
|
||||||
|
ds_topics,
|
||||||
|
"big_number_total",
|
||||||
|
{
|
||||||
|
"metric": {
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "SUM(end_offset)",
|
||||||
|
"label": "Total Offset",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
chart_specs = [
|
||||||
|
("md", "Pipeline & Change Data Capture", "Debezium → Kafka → Spark · Live CDC visibility"),
|
||||||
|
("md", "Debezium Connect", "Connector health on kafka01 :8083"),
|
||||||
|
(charts[0], "Debezium · Connector Status"),
|
||||||
|
(charts[1], "Debezium · RUNNING vs State"),
|
||||||
|
("md", "Apache Kafka", "Topic volume & CDC streams on kafka01"),
|
||||||
|
(charts[2], "Kafka · Topic Offsets"),
|
||||||
|
(charts[3], "Kafka · Partitions per Topic"),
|
||||||
|
(charts[9], "Pipeline · Total Kafka Messages"),
|
||||||
|
("md", "Data Changes (CDC)", "INSERT / UPDATE / DELETE / SNAPSHOT — gewijzigde data"),
|
||||||
|
(charts[4], "CDC · Changes by Type"),
|
||||||
|
(charts[5], "CDC · Changes per Source"),
|
||||||
|
(charts[6], "CDC · Recent Changes (sampled)"),
|
||||||
|
(charts[7], "CDC · Live Stream Sample (PostgreSQL)"),
|
||||||
|
("md", "Apache Spark", "Batch & streaming jobs · lake01:8080"),
|
||||||
|
(charts[8], "Spark · Applications"),
|
||||||
|
]
|
||||||
|
items = chart_specs
|
||||||
|
cids = [c[0] for c in chart_specs if c[0] != "md"]
|
||||||
|
css = open(CSS_PATH).read() if os.path.exists(CSS_PATH) else ""
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"dashboard_title": DASH_TITLE,
|
||||||
|
"published": True,
|
||||||
|
"position_json": json.dumps(layout(items)),
|
||||||
|
"css": css,
|
||||||
|
"json_metadata": json.dumps(
|
||||||
|
{
|
||||||
|
"color_scheme": "palantir_ops",
|
||||||
|
"refresh_frequency": 120,
|
||||||
|
"chart_configuration": {
|
||||||
|
str(c): {"id": c, "crossFilters": {"scope": "global", "chartsInScope": cids}}
|
||||||
|
for c in cids
|
||||||
|
},
|
||||||
|
}
|
||||||
|
),
|
||||||
|
"owners": [1, 2, 3],
|
||||||
|
}
|
||||||
|
|
||||||
|
r = s.get(f"{BASE}/api/v1/dashboard/", headers=h)
|
||||||
|
dash_id = None
|
||||||
|
for d in r.json().get("result", []):
|
||||||
|
if d.get("dashboard_title") == DASH_TITLE:
|
||||||
|
dash_id = d["id"]
|
||||||
|
break
|
||||||
|
if dash_id:
|
||||||
|
r = s.put(f"{BASE}/api/v1/dashboard/{dash_id}", headers=h, json=payload)
|
||||||
|
else:
|
||||||
|
r = s.post(f"{BASE}/api/v1/dashboard/", headers=h, json=payload)
|
||||||
|
dash_id = r.json()["id"]
|
||||||
|
print("Dashboard", dash_id, r.status_code)
|
||||||
|
for cid in cids:
|
||||||
|
s.put(f"{BASE}/api/v1/chart/{cid}", headers=h, json={"dashboards": [dash_id], "owners": [1, 2, 3]})
|
||||||
|
|
||||||
|
# query contexts
|
||||||
|
os.system("python3 /tmp/fix_charts_qc.py 2>/dev/null || true")
|
||||||
|
print(f"URL: {BASE}/superset/dashboard/{dash_id}/")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,264 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Collect Debezium, Kafka CDC, and Spark metrics into postgres monitor schema."""
|
||||||
|
import json
|
||||||
|
import shlex
|
||||||
|
import subprocess
|
||||||
|
import urllib.request
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
import psycopg2
|
||||||
|
|
||||||
|
PG_DSN = "host=10.0.21.51 dbname=postgres user=mo password=Dell2026!"
|
||||||
|
KAFKA = "10.0.21.36:9092"
|
||||||
|
DEBEZIUM = "http://localhost:8083" # Kafka Connect on kafka01; fallback lake01 :8083
|
||||||
|
SPARK_MASTER = "http://10.0.21.50:8080"
|
||||||
|
|
||||||
|
CDC_TOPICS = [
|
||||||
|
("PostgreSQL", "postgres-sales.public.sales_orders"),
|
||||||
|
("MongoDB", "mongodb-supplychain.supplychain.events"),
|
||||||
|
]
|
||||||
|
|
||||||
|
OP_LABELS = {"c": "INSERT", "u": "UPDATE", "d": "DELETE", "r": "SNAPSHOT", "i": "INSERT"}
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_json(url, timeout=10):
|
||||||
|
with urllib.request.urlopen(url, timeout=timeout) as r:
|
||||||
|
return json.loads(r.read().decode())
|
||||||
|
|
||||||
|
|
||||||
|
def collect_debezium(cur):
|
||||||
|
connectors = fetch_json(f"{DEBEZIUM}/connectors")
|
||||||
|
cur.execute("DELETE FROM monitor.debezium_connectors")
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
for name in connectors:
|
||||||
|
try:
|
||||||
|
st = fetch_json(f"{DEBEZIUM}/connectors/{name}/status")
|
||||||
|
except Exception as e:
|
||||||
|
cur.execute(
|
||||||
|
"""INSERT INTO monitor.debezium_connectors
|
||||||
|
(connector_name, state, task_state, worker_id, checked_at)
|
||||||
|
VALUES (%s,%s,%s,%s,%s)""",
|
||||||
|
(name, "ERROR", str(e)[:32], "", now),
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
conn_state = st.get("connector", {}).get("state", "UNKNOWN")
|
||||||
|
tasks = st.get("tasks") or []
|
||||||
|
task_state = tasks[0].get("state", "NONE") if tasks else "NONE"
|
||||||
|
worker = st.get("connector", {}).get("worker_id", "")
|
||||||
|
cur.execute(
|
||||||
|
"""INSERT INTO monitor.debezium_connectors
|
||||||
|
(connector_name, state, task_state, worker_id, checked_at)
|
||||||
|
VALUES (%s,%s,%s,%s,%s)""",
|
||||||
|
(name, conn_state, task_state, worker, now),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
KAFKA_BIN = "/opt/kafka/bin"
|
||||||
|
USE_SSH_KAFKA = False # set True when running off-host
|
||||||
|
|
||||||
|
|
||||||
|
def _kafka_cmd(bin_name, args):
|
||||||
|
parts = [f"{KAFKA_BIN}/{bin_name}"] + list(args)
|
||||||
|
if USE_SSH_KAFKA:
|
||||||
|
remote = " ".join(shlex.quote(p) for p in parts)
|
||||||
|
full = f"ssh -o StrictHostKeyChecking=no root@10.0.21.36 {remote}"
|
||||||
|
return subprocess.check_output(full, shell=True, stderr=subprocess.DEVNULL, timeout=90, text=True)
|
||||||
|
return subprocess.check_output(parts, stderr=subprocess.DEVNULL, timeout=90, text=True)
|
||||||
|
|
||||||
|
|
||||||
|
def kafka_end_offsets(topic):
|
||||||
|
try:
|
||||||
|
out = _kafka_cmd(
|
||||||
|
"kafka-run-class.sh",
|
||||||
|
[
|
||||||
|
"kafka.tools.GetOffsetShell",
|
||||||
|
"--broker-list",
|
||||||
|
"localhost:9092",
|
||||||
|
"--topic",
|
||||||
|
topic,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
rows = []
|
||||||
|
for line in out.strip().splitlines():
|
||||||
|
parts = line.split(":")
|
||||||
|
if len(parts) >= 3:
|
||||||
|
rows.append((int(parts[1]), int(parts[2])))
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def sample_topic_messages(topic, max_msgs=3000, tail=5000):
|
||||||
|
"""Sample recent messages using kafka-console-consumer from tail."""
|
||||||
|
offsets = kafka_end_offsets(topic)
|
||||||
|
if not offsets:
|
||||||
|
return []
|
||||||
|
# Pick partition 0 for sampling
|
||||||
|
part, end = offsets[0]
|
||||||
|
start = max(0, end - tail)
|
||||||
|
try:
|
||||||
|
out = _kafka_cmd(
|
||||||
|
"kafka-console-consumer.sh",
|
||||||
|
[
|
||||||
|
"--bootstrap-server",
|
||||||
|
"localhost:9092",
|
||||||
|
"--topic",
|
||||||
|
topic,
|
||||||
|
"--partition",
|
||||||
|
str(part),
|
||||||
|
"--offset",
|
||||||
|
str(start),
|
||||||
|
"--max-messages",
|
||||||
|
str(min(max_msgs, tail)),
|
||||||
|
"--timeout-ms",
|
||||||
|
"15000",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
return [ln for ln in out.strip().split("\n") if ln.strip()]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_debezium_line(line):
|
||||||
|
try:
|
||||||
|
doc = json.loads(line)
|
||||||
|
payload = doc.get("payload") or doc
|
||||||
|
op = payload.get("op") or payload.get("operationType") or "?"
|
||||||
|
src = payload.get("source") or {}
|
||||||
|
table = src.get("table") or src.get("collection") or ""
|
||||||
|
ts_ms = payload.get("ts_ms") or src.get("ts_ms")
|
||||||
|
after = payload.get("after") or {}
|
||||||
|
before = payload.get("before") or {}
|
||||||
|
row = after if after else before
|
||||||
|
key = str(row.get("order_id") or row.get("event_id") or row.get("_id") or "")[:200]
|
||||||
|
detail = str(row.get("region") or row.get("type") or row.get("department") or "")[:200]
|
||||||
|
event_ts = None
|
||||||
|
if ts_ms:
|
||||||
|
event_ts = datetime.fromtimestamp(int(ts_ms) / 1000, tz=timezone.utc)
|
||||||
|
return op, table, key, detail, event_ts
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def collect_kafka_cdc(cur):
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
cur.execute("DELETE FROM monitor.kafka_topics")
|
||||||
|
cur.execute("DELETE FROM monitor.cdc_operations")
|
||||||
|
cur.execute("DELETE FROM monitor.cdc_recent_events")
|
||||||
|
|
||||||
|
for source, topic in CDC_TOPICS:
|
||||||
|
for part, end in kafka_end_offsets(topic):
|
||||||
|
cur.execute(
|
||||||
|
"""INSERT INTO monitor.kafka_topics (topic, partition_id, end_offset, checked_at)
|
||||||
|
VALUES (%s,%s,%s,%s)""",
|
||||||
|
(topic, part, end, now),
|
||||||
|
)
|
||||||
|
|
||||||
|
lines = sample_topic_messages(topic, max_msgs=2000, tail=3000)
|
||||||
|
ops = Counter()
|
||||||
|
recent = []
|
||||||
|
for line in lines:
|
||||||
|
parsed = parse_debezium_line(line)
|
||||||
|
if not parsed:
|
||||||
|
continue
|
||||||
|
op, table, key, detail, event_ts = parsed
|
||||||
|
ops[op] += 1
|
||||||
|
if len(recent) < 100:
|
||||||
|
recent.append((op, table, key, detail, event_ts))
|
||||||
|
|
||||||
|
for op, cnt in ops.items():
|
||||||
|
cur.execute(
|
||||||
|
"""INSERT INTO monitor.cdc_operations
|
||||||
|
(source_system, topic, operation, operation_label, event_count, checked_at)
|
||||||
|
VALUES (%s,%s,%s,%s,%s,%s)""",
|
||||||
|
(source, topic, op, OP_LABELS.get(op, op), cnt, now),
|
||||||
|
)
|
||||||
|
|
||||||
|
for op, table, key, detail, event_ts in recent[:50]:
|
||||||
|
cur.execute(
|
||||||
|
"""INSERT INTO monitor.cdc_recent_events
|
||||||
|
(source_system, topic, operation, operation_label, table_name,
|
||||||
|
record_key, detail, event_ts, sampled_at)
|
||||||
|
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
|
||||||
|
(
|
||||||
|
source,
|
||||||
|
topic,
|
||||||
|
op,
|
||||||
|
OP_LABELS.get(op, op),
|
||||||
|
table,
|
||||||
|
key,
|
||||||
|
detail,
|
||||||
|
event_ts,
|
||||||
|
now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def collect_spark(cur):
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
cur.execute("DELETE FROM monitor.spark_applications")
|
||||||
|
try:
|
||||||
|
data = fetch_json(f"{SPARK_MASTER}/json/", timeout=5)
|
||||||
|
apps = []
|
||||||
|
if isinstance(data, dict):
|
||||||
|
# Standalone master JSON
|
||||||
|
for a in data.get("activeapps", []) or []:
|
||||||
|
apps.append(a)
|
||||||
|
for a in data.get("completedapps", []) or []:
|
||||||
|
apps.append(a)
|
||||||
|
for a in apps[:20]:
|
||||||
|
cur.execute(
|
||||||
|
"""INSERT INTO monitor.spark_applications
|
||||||
|
(app_id, app_name, state, cores, memory_mb, duration_sec, checked_at)
|
||||||
|
VALUES (%s,%s,%s,%s,%s,%s,%s)
|
||||||
|
ON CONFLICT (app_id) DO UPDATE SET
|
||||||
|
app_name=EXCLUDED.app_name, state=EXCLUDED.state,
|
||||||
|
cores=EXCLUDED.cores, memory_mb=EXCLUDED.memory_mb,
|
||||||
|
duration_sec=EXCLUDED.duration_sec, checked_at=EXCLUDED.checked_at""",
|
||||||
|
(
|
||||||
|
a.get("id", "unknown"),
|
||||||
|
a.get("name", "Spark App"),
|
||||||
|
"RUNNING" if "attempts" not in a else "COMPLETED",
|
||||||
|
int(a.get("cores", 0) or 0),
|
||||||
|
int((a.get("memory", 0) or 0) / 1024 / 1024),
|
||||||
|
int(a.get("duration", 0) / 1000) if a.get("duration") else 0,
|
||||||
|
now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
# Placeholder row so dashboard shows Spark host status
|
||||||
|
cur.execute(
|
||||||
|
"""INSERT INTO monitor.spark_applications
|
||||||
|
(app_id, app_name, state, cores, memory_mb, duration_sec, checked_at)
|
||||||
|
VALUES (%s,%s,%s,%s,%s,%s,%s)
|
||||||
|
ON CONFLICT (app_id) DO UPDATE SET state=EXCLUDED.state, checked_at=EXCLUDED.checked_at""",
|
||||||
|
(
|
||||||
|
"spark-master",
|
||||||
|
f"Spark Master @ {SPARK_MASTER}",
|
||||||
|
"REACHABLE" if "Connection" not in str(e) else "UNREACHABLE",
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
conn = psycopg2.connect(PG_DSN)
|
||||||
|
conn.autocommit = True
|
||||||
|
cur = conn.cursor()
|
||||||
|
print("Collecting Debezium...")
|
||||||
|
collect_debezium(cur)
|
||||||
|
print("Collecting Kafka CDC samples...")
|
||||||
|
collect_kafka_cdc(cur)
|
||||||
|
print("Collecting Spark...")
|
||||||
|
collect_spark(cur)
|
||||||
|
cur.close()
|
||||||
|
conn.close()
|
||||||
|
print("Done.")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
"""Generate and save query_context for API-created Superset charts."""
|
||||||
|
import json
|
||||||
|
|
||||||
|
app = __import__("superset.app", fromlist=["create_app"]).create_app()
|
||||||
|
|
||||||
|
with app.app_context():
|
||||||
|
from flask import g
|
||||||
|
from superset.extensions import db
|
||||||
|
from superset.models.slice import Slice
|
||||||
|
from superset.models.core import Database
|
||||||
|
from superset.charts.schemas import ChartDataQueryContextSchema
|
||||||
|
from superset import security_manager
|
||||||
|
|
||||||
|
admin = security_manager.find_user(username="admin")
|
||||||
|
g.user = admin
|
||||||
|
|
||||||
|
charts = db.session.query(Slice).order_by(Slice.id).all()
|
||||||
|
for sl in charts:
|
||||||
|
try:
|
||||||
|
fd = sl.form_data
|
||||||
|
metric = fd.get("metric")
|
||||||
|
metrics = fd.get("metrics") or ([metric] if metric else [])
|
||||||
|
if not metrics:
|
||||||
|
metrics = [
|
||||||
|
{
|
||||||
|
"expressionType": "SQL",
|
||||||
|
"sqlExpression": "COUNT(*)",
|
||||||
|
"label": "COUNT(*)",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
groupby = fd.get("groupby") or []
|
||||||
|
payload = {
|
||||||
|
"datasource": {
|
||||||
|
"id": sl.datasource_id,
|
||||||
|
"type": sl.datasource_type,
|
||||||
|
},
|
||||||
|
"force": False,
|
||||||
|
"queries": [
|
||||||
|
{
|
||||||
|
"filters": [],
|
||||||
|
"extras": {"having": "", "where": ""},
|
||||||
|
"applied_time_extras": {},
|
||||||
|
"columns": groupby if isinstance(groupby, list) else [],
|
||||||
|
"metrics": metrics,
|
||||||
|
"orderby": [],
|
||||||
|
"annotation_layers": [],
|
||||||
|
"row_limit": int(fd.get("row_limit") or 1000),
|
||||||
|
"series_limit": 0,
|
||||||
|
"order_desc": True,
|
||||||
|
"url_params": {},
|
||||||
|
"custom_params": {},
|
||||||
|
"custom_form_data": {},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"form_data": fd,
|
||||||
|
"result_format": "json",
|
||||||
|
"result_type": "full",
|
||||||
|
}
|
||||||
|
qc = ChartDataQueryContextSchema().load(payload)
|
||||||
|
ctx = qc.cache_values if hasattr(qc, "cache_values") else None
|
||||||
|
if ctx is None:
|
||||||
|
# fallback: store factory input dict
|
||||||
|
from superset.common.query_context_factory import QueryContextFactory
|
||||||
|
|
||||||
|
factory = QueryContextFactory()
|
||||||
|
ctx = {
|
||||||
|
"datasource": {
|
||||||
|
"id": sl.datasource_id,
|
||||||
|
"type": sl.datasource_type,
|
||||||
|
},
|
||||||
|
"force": False,
|
||||||
|
"queries": payload["queries"],
|
||||||
|
"form_data": fd,
|
||||||
|
"result_format": "json",
|
||||||
|
"result_type": "full",
|
||||||
|
}
|
||||||
|
sl.query_context = json.dumps(ctx) if isinstance(ctx, dict) else json.dumps(payload)
|
||||||
|
sl.query_context_generation = True
|
||||||
|
db.session.add(sl)
|
||||||
|
print("OK", sl.id, sl.slice_name[:50])
|
||||||
|
except Exception as e:
|
||||||
|
print("ERR", sl.id, sl.slice_name[:40], e)
|
||||||
|
db.session.commit()
|
||||||
|
print("committed")
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
/* ATC Lakehouse dashboard — Palantir OPS overlay */
|
||||||
|
.dashboard-wrapper,
|
||||||
|
.dashboard,
|
||||||
|
.grid-container,
|
||||||
|
.dashboard-content,
|
||||||
|
.dashboard-component-chart-holder {
|
||||||
|
background: transparent !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dashboard-header-container {
|
||||||
|
background: linear-gradient(135deg, rgba(12, 28, 52, 0.95) 0%, rgba(6, 20, 40, 0.98) 100%) !important;
|
||||||
|
border-bottom: 1px solid rgba(56, 132, 220, 0.25) !important;
|
||||||
|
backdrop-filter: blur(12px);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dashboard-header .dashboard-title {
|
||||||
|
font-family: 'DM Sans', system-ui, sans-serif !important;
|
||||||
|
font-weight: 700 !important;
|
||||||
|
letter-spacing: -0.02em !important;
|
||||||
|
background: linear-gradient(90deg, #e8eef7 0%, #22d3ee 50%, #fb923c 100%);
|
||||||
|
-webkit-background-clip: text;
|
||||||
|
-webkit-text-fill-color: transparent;
|
||||||
|
background-clip: text;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dashboard-component {
|
||||||
|
background: rgba(12, 28, 52, 0.72) !important;
|
||||||
|
border: 1px solid rgba(56, 132, 220, 0.2) !important;
|
||||||
|
border-radius: 12px !important;
|
||||||
|
box-shadow: 0 4px 24px rgba(0, 0, 0, 0.35), inset 0 1px 0 rgba(255, 255, 255, 0.04) !important;
|
||||||
|
backdrop-filter: blur(8px);
|
||||||
|
transition: border-color 0.2s ease, box-shadow 0.2s ease;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dashboard-component:hover {
|
||||||
|
border-color: rgba(251, 146, 60, 0.35) !important;
|
||||||
|
box-shadow: 0 8px 32px rgba(37, 99, 235, 0.2) !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
.chart-header,
|
||||||
|
.header-title,
|
||||||
|
.header-line {
|
||||||
|
color: #e8eef7 !important;
|
||||||
|
font-family: 'DM Sans', system-ui, sans-serif !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
.slice_container,
|
||||||
|
.chart-container,
|
||||||
|
.dashboard-chart-id {
|
||||||
|
background: transparent !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Big number / KPI tiles */
|
||||||
|
.big-number .header-line,
|
||||||
|
.big-number-viz .header-line {
|
||||||
|
color: #22d3ee !important;
|
||||||
|
font-size: 2.5rem !important;
|
||||||
|
font-weight: 700 !important;
|
||||||
|
text-shadow: 0 0 24px rgba(34, 211, 238, 0.35);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Markdown section headers */
|
||||||
|
.dashboard-markdown,
|
||||||
|
.markdown-component {
|
||||||
|
background: linear-gradient(90deg, rgba(59, 130, 246, 0.12), transparent) !important;
|
||||||
|
border-left: 3px solid #3b82f6 !important;
|
||||||
|
padding: 12px 16px !important;
|
||||||
|
border-radius: 0 8px 8px 0 !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dashboard-markdown h1,
|
||||||
|
.dashboard-markdown h2,
|
||||||
|
.markdown-component h1,
|
||||||
|
.markdown-component h2 {
|
||||||
|
color: #e8eef7 !important;
|
||||||
|
font-family: 'DM Sans', sans-serif !important;
|
||||||
|
margin: 0 !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dashboard-markdown p,
|
||||||
|
.markdown-component p {
|
||||||
|
color: #94a3b8 !important;
|
||||||
|
margin: 4px 0 0 !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Filter bar */
|
||||||
|
.filter-status-pane,
|
||||||
|
.dashboard-filters-panel {
|
||||||
|
background: rgba(6, 20, 40, 0.9) !important;
|
||||||
|
border: 1px solid rgba(56, 132, 220, 0.2) !important;
|
||||||
|
border-radius: 10px !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Grid subtle glow */
|
||||||
|
.grid-row {
|
||||||
|
margin-bottom: 8px;
|
||||||
|
}
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
app = __import__("superset.app", fromlist=["create_app"]).create_app()
|
||||||
|
|
||||||
|
with app.app_context():
|
||||||
|
from superset.extensions import db
|
||||||
|
from superset.models.slice import Slice
|
||||||
|
charts = db.session.query(Slice).all()
|
||||||
|
for sl in charts:
|
||||||
|
try:
|
||||||
|
sl.query_context = sl.get_query_context()
|
||||||
|
db.session.add(sl)
|
||||||
|
print("saved", sl.id, (sl.slice_name or "")[:50])
|
||||||
|
except Exception as e:
|
||||||
|
print("err", sl.id, e)
|
||||||
|
db.session.commit()
|
||||||
|
print("committed", len(charts), "charts")
|
||||||
@@ -1,39 +1,117 @@
|
|||||||
import os
|
import os
|
||||||
|
|
||||||
# Secret key for session signing
|
SECRET_KEY = os.environ.get("SUPERSET_SECRET_KEY", "your-secret-key-here")
|
||||||
SECRET_KEY = os.environ.get('SUPERSET_SECRET_KEY', 'your-secret-key-here')
|
SQLALCHEMY_DATABASE_URI = "sqlite:////app/superset_home/superset.db"
|
||||||
|
|
||||||
# Database configuration - use SQLite to avoid psycopg2 issues
|
|
||||||
SQLALCHEMY_DATABASE_URI = 'sqlite:////app/superset_home/superset.db'
|
|
||||||
|
|
||||||
# Redis cache configuration
|
|
||||||
CACHE_CONFIG = {
|
CACHE_CONFIG = {
|
||||||
'CACHE_TYPE': 'redis',
|
"CACHE_TYPE": "redis",
|
||||||
'CACHE_REDIS_URL': 'redis://redis:6379/0',
|
"CACHE_REDIS_URL": "redis://redis:6379/0",
|
||||||
'CACHE_DEFAULT_TIMEOUT': 300
|
"CACHE_DEFAULT_TIMEOUT": 300,
|
||||||
}
|
}
|
||||||
|
|
||||||
# Enable CSRF protection
|
|
||||||
ENABLE_PROXY_FIX = True
|
ENABLE_PROXY_FIX = True
|
||||||
|
TIMEZONE = "Europe/Amsterdam"
|
||||||
# Feature flags
|
|
||||||
FEATURE_FLAGS = {
|
|
||||||
'ENABLE_TEMPLATE_PROCESSING': True,
|
|
||||||
'ALERT_REPORTS': True,
|
|
||||||
}
|
|
||||||
|
|
||||||
# Row limit
|
|
||||||
ROW_LIMIT = 50000
|
ROW_LIMIT = 50000
|
||||||
|
|
||||||
# Viz types
|
# Branding — logo must be same-origin (/static/...) for CSP (img-src 'self')
|
||||||
VIZ_TYPE_DICT = {
|
APP_NAME = "Dell"
|
||||||
'table': {},
|
APP_ICON = "/static/assets/images/dell-logo.svg"
|
||||||
'dist_bar': {},
|
LOGO_TARGET_PATH = "/superset/welcome/"
|
||||||
'line': {},
|
LOGO_TOOLTIP = "Dell · ATC Lakehouse"
|
||||||
'area': {},
|
|
||||||
'pie': {},
|
FEATURE_FLAGS = {
|
||||||
'number': {},
|
"ENABLE_TEMPLATE_PROCESSING": True,
|
||||||
|
"ALERT_REPORTS": True,
|
||||||
|
"DASHBOARD_NATIVE_FILTERS": True,
|
||||||
|
"DASHBOARD_CROSS_FILTERS": True,
|
||||||
|
"ENABLE_ADVANCED_DATA_TYPES": True,
|
||||||
}
|
}
|
||||||
|
|
||||||
# Timezone
|
# Allow icons server if needed for other assets (optional)
|
||||||
TIMEZONE = 'Europe/Amsterdam'
|
TALISMAN_ENABLED = True
|
||||||
|
TALISMAN_CONFIG = {
|
||||||
|
"content_security_policy": {
|
||||||
|
"base-uri": ["'self'"],
|
||||||
|
"default-src": ["'self'"],
|
||||||
|
"img-src": [
|
||||||
|
"'self'",
|
||||||
|
"blob:",
|
||||||
|
"data:",
|
||||||
|
"https://apachesuperset.gateway.scarf.sh",
|
||||||
|
"https://static.scarf.sh/",
|
||||||
|
"http://atc-docker01.dell-atc.lan:8080",
|
||||||
|
"https://atc-docker01.dell-atc.lan:8080",
|
||||||
|
],
|
||||||
|
"worker-src": ["'self'", "blob:"],
|
||||||
|
"connect-src": ["'self'"],
|
||||||
|
"object-src": "'none'",
|
||||||
|
"style-src": ["'self'", "'unsafe-inline'"],
|
||||||
|
"font-src": ["'self'"],
|
||||||
|
"script-src": ["'self'", "'strict-dynamic'"],
|
||||||
|
},
|
||||||
|
"content_security_policy_nonce_in": ["script-src"],
|
||||||
|
"force_https": False,
|
||||||
|
"frame_options": "SAMEORIGIN",
|
||||||
|
}
|
||||||
|
|
||||||
|
EXTRA_CATEGORICAL_COLOR_SCHEMES = [
|
||||||
|
{
|
||||||
|
"id": "palantir_ops",
|
||||||
|
"description": "Palantir OPS — blue, cyan, orange, teal",
|
||||||
|
"label_colors": {},
|
||||||
|
"isDefault": True,
|
||||||
|
"colors": [
|
||||||
|
"#3b82f6", "#22d3ee", "#fb923c", "#2dd4bf", "#fbbf24",
|
||||||
|
"#a78bfa", "#f472b6", "#34d399", "#60a5fa", "#94a3b8",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
EXTRA_SEQUENTIAL_COLOR_SCHEMES = [
|
||||||
|
{
|
||||||
|
"id": "palantir_blue",
|
||||||
|
"description": "Palantir blue gradient",
|
||||||
|
"isDefault": True,
|
||||||
|
"colors": ["#040c18", "#0c1a30", "#1e40af", "#3b82f6", "#22d3ee", "#7dd3fc"],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
PALANTIR_FONTS = [
|
||||||
|
"https://fonts.googleapis.com/css2?family=DM+Sans:ital,opsz,wght@0,9..40,400;0,9..40,500;0,9..40,600;0,9..40,700&family=JetBrains+Mono:wght@400;500;600&display=swap",
|
||||||
|
]
|
||||||
|
|
||||||
|
PALANTIR_TOKENS = {
|
||||||
|
"brandAppName": "Dell",
|
||||||
|
"brandLogoAlt": "Dell",
|
||||||
|
"brandLogoUrl": "/static/assets/images/dell-logo.svg",
|
||||||
|
"brandLogoMargin": "8px 12px 8px 0",
|
||||||
|
"brandLogoHref": "/",
|
||||||
|
"brandLogoHeight": "32px",
|
||||||
|
"colorPrimary": "#007DB8",
|
||||||
|
"colorLink": "#22d3ee",
|
||||||
|
"colorSuccess": "#2dd4bf",
|
||||||
|
"colorWarning": "#fbbf24",
|
||||||
|
"colorError": "#f87171",
|
||||||
|
"colorInfo": "#38bdf8",
|
||||||
|
"colorBgBase": "#040c18",
|
||||||
|
"colorBgLayout": "#061428",
|
||||||
|
"colorBgContainer": "#0c1a30",
|
||||||
|
"colorBgElevated": "#0f2444",
|
||||||
|
"colorBorder": "#1e3a5f",
|
||||||
|
"colorBorderSecondary": "rgba(56, 132, 220, 0.22)",
|
||||||
|
"colorText": "#e8eef7",
|
||||||
|
"colorTextSecondary": "#94a3b8",
|
||||||
|
"colorTextTertiary": "#64748b",
|
||||||
|
"fontUrls": PALANTIR_FONTS,
|
||||||
|
"fontFamily": "'DM Sans', Inter, Helvetica, Arial, sans-serif",
|
||||||
|
"fontFamilyCode": "'JetBrains Mono', 'IBM Plex Mono', monospace",
|
||||||
|
"borderRadius": 8,
|
||||||
|
"borderRadiusLG": 12,
|
||||||
|
}
|
||||||
|
|
||||||
|
THEME_DEFAULT = {
|
||||||
|
"algorithm": "dark",
|
||||||
|
"token": PALANTIR_TOKENS,
|
||||||
|
}
|
||||||
|
THEME_DARK = None
|
||||||
|
ENABLE_UI_THEME_ADMINISTRATION = False
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
connector.name=kafka
|
||||||
|
kafka.nodes=10.0.21.36:9092
|
||||||
|
kafka.table-names=postgres-sales.public.sales_orders,mongodb-supplychain.supplychain.events,schema-changes.hr
|
||||||
|
kafka.hide-internal-columns=false
|
||||||
@@ -1,3 +1,3 @@
|
|||||||
connector.name=mongodb
|
connector.name=mongodb
|
||||||
mongodb.connection-url=mongodb://10.0.21.51:27017/
|
mongodb.connection-url=mongodb://mo:Dell2026%21@10.0.21.51:27017/?authSource=admin
|
||||||
mongodb.schema-collection=__trino_schema
|
mongodb.schema-collection=__trino_schema
|
||||||
|
|||||||
Reference in New Issue
Block a user