diff --git a/infra/hadoop/s3util.py b/infra/hadoop/s3util.py new file mode 100644 index 0000000..f7c52fa --- /dev/null +++ b/infra/hadoop/s3util.py @@ -0,0 +1,44 @@ +"""List/delete objects under a prefix in the Dell ECS S3 bucket. + +Credentials are read from the environment so no secrets are committed: + S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY, S3_BUCKET (default "data"). +ECS rejects bulk delete (Content-MD5), so deletes are done one object at a time. +""" +import os +import sys +import boto3 +from botocore.config import Config + +s3 = boto3.client( + "s3", + endpoint_url=os.environ["S3_ENDPOINT"], + aws_access_key_id=os.environ["S3_ACCESS_KEY"], + aws_secret_access_key=os.environ["S3_SECRET_KEY"], + config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"), +) +BUCKET = os.environ.get("S3_BUCKET", "data") + + +def listp(prefix): + p = s3.get_paginator("list_objects_v2") + n = 0 + for page in p.paginate(Bucket=BUCKET, Prefix=prefix): + for o in page.get("Contents", []): + print(f"{o['Size']:>12} {o['Key']}") + n += 1 + print(f"# total objects under '{prefix}': {n}") + + +def delp(prefix): + p = s3.get_paginator("list_objects_v2") + n = 0 + for page in p.paginate(Bucket=BUCKET, Prefix=prefix): + for o in page.get("Contents", []): + s3.delete_object(Bucket=BUCKET, Key=o["Key"]) + n += 1 + print(f"# deleted {n} objects under '{prefix}'") + + +if __name__ == "__main__": + cmd, prefix = sys.argv[1], sys.argv[2] + (listp if cmd == "ls" else delp)(prefix) diff --git a/infra/hadoop/trino_query.py b/infra/hadoop/trino_query.py new file mode 100644 index 0000000..ed225c4 --- /dev/null +++ b/infra/hadoop/trino_query.py @@ -0,0 +1,37 @@ +import sys, json, time, urllib.request + +TRINO = "http://127.0.0.1:8089" +USER = "mo" + +def run(sql): + req = urllib.request.Request( + TRINO + "/v1/statement", data=sql.encode(), + headers={"X-Trino-User": USER, "X-Trino-Catalog": "iceberg", + "Content-Type": "text/plain"}) + rows, cols = [], None + r = json.loads(urllib.request.urlopen(req).read()) + while True: + if r.get("columns") and cols is None: + cols = [c["name"] for c in r["columns"]] + rows.extend(r.get("data", []) or []) + nxt = r.get("nextUri") + st = r.get("stats", {}).get("state") + err = r.get("error") + if err: + raise RuntimeError(json.dumps(err.get("message", err))) + if not nxt: + break + time.sleep(0.1) + r = json.loads(urllib.request.urlopen(urllib.request.Request( + nxt, headers={"X-Trino-User": USER})).read()) + return cols, rows + +if __name__ == "__main__": + sql = sys.stdin.read() if len(sys.argv) < 2 else sys.argv[1] + for stmt in [s for s in sql.split(";\n") if s.strip()]: + cols, rows = run(stmt.strip()) + print(f"--- {stmt.strip()[:70]} ---") + if cols: + print("\t".join(cols)) + for row in rows[:50]: + print("\t".join(str(x) for x in row)) diff --git a/infra/superset/reset_users.py b/infra/superset/reset_users.py new file mode 100644 index 0000000..72631ef --- /dev/null +++ b/infra/superset/reset_users.py @@ -0,0 +1,23 @@ +"""Ensure Superset users mo+bart exist, are active Admins, and (re)set their +password. Run inside the superset container: docker exec superset python reset_users.py +Password is read from SUPERSET_USER_PASSWORD (no secret committed).""" +import os +from superset.app import create_app + +PW = os.environ.get("SUPERSET_USER_PASSWORD", "change-me") +app = create_app() +with app.app_context(): + from superset import security_manager as sm + from werkzeug.security import generate_password_hash, check_password_hash + + admin_role = sm.find_role("Admin") + for uname, email in [("mo", "mo@lakehouse.local"), ("bart", "bart@lakehouse.local")]: + u = sm.find_user(username=uname) + if not u: + u = sm.add_user(uname, uname.capitalize(), "User", email, admin_role, password=PW) + u.active = True + if admin_role not in u.roles: + u.roles = list({*u.roles, admin_role}) + u.password = generate_password_hash(PW) + sm.update_user(u) + print(uname, "ok:", check_password_hash(u.password, PW)) diff --git a/infra/superset/superset_config.py b/infra/superset/superset_config.py new file mode 100644 index 0000000..cf525fc --- /dev/null +++ b/infra/superset/superset_config.py @@ -0,0 +1,120 @@ +import os + +SECRET_KEY = os.environ.get("SUPERSET_SECRET_KEY", "your-secret-key-here") +SQLALCHEMY_DATABASE_URI = "sqlite:////app/superset_home/superset.db" + +CACHE_CONFIG = { + "CACHE_TYPE": "redis", + "CACHE_REDIS_URL": "redis://redis:6379/0", + "CACHE_DEFAULT_TIMEOUT": 300, +} + +ENABLE_PROXY_FIX = True +SESSION_COOKIE_SECURE = False +WTF_CSRF_SSL_STRICT = False +TIMEZONE = "Europe/Amsterdam" +ROW_LIMIT = 50000 + +# Branding — logo must be same-origin (/static/...) for CSP (img-src 'self') +APP_NAME = "Dell" +APP_ICON = "/static/assets/images/dell-logo.svg" +LOGO_TARGET_PATH = "/superset/welcome/" +LOGO_TOOLTIP = "Dell · ATC Lakehouse" + +FEATURE_FLAGS = { + "ENABLE_TEMPLATE_PROCESSING": True, + "ALERT_REPORTS": True, + "DASHBOARD_NATIVE_FILTERS": True, + "DASHBOARD_CROSS_FILTERS": True, + "ENABLE_ADVANCED_DATA_TYPES": True, +} + +# Allow icons server if needed for other assets (optional) +TALISMAN_ENABLED = True +TALISMAN_CONFIG = { + "content_security_policy": { + "base-uri": ["'self'"], + "default-src": ["'self'"], + "img-src": [ + "'self'", + "blob:", + "data:", + "https://apachesuperset.gateway.scarf.sh", + "https://static.scarf.sh/", + "http://atc-docker01.dell-atc.lan:8080", + "https://atc-docker01.dell-atc.lan:8080", + ], + "worker-src": ["'self'", "blob:"], + "connect-src": ["'self'"], + "object-src": "'none'", + "style-src": ["'self'", "'unsafe-inline'"], + "font-src": ["'self'"], + "script-src": ["'self'", "'strict-dynamic'"], + }, + "content_security_policy_nonce_in": ["script-src"], + "force_https": False, + "session_cookie_secure": False, + "frame_options": "SAMEORIGIN", +} + +EXTRA_CATEGORICAL_COLOR_SCHEMES = [ + { + "id": "palantir_ops", + "description": "Palantir OPS — blue, cyan, orange, teal", + "label_colors": {}, + "isDefault": True, + "colors": [ + "#3b82f6", "#22d3ee", "#fb923c", "#2dd4bf", "#fbbf24", + "#a78bfa", "#f472b6", "#34d399", "#60a5fa", "#94a3b8", + ], + }, +] + +EXTRA_SEQUENTIAL_COLOR_SCHEMES = [ + { + "id": "palantir_blue", + "description": "Palantir blue gradient", + "isDefault": True, + "colors": ["#040c18", "#0c1a30", "#1e40af", "#3b82f6", "#22d3ee", "#7dd3fc"], + }, +] + +PALANTIR_FONTS = [ + "https://fonts.googleapis.com/css2?family=DM+Sans:ital,opsz,wght@0,9..40,400;0,9..40,500;0,9..40,600;0,9..40,700&family=JetBrains+Mono:wght@400;500;600&display=swap", +] + +PALANTIR_TOKENS = { + "brandAppName": "Dell", + "brandLogoAlt": "Dell", + "brandLogoUrl": "/static/assets/images/dell-logo.svg", + "brandLogoMargin": "8px 12px 8px 0", + "brandLogoHref": "/", + "brandLogoHeight": "32px", + "colorPrimary": "#007DB8", + "colorLink": "#22d3ee", + "colorSuccess": "#2dd4bf", + "colorWarning": "#fbbf24", + "colorError": "#f87171", + "colorInfo": "#38bdf8", + "colorBgBase": "#040c18", + "colorBgLayout": "#061428", + "colorBgContainer": "#0c1a30", + "colorBgElevated": "#0f2444", + "colorBorder": "#1e3a5f", + "colorBorderSecondary": "rgba(56, 132, 220, 0.22)", + "colorText": "#e8eef7", + "colorTextSecondary": "#94a3b8", + "colorTextTertiary": "#64748b", + "fontUrls": PALANTIR_FONTS, + "fontFamily": "'DM Sans', Inter, Helvetica, Arial, sans-serif", + "fontFamilyCode": "'JetBrains Mono', 'IBM Plex Mono', monospace", + "borderRadius": 8, + "borderRadiusLG": 12, +} + +THEME_DEFAULT = { + "algorithm": "dark", + "token": PALANTIR_TOKENS, +} +THEME_DARK = None +ENABLE_UI_THEME_ADMINISTRATION = False