ATC/Lakehouse
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Full lab documentation and infrastructure as code

Browse Source 
- Trino catalogs, Grafana, Spark jobs, LDAP LDIF, NPM compose
- Airflow DAG scripts, Proxmox VM inventory, network docs
- Ansible playbook, Gitea CI validate workflow
- Backup and health-check scripts, cron documentation
- Homepage DOCS tab with links to all documentation
- Extended collect-fleet-config.sh and populate-repo.py
This commit is contained in:
Lakehouse Admin
2026-05-19 23:12:41 +02:00
parent 3c8993a41e
commit df5ec93dc3
46 changed files with 1809 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/spark-jobs/README.md
+1
View File
@@ -0,0 +1 @@
# Spark jobs
config/spark-jobs/consumer.py
+38
View File
@@ -0,0 +1,38 @@
from kafka import KafkaConsumer
import boto3
import json
from datetime import datetime
KAFKA_BROKER = "10.0.21.36:9092"
KAFKA_TOPIC = "test-lakehouse"
S3_ENDPOINT = "http://10.0.20.111:9020"
S3_BUCKET = "data"
S3_ACCESS_KEY = "REDACTED"
S3_SECRET_KEY = "REDACTED"
print("Starting consumer...")
print(f"Kafka: {KAFKA_BROKER}, Topic: {KAFKA_TOPIC}")
print(f"S3: {S3_ENDPOINT}, Bucket: {S3_BUCKET}")
s3 = boto3.client(
's3',
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY,
aws_secret_access_key=S3_SECRET_KEY,
use_ssl=False,
verify=False
)
consumer = KafkaConsumer(
KAFKA_TOPIC,
bootstrap_servers=[KAFKA_BROKER],
auto_offset_reset='earliest',
value_deserializer=lambda x: json.loads(x.decode('utf-8'))
)
print("Waiting for messages...")
for msg in consumer:
ts = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
key = f"kafka/{KAFKA_TOPIC}/msg_{ts}.json"
s3.put_object(Bucket=S3_BUCKET, Key=key, Body=json.dumps(msg.value))
print(f"Written: {key}")
config/spark-jobs/consumer_fixed.py
+48
View File
@@ -0,0 +1,48 @@
from kafka import KafkaConsumer
import boto3
import json
from datetime import datetime
KAFKA_BROKER = "10.0.21.36:9092"
KAFKA_TOPIC = "test-lakehouse"
S3_ENDPOINT = "http://10.0.20.111:9020"
S3_BUCKET = "data"
S3_ACCESS_KEY = "REDACTED"
S3_SECRET_KEY = "REDACTED"
print("Starting consumer...")
print(f"Kafka: {KAFKA_BROKER}, Topic: {KAFKA_TOPIC}")
s3 = boto3.client(
's3',
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY,
aws_secret_access_key=S3_SECRET_KEY,
use_ssl=False,
verify=False
)
def deserialize(value):
if value is None or len(value) == 0:
return None
try:
return json.loads(value.decode('utf-8'))
except:
print(f"Skipping non-JSON")
return None
consumer = KafkaConsumer(
KAFKA_TOPIC,
bootstrap_servers=[KAFKA_BROKER],
auto_offset_reset='latest',
value_deserializer=deserialize
)
print("Waiting for messages...")
for msg in consumer:
if msg.value is None:
continue
ts = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
key = f"kafka/{KAFKA_TOPIC}/msg_{ts}.json"
s3.put_object(Bucket=S3_BUCKET, Key=key, Body=json.dumps(msg.value))
print(f"Written: {key}")
config/spark-jobs/kafka_to_s3.py
+34
View File
@@ -0,0 +1,34 @@
from pyspark.sql import SparkSession
from pyspark.sql.functions import col, current_timestamp
spark = SparkSession.builder \
.appName("KafkaToS3") \
.config("spark.hadoop.fs.s3a.endpoint", "http://10.0.20.111:9020") \
.config("spark.hadoop.fs.s3a.access.key", "AKIA38FD4BA7FC1FB43C") \
.config("spark.hadoop.fs.s3a.secret.key", "IK7M3ro+CWb7f4OyNKdK1W2SCvMwJTrPX1NBDwsj") \
.config("spark.hadoop.fs.s3a.path.style.access", "true") \
.config("spark.hadoop.fs.s3a.connection.ssl.enabled", "false") \
.getOrCreate()
spark.sparkContext.setLogLevel("WARN")
df = spark.readStream \
.format("kafka") \
.option("kafka.bootstrap.servers", "10.0.21.36:9092") \
.option("subscribe", "test-lakehouse") \
.option("startingOffsets", "latest") \
.load()
output = df.select(
col("key").cast("string"),
col("value").cast("string"),
current_timestamp().alias("timestamp")
)
query = output.writeStream \
.outputMode("append") \
.format("console") \
.start()
print("Streaming started. Press Ctrl+C to stop.")
query.awaitTermination()
config/spark-jobs/kafka_to_s3_consumer.py
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env python3
from kafka import KafkaConsumer
import boto3
import json
import os
import sys
from datetime import datetime
# Configuratie
KAFKA_BROKER = "10.0.21.36:9092"
KAFKA_TOPIC = "test-lakehouse"
S3_ENDPOINT = "http://10.0.20.111:9020"
S3_BUCKET = "data"
S3_ACCESS_KEY = "REDACTED"
S3_SECRET_KEY = "REDACTED"
print("=" * 60)
print(f"Kafka Consumer starting...")
print(f"Kafka broker: {KAFKA_BROKER}")
print(f"Kafka topic: {KAFKA_TOPIC}")
print(f"S3 endpoint: {S3_ENDPOINT}")
print(f"S3 bucket: {S3_BUCKET}")
print("=" * 60)
# S3 client
s3 = boto3.client(
's3',
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY,
aws_secret_access_key=S3_SECRET_KEY,
use_ssl=False,
verify=False
)
# Kafka consumer
try:
consumer = KafkaConsumer(
KAFKA_TOPIC,
bootstrap_servers=[KAFKA_BROKER],
auto_offset_reset='earliest',
enable_auto_commit=True,
value_deserializer=lambda x: json.loads(x.decode('utf-8'))
)
print("Connected to Kafka successfully!")
except Exception as e:
print(f"Failed to connect to Kafka: {e}")
sys.exit(1)
print(f"Listening for messages on topic '{KAFKA_TOPIC}'...")
print("-" * 60)
message_count = 0
for message in consumer:
message_count += 1
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
key = f"kafka/{KAFKA_TOPIC}/message_{timestamp}.json"
try:
s3.put_object(
Bucket=S3_BUCKET,
Key=key,
Body=json.dumps(message.value, indent=2)
)
print(f"[{message_count}] Written to s3://{S3_BUCKET}/{key}")
print(f" Data: {json.dumps(message.value)[:100]}...")
except Exception as e:
print(f"Error writing to S3: {e}")
config/spark-jobs/s3_consumer_all_topics.py
+69
View File
@@ -0,0 +1,69 @@
from kafka import KafkaConsumer
from kafka import TopicPartition
import boto3
import json
from datetime import datetime
# Configuratie
KAFKA_BROKER = "10.0.21.36:9092"
S3_ENDPOINT = "http://10.0.20.111:9020"
S3_BUCKET = "data"
S3_ACCESS_KEY = "REDACTED"
S3_SECRET_KEY = "REDACTED"
print("Starting S3 consumer for ALL topics...")
print(f"Kafka: {KAFKA_BROKER}")
print(f"S3: {S3_ENDPOINT}, Bucket: {S3_BUCKET}")
# S3 client
s3 = boto3.client(
's3',
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY,
aws_secret_access_key=S3_SECRET_KEY,
use_ssl=False,
verify=False
)
# Consumer zonder specifiek topic
consumer = KafkaConsumer(bootstrap_servers=[KAFKA_BROKER])
# Haal alle topics op
all_topics = consumer.topics()
print(f"Found topics: {list(all_topics)}")
# Wijs alle partitions van alle topics toe
for topic in all_topics:
partitions = consumer.partitions_for_topic(topic)
for partition in partitions:
tp = TopicPartition(topic, partition)
consumer.assign([tp])
consumer.seek_to_beginning()
print(f"Assigned: {topic} - partition {partition}")
print("\nReading all messages from all topics...")
print("-" * 60)
msg_count = 0
topic_count = {}
for msg in consumer:
msg_count += 1
topic = msg.topic
topic_count[topic] = topic_count.get(topic, 0) + 1
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
key = f"kafka/{topic}/message_{timestamp}.json"
try:
data = json.loads(msg.value.decode('utf-8'))
except:
data = {"raw": msg.value.decode('utf-8')}
s3.put_object(
Bucket=S3_BUCKET,
Key=key,
Body=json.dumps(data, indent=2)
)
print(f"[{msg_count}] {topic} -> s3://{S3_BUCKET}/{key}")
config/spark-jobs/s3_consumer_realtime.py
+45
View File
@@ -0,0 +1,45 @@
from kafka import KafkaConsumer
import boto3
import json
from datetime import datetime
KAFKA_BROKER = "10.0.21.36:9092"
S3_ENDPOINT = "http://10.0.20.111:9020"
S3_BUCKET = "data"
S3_ACCESS_KEY = "REDACTED"
S3_SECRET_KEY = "REDACTED"
s3 = boto3.client(
's3',
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY,
aws_secret_access_key=S3_SECRET_KEY,
use_ssl=False,
verify=False
)
# Consumer die naar ALLE topics luistert (alleen nieuwe berichten)
consumer = KafkaConsumer(
bootstrap_servers=[KAFKA_BROKER],
auto_offset_reset='latest'
)
# Subscribe op alle topics
consumer.subscribe(pattern='.*')
print("Listening to ALL topics...")
print("Waiting for new messages...")
msg_count = 0
for msg in consumer:
msg_count += 1
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
key = f"kafka/{msg.topic}/message_{timestamp}.json"
try:
data = json.loads(msg.value.decode('utf-8'))
except:
data = {"raw": msg.value.decode('utf-8')}
s3.put_object(Bucket=S3_BUCKET, Key=key, Body=json.dumps(data, indent=2))
print(f"[{msg_count}] {msg.topic} -> s3://{S3_BUCKET}/{key}")
config/spark-jobs/s3_consumer_working.py
+57
View File
@@ -0,0 +1,57 @@
from kafka import KafkaConsumer
from kafka import TopicPartition
import boto3
import json
from datetime import datetime
# Configuratie
KAFKA_BROKER = "10.0.21.36:9092"
KAFKA_TOPIC = "test-lakehouse"
S3_ENDPOINT = "http://10.0.20.111:9020"
S3_BUCKET = "data"
S3_ACCESS_KEY = "REDACTED"
S3_SECRET_KEY = "REDACTED"
print("Starting S3 consumer...")
print(f"Kafka: {KAFKA_BROKER}, Topic: {KAFKA_TOPIC}")
print(f"S3: {S3_ENDPOINT}, Bucket: {S3_BUCKET}")
# S3 client
s3 = boto3.client(
's3',
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY,
aws_secret_access_key=S3_SECRET_KEY,
use_ssl=False,
verify=False
)
# Kafka consumer met vaste partition
consumer = KafkaConsumer(bootstrap_servers=[KAFKA_BROKER])
tp = TopicPartition(KAFKA_TOPIC, 0)
consumer.assign([tp])
consumer.seek_to_beginning()
print("Reading all messages from beginning...")
print("-" * 50)
msg_count = 0
for msg in consumer:
msg_count += 1
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
key = f"kafka/{KAFKA_TOPIC}/message_{timestamp}.json"
try:
data = json.loads(msg.value.decode('utf-8'))
except:
data = {"raw": msg.value.decode('utf-8')}
s3.put_object(
Bucket=S3_BUCKET,
Key=key,
Body=json.dumps(data, indent=2)
)
print(f"[{msg_count}] Written: s3://{S3_BUCKET}/{key}")
print(f" Data: {str(data)[:80]}...")
print(f"Done! Total {msg_count} messages written to S3")
config/spark-jobs/simple_consumer.py
+12
View File
@@ -0,0 +1,12 @@
from kafka import KafkaConsumer
import json
consumer = KafkaConsumer(
'test-lakehouse',
bootstrap_servers=['10.0.21.36:9092'],
auto_offset_reset='latest'
)
print("Waiting for new messages...")
for msg in consumer:
print(f"Received: {msg.value.decode('utf-8')}")