Full lab documentation and infrastructure as code

- Trino catalogs, Grafana, Spark jobs, LDAP LDIF, NPM compose
- Airflow DAG scripts, Proxmox VM inventory, network docs
- Ansible playbook, Gitea CI validate workflow
- Backup and health-check scripts, cron documentation
- Homepage DOCS tab with links to all documentation
- Extended collect-fleet-config.sh and populate-repo.py

config/spark-jobs/kafka_to_s3.py

@@ -0,0 +1,34 @@
+from pyspark.sql import SparkSession
+from pyspark.sql.functions import col, current_timestamp
+
+spark = SparkSession.builder \
+    .appName("KafkaToS3") \
+    .config("spark.hadoop.fs.s3a.endpoint", "http://10.0.20.111:9020") \
+    .config("spark.hadoop.fs.s3a.access.key", "AKIA38FD4BA7FC1FB43C") \
+    .config("spark.hadoop.fs.s3a.secret.key", "IK7M3ro+CWb7f4OyNKdK1W2SCvMwJTrPX1NBDwsj") \
+    .config("spark.hadoop.fs.s3a.path.style.access", "true") \
+    .config("spark.hadoop.fs.s3a.connection.ssl.enabled", "false") \
+    .getOrCreate()
+
+spark.sparkContext.setLogLevel("WARN")
+
+df = spark.readStream \
+    .format("kafka") \
+    .option("kafka.bootstrap.servers", "10.0.21.36:9092") \
+    .option("subscribe", "test-lakehouse") \
+    .option("startingOffsets", "latest") \
+    .load()
+
+output = df.select(
+    col("key").cast("string"),
+    col("value").cast("string"),
+    current_timestamp().alias("timestamp")
+)
+
+query = output.writeStream \
+    .outputMode("append") \
+    .format("console") \
+    .start()
+
+print("Streaming started. Press Ctrl+C to stop.")
+query.awaitTermination()