Lakehouse Admin
df5ec93dc3
- Trino catalogs, Grafana, Spark jobs, LDAP LDIF, NPM compose - Airflow DAG scripts, Proxmox VM inventory, network docs - Ansible playbook, Gitea CI validate workflow - Backup and health-check scripts, cron documentation - Homepage DOCS tab with links to all documentation - Extended collect-fleet-config.sh and populate-repo.py
35 lines
1.1 KiB
Python
35 lines
1.1 KiB
Python
from pyspark.sql import SparkSession
|
|
from pyspark.sql.functions import col, current_timestamp
|
|
|
|
spark = SparkSession.builder \
|
|
.appName("KafkaToS3") \
|
|
.config("spark.hadoop.fs.s3a.endpoint", "http://10.0.20.111:9020") \
|
|
.config("spark.hadoop.fs.s3a.access.key", "AKIA38FD4BA7FC1FB43C") \
|
|
.config("spark.hadoop.fs.s3a.secret.key", "IK7M3ro+CWb7f4OyNKdK1W2SCvMwJTrPX1NBDwsj") \
|
|
.config("spark.hadoop.fs.s3a.path.style.access", "true") \
|
|
.config("spark.hadoop.fs.s3a.connection.ssl.enabled", "false") \
|
|
.getOrCreate()
|
|
|
|
spark.sparkContext.setLogLevel("WARN")
|
|
|
|
df = spark.readStream \
|
|
.format("kafka") \
|
|
.option("kafka.bootstrap.servers", "10.0.21.36:9092") \
|
|
.option("subscribe", "test-lakehouse") \
|
|
.option("startingOffsets", "latest") \
|
|
.load()
|
|
|
|
output = df.select(
|
|
col("key").cast("string"),
|
|
col("value").cast("string"),
|
|
current_timestamp().alias("timestamp")
|
|
)
|
|
|
|
query = output.writeStream \
|
|
.outputMode("append") \
|
|
.format("console") \
|
|
.start()
|
|
|
|
print("Streaming started. Press Ctrl+C to stop.")
|
|
query.awaitTermination()
|