35 lines
1.1 KiB
Python
35 lines
1.1 KiB
Python
|
from pyspark.sql import SparkSession
|
||
|
|
from pyspark.sql.functions import col, current_timestamp
|
||
|
|
|
||
|
|
spark = SparkSession.builder \
|
||
|
|
.appName("KafkaToS3") \
|
||
|
|
.config("spark.hadoop.fs.s3a.endpoint", "http://10.0.20.111:9020") \
|
||
|
|
.config("spark.hadoop.fs.s3a.access.key", "AKIA38FD4BA7FC1FB43C") \
|
||
|
|
.config("spark.hadoop.fs.s3a.secret.key", "IK7M3ro+CWb7f4OyNKdK1W2SCvMwJTrPX1NBDwsj") \
|
||
|
|
.config("spark.hadoop.fs.s3a.path.style.access", "true") \
|
||
|
|
.config("spark.hadoop.fs.s3a.connection.ssl.enabled", "false") \
|
||
|
|
.getOrCreate()
|
||
|
|
|
||
|
|
spark.sparkContext.setLogLevel("WARN")
|
||
|
|
|
||
|
|
df = spark.readStream \
|
||
|
|
.format("kafka") \
|
||
|
|
.option("kafka.bootstrap.servers", "10.0.21.36:9092") \
|
||
|
|
.option("subscribe", "test-lakehouse") \
|
||
|
|
.option("startingOffsets", "latest") \
|
||
|
|
.load()
|
||
|
|
|
||
|
|
output = df.select(
|
||
|
|
col("key").cast("string"),
|
||
|
|
col("value").cast("string"),
|
||
|
|
current_timestamp().alias("timestamp")
|
||
|
|
)
|
||
|
|
|
||
|
|
query = output.writeStream \
|
||
|
|
.outputMode("append") \
|
||
|
|
.format("console") \
|
||
|
|
.start()
|
||
|
|
|
||
|
|
print("Streaming started. Press Ctrl+C to stop.")
|
||
|
|
query.awaitTermination()
|