from pyspark.sql import SparkSession
from pyspark.sql.functions import col, current_timestamp

spark = SparkSession.builder \
    .appName("KafkaToS3") \
    .config("spark.hadoop.fs.s3a.endpoint", "http://10.0.20.111:9020") \
    .config("spark.hadoop.fs.s3a.access.key", "AKIA38FD4BA7FC1FB43C") \
    .config("spark.hadoop.fs.s3a.secret.key", "IK7M3ro+CWb7f4OyNKdK1W2SCvMwJTrPX1NBDwsj") \
    .config("spark.hadoop.fs.s3a.path.style.access", "true") \
    .config("spark.hadoop.fs.s3a.connection.ssl.enabled", "false") \
    .getOrCreate()

spark.sparkContext.setLogLevel("WARN")

df = spark.readStream \
    .format("kafka") \
    .option("kafka.bootstrap.servers", "10.0.21.36:9092") \
    .option("subscribe", "test-lakehouse") \
    .option("startingOffsets", "latest") \
    .load()

output = df.select(
    col("key").cast("string"),
    col("value").cast("string"),
    current_timestamp().alias("timestamp")
)

query = output.writeStream \
    .outputMode("append") \
    .format("console") \
    .start()

print("Streaming started. Press Ctrl+C to stop.")
query.awaitTermination()