from kafka import KafkaConsumer
from kafka import TopicPartition
import boto3
import json
from datetime import datetime

# Configuratie
KAFKA_BROKER = "10.0.21.36:9092"
KAFKA_TOPIC = "test-lakehouse"
S3_ENDPOINT = "http://10.0.20.111:9020"
S3_BUCKET = "data"
S3_ACCESS_KEY = "REDACTED"
S3_SECRET_KEY = "REDACTED"

print("Starting S3 consumer...")
print(f"Kafka: {KAFKA_BROKER}, Topic: {KAFKA_TOPIC}")
print(f"S3: {S3_ENDPOINT}, Bucket: {S3_BUCKET}")

# S3 client
s3 = boto3.client(
    's3',
    endpoint_url=S3_ENDPOINT,
    aws_access_key_id=S3_ACCESS_KEY,
    aws_secret_access_key=S3_SECRET_KEY,
    use_ssl=False,
    verify=False
)

# Kafka consumer met vaste partition
consumer = KafkaConsumer(bootstrap_servers=[KAFKA_BROKER])
tp = TopicPartition(KAFKA_TOPIC, 0)
consumer.assign([tp])
consumer.seek_to_beginning()

print("Reading all messages from beginning...")
print("-" * 50)

msg_count = 0
for msg in consumer:
    msg_count += 1
    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
    key = f"kafka/{KAFKA_TOPIC}/message_{timestamp}.json"
    
    try:
        data = json.loads(msg.value.decode('utf-8'))
    except:
        data = {"raw": msg.value.decode('utf-8')}
    
    s3.put_object(
        Bucket=S3_BUCKET,
        Key=key,
        Body=json.dumps(data, indent=2)
    )
    print(f"[{msg_count}] Written: s3://{S3_BUCKET}/{key}")
    print(f"    Data: {str(data)[:80]}...")

print(f"Done! Total {msg_count} messages written to S3")