Merge pull request #15 from farbodahm/feature/dockerize-project

Feature/dockerize project

Author: farbodahm

README.md

@@ -1,2 +1,38 @@
 # stateful-stream-processing
 My university proposal on Stateful Stream Processing
+
+- [stateful-stream-processing](#stateful-stream-processing)
+  - [Problem Description](#problem-description)
+  - [Development Environment](#development-environment)
+
+## Problem Description
+TODO
+
+## Development Environment
+To run the project in dev env, make sure you have [Docker]([dasd](https://docs.docker.com/get-docker/))
+and [Docker-Compose](https://docs.docker.com/compose/install/) installed.
+Then, to run the infrastructure for Kafka and Database, run:
+
+```bash
+docker-compose -f infra.docker-compose.yml up
+```
+
+Wait for few minutes and after you see logs related to Kafka working normally, then run the producer and consumer applications:
+
+```bash
+docker-compose -f app.docker-compose.yml up
+```
+
+This will run the applications.
+If you changed any part of the applications, including `producer`, `consumer` or `processor` you need to re-build the Docker images.
+
+```bash
+docker-compose -f app.docker-compose.yml up --build
+```
+
+If you wanted to tear down everything, then simply run:
+```bash
+docker-compose down
+```
+
+**NOTE:** This will also deletes all of the Kafka topics and database, Be careful if you want keep the data produced in it.

app.docker-compose.yml

@@ -0,0 +1,24 @@
+services:
+  producer:
+    build:
+      context: .
+      args:
+        KAFKA_BROKER: "localhost:9092"
+        SCHEMA_REGISTRY_URL: "http://localhost:8081"
+      dockerfile: producer.Dockerfile
+    network_mode: "host"
+
+  consumer:
+    build:
+      context: .
+      args:
+        KAFKA_BROKER: "localhost:9092"
+        SCHEMA_REGISTRY_URL: "http://localhost:8081"
+        CONSUMER_GROUP: "Consumer1"
+        DB_HOST: "127.0.0.1"
+        DB_PORT: "5432"
+        DB_USERNAME: "postgres"
+        DB_PASSWORD: "postgres"
+        DB_NAME: twitter
+      dockerfile: consumer.Dockerfile
+    network_mode: "host"

consumer.Dockerfile

@@ -0,0 +1,41 @@
+FROM python:3.11-slim-bullseye
+ARG KAFKA_BROKER
+ARG SCHEMA_REGISTRY_URL
+ARG CONSUMER_GROUP
+ARG DB_HOST
+ARG DB_PORT
+ARG DB_USERNAME
+ARG DB_PASSWORD
+ARG DB_NAME
+
+WORKDIR /consumer
+
+COPY requirements.txt requirements.txt
+
+RUN apt-get update \
+    && apt-get -y install libpq-dev gcc
+
+RUN pip3 install -r requirements.txt
+
+COPY . .
+
+ENV PYTHONPATH "${PYTHONPATH}:/consumer"
+# Convert build time arguments to runtime env variables
+ENV KAFKA_BROKER ${KAFKA_BROKER}
+ENV SCHEMA_REGISTRY_URL ${SCHEMA_REGISTRY_URL}
+ENV CONSUMER_GROUP ${CONSUMER_GROUP}
+ENV DB_HOST ${DB_HOST}
+ENV DB_PORT ${DB_PORT} 
+ENV DB_USERNAME ${DB_USERNAME}
+ENV DB_PASSWORD ${DB_PASSWORD}
+ENV DB_NAME ${DB_NAME}
+
+CMD exec python3 consumer/main.py \
+    -b ${KAFKA_BROKER} \
+    -s ${SCHEMA_REGISTRY_URL} \
+    -g ${CONSUMER_GROUP} \
+    --db-ip ${DB_HOST} \
+    --db-port ${DB_PORT} \
+    --db-user ${DB_USERNAME} \
+    --db-password ${DB_PASSWORD} \
+    --db-name ${DB_NAME}

cosumer/config.py consumer/config.py

@@ -1,3 +1,5 @@
+from time import sleep
+
 from sqlalchemy.orm import Session
 
 from model.twitter_database_model import Base
@@ -9,6 +11,9 @@
 
 def main() -> None:
     """Starting point of the producer system"""
+    # TODO: Find a good way to wait for the producer to be ready
+    sleep(5)
+
     cli_args_parser = CliArgsParser()
     cli_args = cli_args_parser.parser.parse_args()
 

cosumer/consume.py consumer/consume.py

@@ -0,0 +1,24 @@
+FROM python:3.11-slim-bullseye
+ARG KAFKA_BROKER
+ARG SCHEMA_REGISTRY_URL
+
+WORKDIR /producer
+
+COPY requirements.txt requirements.txt
+
+RUN apt-get update \
+    && apt-get -y install libpq-dev gcc
+
+RUN pip3 install -r requirements.txt
+
+COPY . .
+
+ENV PYTHONPATH "${PYTHONPATH}:/producer"
+# Convert build time arguments to runtime env variables
+ENV KAFKA_BROKER ${KAFKA_BROKER}
+ENV SCHEMA_REGISTRY_URL ${SCHEMA_REGISTRY_URL}
+
+
+CMD exec python3 producer/main.py \
+    -b ${KAFKA_BROKER} \
+    -s ${SCHEMA_REGISTRY_URL}