Merge pull request #522 from Altinity/replacingmergetree_kafka

ReplicatedReplacingmergetree support for kafka

Author: subkanthi

sink-connector/deploy/docker/clickhouse/config.xml

@@ -0,0 +1,14 @@
+<clickhouse replace="true">
+    <timezone>America/Chicago</timezone>
+    <zookeeper>
+        <node index="1">
+            <host>zookeeper</host>
+            <port>2181</port>
+        </node>
+        <session_timeout_ms>15000</session_timeout_ms>
+    </zookeeper>
+    <macros>
+        <replica>clickhouse</replica>
+        <shard>02</shard>
+    </macros>
+</clickhouse>

sink-connector/deploy/docker/docker-compose.yaml

@@ -96,7 +96,7 @@ services:
 
   sink:
     container_name: sink
-    image: altinity/clickhouse-sink-connector:${SINK_VERSION}
+    image: altinity/clickhouse-kafka-sink-connector-on-debezium-base:2024-03-29
     restart: "no"
     ports:
       - "18083:8083"
@@ -124,6 +124,9 @@ services:
     container_name: clickhouse
     image: clickhouse/clickhouse-server:latest
     restart: "no"
+    depends_on:
+      zookeeper:
+        condition: service_healthy
     ports:
       - "8123:8123"
       - "9000:9000"
@@ -138,6 +141,12 @@ services:
     volumes:
       - ../sql/init_clickhouse.sql:/docker-entrypoint-initdb.d/init_clickhouse.sql
       -  ./clickhouse/users.xml:/etc/clickhouse-server/users.xml
+      - ./clickhouse/config.xml:/etc/clickhouse-server/conf.d/config.xml
+
+  zookeeper:
+    extends:
+      file: zookeeper-service.yml
+      service: zookeeper
 
   #### MONITORING #####
 #  https://stackoverflow.com/questions/55473562/configuring-a-jmx-prometheus-exporter-docker-container-to-read-jmx-local-applica
@@ -203,4 +212,4 @@ services:
       - GF_USERS_DEFAULT_THEME=light
     depends_on:
       - prometheus
-  ### END OF MONITORING ###
+  ### END OF MONITORING ###

sink-connector/deploy/docker/zookeeper-service.yml

@@ -0,0 +1,18 @@
+version: "3.4"
+
+services:
+  zookeeper:
+      image: zookeeper:3.6.2
+      expose:
+        - "2181"
+      environment:
+        ZOO_TICK_TIME: 500
+        ZOO_MY_ID: 1
+      healthcheck:
+        test: echo stat | nc localhost 2181
+        interval: 3s
+        timeout: 2s
+        retries: 5
+        start_period: 2s
+      security_opt:
+        - label:disable

sink-connector/deploy/sink-connector-setup-database.sh

@@ -64,7 +64,8 @@ if [[ $1 == "apicurio" ]]; then
         "auto.create.tables": true,
         "schema.evolution": false,
 
-        "deduplication.policy": "off"
+        "deduplication.policy": "off",
+        "auto.create.tables.replicated": "true"
         }
     }
 EOF