From 4ae954125966fd4df56abb20b632103058cef167 Mon Sep 17 00:00:00 2001
From: Reinaldy Rafli <aldy505@proton.me>
Date: Thu, 13 Apr 2023 08:06:41 +0700
Subject: [PATCH] use embedded clickhouse keeper

---
 README.md          |  2 +-
 config.xml         | 12 ++++++++++--
 docker-compose.yml | 26 +++++++-------------------
 keeper1.xml        | 32 ++++++++++++++++++++++++++++++++
 keeper2.xml        | 32 ++++++++++++++++++++++++++++++++
 keeper3.xml        | 32 ++++++++++++++++++++++++++++++++
 6 files changed, 114 insertions(+), 22 deletions(-)
 create mode 100644 keeper1.xml
 create mode 100644 keeper2.xml
 create mode 100644 keeper3.xml

diff --git a/README.md b/README.md
index d0b2a55..117349d 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Local Clickhouse Cluster with Docker Compose
 
-Sets up a local Clickhouse cluster with 3 nodes (and Zookeeper as a bonus).
+Sets up a local Clickhouse cluster with 3 nodes (with embedded Clickhouse Keeper as a replacement for Zookeeper).
 
 To run it:
 
diff --git a/config.xml b/config.xml
index db2d249..31de016 100644
--- a/config.xml
+++ b/config.xml
@@ -23,8 +23,16 @@
 
     <zookeeper>
         <node>
-            <host>zookeeper</host>
-            <port>2181</port>
+            <host>ch1</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>ch2</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>ch3</host>
+            <port>9181</port>
         </node>
     </zookeeper>
 
diff --git a/docker-compose.yml b/docker-compose.yml
index 853ce61..4820d1f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,10 +1,11 @@
 services:
   ch1:
-    image: clickhouse/clickhouse-server:22.8.1
+    image: clickhouse/clickhouse-server:23.1.3
     restart: on-failure:10
     volumes:
       - ./config.xml:/etc/clickhouse-server/config.d/local.xml
       - ./macro1.xml:/etc/clickhouse-server/config.d/macros.xml
+      - ./keeper1.xml:/etc/clickhouse-server/config.d/keeper.xml
       - ch1data:/var/lib/clickhouse
     ports:
       - '18123:8123'
@@ -20,15 +21,13 @@ services:
       timeout: 5s
       retries: 3
       start_period: 30s
-    depends_on:
-      zookeeper:
-        condition: service_healthy
   ch2:
-    image: clickhouse/clickhouse-server:22.8.1
+    image: clickhouse/clickhouse-server:23.1.3
     restart: on-failure:10
     volumes:
       - ./config.xml:/etc/clickhouse-server/config.d/local.xml
       - ./macro2.xml:/etc/clickhouse-server/config.d/macros.xml
+      - ./keeper2.xml:/etc/clickhouse-server/config.d/keeper.xml
       - ch2data:/var/lib/clickhouse
     ports:
       - '28123:8123'
@@ -45,16 +44,15 @@ services:
       retries: 3
       start_period: 30s
     depends_on:
-      zookeeper:
-        condition: service_healthy
       ch1:
         condition: service_started
   ch3:
-    image: clickhouse/clickhouse-server:22.8.1
+    image: clickhouse/clickhouse-server:23.1.3
     restart: on-failure:10
     volumes:
       - ./config.xml:/etc/clickhouse-server/config.d/local.xml
       - ./macro3.xml:/etc/clickhouse-server/config.d/macros.xml
+      - ./keeper3.xml:/etc/clickhouse-server/config.d/keeper.xml
       - ch3data:/var/lib/clickhouse
     ports:
       - '38123:8123'
@@ -71,19 +69,9 @@ services:
       retries: 3
       start_period: 30s
     depends_on:
-      zookeeper:
-        condition: service_healthy
       ch1:
         condition: service_started
-  zookeeper:
-    image: zookeeper:3.8.0
-    restart: on-failure:10
-    healthcheck:
-      test: [ "CMD", "sh", "-c", "nc -nz 127.0.0.1 2181" ]
-      interval: 30s
-      timeout: 5s
-      retries: 3
-      start_period: 30s
+
 volumes:
   ch1data:
   ch2data:
diff --git a/keeper1.xml b/keeper1.xml
new file mode 100644
index 0000000..16358ca
--- /dev/null
+++ b/keeper1.xml
@@ -0,0 +1,32 @@
+<clickhouse>
+  <keeper_server>
+    <tcp_port>9181</tcp_port>
+    <server_id>1</server_id>
+    <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+    <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+
+    <coordination_settings>
+        <operation_timeout_ms>10000</operation_timeout_ms>
+        <session_timeout_ms>30000</session_timeout_ms>
+        <raft_logs_level>trace</raft_logs_level>
+    </coordination_settings>
+
+    <raft_configuration>
+        <server>
+            <id>1</id>
+            <hostname>ch1</hostname>
+            <port>9444</port>
+        </server>
+        <server>
+            <id>2</id>
+            <hostname>ch2</hostname>
+            <port>9444</port>
+        </server>
+        <server>
+            <id>3</id>
+            <hostname>ch3</hostname>
+            <port>9444</port>
+        </server>
+    </raft_configuration>
+  </keeper_server>
+</clickhouse>
\ No newline at end of file
diff --git a/keeper2.xml b/keeper2.xml
new file mode 100644
index 0000000..65aa151
--- /dev/null
+++ b/keeper2.xml
@@ -0,0 +1,32 @@
+<clickhouse>
+  <keeper_server>
+    <tcp_port>9181</tcp_port>
+    <server_id>2</server_id>
+    <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+    <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+
+    <coordination_settings>
+        <operation_timeout_ms>10000</operation_timeout_ms>
+        <session_timeout_ms>30000</session_timeout_ms>
+        <raft_logs_level>trace</raft_logs_level>
+    </coordination_settings>
+
+    <raft_configuration>
+        <server>
+            <id>1</id>
+            <hostname>ch1</hostname>
+            <port>9444</port>
+        </server>
+        <server>
+            <id>2</id>
+            <hostname>ch2</hostname>
+            <port>9444</port>
+        </server>
+        <server>
+            <id>3</id>
+            <hostname>ch3</hostname>
+            <port>9444</port>
+        </server>
+    </raft_configuration>
+  </keeper_server>
+</clickhouse>
\ No newline at end of file
diff --git a/keeper3.xml b/keeper3.xml
new file mode 100644
index 0000000..dc1ac50
--- /dev/null
+++ b/keeper3.xml
@@ -0,0 +1,32 @@
+<clickhouse>
+  <keeper_server>
+    <tcp_port>9181</tcp_port>
+    <server_id>3</server_id>
+    <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+    <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+
+    <coordination_settings>
+        <operation_timeout_ms>10000</operation_timeout_ms>
+        <session_timeout_ms>30000</session_timeout_ms>
+        <raft_logs_level>trace</raft_logs_level>
+    </coordination_settings>
+
+    <raft_configuration>
+        <server>
+            <id>1</id>
+            <hostname>ch1</hostname>
+            <port>9444</port>
+        </server>
+        <server>
+            <id>2</id>
+            <hostname>ch2</hostname>
+            <port>9444</port>
+        </server>
+        <server>
+            <id>3</id>
+            <hostname>ch3</hostname>
+            <port>9444</port>
+        </server>
+    </raft_configuration>
+  </keeper_server>
+</clickhouse>
\ No newline at end of file