feat: removed zookeeper for clickhouse
This commit is contained in:
parent
8db2de35c9
commit
ec47b0033d
GPG Key ID:
1DAB793F100A560A
|
|
@ -4,8 +4,4 @@ COPY local.xml /etc/clickhouse-server/config.d/local.xml
|
||||||
|
|
||||||
COPY macros1.xml /etc/clickhouse-server/config.d/macros.xml
|
COPY macros1.xml /etc/clickhouse-server/config.d/macros.xml
|
||||||
|
|
||||||
COPY wait-for-it.sh /wait-for-it.sh
|
COPY keeper1.xml /etc/clickhouse-server/config.d/keeper.xml
|
||||||
|
|
||||||
RUN chmod +x wait-for-it.sh
|
|
||||||
|
|
||||||
COPY wait-zookeeper.sh /docker-entrypoint-initdb.d/wait-zookeeper.sh
|
|
||||||
|
|
|
||||||
|
|
@ -4,8 +4,4 @@ COPY local.xml /etc/clickhouse-server/config.d/local.xml
|
||||||
|
|
||||||
COPY macros2.xml /etc/clickhouse-server/config.d/macros.xml
|
COPY macros2.xml /etc/clickhouse-server/config.d/macros.xml
|
||||||
|
|
||||||
COPY wait-for-it.sh /wait-for-it.sh
|
COPY keeper2.xml /etc/clickhouse-server/config.d/keeper.xml
|
||||||
|
|
||||||
RUN chmod +x wait-for-it.sh
|
|
||||||
|
|
||||||
COPY wait-zookeeper.sh /docker-entrypoint-initdb.d/wait-zookeeper.sh
|
|
||||||
|
|
|
||||||
|
|
@ -4,8 +4,4 @@ COPY local.xml /etc/clickhouse-server/config.d/local.xml
|
||||||
|
|
||||||
COPY macros3.xml /etc/clickhouse-server/config.d/macros.xml
|
COPY macros3.xml /etc/clickhouse-server/config.d/macros.xml
|
||||||
|
|
||||||
COPY wait-for-it.sh /wait-for-it.sh
|
COPY keeper3.xml /etc/clickhouse-server/config.d/keeper.xml
|
||||||
|
|
||||||
RUN chmod +x wait-for-it.sh
|
|
||||||
|
|
||||||
COPY wait-zookeeper.sh /docker-entrypoint-initdb.d/wait-zookeeper.sh
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
# ClickHouse
|
# ClickHouse
|
||||||
|
|
||||||
Zookeper is required for running the cluster. If you'd like to only run a single node of ClickHouse,
|
If you'd like to only run a single node of ClickHouse, please use the ClickHouse image directly.
|
||||||
please use the ClickHouse image directly.
|
These images does not requires Zookeeper.
|
||||||
|
|
||||||
Github Actions workflow file sample:
|
Github Actions workflow file sample:
|
||||||
|
|
||||||
|
|
@ -15,17 +15,8 @@ jobs:
|
||||||
job-name:
|
job-name:
|
||||||
name: Job Name
|
name: Job Name
|
||||||
services:
|
services:
|
||||||
zookeeper:
|
|
||||||
image: zookeeper:3.8.0
|
|
||||||
options: >-
|
|
||||||
--health-cmd "nc -nz 127.0.0.1 2181"
|
|
||||||
--health-interval 10s
|
|
||||||
--health-timeout 5s
|
|
||||||
--health-retries 10
|
|
||||||
--health-start-period 30s
|
|
||||||
--hostname zookeeper
|
|
||||||
clickhouse_node1:
|
clickhouse_node1:
|
||||||
image: ghcr.io/aldy505-clickhouse-node1
|
image: ghcr.io/aldy505/clickhouse-node1
|
||||||
options: >-
|
options: >-
|
||||||
--health-cmd "wget --spider -q localhost:8123/ping"
|
--health-cmd "wget --spider -q localhost:8123/ping"
|
||||||
--health-interval 15s
|
--health-interval 15s
|
||||||
|
|
@ -35,12 +26,12 @@ jobs:
|
||||||
--hostname clickhouse_node1
|
--hostname clickhouse_node1
|
||||||
--restart on-failure:10
|
--restart on-failure:10
|
||||||
clickhouse_node2:
|
clickhouse_node2:
|
||||||
image: ghcr.io/aldy505-clickhouse-node2
|
image: ghcr.io/aldy505/clickhouse-node2
|
||||||
options: >-
|
options: >-
|
||||||
--hostname clickhouse_node2
|
--hostname clickhouse_node2
|
||||||
--restart on-failure:10
|
--restart on-failure:10
|
||||||
clickhouse_node3:
|
clickhouse_node3:
|
||||||
image: ghcr.io/aldy505-clickhouse-node3
|
image: ghcr.io/aldy505/clickhouse-node3
|
||||||
options: >-
|
options: >-
|
||||||
--hostname clickhouse_node3
|
--hostname clickhouse_node3
|
||||||
--restart on-failure:10
|
--restart on-failure:10
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
<clickhouse>
|
||||||
|
<keeper_server>
|
||||||
|
<tcp_port>9181</tcp_port>
|
||||||
|
<server_id>1</server_id>
|
||||||
|
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
||||||
|
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
||||||
|
|
||||||
|
<coordination_settings>
|
||||||
|
<operation_timeout_ms>10000</operation_timeout_ms>
|
||||||
|
<session_timeout_ms>30000</session_timeout_ms>
|
||||||
|
<raft_logs_level>trace</raft_logs_level>
|
||||||
|
</coordination_settings>
|
||||||
|
|
||||||
|
<raft_configuration>
|
||||||
|
<server>
|
||||||
|
<id>1</id>
|
||||||
|
<hostname>clickhouse_node1</hostname>
|
||||||
|
<port>9444</port>
|
||||||
|
</server>
|
||||||
|
<server>
|
||||||
|
<id>2</id>
|
||||||
|
<hostname>clickhouse_node2</hostname>
|
||||||
|
<port>9444</port>
|
||||||
|
</server>
|
||||||
|
<server>
|
||||||
|
<id>3</id>
|
||||||
|
<hostname>clickhouse_node3</hostname>
|
||||||
|
<port>9444</port>
|
||||||
|
</server>
|
||||||
|
</raft_configuration>
|
||||||
|
</keeper_server>
|
||||||
|
</clickhouse>
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
<clickhouse>
|
||||||
|
<keeper_server>
|
||||||
|
<tcp_port>9181</tcp_port>
|
||||||
|
<server_id>2</server_id>
|
||||||
|
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
||||||
|
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
||||||
|
|
||||||
|
<coordination_settings>
|
||||||
|
<operation_timeout_ms>10000</operation_timeout_ms>
|
||||||
|
<session_timeout_ms>30000</session_timeout_ms>
|
||||||
|
<raft_logs_level>trace</raft_logs_level>
|
||||||
|
</coordination_settings>
|
||||||
|
|
||||||
|
<raft_configuration>
|
||||||
|
<server>
|
||||||
|
<id>1</id>
|
||||||
|
<hostname>clickhouse_node1</hostname>
|
||||||
|
<port>9444</port>
|
||||||
|
</server>
|
||||||
|
<server>
|
||||||
|
<id>2</id>
|
||||||
|
<hostname>clickhouse_node2</hostname>
|
||||||
|
<port>9444</port>
|
||||||
|
</server>
|
||||||
|
<server>
|
||||||
|
<id>3</id>
|
||||||
|
<hostname>clickhouse_node3</hostname>
|
||||||
|
<port>9444</port>
|
||||||
|
</server>
|
||||||
|
</raft_configuration>
|
||||||
|
</keeper_server>
|
||||||
|
</clickhouse>
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
<clickhouse>
|
||||||
|
<keeper_server>
|
||||||
|
<tcp_port>9181</tcp_port>
|
||||||
|
<server_id>3</server_id>
|
||||||
|
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
||||||
|
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
||||||
|
|
||||||
|
<coordination_settings>
|
||||||
|
<operation_timeout_ms>10000</operation_timeout_ms>
|
||||||
|
<session_timeout_ms>30000</session_timeout_ms>
|
||||||
|
<raft_logs_level>trace</raft_logs_level>
|
||||||
|
</coordination_settings>
|
||||||
|
|
||||||
|
<raft_configuration>
|
||||||
|
<server>
|
||||||
|
<id>1</id>
|
||||||
|
<hostname>clickhouse_node1</hostname>
|
||||||
|
<port>9444</port>
|
||||||
|
</server>
|
||||||
|
<server>
|
||||||
|
<id>2</id>
|
||||||
|
<hostname>clickhouse_node2</hostname>
|
||||||
|
<port>9444</port>
|
||||||
|
</server>
|
||||||
|
<server>
|
||||||
|
<id>3</id>
|
||||||
|
<hostname>clickhouse_node3</hostname>
|
||||||
|
<port>9444</port>
|
||||||
|
</server>
|
||||||
|
</raft_configuration>
|
||||||
|
</keeper_server>
|
||||||
|
</clickhouse>
|
||||||
|
|
@ -21,8 +21,16 @@
|
||||||
|
|
||||||
<zookeeper>
|
<zookeeper>
|
||||||
<node>
|
<node>
|
||||||
<host>zookeeper</host>
|
<host>clickhouse_node1</host>
|
||||||
<port>2181</port>
|
<port>9181</port>
|
||||||
|
</node>
|
||||||
|
<node>
|
||||||
|
<host>clickhouse_node2</host>
|
||||||
|
<port>9181</port>
|
||||||
|
</node>
|
||||||
|
<node>
|
||||||
|
<host>clickhouse_node3</host>
|
||||||
|
<port>9181</port>
|
||||||
</node>
|
</node>
|
||||||
</zookeeper>
|
</zookeeper>
|
||||||
</clickhouse>
|
</clickhouse>
|
||||||
|
|
|
||||||
|
|
@ -1,209 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
# wait-for-it (https://github.com/vishnubob/wait-for-it)
|
|
||||||
# wait-for-it.sh is a pure bash script that will wait on the availability of a host and TCP port.
|
|
||||||
# It is useful for synchronizing the spin-up of interdependent services, such as linked docker containers.
|
|
||||||
# Since it is a pure bash script, it does not have any external dependencies.
|
|
||||||
#
|
|
||||||
# The MIT License (MIT)
|
|
||||||
# Copyright (c) 2016 Giles Hall
|
|
||||||
#
|
|
||||||
# Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
||||||
# this software and associated documentation files (the "Software"), to deal in
|
|
||||||
# the Software without restriction, including without limitation the rights to
|
|
||||||
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
|
||||||
# of the Software, and to permit persons to whom the Software is furnished to do
|
|
||||||
# so, subject to the following conditions:
|
|
||||||
#
|
|
||||||
# The above copyright notice and this permission notice shall be included in all
|
|
||||||
# copies or substantial portions of the Software.
|
|
||||||
#
|
|
||||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
# SOFTWARE.
|
|
||||||
|
|
||||||
# Use this script to test if a given TCP host/port are available
|
|
||||||
|
|
||||||
WAITFORIT_cmdname=${0##*/}
|
|
||||||
|
|
||||||
echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
|
|
||||||
|
|
||||||
usage()
|
|
||||||
{
|
|
||||||
cat << USAGE >&2
|
|
||||||
Usage:
|
|
||||||
$WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
|
|
||||||
-h HOST | --host=HOST Host or IP under test
|
|
||||||
-p PORT | --port=PORT TCP port under test
|
|
||||||
Alternatively, you specify the host and port as host:port
|
|
||||||
-s | --strict Only execute subcommand if the test succeeds
|
|
||||||
-q | --quiet Don't output any status messages
|
|
||||||
-t TIMEOUT | --timeout=TIMEOUT
|
|
||||||
Timeout in seconds, zero for no timeout
|
|
||||||
-- COMMAND ARGS Execute command with args after the test finishes
|
|
||||||
USAGE
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
wait_for()
|
|
||||||
{
|
|
||||||
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
|
|
||||||
echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
|
|
||||||
else
|
|
||||||
echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
|
|
||||||
fi
|
|
||||||
WAITFORIT_start_ts=$(date +%s)
|
|
||||||
while :
|
|
||||||
do
|
|
||||||
if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
|
|
||||||
nc -z $WAITFORIT_HOST $WAITFORIT_PORT
|
|
||||||
WAITFORIT_result=$?
|
|
||||||
else
|
|
||||||
(echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
|
|
||||||
WAITFORIT_result=$?
|
|
||||||
fi
|
|
||||||
if [[ $WAITFORIT_result -eq 0 ]]; then
|
|
||||||
WAITFORIT_end_ts=$(date +%s)
|
|
||||||
echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
return $WAITFORIT_result
|
|
||||||
}
|
|
||||||
|
|
||||||
wait_for_wrapper()
|
|
||||||
{
|
|
||||||
# In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
|
|
||||||
if [[ $WAITFORIT_QUIET -eq 1 ]]; then
|
|
||||||
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
|
|
||||||
else
|
|
||||||
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
|
|
||||||
fi
|
|
||||||
WAITFORIT_PID=$!
|
|
||||||
trap "kill -INT -$WAITFORIT_PID" INT
|
|
||||||
wait $WAITFORIT_PID
|
|
||||||
WAITFORIT_RESULT=$?
|
|
||||||
if [[ $WAITFORIT_RESULT -ne 0 ]]; then
|
|
||||||
echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
|
|
||||||
fi
|
|
||||||
return $WAITFORIT_RESULT
|
|
||||||
}
|
|
||||||
|
|
||||||
# process arguments
|
|
||||||
while [[ $# -gt 0 ]]
|
|
||||||
do
|
|
||||||
case "$1" in
|
|
||||||
*:* )
|
|
||||||
WAITFORIT_hostport=(${1//:/ })
|
|
||||||
WAITFORIT_HOST=${WAITFORIT_hostport[0]}
|
|
||||||
WAITFORIT_PORT=${WAITFORIT_hostport[1]}
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
--child)
|
|
||||||
WAITFORIT_CHILD=1
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
-q | --quiet)
|
|
||||||
WAITFORIT_QUIET=1
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
-s | --strict)
|
|
||||||
WAITFORIT_STRICT=1
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
-h)
|
|
||||||
WAITFORIT_HOST="$2"
|
|
||||||
if [[ $WAITFORIT_HOST == "" ]]; then break; fi
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--host=*)
|
|
||||||
WAITFORIT_HOST="${1#*=}"
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
-p)
|
|
||||||
WAITFORIT_PORT="$2"
|
|
||||||
if [[ $WAITFORIT_PORT == "" ]]; then break; fi
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--port=*)
|
|
||||||
WAITFORIT_PORT="${1#*=}"
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
-t)
|
|
||||||
WAITFORIT_TIMEOUT="$2"
|
|
||||||
if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--timeout=*)
|
|
||||||
WAITFORIT_TIMEOUT="${1#*=}"
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
--)
|
|
||||||
shift
|
|
||||||
WAITFORIT_CLI=("$@")
|
|
||||||
break
|
|
||||||
;;
|
|
||||||
--help)
|
|
||||||
usage
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echoerr "Unknown argument: $1"
|
|
||||||
usage
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
|
|
||||||
echoerr "Error: you need to provide a host and port to test."
|
|
||||||
usage
|
|
||||||
fi
|
|
||||||
|
|
||||||
WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
|
|
||||||
WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
|
|
||||||
WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
|
|
||||||
WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
|
|
||||||
|
|
||||||
# Check to see if timeout is from busybox?
|
|
||||||
WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
|
|
||||||
WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
|
|
||||||
|
|
||||||
WAITFORIT_BUSYTIMEFLAG=""
|
|
||||||
if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
|
|
||||||
WAITFORIT_ISBUSY=1
|
|
||||||
# Check if busybox timeout uses -t flag
|
|
||||||
# (recent Alpine versions don't support -t anymore)
|
|
||||||
if timeout &>/dev/stdout | grep -q -e '-t '; then
|
|
||||||
WAITFORIT_BUSYTIMEFLAG="-t"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
WAITFORIT_ISBUSY=0
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ $WAITFORIT_CHILD -gt 0 ]]; then
|
|
||||||
wait_for
|
|
||||||
WAITFORIT_RESULT=$?
|
|
||||||
exit $WAITFORIT_RESULT
|
|
||||||
else
|
|
||||||
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
|
|
||||||
wait_for_wrapper
|
|
||||||
WAITFORIT_RESULT=$?
|
|
||||||
else
|
|
||||||
wait_for
|
|
||||||
WAITFORIT_RESULT=$?
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ $WAITFORIT_CLI != "" ]]; then
|
|
||||||
if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
|
|
||||||
echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
|
|
||||||
exit $WAITFORIT_RESULT
|
|
||||||
fi
|
|
||||||
exec "${WAITFORIT_CLI[@]}"
|
|
||||||
else
|
|
||||||
exit $WAITFORIT_RESULT
|
|
||||||
fi
|
|
||||||
|
|
@ -1,3 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
/wait-for-it.sh zookeeper:2181 --timeout=60 --strict -- echo "Zookeeper is up"
|
|
||||||
Loading…
Reference in New Issue