clickhouse

.github/workflows/clickhouse.yml

@@ -0,0 +1,128 @@
+name: ClickHouse
+
+on:
+  push:
+    branches:
+      - master
+
+
+jobs:
+  node1:
+    name: Node 1
+    runs-on: ubuntu-latest
+    timeout-minutes: 300
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ghcr.io/aldy505-clickhouse-node1
+          flavor: |
+            latest=true
+          tags: |
+            type=sha
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: "{{defaultContext}}:clickhouse-node1"
+          file: "{{defaultContext}}/Dockerfile.node1"
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  node2:
+    name: Node 2
+    runs-on: ubuntu-latest
+    timeout-minutes: 300
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ghcr.io/aldy505-clickhouse-node2
+          flavor: |
+            latest=true
+          tags: |
+            type=sha
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: "{{defaultContext}}:clickhouse-node2"
+          file: "{{defaultContext}}/Dockerfile.node2"
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  node3:
+    name: Node 3
+    runs-on: ubuntu-latest
+    timeout-minutes: 300
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ghcr.io/aldy505-clickhouse-node3
+          flavor: |
+            latest=true
+          tags: |
+            type=sha
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: "{{defaultContext}}:clickhouse-node3"
+          file: "{{defaultContext}}/Dockerfile.node3"
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

clickhouse/Dockerfile.node1

@@ -0,0 +1,11 @@
+FROM clickhouse/clickhouse-server:23.1.3.5
+
+COPY local.xml /etc/clickhouse-server/config.d/local.xml
+
+COPY macros1.xml /etc/clickhouse-server/config.d/macros.xml
+
+COPY wait-for-it.sh /wait-for-it.sh
+
+RUN chmod +x wait-for-it.sh
+
+COPY wait-zookeeper.sh /docker-entrypoint-initdb.d/wait-zookeeper.sh

clickhouse/Dockerfile.node2

@@ -0,0 +1,11 @@
+FROM clickhouse/clickhouse-server:23.1.3.5
+
+COPY local.xml /etc/clickhouse-server/config.d/local.xml
+
+COPY macros2.xml /etc/clickhouse-server/config.d/macros.xml
+
+COPY wait-for-it.sh /wait-for-it.sh
+
+RUN chmod +x wait-for-it.sh
+
+COPY wait-zookeeper.sh /docker-entrypoint-initdb.d/wait-zookeeper.sh

clickhouse/Dockerfile.node3

@@ -0,0 +1,11 @@
+FROM clickhouse/clickhouse-server:23.1.3.5
+
+COPY local.xml /etc/clickhouse-server/config.d/local.xml
+
+COPY macros3.xml /etc/clickhouse-server/config.d/macros.xml
+
+COPY wait-for-it.sh /wait-for-it.sh
+
+RUN chmod +x wait-for-it.sh
+
+COPY wait-zookeeper.sh /docker-entrypoint-initdb.d/wait-zookeeper.sh

clickhouse/README.md

@@ -0,0 +1,51 @@
+# ClickHouse
+
+Zookeper is required for running the cluster. If you'd like to only run a single node of ClickHouse,
+please use the ClickHouse image directly.
+
+Github Actions workflow file sample:
+
+```yaml
+name: Your CI
+
+on:
+    # ...
+
+jobs:
+  job-name:
+    name: Job Name
+    services:  
+      zookeeper:
+        image: zookeeper:3.8.0
+        options: >-
+          --health-cmd "nc -nz 127.0.0.1 2181"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+          --health-start-period 30s
+          --hostname zookeeper
+      clickhouse_node1:
+        image: ghcr.io/aldy505-clickhouse-node1
+        options: >-
+          --health-cmd "wget --spider -q localhost:8123/ping"
+          --health-interval 15s
+          --health-timeout 10s
+          --health-retries 10
+          --health-start-period 60s
+          --hostname clickhouse_node1
+          --restart on-failure:10
+      clickhouse_node2:
+        image: ghcr.io/aldy505-clickhouse-node2
+        options: >-
+          --hostname clickhouse_node2
+          --restart on-failure:10
+      clickhouse_node3:
+        image: ghcr.io/aldy505-clickhouse-node3
+        options: >-
+          --hostname clickhouse_node3
+          --restart on-failure:10
+```
+
+Cluster name: `ci_cluster`
+
+Sample connection URL: `clickhouse://default:@clickhouse_node1:9000,clickhouse_node2:9000,clickhouse_node3:9000/default?dial_timeout=30000ms&max_execution_time=60&debug=false`

clickhouse/local.xml

@@ -0,0 +1,28 @@
+<clickhouse>
+    <remote_servers>
+        <ci_cluster>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse_node1</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse_node2</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse_node3</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </ci_cluster>
+    </remote_servers>
+
+    <zookeeper>
+        <node>
+            <host>zookeeper</host>
+            <port>2181</port>
+        </node>
+    </zookeeper>
+</clickhouse>

clickhouse/macros1.xml

@@ -0,0 +1,6 @@
+<clickhouse>
+    <macros replace="replace">
+        <cluster>ci_cluster</cluster>
+        <replica>clickhouse_node1</replica>
+    </macros>
+</clickhouse>

clickhouse/macros2.xml

@@ -0,0 +1,6 @@
+<clickhouse>
+    <macros replace="replace">
+        <cluster>ci_cluster</cluster>
+        <replica>clickhouse_node2</replica>
+    </macros>
+</clickhouse>

clickhouse/macros3.xml

@@ -0,0 +1,6 @@
+<clickhouse>
+    <macros replace="replace">
+        <cluster>ci_cluster</cluster>
+        <replica>clickhouse_node3</replica>
+    </macros>
+</clickhouse>

clickhouse/wait-for-it.sh

@@ -0,0 +1,209 @@
+#!/usr/bin/env bash
+
+# wait-for-it (https://github.com/vishnubob/wait-for-it)
+# wait-for-it.sh is a pure bash script that will wait on the availability of a host and TCP port.
+# It is useful for synchronizing the spin-up of interdependent services, such as linked docker containers.
+# Since it is a pure bash script, it does not have any external dependencies.
+#
+# The MIT License (MIT)
+# Copyright (c) 2016 Giles Hall
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

clickhouse/wait-zookeeper.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+/wait-for-it.sh zookeeper:2181 --timeout=60 --strict -- echo "Zookeeper is up"