From 342f7ba567140d25dea135348bb7e0eecb22d0c7 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 8 Feb 2022 19:12:04 +0100 Subject: [PATCH] Migrate to nfpm --- docker/packager/binary/Dockerfile | 8 + docker/packager/binary/build.sh | 26 ++- docker/packager/packager | 56 +++-- packages/.gitignore | 1 + packages/build | 156 ++++++++++++++ packages/clickhouse-client.yaml | 57 ++++++ packages/clickhouse-common-static-dbg.yaml | 34 +++ packages/clickhouse-common-static.yaml | 48 +++++ packages/clickhouse-server.init | 227 +++++++++++++++++++++ packages/clickhouse-server.postinstall | 47 +++++ packages/clickhouse-server.service | 27 +++ packages/clickhouse-server.yaml | 68 ++++++ tests/ci/build_check.py | 12 +- 13 files changed, 729 insertions(+), 38 deletions(-) create mode 100644 packages/.gitignore create mode 100755 packages/build create mode 100644 packages/clickhouse-client.yaml create mode 100644 packages/clickhouse-common-static-dbg.yaml create mode 100644 packages/clickhouse-common-static.yaml create mode 100755 packages/clickhouse-server.init create mode 100644 packages/clickhouse-server.postinstall create mode 100644 packages/clickhouse-server.service create mode 100644 packages/clickhouse-server.yaml diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index e3e2e689b17..416b168732a 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -95,6 +95,14 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \ && apt-get install gcc-11 g++-11 --yes \ && apt-get clean +# Architecture of the image when BuildKit/buildx is used +ARG TARGETARCH +ARG NFPM_VERSION=2.15.0 + +RUN arch=${TARGETARCH:-amd64} \ + && curl -Lo /tmp/nfpm.deb "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${arch}.deb" \ + && dpkg -i /tmp/nfpm.deb \ + && rm /tmp/nfpm.deb COPY build.sh / CMD ["bash", "-c", "/build.sh 2>&1 | ts"] diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 2f18b07ffe1..62bbe42e12f 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -2,6 +2,11 @@ set -x -e +cache_status () { + ccache --show-config ||: + ccache --show-stats ||: +} + mkdir -p build/cmake/toolchain/darwin-x86_64 tar xJf MacOSX11.0.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1 ln -sf darwin-x86_64 build/cmake/toolchain/darwin-aarch64 @@ -19,15 +24,23 @@ read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}" env cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" .. -ccache --show-config ||: -ccache --show-stats ||: +cache_status +# clear cache stats ccache --zero-stats ||: -# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty. +# No quotes because I want it to expand to nothing if empty. +# shellcheck disable=SC2086 ninja $NINJA_FLAGS clickhouse-bundle -ccache --show-config ||: -ccache --show-stats ||: +cache_status + +if [ -n "$MAKE_DEB" ]; then + rm -rf /build/packages/root + # No quotes because I want it to expand to nothing if empty. + # shellcheck disable=SC2086 + DESTDIR=/build/packages/root ninja $NINJA_FLAGS install + bash -x /build/packages/build +fi mv ./programs/clickhouse* /output mv ./src/unit_tests_dbms /output ||: # may not exist for some binary builds @@ -84,8 +97,7 @@ fi # ../docker/packager/other/fuzzer.sh # fi -ccache --show-config ||: -ccache --show-stats ||: +cache_status if [ "${CCACHE_DEBUG:-}" == "1" ] then diff --git a/docker/packager/packager b/docker/packager/packager index 4ba03f7633e..9b497314ca0 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -8,11 +8,6 @@ import sys SCRIPT_PATH = os.path.realpath(__file__) -IMAGE_MAP = { - "deb": "clickhouse/deb-builder", - "binary": "clickhouse/binary-builder", -} - def check_image_exists_locally(image_name): try: @@ -66,6 +61,15 @@ def run_docker_image_with_env( subprocess.check_call(cmd, shell=True) +def is_release_build(build_type, package_type, sanitizer, split_binary): + return ( + build_type == "" + and package_type == "deb" + and sanitizer == "" + and not split_binary + ) + + def parse_env_variables( build_type, compiler, @@ -90,6 +94,7 @@ def parse_env_variables( PPC_SUFFIX = "-ppc64le" result = [] + result.append("OUTPUT_DIR=/output") cmake_flags = ["$CMAKE_FLAGS"] is_cross_darwin = compiler.endswith(DARWIN_SUFFIX) @@ -131,7 +136,7 @@ def parse_env_variables( cmake_flags.append( "-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake" ) - result.append("DEB_ARCH_FLAG=-aarm64") + result.append("DEB_ARCH=arm64") elif is_cross_freebsd: cc = compiler[: -len(FREEBSD_SUFFIX)] cmake_flags.append( @@ -144,21 +149,27 @@ def parse_env_variables( ) else: cc = compiler - result.append("DEB_ARCH_FLAG=-aamd64") + result.append("DEB_ARCH=amd64") cxx = cc.replace("gcc", "g++").replace("clang", "clang++") if image_type == "deb": - result.append(f"DEB_CC={cc}") - result.append(f"DEB_CXX={cxx}") - # For building fuzzers - result.append(f"CC={cc}") - result.append(f"CXX={cxx}") - elif image_type == "binary": - result.append(f"CC={cc}") - result.append(f"CXX={cxx}") - cmake_flags.append(f"-DCMAKE_C_COMPILER=`which {cc}`") - cmake_flags.append(f"-DCMAKE_CXX_COMPILER=`which {cxx}`") + result.append("MAKE_DEB=true") + cmake_flags.append("-DENABLE_TESTS=0") + cmake_flags.append("-DENABLE_UTILS=0") + cmake_flags.append("-DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON") + cmake_flags.append("-DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON") + cmake_flags.append("-DCMAKE_AUTOGEN_VERBOSE=ON") + cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr") + cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc") + cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var") + if is_release_build(build_type, package_type, sanitizer, split_binary): + cmake_flags.append("-DINSTALL_STRIPPED_BINARIES=ON") + + result.append(f"CC={cc}") + result.append(f"CXX={cxx}") + cmake_flags.append(f"-DCMAKE_C_COMPILER=`which {cc}`") + cmake_flags.append(f"-DCMAKE_CXX_COMPILER=`which {cxx}`") # Create combined output archive for split build and for performance tests. if package_type == "performance": @@ -170,7 +181,9 @@ def parse_env_variables( if sanitizer: result.append(f"SANITIZER={sanitizer}") if build_type: - result.append(f"BUILD_TYPE={build_type}") + result.append(f"BUILD_TYPE={build_type.capitalize()}") + else: + result.append("BUILD_TYPE=None") if cache == "distcc": result.append(f"CCACHE_PREFIX={cache}") @@ -192,9 +205,8 @@ def parse_env_variables( result.append('DISTCC_HOSTS="localhost/`nproc`"') if alien_pkgs: - result.append( - "ALIEN_PKGS='" + " ".join(["--" + pkg for pkg in alien_pkgs]) + "'" - ) + result.append("MAKE_RPM=true") + result.append("MAKE_TGZ=true") if with_binaries == "programs": result.append("BINARY_OUTPUT=programs") @@ -317,7 +329,7 @@ if __name__ == "__main__": args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir)) image_type = "binary" if args.package_type == "performance" else args.package_type - image_name = IMAGE_MAP[image_type] + image_name = "clickhouse/binary-builder" if not os.path.isabs(args.clickhouse_repo_path): ch_root = os.path.abspath(os.path.join(os.getcwd(), args.clickhouse_repo_path)) diff --git a/packages/.gitignore b/packages/.gitignore new file mode 100644 index 00000000000..355164c1265 --- /dev/null +++ b/packages/.gitignore @@ -0,0 +1 @@ +*/ diff --git a/packages/build b/packages/build new file mode 100755 index 00000000000..53a7538f80e --- /dev/null +++ b/packages/build @@ -0,0 +1,156 @@ +#!/usr/bin/env bash + +set -e + +# Avoid dependency on locale +LC_ALL=C + +# Normalize output directory +if [ -n "$OUTPUT_DIR" ]; then + OUTPUT_DIR=$(realpath -m "$OUTPUT_DIR") +fi + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$CUR_DIR" + +ROOT_DIR=$(readlink -f "$(git rev-parse --show-cdup)") + +PKG_ROOT='root' + +DEB_ARCH=${DEB_ARCH:-amd64} +OUTPUT_DIR=${OUTPUT_DIR:-$ROOT_DIR} +[ -d "${OUTPUT_DIR}" ] || mkdir -p "${OUTPUT_DIR}" +SANITIZER=${SANITIZER:-""} +SOURCE=${SOURCE:-$PKG_ROOT} + +HELP="${0} [--test] [--rpm] [-h|--help] + --test - adds '+test' prefix to version + --apk - build APK packages + --rpm - build RPM packages + --tgz - build tarball package + --help - show this help and exit + +Used envs: + DEB_ARCH='${DEB_ARCH}' + OUTPUT_DIR='${OUTPUT_DIR}' - where the artifact will be placed + SANITIZER='${SANITIZER}' - if any sanitizer is used, affects version string + SOURCE='${SOURCE}' - directory with sources tree + VERSION_STRING='${VERSION_STRING}' - the package version to overwrite +" + +if [ -z "${VERSION_STRING}" ]; then + # Get CLICKHOUSE_VERSION_STRING from the current git repo + eval "$("$ROOT_DIR/tests/ci/version_helper.py" -e)" +else + CLICKHOUSE_VERSION_STRING=${VERSION_STRING} +fi +export CLICKHOUSE_VERSION_STRING + + + +while [[ $1 == --* ]] +do + case "$1" in + --test ) + VERSION_POSTFIX+='+test' + shift ;; + --apk ) + MAKE_APK=1 + shift ;; + --rpm ) + MAKE_RPM=1 + shift ;; + --tgz ) + MAKE_TGZ=1 + shift ;; + --help ) + echo "$HELP" + exit ;; + * ) + echo "Unknown option $1" + exit 2 ;; + esac +done + +function deb2tgz { + local FILE PKG_NAME PKG_DIR PKG_PATH TARBALL + FILE=$1 + PKG_NAME=${FILE##*/}; PKG_NAME=${PKG_NAME%%_*} + PKG_DIR="$PKG_NAME-$CLICKHOUSE_VERSION_STRING" + PKG_PATH="$OUTPUT_DIR/$PKG_NAME-$CLICKHOUSE_VERSION_STRING" + TARBALL="$OUTPUT_DIR/$PKG_NAME-$CLICKHOUSE_VERSION_STRING-$DEB_ARCH.tgz" + rm -rf "$PKG_PATH" + dpkg-deb -R "$FILE" "$PKG_PATH" + mkdir -p "$PKG_PATH/install" + cat > "$PKG_PATH/install/doinst.sh" << 'EOF' +#!/bin/sh +set -e + +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +for filepath in `find $SCRIPTPATH/.. -type f -or -type l | grep -v "\.\./install/"`; do + destpath=${filepath##$SCRIPTPATH/..} + mkdir -p $(dirname "$destpath") + cp -r "$filepath" "$destpath" +done +EOF + chmod +x "$PKG_PATH/install/doinst.sh" + if [ -f "$PKG_PATH/DEBIAN/postinst" ]; then + tail +2 "$PKG_PATH/DEBIAN/postinst" > "$PKG_PATH/install/doinst.sh" + fi + rm -rf "$PKG_PATH/DEBIAN" + if [ -f "/usr/bin/pigz" ]; then + tar --use-compress-program=pigz -cf "$TARBALL" -C "$OUTPUT_DIR" "$PKG_DIR" + else + tar -czf "$TARBALL" -C "$OUTPUT_DIR" "$PKG_DIR" + fi + + rm -r "$PKG_PATH" +} + +# Build options +if [ -n "$SANITIZER" ]; then + if [[ "$SANITIZER" == "address" ]]; then VERSION_POSTFIX+="+asan" + elif [[ "$SANITIZER" == "thread" ]]; then VERSION_POSTFIX+="+tsan" + elif [[ "$SANITIZER" == "memory" ]]; then VERSION_POSTFIX+="+msan" + elif [[ "$SANITIZER" == "undefined" ]]; then VERSION_POSTFIX+="+ubsan" + else + echo "Unknown value of SANITIZER variable: $SANITIZER" + exit 3 + fi +elif [[ $BUILD_TYPE == 'debug' ]]; then + VERSION_POSTFIX+="+debug" +fi + +if [[ "$PKG_ROOT" != "$SOURCE" ]]; then + # packages are built only from PKG_SOURCE + rm -rf "./$PKG_ROOT" + ln -sf "$SOURCE" "$PKG_SOURCE" +fi + +CLICKHOUSE_VERSION_STRING+=$VERSION_POSTFIX +echo -e "\nCurrent version is $CLICKHOUSE_VERSION_STRING" + +for config in clickhouse*.yaml; do + echo "Building deb package for $config" + + # Preserve package path + exec 9>&1 + PKG_PATH=$(nfpm package --target "$OUTPUT_DIR" --config "$config" --packager deb | tee /dev/fd/9) + PKG_PATH=${PKG_PATH##*created package: } + exec 9>&- + + if [ -n "$MAKE_APK" ]; then + echo "Building apk package for $config" + nfpm package --target "$OUTPUT_DIR" --config "$config" --packager apk + fi + if [ -n "$MAKE_RPM" ]; then + echo "Building rpm package for $config" + nfpm package --target "$OUTPUT_DIR" --config "$config" --packager rpm + fi + if [ -n "$MAKE_TGZ" ]; then + echo "Building tarball for $config" + deb2tgz "$PKG_PATH" + fi +done + +# vim: ts=4: sw=4: sts=4: expandtab diff --git a/packages/clickhouse-client.yaml b/packages/clickhouse-client.yaml new file mode 100644 index 00000000000..2a1389b6625 --- /dev/null +++ b/packages/clickhouse-client.yaml @@ -0,0 +1,57 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-client" +arch: "all" +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" + +replaces: +- clickhouse-compressor +conflicts: +- clickhouse-compressor + +maintainer: "ClickHouse Dev Team " +description: | + Client binary for ClickHouse + ClickHouse is a column-oriented database management system + that allows generating analytical data reports in real time. + This package provides clickhouse-client , clickhouse-local and clickhouse-benchmark + +overrides: + deb: + depends: + - clickhouse-common-static (= ${CLICKHOUSE_VERSION_STRING}) + rpm: + depends: + - clickhouse-common-static = ${CLICKHOUSE_VERSION_STRING} + +contents: +- src: root/etc/clickhouse-client/config.xml + dst: /etc/clickhouse-client/config.xml + type: config +- src: root/usr/bin/clickhouse-benchmark + dst: /usr/bin/clickhouse-benchmark +- src: root/usr/bin/clickhouse-compressor + dst: /usr/bin/clickhouse-compressor +- src: root/usr/bin/clickhouse-format + dst: /usr/bin/clickhouse-format +- src: root/usr/bin/clickhouse-client + dst: /usr/bin/clickhouse-client +- src: root/usr/bin/clickhouse-local + dst: /usr/bin/clickhouse-local +- src: root/usr/bin/clickhouse-obfuscator + dst: /usr/bin/clickhouse-obfuscator +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-client/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-client/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-client/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-client/README.md diff --git a/packages/clickhouse-common-static-dbg.yaml b/packages/clickhouse-common-static-dbg.yaml new file mode 100644 index 00000000000..1213f4215c8 --- /dev/null +++ b/packages/clickhouse-common-static-dbg.yaml @@ -0,0 +1,34 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-common-static-dbg" +arch: "${DEB_ARCH}" # amd64, arm64 +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" + +replaces: +- clickhouse-common-dbg +conflicts: +- clickhouse-common-dbg + +maintainer: "ClickHouse Dev Team " +description: | + debugging symbols for clickhouse-common-static + This package contains the debugging symbols for clickhouse-common. + +contents: +- src: root/usr/lib/debug + dst: /usr/lib/debug +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-common-static-dbg/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-common-static-dbg/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-common-static-dbg/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-common-static-dbg/README.md diff --git a/packages/clickhouse-common-static.yaml b/packages/clickhouse-common-static.yaml new file mode 100644 index 00000000000..269d4318e5e --- /dev/null +++ b/packages/clickhouse-common-static.yaml @@ -0,0 +1,48 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-common-static" +arch: "${DEB_ARCH}" # amd64, arm64 +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" + +replaces: +- clickhouse-common +- clickhouse-server-base +provides: +- clickhouse-common +- clickhouse-server-base +suggests: +- clickhouse-common-static-dbg + +maintainer: "ClickHouse Dev Team " +description: | + Common files for ClickHouse + ClickHouse is a column-oriented database management system + that allows generating analytical data reports in real time. + This package provides common files for both clickhouse server and client + +contents: +- src: root/usr/bin/clickhouse + dst: /usr/bin/clickhouse +- src: root/usr/bin/clickhouse-odbc-bridge + dst: /usr/bin/clickhouse-odbc-bridge +- src: root/usr/bin/clickhouse-library-bridge + dst: /usr/bin/clickhouse-library-bridge +- src: root/usr/bin/clickhouse-extract-from-config + dst: /usr/bin/clickhouse-extract-from-config +- src: root/usr/share/bash-completion/completions + dst: /usr/share/bash-completion/completions +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-common-static/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-common-static/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-common-static/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-common-static/README.md diff --git a/packages/clickhouse-server.init b/packages/clickhouse-server.init new file mode 100755 index 00000000000..1695f6286b8 --- /dev/null +++ b/packages/clickhouse-server.init @@ -0,0 +1,227 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: clickhouse-server +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Should-Start: $time $network +# Should-Stop: $network +# Short-Description: clickhouse-server daemon +### END INIT INFO +# +# NOTES: +# - Should-* -- script can start if the listed facilities are missing, unlike Required-* +# +# For the documentation [1]: +# +# [1]: https://wiki.debian.org/LSBInitScripts + +CLICKHOUSE_USER=clickhouse +CLICKHOUSE_GROUP=${CLICKHOUSE_USER} +SHELL=/bin/bash +PROGRAM=clickhouse-server +CLICKHOUSE_GENERIC_PROGRAM=clickhouse +CLICKHOUSE_PROGRAM_ENV="" +EXTRACT_FROM_CONFIG=${CLICKHOUSE_GENERIC_PROGRAM}-extract-from-config +CLICKHOUSE_CONFDIR=/etc/$PROGRAM +CLICKHOUSE_LOGDIR=/var/log/clickhouse-server +CLICKHOUSE_LOGDIR_USER=root +CLICKHOUSE_DATADIR=/var/lib/clickhouse +if [ -d "/var/lock" ]; then + LOCALSTATEDIR=/var/lock +else + LOCALSTATEDIR=/run/lock +fi + +if [ ! -d "$LOCALSTATEDIR" ]; then + mkdir -p "$LOCALSTATEDIR" +fi + +CLICKHOUSE_BINDIR=/usr/bin +CLICKHOUSE_CRONFILE=/etc/cron.d/clickhouse-server +CLICKHOUSE_CONFIG=$CLICKHOUSE_CONFDIR/config.xml +LOCKFILE=$LOCALSTATEDIR/$PROGRAM +CLICKHOUSE_PIDDIR=/var/run/$PROGRAM +CLICKHOUSE_PIDFILE="$CLICKHOUSE_PIDDIR/$PROGRAM.pid" +# CLICKHOUSE_STOP_TIMEOUT=60 # Disabled by default. Place to /etc/default/clickhouse if you need. + +# Some systems lack "flock" +command -v flock >/dev/null && FLOCK=flock + +# Override defaults from optional config file +test -f /etc/default/clickhouse && . /etc/default/clickhouse + + +die() +{ + echo $1 >&2 + exit 1 +} + + +# Check that configuration file is Ok. +check_config() +{ + if [ -x "$CLICKHOUSE_BINDIR/$EXTRACT_FROM_CONFIG" ]; then + su -s $SHELL ${CLICKHOUSE_USER} -c "$CLICKHOUSE_BINDIR/$EXTRACT_FROM_CONFIG --config-file=\"$CLICKHOUSE_CONFIG\" --key=path" >/dev/null || die "Configuration file ${CLICKHOUSE_CONFIG} doesn't parse successfully. Won't restart server. You may use forcerestart if you are sure."; + fi +} + + +initdb() +{ + ${CLICKHOUSE_GENERIC_PROGRAM} install --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" +} + + +start() +{ + ${CLICKHOUSE_GENERIC_PROGRAM} start --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" +} + + +stop() +{ + ${CLICKHOUSE_GENERIC_PROGRAM} stop --pid-path "${CLICKHOUSE_PIDDIR}" +} + + +restart() +{ + ${CLICKHOUSE_GENERIC_PROGRAM} restart --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" +} + + +forcestop() +{ + ${CLICKHOUSE_GENERIC_PROGRAM} stop --force --pid-path "${CLICKHOUSE_PIDDIR}" +} + + +service_or_func() +{ + if [ -x "/bin/systemctl" ] && [ -f /etc/systemd/system/clickhouse-server.service ] && [ -d /run/systemd/system ]; then + systemctl $1 $PROGRAM + else + $1 + fi +} + +forcerestart() +{ + forcestop + # Should not use 'start' function if systemd active + service_or_func start +} + +use_cron() +{ + # 1. running systemd + if [ -x "/bin/systemctl" ] && [ -f /etc/systemd/system/clickhouse-server.service ] && [ -d /run/systemd/system ]; then + return 1 + fi + # 2. disabled by config + if [ -z "$CLICKHOUSE_CRONFILE" ]; then + return 2 + fi + return 0 +} +# returns false if cron disabled (with systemd) +enable_cron() +{ + use_cron && sed -i 's/^#*//' "$CLICKHOUSE_CRONFILE" +} +# returns false if cron disabled (with systemd) +disable_cron() +{ + use_cron && sed -i 's/^#*/#/' "$CLICKHOUSE_CRONFILE" +} + + +is_cron_disabled() +{ + use_cron || return 0 + + # Assumes that either no lines are commented or all lines are commented. + # Also please note, that currently cron file for ClickHouse has only one line (but some time ago there was more). + grep -q -E '^#' "$CLICKHOUSE_CRONFILE"; +} + + +main() +{ + # See how we were called. + EXIT_STATUS=0 + case "$1" in + start) + service_or_func start && enable_cron + ;; + stop) + disable_cron + service_or_func stop + ;; + restart) + service_or_func restart && enable_cron + ;; + forcestop) + disable_cron + forcestop + ;; + forcerestart) + forcerestart && enable_cron + ;; + reload) + service_or_func restart + ;; + condstart) + service_or_func start + ;; + condstop) + service_or_func stop + ;; + condrestart) + service_or_func restart + ;; + condreload) + service_or_func restart + ;; + initdb) + initdb + ;; + enable_cron) + enable_cron + ;; + disable_cron) + disable_cron + ;; + *) + echo "Usage: $0 {start|stop|status|restart|forcestop|forcerestart|reload|condstart|condstop|condrestart|condreload|initdb}" + exit 2 + ;; + esac + + exit $EXIT_STATUS +} + + +status() +{ + ${CLICKHOUSE_GENERIC_PROGRAM} status --pid-path "${CLICKHOUSE_PIDDIR}" +} + + +# Running commands without need of locking +case "$1" in +status) + status + exit 0 + ;; +esac + + +( + if $FLOCK -n 9; then + main "$@" + else + echo "Init script is already running" && exit 1 + fi +) 9> $LOCKFILE diff --git a/packages/clickhouse-server.postinstall b/packages/clickhouse-server.postinstall new file mode 100644 index 00000000000..419c13e3daf --- /dev/null +++ b/packages/clickhouse-server.postinstall @@ -0,0 +1,47 @@ +#!/bin/sh +set -e +# set -x + +PROGRAM=clickhouse-server +CLICKHOUSE_USER=${CLICKHOUSE_USER:=clickhouse} +CLICKHOUSE_GROUP=${CLICKHOUSE_GROUP:=${CLICKHOUSE_USER}} +# Please note that we don't support paths with whitespaces. This is rather ignorant. +CLICKHOUSE_CONFDIR=${CLICKHOUSE_CONFDIR:=/etc/clickhouse-server} +CLICKHOUSE_DATADIR=${CLICKHOUSE_DATADIR:=/var/lib/clickhouse} +CLICKHOUSE_LOGDIR=${CLICKHOUSE_LOGDIR:=/var/log/clickhouse-server} +CLICKHOUSE_BINDIR=${CLICKHOUSE_BINDIR:=/usr/bin} +CLICKHOUSE_GENERIC_PROGRAM=${CLICKHOUSE_GENERIC_PROGRAM:=clickhouse} +EXTRACT_FROM_CONFIG=${CLICKHOUSE_GENERIC_PROGRAM}-extract-from-config +CLICKHOUSE_CONFIG=$CLICKHOUSE_CONFDIR/config.xml +CLICKHOUSE_PIDDIR=/var/run/$PROGRAM + +[ -f /usr/share/debconf/confmodule ] && . /usr/share/debconf/confmodule +[ -f /etc/default/clickhouse ] && . /etc/default/clickhouse + +if [ ! -f "/etc/debian_version" ]; then + not_deb_os=1 +fi + +if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then + + ${CLICKHOUSE_GENERIC_PROGRAM} install --user "${CLICKHOUSE_USER}" --group "${CLICKHOUSE_GROUP}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" --log-path "${CLICKHOUSE_LOGDIR}" --data-path "${CLICKHOUSE_DATADIR}" + + if [ -x "/bin/systemctl" ] && [ -f /etc/systemd/system/clickhouse-server.service ] && [ -d /run/systemd/system ]; then + # if old rc.d service present - remove it + if [ -x "/etc/init.d/clickhouse-server" ] && [ -x "/usr/sbin/update-rc.d" ]; then + /usr/sbin/update-rc.d clickhouse-server remove + fi + + /bin/systemctl daemon-reload + /bin/systemctl enable clickhouse-server + else + # If you downgrading to version older than 1.1.54336 run: systemctl disable clickhouse-server + if [ -x "/etc/init.d/clickhouse-server" ]; then + if [ -x "/usr/sbin/update-rc.d" ]; then + /usr/sbin/update-rc.d clickhouse-server defaults 19 19 >/dev/null || exit $? + else + echo # Other OS + fi + fi + fi +fi diff --git a/packages/clickhouse-server.service b/packages/clickhouse-server.service new file mode 100644 index 00000000000..a9400b24270 --- /dev/null +++ b/packages/clickhouse-server.service @@ -0,0 +1,27 @@ +[Unit] +Description=ClickHouse Server (analytic DBMS for big data) +Requires=network-online.target +# NOTE: that After/Wants=time-sync.target is not enough, you need to ensure +# that the time was adjusted already, if you use systemd-timesyncd you are +# safe, but if you use ntp or some other daemon, you should configure it +# additionaly. +After=time-sync.target network-online.target +Wants=time-sync.target + +[Service] +Type=simple +User=clickhouse +Group=clickhouse +Restart=always +RestartSec=30 +RuntimeDirectory=clickhouse-server +ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.xml --pid-file=/run/clickhouse-server/clickhouse-server.pid +# Minus means that this file is optional. +EnvironmentFile=-/etc/default/clickhouse +LimitCORE=infinity +LimitNOFILE=500000 +CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE + +[Install] +# ClickHouse should not start from the rescue shell (rescue.target). +WantedBy=multi-user.target diff --git a/packages/clickhouse-server.yaml b/packages/clickhouse-server.yaml new file mode 100644 index 00000000000..ed56eb27e54 --- /dev/null +++ b/packages/clickhouse-server.yaml @@ -0,0 +1,68 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-server" +arch: "all" +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" + +conflicts: +- clickhouse-keeper +depends: +- adduser +replaces: +- clickhouse-server-common +- clickhouse-server-base +provides: +- clickhouse-server-common +recommends: +- libcap2-bin + +maintainer: "ClickHouse Dev Team " +description: | + Server binary for ClickHouse + ClickHouse is a column-oriented database management system + that allows generating analytical data reports in real time. + This package provides clickhouse common configuration files + +overrides: + deb: + depends: + - clickhouse-common-static (= ${CLICKHOUSE_VERSION_STRING}) + rpm: + depends: + - clickhouse-common-static = ${CLICKHOUSE_VERSION_STRING} + +contents: +- src: root/etc/clickhouse-server + dst: /etc/clickhouse-server + type: config +- src: clickhouse-server.init + dst: /etc/init.d/clickhouse-server +- src: clickhouse-server.service + dst: /lib/systemd/system/clickhouse-server.service +- src: root/usr/bin/clickhouse-copier + dst: /usr/bin/clickhouse-copier +- src: clickhouse + dst: /usr/bin/clickhouse-keeper + type: symlink +- src: root/usr/bin/clickhouse-report + dst: /usr/bin/clickhouse-report +- src: root/usr/bin/clickhouse-server + dst: /usr/bin/clickhouse-server +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-server/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-server/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-server/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-server/README.md + +scripts: + postinstall: ./clickhouse-server.postinstall diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index c318e163689..55898832f79 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -21,6 +21,8 @@ from ci_config import CI_CONFIG, BuildConfig from docker_pull_helper import get_image_with_version from tee_popen import TeePopen +IMAGE_NAME = "clickhouse/binary-builder" + def get_build_config(build_check_name: str, build_name: str) -> BuildConfig: if build_check_name == "ClickHouse build check (actions)": @@ -86,13 +88,6 @@ def get_packager_cmd( return cmd -def get_image_name(build_config: BuildConfig) -> str: - if build_config["package_type"] != "deb": - return "clickhouse/binary-builder" - else: - return "clickhouse/deb-builder" - - def build_clickhouse( packager_cmd: str, logs_path: str, build_output_path: str ) -> Tuple[str, bool]: @@ -256,8 +251,7 @@ def main(): else: sys.exit(0) - image_name = get_image_name(build_config) - docker_image = get_image_with_version(IMAGES_PATH, image_name) + docker_image = get_image_with_version(IMAGES_PATH, IMAGE_NAME) image_version = docker_image.version logging.info("Got version from repo %s", version.string)