diff --git a/.github/workflows/debug.yml b/.github/workflows/debug.yml index 7bb5ac65140..fa980a95a39 100644 --- a/.github/workflows/debug.yml +++ b/.github/workflows/debug.yml @@ -2,7 +2,7 @@ name: Debug 'on': - [push, pull_request, release] + [push, pull_request, release, workflow_dispatch] jobs: DebugInfo: diff --git a/CMakeLists.txt b/CMakeLists.txt index dad9a25ab26..4f1a6c05730 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -314,6 +314,15 @@ if (ENABLE_BUILD_PATH_MAPPING) set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.") endif () +option (ENABLE_BUILD_PROFILING "Enable profiling of build time" OFF) +if (ENABLE_BUILD_PROFILING) + if (COMPILER_CLANG) + set (COMPILER_FLAGS "${COMPILER_FLAGS} -ftime-trace") + else () + message (${RECONFIGURE_MESSAGE_LEVEL} "Build profiling is only available with CLang") + endif () +endif () + if (${CMAKE_VERSION} VERSION_LESS "3.12.4") # CMake < 3.12 doesn't support setting 20 as a C++ standard version. # We will add C++ standard controlling flag in CMAKE_CXX_FLAGS manually for now. diff --git a/base/base/getAvailableMemoryAmount.cpp b/base/base/getAvailableMemoryAmount.cpp index d2f794e8952..e9bbbd95caf 100644 --- a/base/base/getAvailableMemoryAmount.cpp +++ b/base/base/getAvailableMemoryAmount.cpp @@ -14,8 +14,8 @@ uint64_t getAvailableMemoryAmountOrZero() { -#if defined(_SC_AVPHYS_PAGES) // linux - return getPageSize() * sysconf(_SC_AVPHYS_PAGES); +#if defined(_SC_PHYS_PAGES) // linux + return getPageSize() * sysconf(_SC_PHYS_PAGES); #elif defined(__FreeBSD__) struct vmtotal vmt; size_t vmt_size = sizeof(vmt); diff --git a/base/base/phdr_cache.cpp b/base/base/phdr_cache.cpp index 20a755ed7a4..36a9b4f1f62 100644 --- a/base/base/phdr_cache.cpp +++ b/base/base/phdr_cache.cpp @@ -2,7 +2,7 @@ #pragma clang diagnostic ignored "-Wreserved-identifier" #endif -/// This code was based on the code by Fedor Korotkiy (prime@yandex-team.ru) for YT product in Yandex. +/// This code was based on the code by Fedor Korotkiy https://www.linkedin.com/in/fedor-korotkiy-659a1838/ #include diff --git a/base/base/phdr_cache.h b/base/base/phdr_cache.h index d2854ece0bc..b522710c4c4 100644 --- a/base/base/phdr_cache.h +++ b/base/base/phdr_cache.h @@ -1,6 +1,6 @@ #pragma once -/// This code was based on the code by Fedor Korotkiy (prime@yandex-team.ru) for YT product in Yandex. +/// This code was based on the code by Fedor Korotkiy https://www.linkedin.com/in/fedor-korotkiy-659a1838/ /** Collects all dl_phdr_info items and caches them in a static array. * Also rewrites dl_iterate_phdr with a lock-free version which consults the above cache diff --git a/base/daemon/BaseDaemon.h b/base/daemon/BaseDaemon.h index 54a74369dce..152a431922c 100644 --- a/base/daemon/BaseDaemon.h +++ b/base/daemon/BaseDaemon.h @@ -76,10 +76,10 @@ public: /// return none if daemon doesn't exist, reference to the daemon otherwise static std::optional> tryGetInstance() { return tryGetInstance(); } - /// В Graphite компоненты пути(папки) разделяются точкой. - /// У нас принят путь формата root_path.hostname_yandex_ru.key - /// root_path по умолчанию one_min - /// key - лучше группировать по смыслу. Например "meminfo.cached" или "meminfo.free", "meminfo.total" + /// Graphite metric name has components separated by dots. + /// We used to have the following format: root_path.hostname_clickhouse_com.key + /// root_path - one_min by default + /// key - something that makes sense. Examples: "meminfo.cached" or "meminfo.free", "meminfo.total". template void writeToGraphite(const std::string & key, const T & value, const std::string & config_name = DEFAULT_GRAPHITE_CONFIG_NAME, time_t timestamp = 0, const std::string & custom_root_path = "") { diff --git a/contrib/sysroot b/contrib/sysroot index bbcac834526..e9fb375d0a1 160000 --- a/contrib/sysroot +++ b/contrib/sysroot @@ -1 +1 @@ -Subproject commit bbcac834526d90d1e764164b861be426891d1743 +Subproject commit e9fb375d0a1e5ebfd74c043f088f2342552103f8 diff --git a/debian/.gitignore b/debian/.gitignore deleted file mode 100644 index b4432556de7..00000000000 --- a/debian/.gitignore +++ /dev/null @@ -1,18 +0,0 @@ -control -copyright -tmp/ -clickhouse-benchmark/ -clickhouse-client.docs -clickhouse-client/ -clickhouse-common-static-dbg/ -clickhouse-common-static.docs -clickhouse-common-static/ -clickhouse-server-base/ -clickhouse-server-common/ -clickhouse-server/ -debhelper-build-stamp -files -*.debhelper.log -*.debhelper -*.substvars - diff --git a/debian/.pbuilderrc b/debian/.pbuilderrc deleted file mode 100644 index 485906f6198..00000000000 --- a/debian/.pbuilderrc +++ /dev/null @@ -1,223 +0,0 @@ -# -# sudo apt install pbuilder fakeroot debhelper debian-archive-keyring debian-keyring -# -# ubuntu: -# prepare old (trusty or earlier) host system: - -# sudo ln -s gutsy /usr/share/debootstrap/scripts/eoan -# sudo ln -s gutsy /usr/share/debootstrap/scripts/disco -# sudo ln -s gutsy /usr/share/debootstrap/scripts/cosmic -# sudo ln -s gutsy /usr/share/debootstrap/scripts/artful -# sudo ln -s gutsy /usr/share/debootstrap/scripts/bionic -# sudo ln -s sid /usr/share/debootstrap/scripts/buster -# build ubuntu: -# sudo DIST=bionic pbuilder create --configfile debian/.pbuilderrc && DIST=bionic pdebuild --configfile debian/.pbuilderrc -# sudo DIST=cosmic pbuilder create --configfile debian/.pbuilderrc && DIST=cosmic pdebuild --configfile debian/.pbuilderrc -# sudo DIST=disco pbuilder create --configfile debian/.pbuilderrc && DIST=disco pdebuild --configfile debian/.pbuilderrc -# sudo DIST=eoan pbuilder create --configfile debian/.pbuilderrc && DIST=eoan pdebuild --configfile debian/.pbuilderrc -# sudo DIST=devel pbuilder create --configfile debian/.pbuilderrc && DIST=devel pdebuild --configfile debian/.pbuilderrc -# build debian: -# sudo DIST=stable pbuilder create --configfile debian/.pbuilderrc && DIST=stable pdebuild --configfile debian/.pbuilderrc -# sudo DIST=testing pbuilder create --configfile debian/.pbuilderrc && DIST=testing pdebuild --configfile debian/.pbuilderrc -# sudo DIST=unstable pbuilder create --configfile debian/.pbuilderrc && DIST=unstable pdebuild --configfile debian/.pbuilderrc -# sudo DIST=experimental pbuilder create --configfile debian/.pbuilderrc && DIST=experimental pdebuild --configfile debian/.pbuilderrc -# build i386 experimental: -# sudo DIST=trusty ARCH=i386 pbuilder create --configfile debian/.pbuilderrc && DIST=trusty ARCH=i386 pdebuild --configfile debian/.pbuilderrc -# sudo DIST=xenial ARCH=i386 pbuilder create --configfile debian/.pbuilderrc && DIST=xenial ARCH=i386 pdebuild --configfile debian/.pbuilderrc -# sudo DIST=zesty ARCH=i386 pbuilder create --configfile debian/.pbuilderrc && DIST=zesty ARCH=i386 pdebuild --configfile debian/.pbuilderrc -# sudo DIST=artful ARCH=i386 pbuilder create --configfile debian/.pbuilderrc && DIST=artful ARCH=i386 pdebuild --configfile debian/.pbuilderrc -# sudo DIST=bionic ARCH=i386 pbuilder create --configfile debian/.pbuilderrc && DIST=bionic ARCH=i386 pdebuild --configfile debian/.pbuilderrc -# sudo DIST=stable ARCH=i386 pbuilder create --configfile debian/.pbuilderrc && DIST=stable ARCH=i386 pdebuild --configfile debian/.pbuilderrc -# sudo DIST=testing ARCH=i386 pbuilder create --configfile debian/.pbuilderrc && DIST=testing ARCH=i386 pdebuild --configfile debian/.pbuilderrc -# sudo DIST=experimental ARCH=i386 pbuilder create --configfile debian/.pbuilderrc && DIST=experimental ARCH=i386 pdebuild --configfile debian/.pbuilderrc -# test gcc-9 -# env DEB_CC=gcc-9 DEB_CXX=g++-9 EXTRAPACKAGES="g++-9 gcc-9" DIST=disco pdebuild --configfile debian/.pbuilderrc -# use only clang: -# env DEB_CC=clang-8 DEB_CXX=clang++-8 EXTRAPACKAGES=clang-8 DIST=disco pdebuild --configfile debian/.pbuilderrc -# env DEB_CC=clang-5.0 DEB_CXX=clang++-5.0 EXTRAPACKAGES=clang-5.0 DIST=artful pdebuild --configfile debian/.pbuilderrc -# clang+asan: -# env DEB_CC=clang-5.0 DEB_CXX=clang++-5.0 EXTRAPACKAGES="clang-5.0 libc++abi-dev libc++-dev" CMAKE_FLAGS="-DENABLE_TCMALLOC=0 -DENABLE_UNWIND=0 -DCMAKE_BUILD_TYPE=Asan" DIST=artful pdebuild --configfile debian/.pbuilderrc -# clang+tsan: -# env DEB_CC=clang-5.0 DEB_CXX=clang++-5.0 EXTRAPACKAGES="clang-5.0 libc++abi-dev libc++-dev" CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Tsan" DIST=artful pdebuild --configfile debian/.pbuilderrc -# without sse for old systems and some VM: -# env DH_VERBOSE=1 CMAKE_FLAGS="-DHAVE_SSE41=0 -DHAVE_SSE42=0 -DHAVE_POPCNT=0 -DHAVE_SSE2_INTRIN=0 -DSSE2FLAG=' ' -DHAVE_SSE42_INTRIN=0 -DSSE4FLAG=' ' -DHAVE_PCLMULQDQ_INTRIN=0 -DPCLMULFLAG=' '" DIST=artful pdebuild --configfile debian/.pbuilderrc - -# Note: on trusty host creating some future dists can fail (debootstrap error). - -# Your packages built here: /var/cache/pbuilder/*-*/result - -# from https://wiki.debian.org/PbuilderTricks : - -# Codenames for Debian suites according to their alias. Update these when -# needed. -UNSTABLE_CODENAME="sid" -TESTING_CODENAME="buster" -STABLE_CODENAME="stretch" -STABLE_BACKPORTS_SUITE="$STABLE_CODENAME-backports" - -# List of Debian suites. -DEBIAN_SUITES=($UNSTABLE_CODENAME $TESTING_CODENAME $STABLE_CODENAME $STABLE_BACKPORTS_SUITE - "experimental" "unstable" "testing" "stable") - -# List of Ubuntu suites. Update these when needed. -UBUNTU_SUITES=("eoan" "disco" "cosmic" "bionic" "artful" "zesty" "xenial" "trusty" "devel") - -# Set a default distribution if none is used. Note that you can set your own default (i.e. ${DIST:="unstable"}). -HOST_DIST=`lsb_release --short --codename` -: ${DIST:="$HOST_DIST"} - -# Optionally change Debian codenames in $DIST to their aliases. -case "$DIST" in - $UNSTABLE_CODENAME) - DIST="unstable" - ;; - $TESTING_CODENAME) - DIST="testing" - ;; - $STABLE_CODENAME) - DIST="stable" - ;; -esac - -# Optionally set the architecture to the host architecture if none set. Note -# that you can set your own default (i.e. ${ARCH:="i386"}). -: ${ARCH:="$(dpkg --print-architecture)"} - -NAME="$DIST" -if [ -n "${ARCH}" ]; then - NAME="$NAME-$ARCH" - DEBOOTSTRAPOPTS=("--arch" "$ARCH" "${DEBOOTSTRAPOPTS[@]}") -fi - -BASETGZ=${SET_BASETGZ} -BASETGZ=${BASETGZ:="/var/cache/pbuilder/$NAME-base.tgz"} -DISTRIBUTION="$DIST" -BUILDRESULT=${SET_BUILDRESULT} -BUILDRESULT=${BUILDRESULT:="/var/cache/pbuilder/$NAME/result/"} -APTCACHE="/var/cache/pbuilder/$NAME/aptcache/" -BUILDPLACE="/var/cache/pbuilder/build/" -ALLOWUNTRUSTED=${SET_ALLOWUNTRUSTED:=${ALLOWUNTRUSTED}} - -#DEBOOTSTRAPOPTS=( '--variant=buildd' $SET_DEBOOTSTRAPOPTS ) - - -if $(echo ${DEBIAN_SUITES[@]} | grep -q $DIST); then - # Debian configuration - OSNAME=debian - MIRRORSITE=${SET_MIRRORSITE="http://deb.debian.org/$OSNAME/"} - COMPONENTS="main contrib non-free" - if $(echo "$STABLE_CODENAME stable" | grep -q $DIST); then - OTHERMIRROR="$OTHERMIRROR | deb $MIRRORSITE $STABLE_BACKPORTS_SUITE $COMPONENTS" - fi - # APTKEYRINGS=/usr/share/keyrings/debian-archive-keyring.gpg - - case "$HOST_DIST" in - "trusty" ) - DEBOOTSTRAPOPTS+=( '--no-check-gpg' ) - ;; - *) - DEBOOTSTRAPOPTS+=( '--keyring' '/usr/share/keyrings/debian-archive-keyring.gpg' ) - # DEBOOTSTRAPOPTS+=( '--keyring' '/usr/share/keyrings/debian-keyring.gpg' ) - esac -elif $(echo ${UBUNTU_SUITES[@]} | grep -q $DIST); then - # Ubuntu configuration - OSNAME=ubuntu - - if [[ "$ARCH" == "amd64" || "$ARCH" == "i386" ]]; then - MIRRORSITE=${SET_MIRRORSITE="http://archive.ubuntu.com/$OSNAME/"} - else - MIRRORSITE=${SET_MIRRORSITE="http://ports.ubuntu.com/ubuntu-ports/"} - fi - - COMPONENTS="main restricted universe multiverse" - - OTHERMIRROR="$OTHERMIRROR | deb $MIRRORSITE $DIST-updates main restricted universe multiverse" - OTHERMIRROR="$OTHERMIRROR | deb $MIRRORSITE $DIST-security main restricted universe multiverse" - OTHERMIRROR="$OTHERMIRROR | deb $MIRRORSITE $DIST-proposed main restricted universe multiverse" - - case "$DIST" in - "trusty" | "xenial" ) - OTHERMIRROR="$OTHERMIRROR | deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/$OSNAME $DIST main" - ALLOWUNTRUSTED=yes - ;; - esac - - # deb http://apt.llvm.org/zesty/ llvm-toolchain-zesty-5.0 main -else - echo "Unknown distribution: $DIST" - exit 1 -fi - -echo "using $NAME $OSNAME $DIST $ARCH $LOGNAME $MIRRORSITE" - -case "$DIST" in - "trusty") - # ccache broken - ;; - *) - CCACHEDIR=${SET_CCACHEDIR:="/var/cache/pbuilder/ccache"} - ;; -esac - -# old systems with default gcc <= 6 -case "$DIST" in - "trusty" | "xenial" | "stable" ) - export DEB_CC=gcc-7 - export DEB_CXX=g++-7 - ;; -esac - -if [ "$ARCH" != arm64 ]; then - case "$DIST" in -# TODO: fix llvm-8 and use for "disco" and "eoan" - "experimental") - EXTRAPACKAGES+=" liblld-8-dev libclang-8-dev llvm-8-dev liblld-8 " - export CMAKE_FLAGS="-DLLVM_VERSION=8 $CMAKE_FLAGS" - ;; - "eoan" | "disco" | "cosmic" | "testing" | "unstable") - EXTRAPACKAGES+=" liblld-7-dev libclang-7-dev llvm-7-dev liblld-7 " - export CMAKE_FLAGS="-DLLVM_VERSION=7 $CMAKE_FLAGS" - ;; - "bionic") - EXTRAPACKAGES+=" liblld-6.0-dev libclang-6.0-dev liblld-6.0 " - export CMAKE_FLAGS="-DLLVM_VERSION=6 $CMAKE_FLAGS" - ;; - "artful" ) - EXTRAPACKAGES+=" liblld-5.0-dev libclang-5.0-dev liblld-5.0 " - ;; - esac -else - export CMAKE_FLAGS="-DENABLE_EMBEDDED_COMPILER=0 $CMAKE_FLAGS" -fi - -# Will test symbols -#EXTRAPACKAGES+=" gdb " - -# For killall in pbuilder-hooks: -EXTRAPACKAGES+=" psmisc " - -[[ $CCACHE_PREFIX == 'distcc' ]] && EXTRAPACKAGES+=" $CCACHE_PREFIX " && USENETWORK=yes && export DISTCC_DIR=/var/cache/pbuilder/distcc - -[[ $ARCH == 'i386' ]] && EXTRAPACKAGES+=" libssl-dev " - -export DEB_BUILD_OPTIONS=parallel=`nproc` - -# Floating bug with permissions: -[ -n "$CCACHEDIR" ] && sudo mkdir -p $CCACHEDIR -[ -n "$CCACHEDIR" ] && sudo chmod -R a+rwx $CCACHEDIR || true -# chown -R $BUILDUSERID:$BUILDUSERID $CCACHEDIR - - -# Do not create source package inside pbuilder (-b) -# Use current dir to make package (by default should have src archive) -# echo "3.0 (native)" > debian/source/format -# OR -# pdebuild -b --debbuildopts "--source-option=--format=\"3.0 (native)\"" -# OR -DEBBUILDOPTS="-b --source-option=--format=\"3.0 (native)\"" - -HOOKDIR="debian/pbuilder-hooks" - -#echo "DEBOOTSTRAPOPTS=${DEBOOTSTRAPOPTS[@]}" -#echo "ALLOWUNTRUSTED=${ALLOWUNTRUSTED} OTHERMIRROR=${OTHERMIRROR}" -#echo "EXTRAPACKAGES=${EXTRAPACKAGES}" diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index 43b46f561c8..00000000000 --- a/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -clickhouse (22.1.1.1) unstable; urgency=low - - * Modified source code - - -- clickhouse-release Thu, 09 Dec 2021 00:32:58 +0300 diff --git a/debian/changelog.in b/debian/changelog.in deleted file mode 100644 index ce2ee757999..00000000000 --- a/debian/changelog.in +++ /dev/null @@ -1,5 +0,0 @@ -clickhouse (@VERSION_STRING@) unstable; urgency=low - - * Modified source code - - -- @AUTHOR@ <@EMAIL@> @DATE@ diff --git a/debian/clickhouse-client.install b/debian/clickhouse-client.install deleted file mode 100644 index f19f937b318..00000000000 --- a/debian/clickhouse-client.install +++ /dev/null @@ -1,7 +0,0 @@ -usr/bin/clickhouse-client -usr/bin/clickhouse-local -usr/bin/clickhouse-compressor -usr/bin/clickhouse-benchmark -usr/bin/clickhouse-format -usr/bin/clickhouse-obfuscator -etc/clickhouse-client/config.xml diff --git a/debian/clickhouse-common-static.install b/debian/clickhouse-common-static.install deleted file mode 100644 index 087a6dbba8f..00000000000 --- a/debian/clickhouse-common-static.install +++ /dev/null @@ -1,5 +0,0 @@ -usr/bin/clickhouse -usr/bin/clickhouse-odbc-bridge -usr/bin/clickhouse-library-bridge -usr/bin/clickhouse-extract-from-config -usr/share/bash-completion/completions diff --git a/debian/clickhouse-server.cron.d b/debian/clickhouse-server.cron.d deleted file mode 100644 index 1e5d4aab733..00000000000 --- a/debian/clickhouse-server.cron.d +++ /dev/null @@ -1 +0,0 @@ -#*/10 * * * * root ((which service > /dev/null 2>&1 && (service clickhouse-server condstart ||:)) || /etc/init.d/clickhouse-server condstart) > /dev/null 2>&1 diff --git a/debian/clickhouse-server.docs b/debian/clickhouse-server.docs deleted file mode 100644 index e12d6533be2..00000000000 --- a/debian/clickhouse-server.docs +++ /dev/null @@ -1,4 +0,0 @@ -LICENSE -AUTHORS -README.md -CHANGELOG.md diff --git a/debian/clickhouse-server.init b/debian/clickhouse-server.init deleted file mode 100755 index 1695f6286b8..00000000000 --- a/debian/clickhouse-server.init +++ /dev/null @@ -1,227 +0,0 @@ -#!/bin/sh -### BEGIN INIT INFO -# Provides: clickhouse-server -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Should-Start: $time $network -# Should-Stop: $network -# Short-Description: clickhouse-server daemon -### END INIT INFO -# -# NOTES: -# - Should-* -- script can start if the listed facilities are missing, unlike Required-* -# -# For the documentation [1]: -# -# [1]: https://wiki.debian.org/LSBInitScripts - -CLICKHOUSE_USER=clickhouse -CLICKHOUSE_GROUP=${CLICKHOUSE_USER} -SHELL=/bin/bash -PROGRAM=clickhouse-server -CLICKHOUSE_GENERIC_PROGRAM=clickhouse -CLICKHOUSE_PROGRAM_ENV="" -EXTRACT_FROM_CONFIG=${CLICKHOUSE_GENERIC_PROGRAM}-extract-from-config -CLICKHOUSE_CONFDIR=/etc/$PROGRAM -CLICKHOUSE_LOGDIR=/var/log/clickhouse-server -CLICKHOUSE_LOGDIR_USER=root -CLICKHOUSE_DATADIR=/var/lib/clickhouse -if [ -d "/var/lock" ]; then - LOCALSTATEDIR=/var/lock -else - LOCALSTATEDIR=/run/lock -fi - -if [ ! -d "$LOCALSTATEDIR" ]; then - mkdir -p "$LOCALSTATEDIR" -fi - -CLICKHOUSE_BINDIR=/usr/bin -CLICKHOUSE_CRONFILE=/etc/cron.d/clickhouse-server -CLICKHOUSE_CONFIG=$CLICKHOUSE_CONFDIR/config.xml -LOCKFILE=$LOCALSTATEDIR/$PROGRAM -CLICKHOUSE_PIDDIR=/var/run/$PROGRAM -CLICKHOUSE_PIDFILE="$CLICKHOUSE_PIDDIR/$PROGRAM.pid" -# CLICKHOUSE_STOP_TIMEOUT=60 # Disabled by default. Place to /etc/default/clickhouse if you need. - -# Some systems lack "flock" -command -v flock >/dev/null && FLOCK=flock - -# Override defaults from optional config file -test -f /etc/default/clickhouse && . /etc/default/clickhouse - - -die() -{ - echo $1 >&2 - exit 1 -} - - -# Check that configuration file is Ok. -check_config() -{ - if [ -x "$CLICKHOUSE_BINDIR/$EXTRACT_FROM_CONFIG" ]; then - su -s $SHELL ${CLICKHOUSE_USER} -c "$CLICKHOUSE_BINDIR/$EXTRACT_FROM_CONFIG --config-file=\"$CLICKHOUSE_CONFIG\" --key=path" >/dev/null || die "Configuration file ${CLICKHOUSE_CONFIG} doesn't parse successfully. Won't restart server. You may use forcerestart if you are sure."; - fi -} - - -initdb() -{ - ${CLICKHOUSE_GENERIC_PROGRAM} install --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" -} - - -start() -{ - ${CLICKHOUSE_GENERIC_PROGRAM} start --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" -} - - -stop() -{ - ${CLICKHOUSE_GENERIC_PROGRAM} stop --pid-path "${CLICKHOUSE_PIDDIR}" -} - - -restart() -{ - ${CLICKHOUSE_GENERIC_PROGRAM} restart --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" -} - - -forcestop() -{ - ${CLICKHOUSE_GENERIC_PROGRAM} stop --force --pid-path "${CLICKHOUSE_PIDDIR}" -} - - -service_or_func() -{ - if [ -x "/bin/systemctl" ] && [ -f /etc/systemd/system/clickhouse-server.service ] && [ -d /run/systemd/system ]; then - systemctl $1 $PROGRAM - else - $1 - fi -} - -forcerestart() -{ - forcestop - # Should not use 'start' function if systemd active - service_or_func start -} - -use_cron() -{ - # 1. running systemd - if [ -x "/bin/systemctl" ] && [ -f /etc/systemd/system/clickhouse-server.service ] && [ -d /run/systemd/system ]; then - return 1 - fi - # 2. disabled by config - if [ -z "$CLICKHOUSE_CRONFILE" ]; then - return 2 - fi - return 0 -} -# returns false if cron disabled (with systemd) -enable_cron() -{ - use_cron && sed -i 's/^#*//' "$CLICKHOUSE_CRONFILE" -} -# returns false if cron disabled (with systemd) -disable_cron() -{ - use_cron && sed -i 's/^#*/#/' "$CLICKHOUSE_CRONFILE" -} - - -is_cron_disabled() -{ - use_cron || return 0 - - # Assumes that either no lines are commented or all lines are commented. - # Also please note, that currently cron file for ClickHouse has only one line (but some time ago there was more). - grep -q -E '^#' "$CLICKHOUSE_CRONFILE"; -} - - -main() -{ - # See how we were called. - EXIT_STATUS=0 - case "$1" in - start) - service_or_func start && enable_cron - ;; - stop) - disable_cron - service_or_func stop - ;; - restart) - service_or_func restart && enable_cron - ;; - forcestop) - disable_cron - forcestop - ;; - forcerestart) - forcerestart && enable_cron - ;; - reload) - service_or_func restart - ;; - condstart) - service_or_func start - ;; - condstop) - service_or_func stop - ;; - condrestart) - service_or_func restart - ;; - condreload) - service_or_func restart - ;; - initdb) - initdb - ;; - enable_cron) - enable_cron - ;; - disable_cron) - disable_cron - ;; - *) - echo "Usage: $0 {start|stop|status|restart|forcestop|forcerestart|reload|condstart|condstop|condrestart|condreload|initdb}" - exit 2 - ;; - esac - - exit $EXIT_STATUS -} - - -status() -{ - ${CLICKHOUSE_GENERIC_PROGRAM} status --pid-path "${CLICKHOUSE_PIDDIR}" -} - - -# Running commands without need of locking -case "$1" in -status) - status - exit 0 - ;; -esac - - -( - if $FLOCK -n 9; then - main "$@" - else - echo "Init script is already running" && exit 1 - fi -) 9> $LOCKFILE diff --git a/debian/clickhouse-server.install b/debian/clickhouse-server.install deleted file mode 100644 index b1475fdf162..00000000000 --- a/debian/clickhouse-server.install +++ /dev/null @@ -1,6 +0,0 @@ -usr/bin/clickhouse-server -usr/bin/clickhouse-copier -usr/bin/clickhouse-report -etc/clickhouse-server/config.xml -etc/clickhouse-server/users.xml -etc/systemd/system/clickhouse-server.service diff --git a/debian/clickhouse-server.postinst b/debian/clickhouse-server.postinst deleted file mode 100644 index 419c13e3daf..00000000000 --- a/debian/clickhouse-server.postinst +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh -set -e -# set -x - -PROGRAM=clickhouse-server -CLICKHOUSE_USER=${CLICKHOUSE_USER:=clickhouse} -CLICKHOUSE_GROUP=${CLICKHOUSE_GROUP:=${CLICKHOUSE_USER}} -# Please note that we don't support paths with whitespaces. This is rather ignorant. -CLICKHOUSE_CONFDIR=${CLICKHOUSE_CONFDIR:=/etc/clickhouse-server} -CLICKHOUSE_DATADIR=${CLICKHOUSE_DATADIR:=/var/lib/clickhouse} -CLICKHOUSE_LOGDIR=${CLICKHOUSE_LOGDIR:=/var/log/clickhouse-server} -CLICKHOUSE_BINDIR=${CLICKHOUSE_BINDIR:=/usr/bin} -CLICKHOUSE_GENERIC_PROGRAM=${CLICKHOUSE_GENERIC_PROGRAM:=clickhouse} -EXTRACT_FROM_CONFIG=${CLICKHOUSE_GENERIC_PROGRAM}-extract-from-config -CLICKHOUSE_CONFIG=$CLICKHOUSE_CONFDIR/config.xml -CLICKHOUSE_PIDDIR=/var/run/$PROGRAM - -[ -f /usr/share/debconf/confmodule ] && . /usr/share/debconf/confmodule -[ -f /etc/default/clickhouse ] && . /etc/default/clickhouse - -if [ ! -f "/etc/debian_version" ]; then - not_deb_os=1 -fi - -if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then - - ${CLICKHOUSE_GENERIC_PROGRAM} install --user "${CLICKHOUSE_USER}" --group "${CLICKHOUSE_GROUP}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" --log-path "${CLICKHOUSE_LOGDIR}" --data-path "${CLICKHOUSE_DATADIR}" - - if [ -x "/bin/systemctl" ] && [ -f /etc/systemd/system/clickhouse-server.service ] && [ -d /run/systemd/system ]; then - # if old rc.d service present - remove it - if [ -x "/etc/init.d/clickhouse-server" ] && [ -x "/usr/sbin/update-rc.d" ]; then - /usr/sbin/update-rc.d clickhouse-server remove - fi - - /bin/systemctl daemon-reload - /bin/systemctl enable clickhouse-server - else - # If you downgrading to version older than 1.1.54336 run: systemctl disable clickhouse-server - if [ -x "/etc/init.d/clickhouse-server" ]; then - if [ -x "/usr/sbin/update-rc.d" ]; then - /usr/sbin/update-rc.d clickhouse-server defaults 19 19 >/dev/null || exit $? - else - echo # Other OS - fi - fi - fi -fi diff --git a/debian/clickhouse-server.service b/debian/clickhouse-server.service deleted file mode 100644 index 028b4fbf8ab..00000000000 --- a/debian/clickhouse-server.service +++ /dev/null @@ -1,27 +0,0 @@ -[Unit] -Description=ClickHouse Server (analytic DBMS for big data) -Requires=network-online.target -# NOTE: that After/Wants=time-sync.target is not enough, you need to ensure -# that the time was adjusted already, if you use systemd-timesyncd you are -# safe, but if you use ntp or some other daemon, you should configure it -# additionaly. -After=time-sync.target network-online.target -Wants=time-sync.target - -[Service] -Type=simple -User=clickhouse -Group=clickhouse -Restart=always -RestartSec=30 -RuntimeDirectory=clickhouse-server -ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.xml --pid-file=/run/clickhouse-server/clickhouse-server.pid -# Minus means that this file is optional. -EnvironmentFile=-/etc/default/clickhouse -LimitCORE=infinity -LimitNOFILE=500000 -CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE CAP_NET_BIND_SERVICE - -[Install] -# ClickHouse should not start from the rescue shell (rescue.target). -WantedBy=multi-user.target diff --git a/debian/compat b/debian/compat deleted file mode 100644 index f599e28b8ab..00000000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -10 diff --git a/debian/control b/debian/control deleted file mode 100644 index c5d98d98f41..00000000000 --- a/debian/control +++ /dev/null @@ -1,58 +0,0 @@ -Source: clickhouse -Section: database -Priority: optional -Maintainer: Alexey Milovidov -Build-Depends: debhelper (>= 9), - cmake | cmake3, - ninja-build, - clang-13, - llvm-13, - lld-13, - libc6-dev, - tzdata -Standards-Version: 3.9.8 - -Package: clickhouse-client -Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-common-static (= ${binary:Version}) -Replaces: clickhouse-compressor -Conflicts: clickhouse-compressor -Description: Client binary for ClickHouse - ClickHouse is a column-oriented database management system - that allows generating analytical data reports in real time. - . - This package provides clickhouse-client , clickhouse-local and clickhouse-benchmark - -Package: clickhouse-common-static -Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends} -Suggests: clickhouse-common-static-dbg -Replaces: clickhouse-common, clickhouse-server-base -Provides: clickhouse-common, clickhouse-server-base -Description: Common files for ClickHouse - ClickHouse is a column-oriented database management system - that allows generating analytical data reports in real time. - . - This package provides common files for both clickhouse server and client - -Package: clickhouse-server -Architecture: all -Depends: ${shlibs:Depends}, ${misc:Depends}, clickhouse-common-static (= ${binary:Version}), adduser -Recommends: libcap2-bin -Replaces: clickhouse-server-common, clickhouse-server-base -Provides: clickhouse-server-common -Description: Server binary for ClickHouse - ClickHouse is a column-oriented database management system - that allows generating analytical data reports in real time. - . - This package provides clickhouse common configuration files - -Package: clickhouse-common-static-dbg -Architecture: any -Section: debug -Priority: optional -Depends: ${misc:Depends} -Replaces: clickhouse-common-dbg -Conflicts: clickhouse-common-dbg -Description: debugging symbols for clickhouse-common-static - This package contains the debugging symbols for clickhouse-common. diff --git a/debian/rules b/debian/rules deleted file mode 100755 index 414d472c13d..00000000000 --- a/debian/rules +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/make -f -# -*- makefile -*- - -# Uncomment this to turn on verbose mode. -export DH_VERBOSE=1 - -# -pie only for static mode -export DEB_BUILD_MAINT_OPTIONS=hardening=-all - -# because copy_headers.sh have hardcoded path to build/include_directories.txt -BUILDDIR = obj-$(DEB_HOST_GNU_TYPE) -CURDIR = $(shell pwd) -DESTDIR = $(CURDIR)/debian/tmp - -DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH) - -ifeq ($(CCACHE_PREFIX),distcc) - THREADS_COUNT=$(shell distcc -j) -endif -ifeq ($(THREADS_COUNT),) - THREADS_COUNT=$(shell nproc || grep -c ^processor /proc/cpuinfo || sysctl -n hw.ncpu || echo 4) -endif -DEB_BUILD_OPTIONS+=parallel=$(THREADS_COUNT) - -ifndef ENABLE_TESTS - CMAKE_FLAGS += -DENABLE_TESTS=0 -else -# To export binaries and from deb build we do not strip them. No need to run tests in deb build as we run them in CI - DEB_BUILD_OPTIONS+= nocheck - DEB_BUILD_OPTIONS+= nostrip -endif - -ifndef MAKE_TARGET - MAKE_TARGET = clickhouse-bundle -endif - -CMAKE_FLAGS += -DENABLE_UTILS=0 - -DEB_CC ?= $(shell which gcc-11 gcc-10 gcc-9 gcc | head -n1) -DEB_CXX ?= $(shell which g++-11 g++-10 g++-9 g++ | head -n1) - -ifdef DEB_CXX - DEB_BUILD_GNU_TYPE := $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE) - DEB_HOST_GNU_TYPE := $(shell dpkg-architecture -qDEB_HOST_GNU_TYPE) -ifeq ($(DEB_BUILD_GNU_TYPE),$(DEB_HOST_GNU_TYPE)) - CC := $(DEB_CC) - CXX := $(DEB_CXX) -else ifeq (clang,$(findstring clang,$(DEB_CXX))) -# If we crosscompile with clang, it knows what to do - CC := $(DEB_CC) - CXX := $(DEB_CXX) -else - CC := $(DEB_HOST_GNU_TYPE)-$(DEB_CC) - CXX := $(DEB_HOST_GNU_TYPE)-$(DEB_CXX) -endif -endif - -ifdef CXX - CMAKE_FLAGS += -DCMAKE_CXX_COMPILER=`which $(CXX)` -endif -ifdef CC - CMAKE_FLAGS += -DCMAKE_C_COMPILER=`which $(CC)` -endif - -ifndef DISABLE_NINJA - NINJA=$(shell which ninja) -ifneq ($(NINJA),) - CMAKE_FLAGS += -GNinja - export MAKE=$(NINJA) $(NINJA_FLAGS) -endif -endif - -ifndef DH_VERBOSE - CMAKE_FLAGS += -DCMAKE_VERBOSE_MAKEFILE=0 -endif - -# Useful for bulding on low memory systems -ifndef DISABLE_PARALLEL - DH_FLAGS += --parallel -else - THREADS_COUNT = 1 -endif - -%: - dh $@ $(DH_FLAGS) --buildsystem=cmake - -override_dh_auto_configure: - dh_auto_configure -- $(CMAKE_FLAGS) - -override_dh_auto_build: - # Fix for ninja. Do not add -O. - $(MAKE) -j$(THREADS_COUNT) -C $(BUILDDIR) $(MAKE_TARGET) - -override_dh_auto_test: -ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) - cd $(BUILDDIR) && ctest -j$(THREADS_COUNT) -V -endif - -# Disable config.guess and config.sub update -override_dh_update_autotools_config: - -override_dh_clean: - rm -rf debian/copyright debian/clickhouse-client.docs debian/clickhouse-common-static.docs - dh_clean # -X contrib - -override_dh_strip: - #https://www.debian.org/doc/debian-policy/ch-source.html#debian-rules-and-deb-build-options -ifeq (,$(filter nostrip,$(DEB_BUILD_OPTIONS))) - dh_strip -pclickhouse-common-static --dbg-package=clickhouse-common-static-dbg -endif - -override_dh_install: - # Making docs - cp LICENSE debian/copyright - - ln -sf clickhouse-server.docs debian/clickhouse-client.docs - ln -sf clickhouse-server.docs debian/clickhouse-common-static.docs - - # systemd compatibility - mkdir -p $(DESTDIR)/etc/systemd/system/ - cp debian/clickhouse-server.service $(DESTDIR)/etc/systemd/system/ - - dh_install --list-missing --sourcedir=$(DESTDIR) - -override_dh_auto_install: - env DESTDIR=$(DESTDIR) $(MAKE) -j$(THREADS_COUNT) -C $(BUILDDIR) install - -override_dh_shlibdeps: - true # We depend only on libc and dh_shlibdeps gives us wrong (too strict) dependency. - -override_dh_builddeb: - dh_builddeb -- -Z gzip # Older systems don't have "xz", so use "gzip" instead. diff --git a/debian/source/format b/debian/source/format deleted file mode 100644 index 163aaf8d82b..00000000000 --- a/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (quilt) diff --git a/debian/source/options b/debian/source/options deleted file mode 100644 index 0ceb3b9e28b..00000000000 --- a/debian/source/options +++ /dev/null @@ -1,9 +0,0 @@ -tar-ignore -tar-ignore="build_*/*" -tar-ignore="workspace/*" -tar-ignore="contrib/poco/openssl/*" -tar-ignore="contrib/poco/gradle/*" -tar-ignore="contrib/poco/Data/SQLite/*" -tar-ignore="contrib/poco/PDF/*" -compression-level=3 -compression=gzip diff --git a/debian/watch b/debian/watch deleted file mode 100644 index ed3cab97ade..00000000000 --- a/debian/watch +++ /dev/null @@ -1,6 +0,0 @@ -version=4 - -opts="filenamemangle=s%(?:.*?)?v?(\d[\d.]*)-stable\.tar\.gz%clickhouse-$1.tar.gz%" \ - https://github.com/ClickHouse/ClickHouse/tags \ - (?:.*?/)?v?(\d[\d.]*)-stable\.tar\.gz debian uupdate - diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index cdfa080a475..6297bbead70 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -362,19 +362,6 @@ function get_profiles clickhouse-client --port $RIGHT_SERVER_PORT --query "select 1" } -function build_log_column_definitions -{ -# FIXME This loop builds column definitons from TSVWithNamesAndTypes in an -# absolutely atrocious way. This should be done by the file() function itself. -for x in {right,left}-{addresses,{query,query-thread,trace,{async-,}metric}-log}.tsv -do - paste -d' ' \ - <(sed -n '1{s/\t/\n/g;p;q}' "$x" | sed 's/\(^.*$\)/"\1"/') \ - <(sed -n '2{s/\t/\n/g;p;q}' "$x" ) \ - | tr '\n' ', ' | sed 's/,$//' > "$x.columns" -done -} - # Build and analyze randomization distribution for all queries. function analyze_queries { @@ -382,8 +369,6 @@ rm -v analyze-commands.txt analyze-errors.log all-queries.tsv unstable-queries.t rm -rf analyze ||: mkdir analyze analyze/tmp ||: -build_log_column_definitions - # Split the raw test output into files suitable for analysis. # To debug calculations only for a particular test, substitute a suitable # wildcard here, e.g. `for test_file in modulo-raw.tsv`. @@ -422,12 +407,10 @@ create table partial_query_times engine File(TSVWithNamesAndTypes, -- Process queries that were run normally, on both servers. create view left_query_log as select * - from file('left-query-log.tsv', TSVWithNamesAndTypes, - '$(cat "left-query-log.tsv.columns")'); + from file('left-query-log.tsv', TSVWithNamesAndTypes); create view right_query_log as select * - from file('right-query-log.tsv', TSVWithNamesAndTypes, - '$(cat "right-query-log.tsv.columns")'); + from file('right-query-log.tsv', TSVWithNamesAndTypes); create view query_logs as select 0 version, query_id, ProfileEvents, @@ -645,8 +628,6 @@ mkdir report report/tmp ||: rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.tsv unstable-query-metrics.tsv changed-perf.tsv unstable-tests.tsv unstable-queries.tsv bad-tests.tsv slow-on-client.tsv all-queries.tsv run-errors.tsv ||: -build_log_column_definitions - cat analyze/errors.log >> report/errors.log ||: cat profile-errors.log >> report/errors.log ||: @@ -1028,8 +1009,7 @@ create table unstable_query_runs engine File(TSVWithNamesAndTypes, ; create view query_log as select * - from file('$version-query-log.tsv', TSVWithNamesAndTypes, - '$(cat "$version-query-log.tsv.columns")'); + from file('$version-query-log.tsv', TSVWithNamesAndTypes); create table unstable_run_metrics engine File(TSVWithNamesAndTypes, 'unstable-run-metrics.$version.rep') as @@ -1057,8 +1037,7 @@ create table unstable_run_metrics_2 engine File(TSVWithNamesAndTypes, array join v, n; create view trace_log as select * - from file('$version-trace-log.tsv', TSVWithNamesAndTypes, - '$(cat "$version-trace-log.tsv.columns")'); + from file('$version-trace-log.tsv', TSVWithNamesAndTypes); create view addresses_src as select addr, -- Some functions change name between builds, e.g. '__clone' or 'clone' or @@ -1067,8 +1046,7 @@ create view addresses_src as select addr, [name, 'clone.S (filtered by script)', 'pthread_cond_timedwait (filtered by script)'] -- this line is a subscript operator of the above array [1 + multiSearchFirstIndex(name, ['clone.S', 'pthread_cond_timedwait'])] name - from file('$version-addresses.tsv', TSVWithNamesAndTypes, - '$(cat "$version-addresses.tsv.columns")'); + from file('$version-addresses.tsv', TSVWithNamesAndTypes); create table addresses_join_$version engine Join(any, left, address) as select addr address, name from addresses_src; @@ -1195,15 +1173,12 @@ done function report_metrics { -build_log_column_definitions - rm -rf metrics ||: mkdir metrics clickhouse-local --query " create view right_async_metric_log as - select * from file('right-async-metric-log.tsv', TSVWithNamesAndTypes, - '$(cat right-async-metric-log.tsv.columns)') + select * from file('right-async-metric-log.tsv', TSVWithNamesAndTypes) ; -- Use the right log as time reference because it may have higher precision. @@ -1211,8 +1186,7 @@ create table metrics engine File(TSV, 'metrics/metrics.tsv') as with (select min(event_time) from right_async_metric_log) as min_time select metric, r.event_time - min_time event_time, l.value as left, r.value as right from right_async_metric_log r - asof join file('left-async-metric-log.tsv', TSVWithNamesAndTypes, - '$(cat left-async-metric-log.tsv.columns)') l + asof join file('left-async-metric-log.tsv', TSVWithNamesAndTypes) l on l.metric = r.metric and r.event_time <= l.event_time order by metric, event_time ; diff --git a/docker/test/stress/stress b/docker/test/stress/stress index e42587e9a4d..d78de84f60d 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -83,15 +83,15 @@ def make_query_command(query): def prepare_for_hung_check(drop_databases): # FIXME this function should not exist, but... - # ThreadFuzzer significantly slows down server and causes false-positive hung check failures - call_with_retry("clickhouse client -q 'SYSTEM STOP THREAD FUZZER'") - # We attach gdb to clickhouse-server before running tests # to print stacktraces of all crashes even if clickhouse cannot print it for some reason. # However, it obstruct checking for hung queries. logging.info("Will terminate gdb (if any)") call_with_retry("kill -TERM $(pidof gdb)") + # ThreadFuzzer significantly slows down server and causes false-positive hung check failures + call_with_retry("clickhouse client -q 'SYSTEM STOP THREAD FUZZER'") + call_with_retry(make_query_command('SELECT 1 FORMAT Null')) # Some tests execute SYSTEM STOP MERGES or similar queries. diff --git a/docs/en/getting-started/example-datasets/ontime.md b/docs/en/getting-started/example-datasets/ontime.md index efc807b75fa..87096354f1f 100644 --- a/docs/en/getting-started/example-datasets/ontime.md +++ b/docs/en/getting-started/example-datasets/ontime.md @@ -159,6 +159,10 @@ $ clickhouse-client --query "select count(*) from datasets.ontime" !!! info "Info" If you will run the queries described below, you have to use the full table name, `datasets.ontime`. + +!!! info "Info" + If you are using the prepared partitions or the Online Playground replace any occurrence of `IATA_CODE_Reporting_Airline` or `IATA_CODE_Reporting_Airline AS Carrier` in the following queries with `Carrier` (see `describe ontime`). + ## Queries {#queries} Q0. diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md index eaf7a96ce42..a252f55de2c 100644 --- a/docs/en/interfaces/cli.md +++ b/docs/en/interfaces/cli.md @@ -124,7 +124,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va - `--time, -t` – If specified, print the query execution time to ‘stderr’ in non-interactive mode. - `--stacktrace` – If specified, also print the stack trace if an exception occurs. - `--config-file` – The name of the configuration file. -- `--secure` – If specified, will connect to server over secure connection. +- `--secure` – If specified, will connect to server over secure connection (TLS). You might need to configure your CA certificates in the [configuration file](#configuration_files). The available configuration settings are the same as for [server-side TLS configuration](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-openssl). - `--history_file` — Path to a file containing command history. - `--param_` — Value for a [query with parameters](#cli-queries-with-parameters). - `--hardware-utilization` — Print hardware utilization information in progress bar. @@ -148,7 +148,12 @@ Example of a config file: username password - False + true + + + /etc/ssl/cert.pem + + ``` diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index d72fb4d6f17..ca2a165bbd8 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -5,11 +5,10 @@ toc_title: HTTP Interface # HTTP Interface {#http-interface} -The HTTP interface lets you use ClickHouse on any platform from any programming language. We use it for working from Java and Perl, as well as shell scripts. In other departments, the HTTP interface is used from Perl, Python, and Go. The HTTP interface is more limited than the native interface, but it has better compatibility. +The HTTP interface lets you use ClickHouse on any platform from any programming language in a form of REST API. The HTTP interface is more limited than the native interface, but it has better language support. By default, `clickhouse-server` listens for HTTP on port 8123 (this can be changed in the config). - -Sometimes, `curl` command is not available on user operating systems. On Ubuntu or Debian, run `sudo apt install curl`. Please refer this [documentation](https://curl.se/download.html) to install it before running the examples. +HTTPS can be enabled as well with port 8443 by default. If you make a `GET /` request without parameters, it returns 200 response code and the string which defined in [http_server_default_response](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-http_server_default_response) default value “Ok.” (with a line feed at the end) @@ -18,11 +17,12 @@ $ curl 'http://localhost:8123/' Ok. ``` +Sometimes, `curl` command is not available on user operating systems. On Ubuntu or Debian, run `sudo apt install curl`. Please refer this [documentation](https://curl.se/download.html) to install it before running the examples. + Web UI can be accessed here: `http://localhost:8123/play`. ![Web UI](../images/play.png) - In health-check scripts use `GET /ping` request. This handler always returns “Ok.” (with a line feed at the end). Available from version 18.12.13. See also `/replicas_status` to check replica's delay. ``` bash @@ -32,7 +32,7 @@ $ curl 'http://localhost:8123/replicas_status' Ok. ``` -Send the request as a URL ‘query’ parameter, or as a POST. Or send the beginning of the query in the ‘query’ parameter, and the rest in the POST (we’ll explain later why this is necessary). The size of the URL is limited to 16 KB, so keep this in mind when sending large queries. +Send the request as a URL ‘query’ parameter, or as a POST. Or send the beginning of the query in the ‘query’ parameter, and the rest in the POST (we’ll explain later why this is necessary). The size of the URL is limited to 1 MiB by default, this can be changed with the `http_max_uri_size` setting. If successful, you receive the 200 response code and the result in the response body. If an error occurs, you receive the 500 response code and an error description text in the response body. diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index 98eea85bbfa..2c5c0b7cd52 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -28,6 +28,7 @@ toc_title: Adopters | Badoo | Dating | Timeseries | — | 1.6 mln events/sec (2018) | [Slides in Russian, December 2019](https://presentations.clickhouse.com/meetup38/forecast.pdf) | | Beeline | Telecom | Data Platform | — | — | [Blog post, July 2021](https://habr.com/en/company/beeline/blog/567508/) | | Benocs | Network Telemetry and Analytics | Main Product | — | — | [Slides in English, October 2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | +| Better Stack | Cloud, SaaS | Log Management | - | - | [Official Website](https://betterstack.com/logtail) | | BIGO | Video | Computing Platform | — | — | [Blog Article, August 2020](https://www.programmersought.com/article/44544895251/) | | BiliBili | Video sharing | — | — | — | [Blog post, June 2021](https://chowdera.com/2021/06/20210622012241476b.html) | | Bloomberg | Finance, Media | Monitoring | — | — | [Job opening, September 2021](https://careers.bloomberg.com/job/detail/94913), [slides, May 2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) | @@ -112,7 +113,7 @@ toc_title: Adopters | NLMK | Steel | Monitoring | — | — | [Article in Russian, Jan 2022](https://habr.com/en/company/nlmk/blog/645943/) | | NOC Project | Network Monitoring | Analytics | Main Product | — | [Official Website](https://getnoc.com/features/big-data/) | | Noction | Network Technology | Main Product | — | — | [Official Website](https://www.noction.com/news/irp-3-11-remote-triggered-blackholing-capability) -| ntop | Network Monitoning | Monitoring | — | — | [Official website, Jan 2022](https://www.ntop.org/ntop/historical-traffic-analysis-at-scale-using-clickhouse-with-ntopng/) | +| ntop | Network Monitoning | Monitoring | — | — | [Official website, January 2022](https://www.ntop.org/ntop/historical-traffic-analysis-at-scale-using-clickhouse-with-ntopng/) | | Nuna Inc. | Health Data Analytics | — | — | — | [Talk in English, July 2020](https://youtu.be/GMiXCMFDMow?t=170) | | Ok.ru | Social Network | — | 72 servers | 810 TB compressed, 50bn rows/day, 1.5 TB/day | [SmartData conference, October 2021](https://assets.ctfassets.net/oxjq45e8ilak/4JPHkbJenLgZhBGGyyonFP/57472ec6987003ec4078d0941740703b/____________________ClickHouse_______________________.pdf) | | Omnicomm | Transportation Monitoring | — | — | — | [Facebook post, October 2021](https://www.facebook.com/OmnicommTeam/posts/2824479777774500) | @@ -123,6 +124,7 @@ toc_title: Adopters | Panelbear | Analytics | Monitoring and Analytics | — | — | [Tech Stack, November 2020](https://panelbear.com/blog/tech-stack/) | | Percent 百分点 | Analytics | Main Product | — | — | [Slides in Chinese, June 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) | | Percona | Performance analysis | Percona Monitoring and Management | — | — | [Official website, Mar 2020](https://www.percona.com/blog/2020/03/30/advanced-query-analysis-in-percona-monitoring-and-management-with-direct-clickhouse-access/) | +| PingCAP | Analytics | Real-Time Transactional and Analytical Processing | - | - | [GitHub, TiFlash/TiDB](https://github.com/pingcap/tiflash) | | Plausible | Analytics | Main Product | — | — | [Blog post, June 2020](https://twitter.com/PlausibleHQ/status/1273889629087969280) | | PostHog | Product Analytics | Main Product | — | — | [Release Notes, October 2020](https://posthog.com/blog/the-posthog-array-1-15-0), [Blog, November 2021](https://posthog.com/blog/how-we-turned-clickhouse-into-our-eventmansion) | | Postmates | Delivery | — | — | — | [Talk in English, July 2020](https://youtu.be/GMiXCMFDMow?t=188) | @@ -159,6 +161,7 @@ toc_title: Adopters | Suning | E-Commerce | User behaviour analytics | — | — | [Blog article](https://www.sohu.com/a/434152235_411876) | | Superwall | Monetization Tooling | Main product | — | — | [Word of mouth, Jan 2022](https://github.com/ClickHouse/ClickHouse/pull/33573) | | Swetrix | Analytics | Main Product | — | — | [Source code](https://github.com/swetrix/swetrix-api) | +| Synpse | Application Management | Main Product | - | - | [Tweet, January 2022](https://twitter.com/KRusenas/status/1483571168363880455) | | Teralytics | Mobility | Analytics | — | — | [Tech blog](https://www.teralytics.net/knowledge-hub/visualizing-mobility-data-the-scalability-challenge) | | Tencent | Big Data | Data processing | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/5.%20ClickHouse大数据集群应用_李俊飞腾讯网媒事业部.pdf) | | Tencent | Messaging | Logging | — | — | [Talk in Chinese, November 2019](https://youtu.be/T-iVQRuw-QY?t=5050) | @@ -172,6 +175,7 @@ toc_title: Adopters | UTMSTAT | Analytics | Main product | — | — | [Blog post, June 2020](https://vc.ru/tribuna/133956-striming-dannyh-iz-servisa-skvoznoy-analitiki-v-clickhouse) | | Vercel | Traffic and Performance Analytics | — | — | — | Direct reference, October 2021 | | VKontakte | Social Network | Statistics, Logging | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) | +| VKontech | Distributed Systems | Migrating from MongoDB | - | - | [Blog, January 2022](https://vkontech.com/migrating-your-reporting-queries-from-a-general-purpose-db-mongodb-to-a-data-warehouse-clickhouse-performance-overview/) | | VMware | Cloud | VeloCloud, SDN | — | — | [Product documentation](https://docs.vmware.com/en/vRealize-Operations-Manager/8.3/com.vmware.vcom.metrics.doc/GUID-A9AD72E1-C948-4CA2-971B-919385AB3CA8.html) | | Walmart Labs | Internet, Retail | — | — | — | [Talk in English, July 2020](https://youtu.be/GMiXCMFDMow?t=144) | | Wargaming | Games | | — | — | [Interview](https://habr.com/en/post/496954/) | @@ -197,5 +201,6 @@ toc_title: Adopters | ДомКлик | Real Estate | — | — | — | [Article in Russian, October 2021](https://habr.com/ru/company/domclick/blog/585936/) | | АС "Стрела" | Transportation | — | — | — | [Job posting, Jan 2022](https://vk.com/topic-111905078_35689124?post=3553) | | Piwik PRO | Web Analytics | — | — | — | [Official website, Dec 2018](https://piwik.pro/blog/piwik-pro-clickhouse-faster-efficient-reports/) | +| Deepglint 格灵深瞳 | AI, Computer Vision | OLAP | — | — | [Official Website](https://www.deepglint.com/) | [Original article](https://clickhouse.com/docs/en/introduction/adopters/) diff --git a/docs/en/operations/clickhouse-keeper.md b/docs/en/operations/clickhouse-keeper.md index a8ca2079070..26d61dabaf9 100644 --- a/docs/en/operations/clickhouse-keeper.md +++ b/docs/en/operations/clickhouse-keeper.md @@ -3,13 +3,10 @@ toc_priority: 66 toc_title: ClickHouse Keeper --- -# [pre-production] ClickHouse Keeper {#clickHouse-keeper} +# ClickHouse Keeper {#clickHouse-keeper} ClickHouse server uses [ZooKeeper](https://zookeeper.apache.org/) coordination system for data [replication](../engines/table-engines/mergetree-family/replication.md) and [distributed DDL](../sql-reference/distributed-ddl.md) queries execution. ClickHouse Keeper is an alternative coordination system compatible with ZooKeeper. -!!! warning "Warning" - This feature is currently in the pre-production stage. We test it in our CI and on small internal installations. - ## Implementation details {#implementation-details} ZooKeeper is one of the first well-known open-source coordination systems. It's implemented in Java, has quite a simple and powerful data model. ZooKeeper's coordination algorithm called ZAB (ZooKeeper Atomic Broadcast) doesn't provide linearizability guarantees for reads, because each ZooKeeper node serves reads locally. Unlike ZooKeeper ClickHouse Keeper is written in C++ and uses [RAFT algorithm](https://raft.github.io/) [implementation](https://github.com/eBay/NuRaft). This algorithm allows to have linearizability for reads and writes, has several open-source implementations in different languages. diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 301b348925f..fb53fd38fe3 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -366,12 +366,12 @@ Opens `https://tabix.io/` when accessing `http://localhost: http_port`.
]]> -``` +``` ## hsts_max_age {#hsts-max-age} - -Expired time for HSTS in seconds. The default value is 0 means clickhouse disabled HSTS. If you set a positive number, the HSTS will be enabled and the max-age is the number you set. - -**Example** + +Expired time for HSTS in seconds. The default value is 0 means clickhouse disabled HSTS. If you set a positive number, the HSTS will be enabled and the max-age is the number you set. + +**Example** ```xml 600000 @@ -468,7 +468,7 @@ To enable authentication, set `interserver_http_credentials.allow_empty` to `tru After configuring all replicas set `allow_empty` to `false` or remove this setting. It makes authentication with new credentials mandatory. -To change existing credentials, move the username and the password to `interserver_http_credentials.old` section and update `user` and `password` with new values. At this point the server uses new credentials to connect to other replicas and accepts connections with either new or old credentials. +To change existing credentials, move the username and the password to `interserver_http_credentials.old` section and update `user` and `password` with new values. At this point the server uses new credentials to connect to other replicas and accepts connections with either new or old credentials. ``` xml @@ -834,7 +834,7 @@ The value 0 means that you can delete all tables without any restrictions. ClickHouse uses threads from the Global Thread pool to process queries. If there is no idle thread to process a query, then a new thread is created in the pool. `max_thread_pool_size` limits the maximum number of threads in the pool. -Possible values: +Possible values: - Positive integer. @@ -850,7 +850,7 @@ Default value: `10000`. If the number of **idle** threads in the Global Thread pool is greater than `max_thread_pool_free_size`, then ClickHouse releases resources occupied by some threads and the pool size is decreased. Threads can be created again if necessary. -Possible values: +Possible values: - Positive integer. @@ -866,7 +866,7 @@ Default value: `1000`. The maximum number of jobs that can be scheduled on the Global Thread pool. Increasing queue size leads to larger memory usage. It is recommended to keep this value equal to [max_thread_pool_size](#max-thread-pool-size). -Possible values: +Possible values: - Positive integer. @@ -941,30 +941,30 @@ For more information, see the MergeTreeSettings.h header file. SSL client/server configuration. -Support for SSL is provided by the `libpoco` library. The interface is described in the file [SSLManager.h](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h) +Support for SSL is provided by the `libpoco` library. The available configuration options are explained in [SSLManager.h](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h). Default values can be found in [SSLManager.cpp](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/src/SSLManager.cpp). Keys for server/client settings: - privateKeyFile – The path to the file with the secret key of the PEM certificate. The file may contain a key and certificate at the same time. - certificateFile – The path to the client/server certificate file in PEM format. You can omit it if `privateKeyFile` contains the certificate. -- caConfig – The path to the file or directory that contains trusted root certificates. -- verificationMode – The method for checking the node’s certificates. Details are in the description of the [Context](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Possible values: `none`, `relaxed`, `strict`, `once`. -- verificationDepth – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value. -- loadDefaultCAFile – Indicates that built-in CA certificates for OpenSSL will be used. Acceptable values: `true`, `false`. \| -- cipherList – Supported OpenSSL encryptions. For example: `ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH`. -- cacheSessions – Enables or disables caching sessions. Must be used in combination with `sessionIdContext`. Acceptable values: `true`, `false`. -- sessionIdContext – A unique set of random characters that the server appends to each generated identifier. The length of the string must not exceed `SSL_MAX_SSL_SESSION_ID_LENGTH`. This parameter is always recommended since it helps avoid problems both if the server caches the session and if the client requested caching. Default value: `${application.name}`. -- sessionCacheSize – The maximum number of sessions that the server caches. Default value: 1024\*20. 0 – Unlimited sessions. -- sessionTimeout – Time for caching the session on the server. -- extendedVerification – Automatically extended verification of certificates after the session ends. Acceptable values: `true`, `false`. -- requireTLSv1 – Require a TLSv1 connection. Acceptable values: `true`, `false`. -- requireTLSv1_1 – Require a TLSv1.1 connection. Acceptable values: `true`, `false`. -- requireTLSv1_2 – Require a TLSv1.2 connection. Acceptable values: `true`, `false`. -- fips – Activates OpenSSL FIPS mode. Supported if the library’s OpenSSL version supports FIPS. -- privateKeyPassphraseHandler – Class (PrivateKeyPassphraseHandler subclass) that requests the passphrase for accessing the private key. For example: ``, `KeyFileHandler`, `test`, ``. -- invalidCertificateHandler – Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` ConsoleCertificateHandler ` . -- disableProtocols – Protocols that are not allowed to use. -- preferServerCiphers – Preferred server ciphers on the client. +- caConfig (default: none) – The path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://www.openssl.org/docs/man3.0/man3/SSL_CTX_load_verify_locations.html). +- verificationMode (default: relaxed) – The method for checking the node’s certificates. Details are in the description of the [Context](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Possible values: `none`, `relaxed`, `strict`, `once`. +- verificationDepth (default: 9) – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value. +- loadDefaultCAFile (default: true) – Wether built-in CA certificates for OpenSSL will be used. ClickHouse assumes that builtin CA certificates are in the file `/etc/ssl/cert.pem` (resp. the directory `/etc/ssl/certs`) or in file (resp. directory) specified by the environment variable `SSL_CERT_FILE` (resp. `SSL_CERT_DIR`). +- cipherList (default: `ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH`) - Supported OpenSSL encryptions. +- cacheSessions (default: false) – Enables or disables caching sessions. Must be used in combination with `sessionIdContext`. Acceptable values: `true`, `false`. +- sessionIdContext (default: `${application.name}`) – A unique set of random characters that the server appends to each generated identifier. The length of the string must not exceed `SSL_MAX_SSL_SESSION_ID_LENGTH`. This parameter is always recommended since it helps avoid problems both if the server caches the session and if the client requested caching. Default value: `${application.name}`. +- sessionCacheSize (default: [1024\*20](https://github.com/ClickHouse/boringssl/blob/master/include/openssl/ssl.h#L1978)) – The maximum number of sessions that the server caches. A value of 0 means unlimited sessions. +- sessionTimeout (default: [2h](https://github.com/ClickHouse/boringssl/blob/master/include/openssl/ssl.h#L1926)) – Time for caching the session on the server. +- extendedVerification (default: false) – If enabled, verify that the certificate CN or SAN matches the peer hostname. +- requireTLSv1 (default: false) – Require a TLSv1 connection. Acceptable values: `true`, `false`. +- requireTLSv1_1 (default: false) – Require a TLSv1.1 connection. Acceptable values: `true`, `false`. +- requireTLSv1_2 (default: false) – Require a TLSv1.2 connection. Acceptable values: `true`, `false`. +- fips (default: false) – Activates OpenSSL FIPS mode. Supported if the library’s OpenSSL version supports FIPS. +- privateKeyPassphraseHandler (default: `KeyConsoleHandler`)– Class (PrivateKeyPassphraseHandler subclass) that requests the passphrase for accessing the private key. For example: ``, `KeyFileHandler`, `test`, ``. +- invalidCertificateHandler (default: `ConsoleCertificateHandler`) – Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: ` ConsoleCertificateHandler ` . +- disableProtocols (default: "") – Protocols that are not allowed to use. +- preferServerCiphers (default: false) – Preferred server ciphers on the client. **Example of settings:** diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md index beffd45bcbd..6cda47ab9fb 100644 --- a/docs/en/operations/storing-data.md +++ b/docs/en/operations/storing-data.md @@ -11,10 +11,6 @@ To work with data stored on `Amazon S3` disks use [S3](../engines/table-engines/ To load data from a web server with static files use a disk with type [web](#storing-data-on-webserver). -## Zero-copy Replication {#zero-copy} - -ClickHouse supports zero-copy replication for `S3` and `HDFS` disks, which means that if the data is stored remotely on several machines and needs to be synchronized, then only the metadata is replicated (paths to the data parts), but not the data itself. - ## Configuring HDFS {#configuring-hdfs} [MergeTree](../engines/table-engines/mergetree-family/mergetree.md) and [Log](../engines/table-engines/log-family/log.md) family table engines can store data to HDFS using a disk with type `HDFS`. @@ -316,3 +312,8 @@ When loading files by `endpoint`, they must be loaded into `/store/` p If URL is not reachable on disk load when the server is starting up tables, then all errors are caught. If in this case there were errors, tables can be reloaded (become visible) via `DETACH TABLE table_name` -> `ATTACH TABLE table_name`. If metadata was successfully loaded at server startup, then tables are available straight away. Use [http_max_single_read_retries](../operations/settings/settings.md#http-max-single-read-retries) setting to limit the maximum number of retries during a single HTTP read. + + +## Zero-copy Replication (not ready for production) {#zero-copy} + +ClickHouse supports zero-copy replication for `S3` and `HDFS` disks, which means that if the data is stored remotely on several machines and needs to be synchronized, then only the metadata is replicated (paths to the data parts), but not the data itself. diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index c75fa8e92ce..f2a13569c23 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -182,7 +182,7 @@ Marks numbers: 0 1 2 3 4 5 6 7 8 Разреженный индекс допускает чтение лишних строк. При чтении одного диапазона первичного ключа, может быть прочитано до `index_granularity * 2` лишних строк в каждом блоке данных. -Разреженный индекс почти всегда помещаеся в оперативную память и позволяет работать с очень большим количеством строк в таблицах. +Разреженный индекс почти всегда помещается в оперативную память и позволяет работать с очень большим количеством строк в таблицах. ClickHouse не требует уникального первичного ключа. Можно вставить много строк с одинаковым первичным ключом. diff --git a/docs/ru/sql-reference/data-types/special-data-types/nothing.md b/docs/ru/sql-reference/data-types/special-data-types/nothing.md index 30d425461e1..7a58d52573f 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/nothing.md +++ b/docs/ru/sql-reference/data-types/special-data-types/nothing.md @@ -5,9 +5,9 @@ toc_title: Nothing # Nothing {#nothing} -Этот тип данных предназначен только для того, чтобы представлять [NULL](../../../sql-reference/data-types/special-data-types/nothing.md), т.е. отсутствие значения. +Этот тип данных предназначен только для того, чтобы представлять [NULL](../../../sql-reference/syntax.md#null-literal), т.е. отсутствие значения. -Невозможно создать значение типа `Nothing`, поэтому он используется там, где значение не подразумевается. Например, `NULL` записывается как `Nullable(Nothing)` ([Nullable](../../../sql-reference/data-types/special-data-types/nothing.md) — это тип данных, позволяющий хранить `NULL` в таблицах). Также тип `Nothing` используется для обозначения пустых массивов: +Невозможно создать значение типа `Nothing`, поэтому он используется там, где значение не подразумевается. Например, `NULL` записывается как `Nullable(Nothing)` ([Nullable](../../../sql-reference/data-types/nullable.md) — это тип данных, позволяющий хранить `NULL` в таблицах). Также тип `Nothing` используется для обозначения пустых массивов: ``` sql SELECT toTypeName(Array()) diff --git a/docs/zh/sql-reference/data-types/special-data-types/nothing.md b/docs/zh/sql-reference/data-types/special-data-types/nothing.md index 52c705522e5..64f656185c1 100644 --- a/docs/zh/sql-reference/data-types/special-data-types/nothing.md +++ b/docs/zh/sql-reference/data-types/special-data-types/nothing.md @@ -2,7 +2,7 @@ 此数据类型的唯一目的是表示不是期望值的情况。 所以不能创建一个 `Nothing` 类型的值。 -例如，文本 [NULL](../../../sql-reference/data-types/special-data-types/nothing.md#null-literal) 的类型为 `Nullable(Nothing)`。详情请见 [可为空](../../../sql-reference/data-types/special-data-types/nothing.md)。 +例如，字面量 [NULL](../../../sql-reference/syntax.md#null-literal) 的类型为 `Nullable(Nothing)`。详情请见 [可为空](../../../sql-reference/data-types/nullable.md)。 `Nothing` 类型也可以用来表示空数组： diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index d47372631fe..d0fc3528473 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -66,40 +66,40 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) using namespace DB; namespace po = boost::program_options; - po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); - desc.add_options() - ("help,h", "produce help message") - ("input", po::value()->value_name("INPUT"), "input file") - ("output", po::value()->value_name("OUTPUT"), "output file") - ("decompress,d", "decompress") - ("offset-in-compressed-file", po::value()->default_value(0ULL), "offset to the compressed block (i.e. physical file offset)") - ("offset-in-decompressed-block", po::value()->default_value(0ULL), "offset to the decompressed block (i.e. virtual offset)") - ("block-size,b", po::value()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size") - ("hc", "use LZ4HC instead of LZ4") - ("zstd", "use ZSTD instead of LZ4") - ("codec", po::value>()->multitoken(), "use codecs combination instead of LZ4") - ("level", po::value(), "compression level for codecs specified via flags") - ("none", "use no compression instead of LZ4") - ("stat", "print block statistics of compressed data") - ; - - po::positional_options_description positional_desc; - positional_desc.add("input", 1); - positional_desc.add("output", 1); - - po::variables_map options; - po::store(po::command_line_parser(argc, argv).options(desc).positional(positional_desc).run(), options); - - if (options.count("help")) - { - std::cout << "Usage: " << argv[0] << " [options] < INPUT > OUTPUT" << std::endl; - std::cout << "Usage: " << argv[0] << " [options] INPUT OUTPUT" << std::endl; - std::cout << desc << std::endl; - return 0; - } - try { + po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); + desc.add_options() + ("help,h", "produce help message") + ("input", po::value()->value_name("INPUT"), "input file") + ("output", po::value()->value_name("OUTPUT"), "output file") + ("decompress,d", "decompress") + ("offset-in-compressed-file", po::value()->default_value(0ULL), "offset to the compressed block (i.e. physical file offset)") + ("offset-in-decompressed-block", po::value()->default_value(0ULL), "offset to the decompressed block (i.e. virtual offset)") + ("block-size,b", po::value()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size") + ("hc", "use LZ4HC instead of LZ4") + ("zstd", "use ZSTD instead of LZ4") + ("codec", po::value>()->multitoken(), "use codecs combination instead of LZ4") + ("level", po::value(), "compression level for codecs specified via flags") + ("none", "use no compression instead of LZ4") + ("stat", "print block statistics of compressed data") + ; + + po::positional_options_description positional_desc; + positional_desc.add("input", 1); + positional_desc.add("output", 1); + + po::variables_map options; + po::store(po::command_line_parser(argc, argv).options(desc).positional(positional_desc).run(), options); + + if (options.count("help")) + { + std::cout << "Usage: " << argv[0] << " [options] < INPUT > OUTPUT" << std::endl; + std::cout << "Usage: " << argv[0] << " [options] INPUT OUTPUT" << std::endl; + std::cout << desc << std::endl; + return 0; + } + bool decompress = options.count("decompress"); bool use_lz4hc = options.count("hc"); bool use_zstd = options.count("zstd"); diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index 50d85cdd43d..d5206da00f5 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -44,40 +44,40 @@ int mainEntryClickHouseFormat(int argc, char ** argv) { using namespace DB; - boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); - desc.add_options() - ("query", po::value(), "query to format") - ("help,h", "produce help message") - ("hilite", "add syntax highlight with ANSI terminal escape sequences") - ("oneline", "format in single line") - ("quiet,q", "just check syntax, no output on success") - ("multiquery,n", "allow multiple queries in the same file") - ("obfuscate", "obfuscate instead of formatting") - ("backslash", "add a backslash at the end of each line of the formatted query") - ("allow_settings_after_format_in_insert", "Allow SETTINGS after FORMAT, but note, that this is not always safe") - ("seed", po::value(), "seed (arbitrary string) that determines the result of obfuscation") - ; - - Settings cmd_settings; - for (const auto & field : cmd_settings.all()) - { - if (field.getName() == "max_parser_depth" || field.getName() == "max_query_size") - cmd_settings.addProgramOption(desc, field); - } - - boost::program_options::variables_map options; - boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); - po::notify(options); - - if (options.count("help")) - { - std::cout << "Usage: " << argv[0] << " [options] < query" << std::endl; - std::cout << desc << std::endl; - return 1; - } - try { + boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); + desc.add_options() + ("query", po::value(), "query to format") + ("help,h", "produce help message") + ("hilite", "add syntax highlight with ANSI terminal escape sequences") + ("oneline", "format in single line") + ("quiet,q", "just check syntax, no output on success") + ("multiquery,n", "allow multiple queries in the same file") + ("obfuscate", "obfuscate instead of formatting") + ("backslash", "add a backslash at the end of each line of the formatted query") + ("allow_settings_after_format_in_insert", "Allow SETTINGS after FORMAT, but note, that this is not always safe") + ("seed", po::value(), "seed (arbitrary string) that determines the result of obfuscation") + ; + + Settings cmd_settings; + for (const auto & field : cmd_settings.all()) + { + if (field.getName() == "max_parser_depth" || field.getName() == "max_query_size") + cmd_settings.addProgramOption(desc, field); + } + + boost::program_options::variables_map options; + boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); + po::notify(options); + + if (options.count("help")) + { + std::cout << "Usage: " << argv[0] << " [options] < query" << std::endl; + std::cout << desc << std::endl; + return 1; + } + bool hilite = options.count("hilite"); bool oneline = options.count("oneline"); bool quiet = options.count("quiet"); diff --git a/programs/git-import/git-import.cpp b/programs/git-import/git-import.cpp index 749dcbfee5f..18a9bb2627c 100644 --- a/programs/git-import/git-import.cpp +++ b/programs/git-import/git-import.cpp @@ -1231,5 +1231,5 @@ try catch (...) { std::cerr << DB::getCurrentExceptionMessage(true) << '\n'; - throw; + return DB::getCurrentExceptionCode(); } diff --git a/programs/main.cpp b/programs/main.cpp index 2cdda075ca7..62f2f2f3150 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -13,6 +13,8 @@ #include #include /// pair +#include + #include "config_tools.h" #include @@ -332,6 +334,20 @@ struct Checker #endif ; +void checkHarmfulEnvironmentVariables() +{ + /// The list is a selection from "man ld-linux". And one variable that is Mac OS X specific. + /// NOTE: We will migrate to full static linking or our own dynamic loader to make this code obsolete. + for (const auto * var : {"LD_PRELOAD", "LD_LIBRARY_PATH", "LD_ORIGIN_PATH", "LD_AUDIT", "LD_DYNAMIC_WEAK", "DYLD_INSERT_LIBRARIES"}) + { + if (const char * value = getenv(var); value && value[0]) + { + std::cerr << fmt::format("Environment variable {} is set to {}. It can compromise security.\n", var, value); + _exit(1); + } + } +} + } @@ -352,6 +368,8 @@ int main(int argc_, char ** argv_) inside_main = true; SCOPE_EXIT({ inside_main = false; }); + checkHarmfulEnvironmentVariables(); + /// Reset new handler to default (that throws std::bad_alloc) /// It is needed because LLVM library clobbers it. std::set_new_handler(nullptr); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index fc9187cb622..0120564e0b1 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1716,6 +1716,7 @@ int Server::main(const std::vector & /*args*/) return Application::EXIT_OK; } + void Server::createServers( Poco::Util::AbstractConfiguration & config, const std::vector & listen_hosts, diff --git a/programs/server/play.html b/programs/server/play.html index ef45ee1476b..146316ef61f 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -589,7 +589,7 @@ stats.innerText = `Elapsed: ${seconds} sec, read ${formatted_rows} rows, ${formatted_bytes}.`; /// We can also render graphs if user performed EXPLAIN PIPELINE graph=1 or EXPLAIN AST graph = 1 - if (response.data.length > 3 && response.data[0][0].startsWith("digraph") && document.getElementById('query').value.match(/^\s*EXPLAIN/i)) { + if (response.data.length > 3 && document.getElementById('query').value.match(/^\s*EXPLAIN/i) && typeof(response.data[0][0]) === "string" && response.data[0][0].startsWith("digraph")) { renderGraph(response); } else { renderTable(response); diff --git a/release b/release deleted file mode 100755 index 3eb5591fe2c..00000000000 --- a/release +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash - -# If you have "no space left" error, you can change the location of temporary files with BUILDPLACE environment variable. - -# Version increment: -# Default release: 18.1.2 -> 18.2.0: -# ./release --version -# or -# ./release --version minor -# Bugfix release (only with small patches to previous release): 18.1.2 -> 18.1.3: -# ./release --version patch -# Do this once per year: 18.1.2 -> 19.0.0: -# ./release --version major - -set -e - -# Avoid dependency on locale -LC_ALL=C - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -cd $CUR_DIR - -source "./utils/release/release_lib.sh" - -DEBUILD_NOSIGN_OPTIONS="-us -uc" -DEBUILD_NODEPS_OPTIONS="-d" - -if [ -z "$VERSION_STRING" ] ; then - get_revision_author -fi - -while [[ $1 == --* ]] -do - if [[ $1 == '--test' ]]; then - TEST='yes' - VERSION_POSTFIX+=+test - shift - elif [[ $1 == '--check-build-dependencies' ]]; then - DEBUILD_NODEPS_OPTIONS="" - shift - elif [[ $1 == '--version' ]]; then - gen_revision_author $2 - exit 0 - elif [[ $1 == '--rpm' ]]; then - MAKE_RPM=1 - shift - elif [[ $1 == '--tgz' ]]; then - MAKE_TGZ=1 - shift - else - echo "Unknown option $1" - exit 2 - fi -done - -# Build options -if [ -n "$SANITIZER" ] -then - if [[ "$SANITIZER" == "address" ]]; then VERSION_POSTFIX+="+asan" - elif [[ "$SANITIZER" == "thread" ]]; then VERSION_POSTFIX+="+tsan" - elif [[ "$SANITIZER" == "memory" ]]; then VERSION_POSTFIX+="+msan" - elif [[ "$SANITIZER" == "undefined" ]]; then VERSION_POSTFIX+="+ubsan" - else - echo "Unknown value of SANITIZER variable: $SANITIZER" - exit 3 - fi - - export DEB_CC=${DEB_CC=clang-10} - export DEB_CXX=${DEB_CXX=clang++-10} - EXTRAPACKAGES="$EXTRAPACKAGES clang-10 lld-10" -elif [[ $BUILD_TYPE == 'debug' ]]; then - CMAKE_BUILD_TYPE=Debug - VERSION_POSTFIX+="+debug" -fi - -CMAKE_FLAGS=" $MALLOC_OPTS -DSANITIZE=$SANITIZER -DENABLE_CHECK_HEAVY_BUILDS=1 $CMAKE_FLAGS" -[[ -n "$CMAKE_BUILD_TYPE" ]] && CMAKE_FLAGS=" -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE $CMAKE_FLAGS" - -export CMAKE_FLAGS -export EXTRAPACKAGES - -VERSION_STRING+=$VERSION_POSTFIX -echo -e "\nCurrent version is $VERSION_STRING" - -if [ -z "$NO_BUILD" ] ; then - gen_changelog "$VERSION_STRING" "" "$AUTHOR" "" - # Build (only binary packages). - debuild --preserve-env -e PATH \ - -e DEB_CC=$DEB_CC -e DEB_CXX=$DEB_CXX -e CMAKE_FLAGS="$CMAKE_FLAGS" \ - -b ${DEBUILD_NOSIGN_OPTIONS} ${DEBUILD_NODEPS_OPTIONS} ${DEB_ARCH_FLAG} -fi - -if [ -n "$MAKE_RPM" ]; then - make_rpm -fi - -if [ -n "$MAKE_TGZ" ]; then - make_tgz -fi diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 7393fcd8d36..a9eb27c291c 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -327,7 +327,8 @@ void DiskAccessStorage::scheduleWriteLists(AccessEntityType type) /// Create the 'need_rebuild_lists.mark' file. /// This file will be used later to find out if writing lists is successful or not. - std::ofstream{getNeedRebuildListsMarkFilePath(directory_path)}; + std::ofstream out{getNeedRebuildListsMarkFilePath(directory_path)}; + out.close(); lists_writing_thread = ThreadFromGlobalPool{&DiskAccessStorage::listsWritingThreadFunc, this}; lists_writing_thread_is_waiting = true; diff --git a/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp b/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp index ab6fdc8fd7e..4d7901a7fac 100644 --- a/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp +++ b/src/AggregateFunctions/AggregateFunctionAvgWeighted.cpp @@ -39,7 +39,7 @@ bool allowTypes(const DataTypePtr& left, const DataTypePtr& right) noexcept } template -static IAggregateFunction * create(const IDataType & second_type, TArgs && ... args) +IAggregateFunction * create(const IDataType & second_type, TArgs && ... args) { const WhichDataType which(second_type); @@ -51,7 +51,7 @@ static IAggregateFunction * create(const IDataType & second_type, TArgs && ... a // Not using helper functions because there are no templates for binary decimal/numeric function. template -static IAggregateFunction * create(const IDataType & first_type, const IDataType & second_type, TArgs && ... args) +IAggregateFunction * create(const IDataType & first_type, const IDataType & second_type, TArgs && ... args) { const WhichDataType which(first_type); diff --git a/src/AggregateFunctions/AggregateFunctionDeltaSum.cpp b/src/AggregateFunctions/AggregateFunctionDeltaSum.cpp index f1c6e7c6112..3b43d9a85f8 100644 --- a/src/AggregateFunctions/AggregateFunctionDeltaSum.cpp +++ b/src/AggregateFunctions/AggregateFunctionDeltaSum.cpp @@ -30,7 +30,7 @@ AggregateFunctionPtr createAggregateFunctionDeltaSum( throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - DataTypePtr data_type = arguments[0]; + const DataTypePtr & data_type = arguments[0]; if (isInteger(data_type) || isFloat(data_type)) return AggregateFunctionPtr(createWithNumericType( diff --git a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp index 5a9fd778277..85075d5a4d6 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp @@ -20,7 +20,7 @@ namespace { template