From 1e0ad94d6731f1c65dbc5a522451d8d08de5ddda Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 15 Mar 2022 14:08:30 +0100 Subject: [PATCH 01/11] Add a stand alone clickhouse-keeper package --- docker/packager/packager | 1 + packages/clickhouse-keeper.yaml | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 packages/clickhouse-keeper.yaml diff --git a/docker/packager/packager b/docker/packager/packager index a5763273f5f..18f56ee93ad 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -163,6 +163,7 @@ def parse_env_variables( cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr") cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc") cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var") + cmake_flags.append("-DBUILD_STANDALONE_KEEPER=ON") if is_release_build(build_type, package_type, sanitizer, split_binary): cmake_flags.append("-DINSTALL_STRIPPED_BINARIES=ON") diff --git a/packages/clickhouse-keeper.yaml b/packages/clickhouse-keeper.yaml new file mode 100644 index 00000000000..e717ba79c5b --- /dev/null +++ b/packages/clickhouse-keeper.yaml @@ -0,0 +1,38 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-keeper" +arch: "${DEB_ARCH}" # amd64, arm64 +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" + +conflicts: +- clickhouse-server +depends: +- adduser + +maintainer: "ClickHouse Dev Team " +description: | + Static clickhouse-keeper binary + A stand-alone clickhouse-keeper package + + +contents: +- src: root/etc/clickhouse-keeper + dst: /etc/clickhouse-keeper + type: config +- src: root/usr/bin/clickhouse-keeper + dst: /usr/bin/clickhouse-keeper +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-keeper/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-keeper/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-keeper/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-keeper/README.md From a4ab73619f012acf2db96b64342829b3be4b804c Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 16 Mar 2022 11:37:18 +0100 Subject: [PATCH 02/11] Fix UBSan build --- programs/keeper/CMakeLists.txt | 6 ------ src/Compression/CompressionFactory.cpp | 17 ++++++++++++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt index 92bb5dc45a3..b1132e3aaea 100644 --- a/programs/keeper/CMakeLists.txt +++ b/programs/keeper/CMakeLists.txt @@ -71,17 +71,11 @@ if (BUILD_STANDALONE_KEEPER) ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressedReadBuffer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressedReadBufferFromFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressedWriteBuffer.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecDelta.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecDoubleDelta.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecEncrypted.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecGorilla.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecLZ4.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecMultiple.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecNone.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecT64.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecZSTD.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionFactory.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/getCompressionCodecForFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/ICompressionCodec.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/LZ4_decompress_faster.cpp diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index ca5e5176d13..8dfc894e15b 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -165,25 +165,36 @@ void registerCodecNone(CompressionCodecFactory & factory); void registerCodecLZ4(CompressionCodecFactory & factory); void registerCodecLZ4HC(CompressionCodecFactory & factory); void registerCodecZSTD(CompressionCodecFactory & factory); +void registerCodecMultiple(CompressionCodecFactory & factory); + + +/// Keeper use only general-purpose codes, so we don't need these special codecs +/// in standalone build +#ifndef KEEPER_STANDALONE_BUILD + void registerCodecDelta(CompressionCodecFactory & factory); void registerCodecT64(CompressionCodecFactory & factory); void registerCodecDoubleDelta(CompressionCodecFactory & factory); void registerCodecGorilla(CompressionCodecFactory & factory); void registerCodecEncrypted(CompressionCodecFactory & factory); -void registerCodecMultiple(CompressionCodecFactory & factory); + +#endif CompressionCodecFactory::CompressionCodecFactory() { - registerCodecLZ4(*this); registerCodecNone(*this); + registerCodecLZ4(*this); registerCodecZSTD(*this); registerCodecLZ4HC(*this); + registerCodecMultiple(*this); + +#ifndef KEEPER_STANDALONE_BUILD registerCodecDelta(*this); registerCodecT64(*this); registerCodecDoubleDelta(*this); registerCodecGorilla(*this); registerCodecEncrypted(*this); - registerCodecMultiple(*this); +#endif default_codec = get("LZ4", {}); } From 86dcbe100c94eb802318ef29de2da3487881fed6 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 25 Mar 2022 11:25:42 +0100 Subject: [PATCH 03/11] Add dbg symbols --- packages/clickhouse-keeper-dbg.yaml | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 packages/clickhouse-keeper-dbg.yaml diff --git a/packages/clickhouse-keeper-dbg.yaml b/packages/clickhouse-keeper-dbg.yaml new file mode 100644 index 00000000000..e5d9419eb89 --- /dev/null +++ b/packages/clickhouse-keeper-dbg.yaml @@ -0,0 +1,32 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-keeper-dbg" +arch: "${DEB_ARCH}" # amd64, arm64 +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" + +conflicts: +- clickhouse-server + +maintainer: "ClickHouse Dev Team " +description: | + debugging symbols for clickhouse-keeper + This package contains the debugging symbols for clickhouse-keeper. + +contents: +- src: root/usr/lib/debug/clickhouse-keeper.debug + dst: /usr/lib/debug/clickhouse-keeper.debug +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-keeper-dbg/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-keeper-dbg/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-keeper-dbg/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-keeper-dbg/README.md From 543c46d2fda34d2c61720a3ba6455f09e19cbcb4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 25 Mar 2022 17:29:15 +0100 Subject: [PATCH 04/11] Fix dbg package --- packages/clickhouse-keeper-dbg.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/clickhouse-keeper-dbg.yaml b/packages/clickhouse-keeper-dbg.yaml index e5d9419eb89..685bff11080 100644 --- a/packages/clickhouse-keeper-dbg.yaml +++ b/packages/clickhouse-keeper-dbg.yaml @@ -19,8 +19,8 @@ description: | This package contains the debugging symbols for clickhouse-keeper. contents: -- src: root/usr/lib/debug/clickhouse-keeper.debug - dst: /usr/lib/debug/clickhouse-keeper.debug +- src: root/usr/lib/debug/usr/bin/clickhouse-keeper.debug + dst: /usr/lib/debug/usr/bin/clickhouse-keeper.debug # docs - src: ../AUTHORS dst: /usr/share/doc/clickhouse-keeper-dbg/AUTHORS From b3d80c7822cc47104012b0409d8d8ce091592b3b Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 25 Mar 2022 20:12:19 +0100 Subject: [PATCH 05/11] test From 6780defb42014348eab758f2fb4cfffa93564584 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 26 Mar 2022 22:45:45 +0100 Subject: [PATCH 06/11] Fix build report check --- tests/ci/build_report_check.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/ci/build_report_check.py b/tests/ci/build_report_check.py index 5afe2991073..5946c00cc8a 100644 --- a/tests/ci/build_report_check.py +++ b/tests/ci/build_report_check.py @@ -148,6 +148,13 @@ if __name__ == "__main__": build_name, ) + some_builds_are_missing = len(build_reports_map) < len(reports_order) + + if some_builds_are_missing: + logging.info("Expected to get %s build results, got %s", len(reports_order), len(build_reports_map)) + else: + logging.info("Got exactly %s builds", len(build_reports_map)) + build_reports = [ build_reports_map[build_name] for build_name in reports_order @@ -219,10 +226,10 @@ if __name__ == "__main__": if build_result.status == "success": ok_builds += 1 - if ok_builds == 0: + if ok_builds == 0 or some_builds_are_missing: summary_status = "error" - description = "{}/{} builds are OK".format(ok_builds, total_builds) + description = f"{ok_builds}/{total_builds} builds are OK" print("::notice ::Report url: {}".format(url)) From 08db628cc0269501d222d9c89bfa9b96213ca261 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 28 Mar 2022 11:18:11 +0200 Subject: [PATCH 07/11] Update packages/clickhouse-keeper.yaml Co-authored-by: Mikhail f. Shiryaev --- packages/clickhouse-keeper.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/clickhouse-keeper.yaml b/packages/clickhouse-keeper.yaml index e717ba79c5b..e99ac30f944 100644 --- a/packages/clickhouse-keeper.yaml +++ b/packages/clickhouse-keeper.yaml @@ -14,6 +14,8 @@ conflicts: - clickhouse-server depends: - adduser +suggests: +- clickhouse-keeper-dbg maintainer: "ClickHouse Dev Team " description: | From 37e1160c678bfacf220880aaf560b71a49d4d7fa Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 28 Mar 2022 11:18:16 +0200 Subject: [PATCH 08/11] Update packages/clickhouse-keeper-dbg.yaml Co-authored-by: Mikhail f. Shiryaev --- packages/clickhouse-keeper-dbg.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/clickhouse-keeper-dbg.yaml b/packages/clickhouse-keeper-dbg.yaml index 685bff11080..2c70b7ad4aa 100644 --- a/packages/clickhouse-keeper-dbg.yaml +++ b/packages/clickhouse-keeper-dbg.yaml @@ -9,10 +9,6 @@ homepage: "https://clickhouse.com" license: "Apache" section: "database" priority: "optional" - -conflicts: -- clickhouse-server - maintainer: "ClickHouse Dev Team " description: | debugging symbols for clickhouse-keeper From 77e700b1cfa293ad95f06e3ab4a3e2da040a616c Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 28 Mar 2022 11:25:08 +0200 Subject: [PATCH 09/11] Update src/Compression/CompressionFactory.cpp --- src/Compression/CompressionFactory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index 8dfc894e15b..abf5e38a8c3 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -168,7 +168,7 @@ void registerCodecZSTD(CompressionCodecFactory & factory); void registerCodecMultiple(CompressionCodecFactory & factory); -/// Keeper use only general-purpose codes, so we don't need these special codecs +/// Keeper use only general-purpose codecs, so we don't need these special codecs /// in standalone build #ifndef KEEPER_STANDALONE_BUILD From 47517970cef7cc23dea58e5051d99196d27fe169 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 28 Mar 2022 11:56:37 +0200 Subject: [PATCH 10/11] Bump CI From 8acb5de5b5ef3512ad1e0bc1fd2ccd663380c5d9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 28 Mar 2022 14:43:15 +0200 Subject: [PATCH 11/11] Fix build report black --- tests/ci/build_report_check.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/ci/build_report_check.py b/tests/ci/build_report_check.py index 5946c00cc8a..592e905bcb5 100644 --- a/tests/ci/build_report_check.py +++ b/tests/ci/build_report_check.py @@ -151,7 +151,11 @@ if __name__ == "__main__": some_builds_are_missing = len(build_reports_map) < len(reports_order) if some_builds_are_missing: - logging.info("Expected to get %s build results, got %s", len(reports_order), len(build_reports_map)) + logging.info( + "Expected to get %s build results, got %s", + len(reports_order), + len(build_reports_map), + ) else: logging.info("Got exactly %s builds", len(build_reports_map))