mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 01:25:21 +00:00
Merge remote-tracking branch 'upstream/master' into cache-better-locks
This commit is contained in:
kssenii
commit
4cf2862a0e
@ -41,6 +41,8 @@ Checks: '*,
|
||||
-clang-analyzer-security.insecureAPI.strcpy,
|
||||
|
||||
-cppcoreguidelines-avoid-c-arrays,
|
||||
-cppcoreguidelines-avoid-const-or-ref-data-members,
|
||||
-cppcoreguidelines-avoid-do-while,
|
||||
-cppcoreguidelines-avoid-goto,
|
||||
-cppcoreguidelines-avoid-magic-numbers,
|
||||
-cppcoreguidelines-avoid-non-const-global-variables,
|
||||
@ -128,6 +130,7 @@ Checks: '*,
|
||||
-portability-simd-intrinsics,
|
||||
|
||||
-readability-braces-around-statements,
|
||||
-readability-convert-member-functions-to-static,
|
||||
-readability-else-after-return,
|
||||
-readability-function-cognitive-complexity,
|
||||
-readability-function-size,
|
||||
|
||||
2
.github/PULL_REQUEST_TEMPLATE.md
vendored
2
.github/PULL_REQUEST_TEMPLATE.md
vendored
@ -7,7 +7,7 @@ tests/ci/run_check.py
|
||||
### Changelog category (leave one):
|
||||
- New Feature
|
||||
- Improvement
|
||||
- Bug Fix (user-visible misbehavior in official stable or prestable release)
|
||||
- Bug Fix (user-visible misbehavior in an official stable release)
|
||||
- Performance Improvement
|
||||
- Backward Incompatible Change
|
||||
- Build/Testing/Packaging Improvement
|
||||
|
||||
16
.github/workflows/backport_branches.yml
vendored
16
.github/workflows/backport_branches.yml
vendored
@ -9,8 +9,22 @@ on: # yamllint disable-line rule:truthy
|
||||
branches:
|
||||
- 'backport/**'
|
||||
jobs:
|
||||
CheckLabels:
|
||||
runs-on: [self-hosted, style-checker]
|
||||
# Run the first check always, even if the CI is cancelled
|
||||
if: ${{ always() }}
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
with:
|
||||
clear-repository: true
|
||||
- name: Labels check
|
||||
run: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
python3 run_check.py
|
||||
PythonUnitTests:
|
||||
runs-on: [self-hosted, style-checker]
|
||||
needs: CheckLabels
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
@ -22,6 +36,7 @@ jobs:
|
||||
python3 -m unittest discover -s . -p '*_test.py'
|
||||
DockerHubPushAarch64:
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
needs: CheckLabels
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
@ -38,6 +53,7 @@ jobs:
|
||||
path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
|
||||
DockerHubPushAmd64:
|
||||
runs-on: [self-hosted, style-checker]
|
||||
needs: CheckLabels
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
|
||||
@ -140,7 +140,7 @@
|
||||
* Upgrade Intel QPL from v0.3.0 to v1.0.0 2. Build libaccel-config and link it statically to QPL library instead of dynamically. [#45809](https://github.com/ClickHouse/ClickHouse/pull/45809) ([jasperzhu](https://github.com/jinjunzh)).
|
||||
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehavior in official stable release)
|
||||
|
||||
* Flush data exactly by `rabbitmq_flush_interval_ms` or by `rabbitmq_max_block_size` in `StorageRabbitMQ`. Closes [#42389](https://github.com/ClickHouse/ClickHouse/issues/42389). Closes [#45160](https://github.com/ClickHouse/ClickHouse/issues/45160). [#44404](https://github.com/ClickHouse/ClickHouse/pull/44404) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Use PODArray to render in sparkBar function, so we can control the memory usage. Close [#44467](https://github.com/ClickHouse/ClickHouse/issues/44467). [#44489](https://github.com/ClickHouse/ClickHouse/pull/44489) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
|
||||
@ -121,6 +121,7 @@ if (ENABLE_COLORED_BUILD AND CMAKE_GENERATOR STREQUAL "Ninja")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always")
|
||||
# ... such manually setting of flags can be removed once CMake supports a variable to
|
||||
# activate colors in *all* build systems: https://gitlab.kitware.com/cmake/cmake/-/issues/15502
|
||||
# --> available since CMake 3.24: https://stackoverflow.com/a/73349744
|
||||
endif ()
|
||||
|
||||
include (cmake/check_flags.cmake)
|
||||
@ -134,24 +135,15 @@ if (COMPILER_CLANG)
|
||||
set(COMPILER_FLAGS "${COMPILER_FLAGS} -gdwarf-aranges")
|
||||
endif ()
|
||||
|
||||
if (HAS_USE_CTOR_HOMING)
|
||||
# For more info see https://blog.llvm.org/posts/2021-04-05-constructor-homing-for-debug-info/
|
||||
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fuse-ctor-homing")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing")
|
||||
endif()
|
||||
# See https://blog.llvm.org/posts/2021-04-05-constructor-homing-for-debug-info/
|
||||
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fuse-ctor-homing")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing")
|
||||
endif()
|
||||
|
||||
no_warning(enum-constexpr-conversion) # breaks Protobuf in clang-16
|
||||
endif ()
|
||||
|
||||
# If compiler has support for -Wreserved-identifier. It is difficult to detect by clang version,
|
||||
# because there are two different branches of clang: clang and AppleClang.
|
||||
# (AppleClang is not supported by ClickHouse, but some developers have misfortune to use it).
|
||||
if (HAS_RESERVED_IDENTIFIER)
|
||||
add_compile_definitions (HAS_RESERVED_IDENTIFIER)
|
||||
endif ()
|
||||
|
||||
option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON)
|
||||
option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF)
|
||||
option(ENABLE_BENCHMARKS "Build all benchmark programs in 'benchmarks' subdirectories" OFF)
|
||||
@ -184,26 +176,11 @@ if (OS_DARWIN)
|
||||
set (ENABLE_CURL_BUILD OFF)
|
||||
endif ()
|
||||
|
||||
# Ignored if `lld` is used
|
||||
option(ADD_GDB_INDEX_FOR_GOLD "Add .gdb-index to resulting binaries for gold linker.")
|
||||
|
||||
if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
|
||||
# Can be lld or ld-lld or lld-13 or /path/to/lld.
|
||||
if (LINKER_NAME MATCHES "lld" AND OS_LINUX)
|
||||
if (LINKER_NAME MATCHES "lld")
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index")
|
||||
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gdb-index")
|
||||
message (STATUS "Adding .gdb-index via --gdb-index linker option.")
|
||||
# we use another tool for gdb-index, because gold linker removes section .debug_aranges, which used inside clickhouse stacktraces
|
||||
# http://sourceware-org.1504.n7.nabble.com/gold-No-debug-aranges-section-when-linking-with-gdb-index-td540965.html#a556932
|
||||
elseif (LINKER_NAME MATCHES "gold$" AND ADD_GDB_INDEX_FOR_GOLD)
|
||||
find_program (GDB_ADD_INDEX_EXE NAMES "gdb-add-index" DOC "Path to gdb-add-index executable")
|
||||
if (NOT GDB_ADD_INDEX_EXE)
|
||||
set (USE_GDB_ADD_INDEX 0)
|
||||
message (WARNING "Cannot add gdb index to binaries, because gold linker is used, but gdb-add-index executable not found.")
|
||||
else()
|
||||
set (USE_GDB_ADD_INDEX 1)
|
||||
message (STATUS "gdb-add-index found: ${GDB_ADD_INDEX_EXE}")
|
||||
endif()
|
||||
endif ()
|
||||
endif()
|
||||
|
||||
@ -235,7 +212,7 @@ endif ()
|
||||
|
||||
# Create BuildID when using lld. For other linkers it is created by default.
|
||||
# (NOTE: LINKER_NAME can be either path or name, and in different variants)
|
||||
if (LINKER_NAME MATCHES "lld" AND OS_LINUX)
|
||||
if (LINKER_NAME MATCHES "lld")
|
||||
# SHA1 is not cryptographically secure but it is the best what lld is offering.
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id=sha1")
|
||||
endif ()
|
||||
@ -311,6 +288,7 @@ set (CMAKE_C_STANDARD_REQUIRED ON)
|
||||
|
||||
if (COMPILER_GCC OR COMPILER_CLANG)
|
||||
# Enable C++14 sized global deallocation functions. It should be enabled by setting -std=c++14 but I'm not sure.
|
||||
# See https://reviews.llvm.org/D112921
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation")
|
||||
endif ()
|
||||
|
||||
@ -329,11 +307,7 @@ if (ARCH_AMD64)
|
||||
set(BRANCHES_WITHIN_32B_BOUNDARIES "-Wa,${BRANCHES_WITHIN_32B_BOUNDARIES}")
|
||||
endif()
|
||||
|
||||
include(CheckCXXCompilerFlag)
|
||||
check_cxx_compiler_flag("${BRANCHES_WITHIN_32B_BOUNDARIES}" HAS_BRANCHES_WITHIN_32B_BOUNDARIES)
|
||||
if (HAS_BRANCHES_WITHIN_32B_BOUNDARIES)
|
||||
set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}")
|
||||
endif()
|
||||
set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}")
|
||||
endif()
|
||||
|
||||
if (COMPILER_GCC)
|
||||
@ -445,6 +419,7 @@ option(WERROR "Enable -Werror compiler option" ON)
|
||||
if (WERROR)
|
||||
# Don't pollute CMAKE_CXX_FLAGS with -Werror as it will break some CMake checks.
|
||||
# Instead, adopt modern cmake usage requirement.
|
||||
# TODO: Set CMAKE_COMPILE_WARNING_AS_ERROR (cmake 3.24)
|
||||
target_compile_options(global-group INTERFACE "-Werror")
|
||||
endif ()
|
||||
|
||||
|
||||
@ -19,8 +19,8 @@ endif()
|
||||
if (NOT "$ENV{CFLAGS}" STREQUAL ""
|
||||
OR NOT "$ENV{CXXFLAGS}" STREQUAL ""
|
||||
OR NOT "$ENV{LDFLAGS}" STREQUAL ""
|
||||
OR CMAKE_C_FLAGS OR CMAKE_CXX_FLAGS OR CMAKE_EXE_LINKER_FLAGS OR CMAKE_SHARED_LINKER_FLAGS OR CMAKE_MODULE_LINKER_FLAGS
|
||||
OR CMAKE_C_FLAGS_INIT OR CMAKE_CXX_FLAGS_INIT OR CMAKE_EXE_LINKER_FLAGS_INIT OR CMAKE_SHARED_LINKER_FLAGS_INIT OR CMAKE_MODULE_LINKER_FLAGS_INIT)
|
||||
OR CMAKE_C_FLAGS OR CMAKE_CXX_FLAGS OR CMAKE_EXE_LINKER_FLAGS OR CMAKE_MODULE_LINKER_FLAGS
|
||||
OR CMAKE_C_FLAGS_INIT OR CMAKE_CXX_FLAGS_INIT OR CMAKE_EXE_LINKER_FLAGS_INIT OR CMAKE_MODULE_LINKER_FLAGS_INIT)
|
||||
|
||||
# if $ENV
|
||||
message("CFLAGS: $ENV{CFLAGS}")
|
||||
@ -36,7 +36,6 @@ if (NOT "$ENV{CFLAGS}" STREQUAL ""
|
||||
message("CMAKE_C_FLAGS_INIT: ${CMAKE_C_FLAGS_INIT}")
|
||||
message("CMAKE_CXX_FLAGS_INIT: ${CMAKE_CXX_FLAGS_INIT}")
|
||||
message("CMAKE_EXE_LINKER_FLAGS_INIT: ${CMAKE_EXE_LINKER_FLAGS_INIT}")
|
||||
message("CMAKE_SHARED_LINKER_FLAGS_INIT: ${CMAKE_SHARED_LINKER_FLAGS_INIT}")
|
||||
message("CMAKE_MODULE_LINKER_FLAGS_INIT: ${CMAKE_MODULE_LINKER_FLAGS_INIT}")
|
||||
|
||||
message(FATAL_ERROR "
|
||||
|
||||
@ -21,11 +21,10 @@ curl https://clickhouse.com/ | sh
|
||||
* [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
|
||||
|
||||
## Upcoming Events
|
||||
* [**v23.2 Release Webinar**](https://clickhouse.com/company/events/v23-2-release-webinar?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-02) - Feb 23 - 23.2 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
|
||||
* [**ClickHouse Meetup in Amsterdam**](https://www.meetup.com/clickhouse-netherlands-user-group/events/291485868/) - Mar 9 - The first ClickHouse Amsterdam Meetup of 2023 is here! 🎉 Join us for short lightning talks and long discussions. Food, drinks & good times on us.
|
||||
* [**ClickHouse Meetup in SF Bay Area**](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/291490121/) - Mar 14 - A night to meet with ClickHouse team in the San Francisco area! Food and drink are a given...but networking is the primary focus.
|
||||
* [**ClickHouse Meetup in Austin**](https://www.meetup.com/clickhouse-austin-user-group/events/291486654/) - Mar 16 - The first ClickHouse Meetup in Austin is happening soon! Interested in speaking, let us know!
|
||||
* [**ClickHouse Meetup in Austin**](https://www.meetup.com/clickhouse-austin-user-group/events/291486654/) - Mar 30 - The first ClickHouse Meetup in Austin is happening soon! Interested in speaking, let us know!
|
||||
* [**v23.3 Release Webinar**](https://clickhouse.com/company/events/v23-3-release-webinar?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-02) - Mar 30 - 23.3 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
|
||||
|
||||
## Recent Recordings
|
||||
* **FOSDEM 2023**: In the "Fast and Streaming Data" room Alexey gave a talk entitled "Building Analytical Apps With ClickHouse" that looks at the landscape of data tools, an interesting data set, and how you can interact with data quickly. Check out the recording on **[YouTube](https://www.youtube.com/watch?v=JlcI2Vfz_uk)**.
|
||||
* **Recording available**: [**v23.1 Release Webinar**](https://www.youtube.com/watch?v=zYSZXBnTMSE) 23.1 is the ClickHouse New Year release. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release. Inverted indices, query cache, and so -- very -- much more.
|
||||
* **Recording available**: [**v23.2 Release Webinar**](https://www.youtube.com/watch?v=2o0vRMMIrkY) NTILE Window Function support, Partition Key for GROUP By, io_uring, Apache Iceberg support, Dynamic Disks, integrations updates! Watch it now!
|
||||
* **All release webinar recordings**: [YouTube playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3jAlSy1JxyP8zluvXaN3nxU)
|
||||
|
||||
@ -466,9 +466,8 @@ JSON::Pos JSON::searchField(const char * data, size_t size) const
|
||||
{
|
||||
if (!it->hasEscapes())
|
||||
{
|
||||
if (static_cast<int>(size) + 2 > it->dataEnd() - it->data())
|
||||
continue;
|
||||
if (!strncmp(data, it->data() + 1, size))
|
||||
const auto current_name = it->getRawName();
|
||||
if (current_name.size() == size && 0 == memcmp(current_name.data(), data, size))
|
||||
break;
|
||||
}
|
||||
else
|
||||
|
||||
@ -2,6 +2,8 @@
|
||||
|
||||
#if WITH_COVERAGE
|
||||
|
||||
#pragma GCC diagnostic ignored "-Wreserved-identifier"
|
||||
|
||||
# include <mutex>
|
||||
# include <unistd.h>
|
||||
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <bit>
|
||||
#include <cstring>
|
||||
#include "types.h"
|
||||
|
||||
|
||||
@ -1,6 +1,4 @@
|
||||
#ifdef HAS_RESERVED_IDENTIFIER
|
||||
#pragma clang diagnostic ignored "-Wreserved-identifier"
|
||||
#endif
|
||||
|
||||
/// This code was based on the code by Fedor Korotkiy https://www.linkedin.com/in/fedor-korotkiy-659a1838/
|
||||
|
||||
|
||||
@ -5,10 +5,8 @@ constexpr size_t KiB = 1024;
|
||||
constexpr size_t MiB = 1024 * KiB;
|
||||
constexpr size_t GiB = 1024 * MiB;
|
||||
|
||||
#ifdef HAS_RESERVED_IDENTIFIER
|
||||
# pragma clang diagnostic push
|
||||
# pragma clang diagnostic ignored "-Wreserved-identifier"
|
||||
#endif
|
||||
#pragma clang diagnostic push
|
||||
#pragma clang diagnostic ignored "-Wreserved-identifier"
|
||||
|
||||
// NOLINTBEGIN(google-runtime-int)
|
||||
constexpr size_t operator"" _KiB(unsigned long long val) { return val * KiB; }
|
||||
@ -16,6 +14,4 @@ constexpr size_t operator"" _MiB(unsigned long long val) { return val * MiB; }
|
||||
constexpr size_t operator"" _GiB(unsigned long long val) { return val * GiB; }
|
||||
// NOLINTEND(google-runtime-int)
|
||||
|
||||
#ifdef HAS_RESERVED_IDENTIFIER
|
||||
# pragma clang diagnostic pop
|
||||
#endif
|
||||
#pragma clang diagnostic pop
|
||||
|
||||
@ -732,9 +732,10 @@ public:
|
||||
if (std::numeric_limits<T>::is_signed && (is_negative(lhs) != is_negative(rhs)))
|
||||
return is_negative(rhs);
|
||||
|
||||
integer<Bits, Signed> t = rhs;
|
||||
for (unsigned i = 0; i < item_count; ++i)
|
||||
{
|
||||
base_type rhs_item = get_item(rhs, big(i));
|
||||
base_type rhs_item = get_item(t, big(i));
|
||||
|
||||
if (lhs.items[big(i)] != rhs_item)
|
||||
return lhs.items[big(i)] > rhs_item;
|
||||
@ -757,9 +758,10 @@ public:
|
||||
if (std::numeric_limits<T>::is_signed && (is_negative(lhs) != is_negative(rhs)))
|
||||
return is_negative(lhs);
|
||||
|
||||
integer<Bits, Signed> t = rhs;
|
||||
for (unsigned i = 0; i < item_count; ++i)
|
||||
{
|
||||
base_type rhs_item = get_item(rhs, big(i));
|
||||
base_type rhs_item = get_item(t, big(i));
|
||||
|
||||
if (lhs.items[big(i)] != rhs_item)
|
||||
return lhs.items[big(i)] < rhs_item;
|
||||
@ -779,9 +781,10 @@ public:
|
||||
{
|
||||
if constexpr (should_keep_size<T>())
|
||||
{
|
||||
integer<Bits, Signed> t = rhs;
|
||||
for (unsigned i = 0; i < item_count; ++i)
|
||||
{
|
||||
base_type rhs_item = get_item(rhs, any(i));
|
||||
base_type rhs_item = get_item(t, any(i));
|
||||
|
||||
if (lhs.items[any(i)] != rhs_item)
|
||||
return false;
|
||||
@ -1239,7 +1242,7 @@ constexpr integer<Bits, Signed>::operator long double() const noexcept
|
||||
for (unsigned i = 0; i < _impl::item_count; ++i)
|
||||
{
|
||||
long double t = res;
|
||||
res *= std::numeric_limits<base_type>::max();
|
||||
res *= static_cast<long double>(std::numeric_limits<base_type>::max());
|
||||
res += t;
|
||||
res += tmp.items[_impl::big(i)];
|
||||
}
|
||||
|
||||
@ -64,6 +64,6 @@ struct fmt::formatter<wide::integer<Bits, Signed>>
|
||||
template <typename FormatContext>
|
||||
auto format(const wide::integer<Bits, Signed> & value, FormatContext & ctx)
|
||||
{
|
||||
return format_to(ctx.out(), "{}", to_string(value));
|
||||
return fmt::format_to(ctx.out(), "{}", to_string(value));
|
||||
}
|
||||
};
|
||||
|
||||
@ -27,7 +27,7 @@
|
||||
#include "Poco/Exception.h"
|
||||
#include "Poco/NumberParser.h"
|
||||
#include "Poco/NumberFormatter.h"
|
||||
#include <set>
|
||||
#include <unordered_map>
|
||||
|
||||
|
||||
namespace Poco {
|
||||
|
||||
@ -27,9 +27,7 @@
|
||||
#define _PATH_TTY "/dev/tty"
|
||||
#endif
|
||||
|
||||
#ifdef HAS_RESERVED_IDENTIFIER
|
||||
#pragma clang diagnostic ignored "-Wreserved-identifier"
|
||||
#endif
|
||||
|
||||
#include <termios.h>
|
||||
#include <signal.h>
|
||||
|
||||
@ -1,7 +1,5 @@
|
||||
include (CheckCXXCompilerFlag)
|
||||
include (CheckCCompilerFlag)
|
||||
|
||||
check_cxx_compiler_flag("-Wreserved-identifier" HAS_RESERVED_IDENTIFIER)
|
||||
check_cxx_compiler_flag("-Wsuggest-destructor-override" HAS_SUGGEST_DESTRUCTOR_OVERRIDE)
|
||||
check_cxx_compiler_flag("-Wsuggest-override" HAS_SUGGEST_OVERRIDE)
|
||||
check_cxx_compiler_flag("-Xclang -fuse-ctor-homing" HAS_USE_CTOR_HOMING)
|
||||
# Set/unset variable based on existence of compiler flags. Example:
|
||||
# check_cxx_compiler_flag("-Wreserved-identifier" HAS_RESERVED_IDENTIFIER)
|
||||
|
||||
@ -5,14 +5,14 @@ if (ENABLE_CLANG_TIDY)
|
||||
|
||||
find_program (CLANG_TIDY_CACHE_PATH NAMES "clang-tidy-cache")
|
||||
if (CLANG_TIDY_CACHE_PATH)
|
||||
find_program (_CLANG_TIDY_PATH NAMES "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12" "clang-tidy")
|
||||
find_program (_CLANG_TIDY_PATH NAMES "clang-tidy-16" "clang-tidy-15" "clang-tidy-14" "clang-tidy")
|
||||
|
||||
# Why do we use ';' here?
|
||||
# It's a cmake black magic: https://cmake.org/cmake/help/latest/prop_tgt/LANG_CLANG_TIDY.html#prop_tgt:%3CLANG%3E_CLANG_TIDY
|
||||
# The CLANG_TIDY_PATH is passed to CMAKE_CXX_CLANG_TIDY, which follows CXX_CLANG_TIDY syntax.
|
||||
set (CLANG_TIDY_PATH "${CLANG_TIDY_CACHE_PATH};${_CLANG_TIDY_PATH}" CACHE STRING "A combined command to run clang-tidy with caching wrapper")
|
||||
else ()
|
||||
find_program (CLANG_TIDY_PATH NAMES "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12" "clang-tidy")
|
||||
find_program (CLANG_TIDY_PATH NAMES "clang-tidy-16" "clang-tidy-15" "clang-tidy-14" "clang-tidy")
|
||||
endif ()
|
||||
|
||||
if (CLANG_TIDY_PATH)
|
||||
|
||||
@ -22,7 +22,6 @@ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
|
||||
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=bfd")
|
||||
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=bfd")
|
||||
|
||||
# Currently, lld does not work with the error:
|
||||
# ld.lld: error: section size decrease is too large
|
||||
|
||||
@ -30,7 +30,6 @@ set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/x86_64-linux-gnu/libc")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
|
||||
|
||||
|
||||
@ -57,52 +57,42 @@ if (LINKER_NAME MATCHES "gold")
|
||||
message (FATAL_ERROR "Linking with gold is unsupported. Please use lld.")
|
||||
endif ()
|
||||
|
||||
# s390x doesnt support lld
|
||||
if (NOT ARCH_S390X)
|
||||
if (NOT LINKER_NAME)
|
||||
if (COMPILER_GCC)
|
||||
find_program (LLD_PATH NAMES "ld.lld")
|
||||
elseif (COMPILER_CLANG)
|
||||
# llvm lld is a generic driver.
|
||||
# Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-ld (WebAssembly) instead
|
||||
if (OS_LINUX)
|
||||
if (NOT LINKER_NAME)
|
||||
if (COMPILER_GCC)
|
||||
find_program (LLD_PATH NAMES "ld.lld")
|
||||
elseif (COMPILER_CLANG)
|
||||
if (OS_LINUX)
|
||||
if (NOT ARCH_S390X) # s390x doesnt support lld
|
||||
find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "ld.lld")
|
||||
elseif (OS_DARWIN)
|
||||
find_program (LLD_PATH NAMES "ld64.lld-${COMPILER_VERSION_MAJOR}" "ld64.lld")
|
||||
endif ()
|
||||
endif ()
|
||||
endif ()
|
||||
if (OS_LINUX)
|
||||
if (LLD_PATH)
|
||||
if (COMPILER_GCC)
|
||||
# GCC driver requires one of supported linker names like "lld".
|
||||
set (LINKER_NAME "lld")
|
||||
else ()
|
||||
# Clang driver simply allows full linker path.
|
||||
set (LINKER_NAME ${LLD_PATH})
|
||||
endif ()
|
||||
endif ()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if ((OS_LINUX OR OS_DARWIN) AND NOT LINKER_NAME)
|
||||
if (LLD_PATH)
|
||||
if (COMPILER_GCC)
|
||||
# GCC driver requires one of supported linker names like "lld".
|
||||
set (LINKER_NAME "lld")
|
||||
else ()
|
||||
# Clang driver simply allows full linker path.
|
||||
set (LINKER_NAME ${LLD_PATH})
|
||||
endif ()
|
||||
endif ()
|
||||
endif ()
|
||||
# TODO: allow different linker on != OS_LINUX
|
||||
|
||||
if (LINKER_NAME)
|
||||
find_program (LLD_PATH NAMES ${LINKER_NAME})
|
||||
if (NOT LLD_PATH)
|
||||
message (FATAL_ERROR "Using linker ${LINKER_NAME} but can't find its path.")
|
||||
endif ()
|
||||
if (COMPILER_CLANG)
|
||||
find_program (LLD_PATH NAMES ${LINKER_NAME})
|
||||
if (NOT LLD_PATH)
|
||||
message (FATAL_ERROR "Using linker ${LINKER_NAME} but can't find its path.")
|
||||
endif ()
|
||||
|
||||
# This a temporary quirk to emit .debug_aranges with ThinLTO
|
||||
# This a temporary quirk to emit .debug_aranges with ThinLTO, can be removed after upgrade to clang-16
|
||||
set (LLD_WRAPPER "${CMAKE_CURRENT_BINARY_DIR}/ld.lld")
|
||||
configure_file ("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ld.lld.in" "${LLD_WRAPPER}" @ONLY)
|
||||
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --ld-path=${LLD_WRAPPER}")
|
||||
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LLD_WRAPPER}")
|
||||
else ()
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
|
||||
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
|
||||
endif ()
|
||||
|
||||
endif ()
|
||||
|
||||
2
contrib/croaring
vendored
2
contrib/croaring
vendored
@ -1 +1 @@
|
||||
Subproject commit 2c867e9f9c9e2a3a7032791f94c4c7ae3013f6e0
|
||||
Subproject commit f40ed52bcdd635840a79877cef4857315dba817c
|
||||
@ -17,7 +17,8 @@ set(SRCS
|
||||
"${LIBRARY_DIR}/src/containers/run.c"
|
||||
"${LIBRARY_DIR}/src/roaring.c"
|
||||
"${LIBRARY_DIR}/src/roaring_priority_queue.c"
|
||||
"${LIBRARY_DIR}/src/roaring_array.c")
|
||||
"${LIBRARY_DIR}/src/roaring_array.c"
|
||||
"${LIBRARY_DIR}/src/memory.c")
|
||||
|
||||
add_library(_roaring ${SRCS})
|
||||
|
||||
|
||||
2
contrib/llvm-project
vendored
2
contrib/llvm-project
vendored
@ -1 +1 @@
|
||||
Subproject commit a8bf69e9cd39a23140a2b633c172d201484172da
|
||||
Subproject commit 4bfaeb31dd0ef13f025221f93c138974a3e0a22a
|
||||
@ -31,6 +31,40 @@
|
||||
#define BIG_CONSTANT(x) (x##LLU)
|
||||
|
||||
#endif // !defined(_MSC_VER)
|
||||
//
|
||||
//-----------------------------------------------------------------------------
|
||||
// Block read - on little-endian machines this is a single load,
|
||||
// while on big-endian or unknown machines the byte accesses should
|
||||
// still get optimized into the most efficient instruction.
|
||||
static inline uint32_t getblock ( const uint32_t * p )
|
||||
{
|
||||
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
||||
return *p;
|
||||
#else
|
||||
const uint8_t *c = (const uint8_t *)p;
|
||||
return (uint32_t)c[0] |
|
||||
(uint32_t)c[1] << 8 |
|
||||
(uint32_t)c[2] << 16 |
|
||||
(uint32_t)c[3] << 24;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline uint64_t getblock ( const uint64_t * p )
|
||||
{
|
||||
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
||||
return *p;
|
||||
#else
|
||||
const uint8_t *c = (const uint8_t *)p;
|
||||
return (uint64_t)c[0] |
|
||||
(uint64_t)c[1] << 8 |
|
||||
(uint64_t)c[2] << 16 |
|
||||
(uint64_t)c[3] << 24 |
|
||||
(uint64_t)c[4] << 32 |
|
||||
(uint64_t)c[5] << 40 |
|
||||
(uint64_t)c[6] << 48 |
|
||||
(uint64_t)c[7] << 56;
|
||||
#endif
|
||||
}
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
@ -52,7 +86,7 @@ uint32_t MurmurHash2 ( const void * key, size_t len, uint32_t seed )
|
||||
|
||||
while(len >= 4)
|
||||
{
|
||||
uint32_t k = *(uint32_t*)data;
|
||||
uint32_t k = getblock((const uint32_t *)data);
|
||||
|
||||
k *= m;
|
||||
k ^= k >> r;
|
||||
@ -105,7 +139,7 @@ uint64_t MurmurHash64A ( const void * key, size_t len, uint64_t seed )
|
||||
|
||||
while(data != end)
|
||||
{
|
||||
uint64_t k = *data++;
|
||||
uint64_t k = getblock(data++);
|
||||
|
||||
k *= m;
|
||||
k ^= k >> r;
|
||||
@ -151,12 +185,12 @@ uint64_t MurmurHash64B ( const void * key, size_t len, uint64_t seed )
|
||||
|
||||
while(len >= 8)
|
||||
{
|
||||
uint32_t k1 = *data++;
|
||||
uint32_t k1 = getblock(data++);
|
||||
k1 *= m; k1 ^= k1 >> r; k1 *= m;
|
||||
h1 *= m; h1 ^= k1;
|
||||
len -= 4;
|
||||
|
||||
uint32_t k2 = *data++;
|
||||
uint32_t k2 = getblock(data++);
|
||||
k2 *= m; k2 ^= k2 >> r; k2 *= m;
|
||||
h2 *= m; h2 ^= k2;
|
||||
len -= 4;
|
||||
@ -164,7 +198,7 @@ uint64_t MurmurHash64B ( const void * key, size_t len, uint64_t seed )
|
||||
|
||||
if(len >= 4)
|
||||
{
|
||||
uint32_t k1 = *data++;
|
||||
uint32_t k1 = getblock(data++);
|
||||
k1 *= m; k1 ^= k1 >> r; k1 *= m;
|
||||
h1 *= m; h1 ^= k1;
|
||||
len -= 4;
|
||||
@ -215,7 +249,7 @@ uint32_t MurmurHash2A ( const void * key, size_t len, uint32_t seed )
|
||||
|
||||
while(len >= 4)
|
||||
{
|
||||
uint32_t k = *(uint32_t*)data;
|
||||
uint32_t k = getblock((const uint32_t *)data);
|
||||
|
||||
mmix(h,k);
|
||||
|
||||
@ -278,7 +312,7 @@ public:
|
||||
|
||||
while(len >= 4)
|
||||
{
|
||||
uint32_t k = *(uint32_t*)data;
|
||||
uint32_t k = getblock((const uint32_t *)data);
|
||||
|
||||
mmix(m_hash,k);
|
||||
|
||||
@ -427,7 +461,7 @@ uint32_t MurmurHashAligned2 ( const void * key, size_t len, uint32_t seed )
|
||||
|
||||
while(len >= 4)
|
||||
{
|
||||
d = *(uint32_t *)data;
|
||||
d = getblock((const uint32_t *)data);
|
||||
t = (t >> sr) | (d << sl);
|
||||
|
||||
uint32_t k = t;
|
||||
@ -492,7 +526,7 @@ uint32_t MurmurHashAligned2 ( const void * key, size_t len, uint32_t seed )
|
||||
{
|
||||
while(len >= 4)
|
||||
{
|
||||
uint32_t k = *(uint32_t *)data;
|
||||
uint32_t k = getblock((const uint32_t *)data);
|
||||
|
||||
MIX(h,k,m);
|
||||
|
||||
|
||||
@ -55,14 +55,32 @@ inline uint64_t rotl64 ( uint64_t x, int8_t r )
|
||||
|
||||
FORCE_INLINE uint32_t getblock32 ( const uint32_t * p, int i )
|
||||
{
|
||||
uint32_t res;
|
||||
memcpy(&res, p + i, sizeof(res));
|
||||
return res;
|
||||
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
||||
return p[i];
|
||||
#else
|
||||
const uint8_t *c = (const uint8_t *)&p[i];
|
||||
return (uint32_t)c[0] |
|
||||
(uint32_t)c[1] << 8 |
|
||||
(uint32_t)c[2] << 16 |
|
||||
(uint32_t)c[3] << 24;
|
||||
#endif
|
||||
}
|
||||
|
||||
FORCE_INLINE uint64_t getblock64 ( const uint64_t * p, int i )
|
||||
{
|
||||
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
||||
return p[i];
|
||||
#else
|
||||
const uint8_t *c = (const uint8_t *)&p[i];
|
||||
return (uint64_t)c[0] |
|
||||
(uint64_t)c[1] << 8 |
|
||||
(uint64_t)c[2] << 16 |
|
||||
(uint64_t)c[3] << 24 |
|
||||
(uint64_t)c[4] << 32 |
|
||||
(uint64_t)c[5] << 40 |
|
||||
(uint64_t)c[6] << 48 |
|
||||
(uint64_t)c[7] << 56;
|
||||
#endif
|
||||
}
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
@ -329,9 +347,13 @@ void MurmurHash3_x64_128 ( const void * key, const size_t len,
|
||||
|
||||
h1 += h2;
|
||||
h2 += h1;
|
||||
|
||||
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
||||
((uint64_t*)out)[0] = h1;
|
||||
((uint64_t*)out)[1] = h2;
|
||||
#else
|
||||
((uint64_t*)out)[0] = h2;
|
||||
((uint64_t*)out)[1] = h1;
|
||||
#endif
|
||||
}
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
530
contrib/qpl-cmake/benchmark_sample/client_scripts/allin1_ssb.sh
Normal file
530
contrib/qpl-cmake/benchmark_sample/client_scripts/allin1_ssb.sh
Normal file
@ -0,0 +1,530 @@
|
||||
#!/bin/bash
|
||||
ckhost="localhost"
|
||||
ckport=("9000" "9001" "9002" "9003")
|
||||
WORKING_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.."
|
||||
OUTPUT_DIR="${WORKING_DIR}/output"
|
||||
LOG_DIR="${OUTPUT_DIR}/log"
|
||||
RAWDATA_DIR="${WORKING_DIR}/rawdata_dir"
|
||||
database_dir="${WORKING_DIR}/database_dir"
|
||||
CLIENT_SCRIPTS_DIR="${WORKING_DIR}/client_scripts"
|
||||
LOG_PACK_FILE="$(date +%Y-%m-%d-%H-%M-%S)"
|
||||
QUERY_FILE="queries_ssb.sql"
|
||||
SERVER_BIND_CMD[0]="numactl -m 0 -N 0"
|
||||
SERVER_BIND_CMD[1]="numactl -m 0 -N 0"
|
||||
SERVER_BIND_CMD[2]="numactl -m 1 -N 1"
|
||||
SERVER_BIND_CMD[3]="numactl -m 1 -N 1"
|
||||
CLIENT_BIND_CMD=""
|
||||
SSB_GEN_FACTOR=20
|
||||
TABLE_NAME="lineorder_flat"
|
||||
TALBE_ROWS="119994608"
|
||||
CODEC_CONFIG="lz4 deflate zstd"
|
||||
|
||||
# define instance number
|
||||
inst_num=$1
|
||||
if [ ! -n "$1" ]; then
|
||||
echo "Please clarify instance number from 1,2,3 or 4"
|
||||
exit 1
|
||||
else
|
||||
echo "Benchmarking with instance number:$1"
|
||||
fi
|
||||
|
||||
if [ ! -d "$OUTPUT_DIR" ]; then
|
||||
mkdir $OUTPUT_DIR
|
||||
fi
|
||||
if [ ! -d "$LOG_DIR" ]; then
|
||||
mkdir $LOG_DIR
|
||||
fi
|
||||
if [ ! -d "$RAWDATA_DIR" ]; then
|
||||
mkdir $RAWDATA_DIR
|
||||
fi
|
||||
|
||||
# define different directories
|
||||
dir_server=("" "_s2" "_s3" "_s4")
|
||||
ckreadSql="
|
||||
CREATE TABLE customer
|
||||
(
|
||||
C_CUSTKEY UInt32,
|
||||
C_NAME String,
|
||||
C_ADDRESS String,
|
||||
C_CITY LowCardinality(String),
|
||||
C_NATION LowCardinality(String),
|
||||
C_REGION LowCardinality(String),
|
||||
C_PHONE String,
|
||||
C_MKTSEGMENT LowCardinality(String)
|
||||
)
|
||||
ENGINE = MergeTree ORDER BY (C_CUSTKEY);
|
||||
|
||||
CREATE TABLE lineorder
|
||||
(
|
||||
LO_ORDERKEY UInt32,
|
||||
LO_LINENUMBER UInt8,
|
||||
LO_CUSTKEY UInt32,
|
||||
LO_PARTKEY UInt32,
|
||||
LO_SUPPKEY UInt32,
|
||||
LO_ORDERDATE Date,
|
||||
LO_ORDERPRIORITY LowCardinality(String),
|
||||
LO_SHIPPRIORITY UInt8,
|
||||
LO_QUANTITY UInt8,
|
||||
LO_EXTENDEDPRICE UInt32,
|
||||
LO_ORDTOTALPRICE UInt32,
|
||||
LO_DISCOUNT UInt8,
|
||||
LO_REVENUE UInt32,
|
||||
LO_SUPPLYCOST UInt32,
|
||||
LO_TAX UInt8,
|
||||
LO_COMMITDATE Date,
|
||||
LO_SHIPMODE LowCardinality(String)
|
||||
)
|
||||
ENGINE = MergeTree PARTITION BY toYear(LO_ORDERDATE) ORDER BY (LO_ORDERDATE, LO_ORDERKEY);
|
||||
|
||||
CREATE TABLE part
|
||||
(
|
||||
P_PARTKEY UInt32,
|
||||
P_NAME String,
|
||||
P_MFGR LowCardinality(String),
|
||||
P_CATEGORY LowCardinality(String),
|
||||
P_BRAND LowCardinality(String),
|
||||
P_COLOR LowCardinality(String),
|
||||
P_TYPE LowCardinality(String),
|
||||
P_SIZE UInt8,
|
||||
P_CONTAINER LowCardinality(String)
|
||||
)
|
||||
ENGINE = MergeTree ORDER BY P_PARTKEY;
|
||||
|
||||
CREATE TABLE supplier
|
||||
(
|
||||
S_SUPPKEY UInt32,
|
||||
S_NAME String,
|
||||
S_ADDRESS String,
|
||||
S_CITY LowCardinality(String),
|
||||
S_NATION LowCardinality(String),
|
||||
S_REGION LowCardinality(String),
|
||||
S_PHONE String
|
||||
)
|
||||
ENGINE = MergeTree ORDER BY S_SUPPKEY;
|
||||
"
|
||||
supplier_table="
|
||||
CREATE TABLE supplier
|
||||
(
|
||||
S_SUPPKEY UInt32,
|
||||
S_NAME String,
|
||||
S_ADDRESS String,
|
||||
S_CITY LowCardinality(String),
|
||||
S_NATION LowCardinality(String),
|
||||
S_REGION LowCardinality(String),
|
||||
S_PHONE String
|
||||
)
|
||||
ENGINE = MergeTree ORDER BY S_SUPPKEY;
|
||||
"
|
||||
part_table="
|
||||
CREATE TABLE part
|
||||
(
|
||||
P_PARTKEY UInt32,
|
||||
P_NAME String,
|
||||
P_MFGR LowCardinality(String),
|
||||
P_CATEGORY LowCardinality(String),
|
||||
P_BRAND LowCardinality(String),
|
||||
P_COLOR LowCardinality(String),
|
||||
P_TYPE LowCardinality(String),
|
||||
P_SIZE UInt8,
|
||||
P_CONTAINER LowCardinality(String)
|
||||
)
|
||||
ENGINE = MergeTree ORDER BY P_PARTKEY;
|
||||
"
|
||||
lineorder_table="
|
||||
CREATE TABLE lineorder
|
||||
(
|
||||
LO_ORDERKEY UInt32,
|
||||
LO_LINENUMBER UInt8,
|
||||
LO_CUSTKEY UInt32,
|
||||
LO_PARTKEY UInt32,
|
||||
LO_SUPPKEY UInt32,
|
||||
LO_ORDERDATE Date,
|
||||
LO_ORDERPRIORITY LowCardinality(String),
|
||||
LO_SHIPPRIORITY UInt8,
|
||||
LO_QUANTITY UInt8,
|
||||
LO_EXTENDEDPRICE UInt32,
|
||||
LO_ORDTOTALPRICE UInt32,
|
||||
LO_DISCOUNT UInt8,
|
||||
LO_REVENUE UInt32,
|
||||
LO_SUPPLYCOST UInt32,
|
||||
LO_TAX UInt8,
|
||||
LO_COMMITDATE Date,
|
||||
LO_SHIPMODE LowCardinality(String)
|
||||
)
|
||||
ENGINE = MergeTree PARTITION BY toYear(LO_ORDERDATE) ORDER BY (LO_ORDERDATE, LO_ORDERKEY);
|
||||
"
|
||||
customer_table="
|
||||
CREATE TABLE customer
|
||||
(
|
||||
C_CUSTKEY UInt32,
|
||||
C_NAME String,
|
||||
C_ADDRESS String,
|
||||
C_CITY LowCardinality(String),
|
||||
C_NATION LowCardinality(String),
|
||||
C_REGION LowCardinality(String),
|
||||
C_PHONE String,
|
||||
C_MKTSEGMENT LowCardinality(String)
|
||||
)
|
||||
ENGINE = MergeTree ORDER BY (C_CUSTKEY);
|
||||
"
|
||||
|
||||
lineorder_flat_table="
|
||||
SET max_memory_usage = 20000000000;
|
||||
CREATE TABLE lineorder_flat
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYear(LO_ORDERDATE)
|
||||
ORDER BY (LO_ORDERDATE, LO_ORDERKEY) AS
|
||||
SELECT
|
||||
l.LO_ORDERKEY AS LO_ORDERKEY,
|
||||
l.LO_LINENUMBER AS LO_LINENUMBER,
|
||||
l.LO_CUSTKEY AS LO_CUSTKEY,
|
||||
l.LO_PARTKEY AS LO_PARTKEY,
|
||||
l.LO_SUPPKEY AS LO_SUPPKEY,
|
||||
l.LO_ORDERDATE AS LO_ORDERDATE,
|
||||
l.LO_ORDERPRIORITY AS LO_ORDERPRIORITY,
|
||||
l.LO_SHIPPRIORITY AS LO_SHIPPRIORITY,
|
||||
l.LO_QUANTITY AS LO_QUANTITY,
|
||||
l.LO_EXTENDEDPRICE AS LO_EXTENDEDPRICE,
|
||||
l.LO_ORDTOTALPRICE AS LO_ORDTOTALPRICE,
|
||||
l.LO_DISCOUNT AS LO_DISCOUNT,
|
||||
l.LO_REVENUE AS LO_REVENUE,
|
||||
l.LO_SUPPLYCOST AS LO_SUPPLYCOST,
|
||||
l.LO_TAX AS LO_TAX,
|
||||
l.LO_COMMITDATE AS LO_COMMITDATE,
|
||||
l.LO_SHIPMODE AS LO_SHIPMODE,
|
||||
c.C_NAME AS C_NAME,
|
||||
c.C_ADDRESS AS C_ADDRESS,
|
||||
c.C_CITY AS C_CITY,
|
||||
c.C_NATION AS C_NATION,
|
||||
c.C_REGION AS C_REGION,
|
||||
c.C_PHONE AS C_PHONE,
|
||||
c.C_MKTSEGMENT AS C_MKTSEGMENT,
|
||||
s.S_NAME AS S_NAME,
|
||||
s.S_ADDRESS AS S_ADDRESS,
|
||||
s.S_CITY AS S_CITY,
|
||||
s.S_NATION AS S_NATION,
|
||||
s.S_REGION AS S_REGION,
|
||||
s.S_PHONE AS S_PHONE,
|
||||
p.P_NAME AS P_NAME,
|
||||
p.P_MFGR AS P_MFGR,
|
||||
p.P_CATEGORY AS P_CATEGORY,
|
||||
p.P_BRAND AS P_BRAND,
|
||||
p.P_COLOR AS P_COLOR,
|
||||
p.P_TYPE AS P_TYPE,
|
||||
p.P_SIZE AS P_SIZE,
|
||||
p.P_CONTAINER AS P_CONTAINER
|
||||
FROM lineorder AS l
|
||||
INNER JOIN customer AS c ON c.C_CUSTKEY = l.LO_CUSTKEY
|
||||
INNER JOIN supplier AS s ON s.S_SUPPKEY = l.LO_SUPPKEY
|
||||
INNER JOIN part AS p ON p.P_PARTKEY = l.LO_PARTKEY;
|
||||
show settings ilike 'max_memory_usage';
|
||||
"
|
||||
|
||||
function insert_data(){
|
||||
echo "insert_data:$1"
|
||||
create_table_prefix="clickhouse client --host ${ckhost} --port $2 --multiquery -q"
|
||||
insert_data_prefix="clickhouse client --query "
|
||||
case $1 in
|
||||
all)
|
||||
clickhouse client --host ${ckhost} --port $2 --multiquery -q"$ckreadSql" && {
|
||||
${insert_data_prefix} "INSERT INTO customer FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/customer.tbl --port=$2
|
||||
${insert_data_prefix} "INSERT INTO part FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/part.tbl --port=$2
|
||||
${insert_data_prefix} "INSERT INTO supplier FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/supplier.tbl --port=$2
|
||||
${insert_data_prefix} "INSERT INTO lineorder FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/lineorder.tbl --port=$2
|
||||
}
|
||||
${create_table_prefix}"${lineorder_flat_table}"
|
||||
;;
|
||||
customer)
|
||||
echo ${create_table_prefix}\"${customer_table}\"
|
||||
${create_table_prefix}"${customer_table}" && {
|
||||
echo "${insert_data_prefix} \"INSERT INTO $1 FORMAT CSV\" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2"
|
||||
${insert_data_prefix} "INSERT INTO $1 FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2
|
||||
}
|
||||
;;
|
||||
part)
|
||||
echo ${create_table_prefix}\"${part_table}\"
|
||||
${create_table_prefix}"${part_table}" && {
|
||||
echo "${insert_data_prefix} \"INSERT INTO $1 FORMAT CSV\" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2"
|
||||
${insert_data_prefix} "INSERT INTO $1 FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2
|
||||
}
|
||||
;;
|
||||
supplier)
|
||||
echo ${create_table_prefix}"${supplier_table}"
|
||||
${create_table_prefix}"${supplier_table}" && {
|
||||
echo "${insert_data_prefix} \"INSERT INTO $1 FORMAT CSV\" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2"
|
||||
${insert_data_prefix} "INSERT INTO $1 FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2
|
||||
}
|
||||
;;
|
||||
lineorder)
|
||||
echo ${create_table_prefix}"${lineorder_table}"
|
||||
${create_table_prefix}"${lineorder_table}" && {
|
||||
echo "${insert_data_prefix} \"INSERT INTO $1 FORMAT CSV\" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2"
|
||||
${insert_data_prefix} "INSERT INTO $1 FORMAT CSV" < ${RAWDATA_DIR}/ssb-dbgen/$1.tbl --port=$2
|
||||
}
|
||||
;;
|
||||
lineorder_flat)
|
||||
echo ${create_table_prefix}"${lineorder_flat_table}"
|
||||
${create_table_prefix}"${lineorder_flat_table}"
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
exit 0
|
||||
;;
|
||||
|
||||
esac
|
||||
}
|
||||
|
||||
function check_sql(){
|
||||
select_sql="select * from "$1" limit 1"
|
||||
clickhouse client --host ${ckhost} --port $2 --multiquery -q"${select_sql}"
|
||||
}
|
||||
|
||||
function check_table(){
|
||||
checknum=0
|
||||
source_tables="customer part supplier lineorder lineorder_flat"
|
||||
test_tables=${1:-${source_tables}}
|
||||
echo "Checking table data required in server..."
|
||||
for i in $(seq 0 $[inst_num-1])
|
||||
do
|
||||
for j in `echo ${test_tables}`
|
||||
do
|
||||
check_sql $j ${ckport[i]} &> /dev/null || {
|
||||
let checknum+=1 && insert_data "$j" ${ckport[i]}
|
||||
}
|
||||
done
|
||||
done
|
||||
|
||||
for i in $(seq 0 $[inst_num-1])
|
||||
do
|
||||
echo "clickhouse client --host ${ckhost} --port ${ckport[i]} -m -q\"select count() from ${TABLE_NAME};\""
|
||||
var=$(clickhouse client --host ${ckhost} --port ${ckport[i]} -m -q"select count() from ${TABLE_NAME};")
|
||||
if [ $var -eq $TALBE_ROWS ];then
|
||||
echo "Instance_${i} Table data integrity check OK -> Rows:$var"
|
||||
else
|
||||
echo "Instance_${i} Table data integrity check Failed -> Rows:$var"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if [ $checknum -gt 0 ];then
|
||||
echo "Need sleep 10s after first table data insertion...$checknum"
|
||||
sleep 10
|
||||
fi
|
||||
}
|
||||
|
||||
function check_instance(){
|
||||
instance_alive=0
|
||||
for i in {1..10}
|
||||
do
|
||||
sleep 1
|
||||
netstat -nltp | grep ${1} > /dev/null
|
||||
if [ $? -ne 1 ];then
|
||||
instance_alive=1
|
||||
break
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
if [ $instance_alive -eq 0 ];then
|
||||
echo "check_instance -> clickhouse server instance faild to launch due to 10s timeout!"
|
||||
exit 1
|
||||
else
|
||||
echo "check_instance -> clickhouse server instance launch successfully!"
|
||||
fi
|
||||
}
|
||||
|
||||
function start_clickhouse_for_insertion(){
|
||||
echo "start_clickhouse_for_insertion"
|
||||
for i in $(seq 0 $[inst_num-1])
|
||||
do
|
||||
echo "cd ${database_dir}/$1${dir_server[i]}"
|
||||
echo "${SERVER_BIND_CMD[i]} clickhouse server -C config_${1}${dir_server[i]}.xml >&${LOG_DIR}/${1}_${i}_server_log& > /dev/null"
|
||||
|
||||
cd ${database_dir}/$1${dir_server[i]}
|
||||
${SERVER_BIND_CMD[i]} clickhouse server -C config_${1}${dir_server[i]}.xml >&${LOG_DIR}/${1}_${i}_server_log& > /dev/null
|
||||
check_instance ${ckport[i]}
|
||||
done
|
||||
}
|
||||
|
||||
function start_clickhouse_for_stressing(){
|
||||
echo "start_clickhouse_for_stressing"
|
||||
for i in $(seq 0 $[inst_num-1])
|
||||
do
|
||||
echo "cd ${database_dir}/$1${dir_server[i]}"
|
||||
echo "${SERVER_BIND_CMD[i]} clickhouse server -C config_${1}${dir_server[i]}.xml >&/dev/null&"
|
||||
|
||||
cd ${database_dir}/$1${dir_server[i]}
|
||||
${SERVER_BIND_CMD[i]} clickhouse server -C config_${1}${dir_server[i]}.xml >&/dev/null&
|
||||
check_instance ${ckport[i]}
|
||||
done
|
||||
}
|
||||
yum -y install git make gcc sudo net-tools &> /dev/null
|
||||
pip3 install clickhouse_driver numpy &> /dev/null
|
||||
test -d ${RAWDATA_DIR}/ssb-dbgen || git clone https://github.com/vadimtk/ssb-dbgen.git ${RAWDATA_DIR}/ssb-dbgen && cd ${RAWDATA_DIR}/ssb-dbgen
|
||||
|
||||
if [ ! -f ${RAWDATA_DIR}/ssb-dbgen/dbgen ];then
|
||||
make && {
|
||||
test -f ${RAWDATA_DIR}/ssb-dbgen/customer.tbl || echo y |./dbgen -s ${SSB_GEN_FACTOR} -T c
|
||||
test -f ${RAWDATA_DIR}/ssb-dbgen/part.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T p
|
||||
test -f ${RAWDATA_DIR}/ssb-dbgen/supplier.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T s
|
||||
test -f ${RAWDATA_DIR}/ssb-dbgen/date.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T d
|
||||
test -f ${RAWDATA_DIR}/ssb-dbgen/lineorder.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T l
|
||||
}
|
||||
else
|
||||
test -f ${RAWDATA_DIR}/ssb-dbgen/customer.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T c
|
||||
test -f ${RAWDATA_DIR}/ssb-dbgen/part.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T p
|
||||
test -f ${RAWDATA_DIR}/ssb-dbgen/supplier.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T s
|
||||
test -f ${RAWDATA_DIR}/ssb-dbgen/date.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T d
|
||||
test -f ${RAWDATA_DIR}/ssb-dbgen/lineorder.tbl || echo y | ./dbgen -s ${SSB_GEN_FACTOR} -T l
|
||||
|
||||
fi
|
||||
|
||||
filenum=`find ${RAWDATA_DIR}/ssb-dbgen/ -name "*.tbl" | wc -l`
|
||||
|
||||
if [ $filenum -ne 5 ];then
|
||||
echo "generate ssb data file *.tbl faild"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
function kill_instance(){
|
||||
instance_alive=1
|
||||
for i in {1..2}
|
||||
do
|
||||
pkill clickhouse && sleep 5
|
||||
instance_alive=0
|
||||
for i in $(seq 0 $[inst_num-1])
|
||||
do
|
||||
netstat -nltp | grep ${ckport[i]} > /dev/null
|
||||
if [ $? -ne 1 ];then
|
||||
instance_alive=1
|
||||
break;
|
||||
fi
|
||||
done
|
||||
if [ $instance_alive -eq 0 ];then
|
||||
break;
|
||||
fi
|
||||
done
|
||||
if [ $instance_alive -eq 0 ];then
|
||||
echo "kill_instance OK!"
|
||||
else
|
||||
echo "kill_instance Failed -> clickhouse server instance still alive due to 10s timeout"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function run_test(){
|
||||
is_xml=0
|
||||
for i in $(seq 0 $[inst_num-1])
|
||||
do
|
||||
if [ -f ${database_dir}/${1}${dir_server[i]}/config_${1}${dir_server[i]}.xml ]; then
|
||||
is_xml=$[is_xml+1]
|
||||
fi
|
||||
done
|
||||
if [ $is_xml -eq $inst_num ];then
|
||||
echo "Benchmark with $inst_num instance"
|
||||
start_clickhouse_for_insertion ${1}
|
||||
|
||||
for i in $(seq 0 $[inst_num-1])
|
||||
do
|
||||
clickhouse client --host ${ckhost} --port ${ckport[i]} -m -q"show databases;" >/dev/null
|
||||
done
|
||||
|
||||
if [ $? -eq 0 ];then
|
||||
check_table
|
||||
fi
|
||||
kill_instance
|
||||
|
||||
if [ $1 == "deflate" ];then
|
||||
test -f ${LOG_DIR}/${1}_server_log && deflatemsg=`cat ${LOG_DIR}/${1}_server_log | grep DeflateJobHWPool`
|
||||
if [ -n "$deflatemsg" ];then
|
||||
echo ------------------------------------------------------
|
||||
echo $deflatemsg
|
||||
echo ------------------------------------------------------
|
||||
fi
|
||||
fi
|
||||
echo "Check table data required in server_${1} -> Done! "
|
||||
|
||||
start_clickhouse_for_stressing ${1}
|
||||
for i in $(seq 0 $[inst_num-1])
|
||||
do
|
||||
clickhouse client --host ${ckhost} --port ${ckport[i]} -m -q"show databases;" >/dev/null
|
||||
done
|
||||
if [ $? -eq 0 ];then
|
||||
test -d ${CLIENT_SCRIPTS_DIR} && cd ${CLIENT_SCRIPTS_DIR}
|
||||
echo "Client stressing... "
|
||||
echo "${CLIENT_BIND_CMD} python3 client_stressing_test.py ${QUERY_FILE} $inst_num &> ${LOG_DIR}/${1}.log"
|
||||
${CLIENT_BIND_CMD} python3 client_stressing_test.py ${QUERY_FILE} $inst_num &> ${LOG_DIR}/${1}.log
|
||||
echo "Completed client stressing, checking log... "
|
||||
finish_log=`grep "Finished" ${LOG_DIR}/${1}.log | wc -l`
|
||||
if [ $finish_log -eq 1 ] ;then
|
||||
kill_instance
|
||||
test -f ${LOG_DIR}/${1}.log && echo "${1}.log ===> ${LOG_DIR}/${1}.log"
|
||||
else
|
||||
kill_instance
|
||||
echo "No find 'Finished' in client log -> Performance test may fail"
|
||||
exit 1
|
||||
|
||||
fi
|
||||
|
||||
else
|
||||
echo "${1} clickhouse server start fail"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "clickhouse server start fail -> Please check xml files required in ${database_dir} for each instance"
|
||||
exit 1
|
||||
|
||||
fi
|
||||
}
|
||||
function clear_log(){
|
||||
if [ -d "$LOG_DIR" ]; then
|
||||
cd ${LOG_DIR} && rm -rf *
|
||||
fi
|
||||
}
|
||||
|
||||
function gather_log_for_codec(){
|
||||
cd ${OUTPUT_DIR} && mkdir -p ${LOG_PACK_FILE}/${1}
|
||||
cp -rf ${LOG_DIR} ${OUTPUT_DIR}/${LOG_PACK_FILE}/${1}
|
||||
}
|
||||
|
||||
function pack_log(){
|
||||
if [ -e "${OUTPUT_DIR}/run.log" ]; then
|
||||
cp ${OUTPUT_DIR}/run.log ${OUTPUT_DIR}/${LOG_PACK_FILE}/
|
||||
fi
|
||||
echo "Please check all log information in ${OUTPUT_DIR}/${LOG_PACK_FILE}"
|
||||
}
|
||||
|
||||
function setup_check(){
|
||||
|
||||
iax_dev_num=`accel-config list | grep iax | wc -l`
|
||||
if [ $iax_dev_num -eq 0 ] ;then
|
||||
iax_dev_num=`accel-config list | grep iax | wc -l`
|
||||
if [ $iax_dev_num -eq 0 ] ;then
|
||||
echo "No IAA devices available -> Please check IAA hardware setup manually!"
|
||||
exit 1
|
||||
else
|
||||
echo "IAA enabled devices number:$iax_dev_num"
|
||||
fi
|
||||
else
|
||||
echo "IAA enabled devices number:$iax_dev_num"
|
||||
fi
|
||||
libaccel_version=`accel-config -v`
|
||||
clickhouser_version=`clickhouse server --version`
|
||||
kernel_dxd_log=`dmesg | grep dxd`
|
||||
echo "libaccel_version:$libaccel_version"
|
||||
echo "clickhouser_version:$clickhouser_version"
|
||||
echo -e "idxd section in kernel log:\n$kernel_dxd_log"
|
||||
}
|
||||
|
||||
setup_check
|
||||
export CLICKHOUSE_WATCHDOG_ENABLE=0
|
||||
for i in ${CODEC_CONFIG[@]}
|
||||
do
|
||||
clear_log
|
||||
codec=${i}
|
||||
echo "run test------------$codec"
|
||||
run_test $codec
|
||||
gather_log_for_codec $codec
|
||||
done
|
||||
|
||||
pack_log
|
||||
echo "Done."
|
||||
@ -0,0 +1,278 @@
|
||||
from operator import eq
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
import sys
|
||||
from clickhouse_driver import Client
|
||||
import numpy as np
|
||||
import subprocess
|
||||
import multiprocessing
|
||||
from multiprocessing import Manager
|
||||
|
||||
warmup_runs = 10
|
||||
calculated_runs = 10
|
||||
seconds = 30
|
||||
max_instances_number = 8
|
||||
retest_number = 3
|
||||
retest_tolerance = 10
|
||||
|
||||
|
||||
def checkInt(str):
|
||||
try:
|
||||
int(str)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def setup_client(index):
|
||||
if index < 4:
|
||||
port_idx = index
|
||||
else:
|
||||
port_idx = index + 4
|
||||
client = Client(
|
||||
host="localhost",
|
||||
database="default",
|
||||
user="default",
|
||||
password="",
|
||||
port="900%d" % port_idx,
|
||||
)
|
||||
union_mode_query = "SET union_default_mode='DISTINCT'"
|
||||
client.execute(union_mode_query)
|
||||
return client
|
||||
|
||||
|
||||
def warm_client(clientN, clientL, query, loop):
|
||||
for c_idx in range(clientN):
|
||||
for _ in range(loop):
|
||||
clientL[c_idx].execute(query)
|
||||
|
||||
|
||||
def read_queries(queries_list):
|
||||
queries = list()
|
||||
queries_id = list()
|
||||
with open(queries_list, "r") as f:
|
||||
for line in f:
|
||||
line = line.rstrip()
|
||||
line = line.split("$")
|
||||
queries_id.append(line[0])
|
||||
queries.append(line[1])
|
||||
return queries_id, queries
|
||||
|
||||
|
||||
def run_task(client, cname, query, loop, query_latency):
|
||||
start_time = time.time()
|
||||
for i in range(loop):
|
||||
client.execute(query)
|
||||
query_latency.append(client.last_query.elapsed)
|
||||
|
||||
end_time = time.time()
|
||||
p95 = np.percentile(query_latency, 95)
|
||||
print(
|
||||
"CLIENT: {0} end. -> P95: %f, qps: %f".format(cname)
|
||||
% (p95, loop / (end_time - start_time))
|
||||
)
|
||||
|
||||
|
||||
def run_multi_clients(clientN, clientList, query, loop):
|
||||
client_pids = {}
|
||||
start_time = time.time()
|
||||
manager = multiprocessing.Manager()
|
||||
query_latency_list0 = manager.list()
|
||||
query_latency_list1 = manager.list()
|
||||
query_latency_list2 = manager.list()
|
||||
query_latency_list3 = manager.list()
|
||||
query_latency_list4 = manager.list()
|
||||
query_latency_list5 = manager.list()
|
||||
query_latency_list6 = manager.list()
|
||||
query_latency_list7 = manager.list()
|
||||
|
||||
for c_idx in range(clientN):
|
||||
client_name = "Role_%d" % c_idx
|
||||
if c_idx == 0:
|
||||
client_pids[c_idx] = multiprocessing.Process(
|
||||
target=run_task,
|
||||
args=(clientList[c_idx], client_name, query, loop, query_latency_list0),
|
||||
)
|
||||
elif c_idx == 1:
|
||||
client_pids[c_idx] = multiprocessing.Process(
|
||||
target=run_task,
|
||||
args=(clientList[c_idx], client_name, query, loop, query_latency_list1),
|
||||
)
|
||||
elif c_idx == 2:
|
||||
client_pids[c_idx] = multiprocessing.Process(
|
||||
target=run_task,
|
||||
args=(clientList[c_idx], client_name, query, loop, query_latency_list2),
|
||||
)
|
||||
elif c_idx == 3:
|
||||
client_pids[c_idx] = multiprocessing.Process(
|
||||
target=run_task,
|
||||
args=(clientList[c_idx], client_name, query, loop, query_latency_list3),
|
||||
)
|
||||
elif c_idx == 4:
|
||||
client_pids[c_idx] = multiprocessing.Process(
|
||||
target=run_task,
|
||||
args=(clientList[c_idx], client_name, query, loop, query_latency_list4),
|
||||
)
|
||||
elif c_idx == 5:
|
||||
client_pids[c_idx] = multiprocessing.Process(
|
||||
target=run_task,
|
||||
args=(clientList[c_idx], client_name, query, loop, query_latency_list5),
|
||||
)
|
||||
elif c_idx == 6:
|
||||
client_pids[c_idx] = multiprocessing.Process(
|
||||
target=run_task,
|
||||
args=(clientList[c_idx], client_name, query, loop, query_latency_list6),
|
||||
)
|
||||
elif c_idx == 7:
|
||||
client_pids[c_idx] = multiprocessing.Process(
|
||||
target=run_task,
|
||||
args=(clientList[c_idx], client_name, query, loop, query_latency_list7),
|
||||
)
|
||||
else:
|
||||
print("ERROR: CLIENT number dismatch!!")
|
||||
exit()
|
||||
print("CLIENT: %s start" % client_name)
|
||||
client_pids[c_idx].start()
|
||||
|
||||
for c_idx in range(clientN):
|
||||
client_pids[c_idx].join()
|
||||
end_time = time.time()
|
||||
totalT = end_time - start_time
|
||||
|
||||
query_latencyTotal = list()
|
||||
for item in query_latency_list0:
|
||||
query_latencyTotal.append(item)
|
||||
for item in query_latency_list1:
|
||||
query_latencyTotal.append(item)
|
||||
for item in query_latency_list2:
|
||||
query_latencyTotal.append(item)
|
||||
for item in query_latency_list3:
|
||||
query_latencyTotal.append(item)
|
||||
for item in query_latency_list4:
|
||||
query_latencyTotal.append(item)
|
||||
for item in query_latency_list5:
|
||||
query_latencyTotal.append(item)
|
||||
for item in query_latency_list6:
|
||||
query_latencyTotal.append(item)
|
||||
for item in query_latency_list7:
|
||||
query_latencyTotal.append(item)
|
||||
|
||||
totalP95 = np.percentile(query_latencyTotal, 95) * 1000
|
||||
return totalT, totalP95
|
||||
|
||||
|
||||
def run_task_caculated(client, cname, query, loop):
|
||||
query_latency = list()
|
||||
start_time = time.time()
|
||||
for i in range(loop):
|
||||
client.execute(query)
|
||||
query_latency.append(client.last_query.elapsed)
|
||||
end_time = time.time()
|
||||
p95 = np.percentile(query_latency, 95)
|
||||
|
||||
|
||||
def run_multi_clients_caculated(clientN, clientList, query, loop):
|
||||
client_pids = {}
|
||||
start_time = time.time()
|
||||
for c_idx in range(clientN):
|
||||
client_name = "Role_%d" % c_idx
|
||||
client_pids[c_idx] = multiprocessing.Process(
|
||||
target=run_task_caculated,
|
||||
args=(clientList[c_idx], client_name, query, loop),
|
||||
)
|
||||
client_pids[c_idx].start()
|
||||
for c_idx in range(clientN):
|
||||
client_pids[c_idx].join()
|
||||
end_time = time.time()
|
||||
totalT = end_time - start_time
|
||||
return totalT
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
client_number = 1
|
||||
queries = list()
|
||||
queries_id = list()
|
||||
|
||||
if len(sys.argv) != 3:
|
||||
print(
|
||||
"usage: python3 client_stressing_test.py [queries_file_path] [client_number]"
|
||||
)
|
||||
sys.exit()
|
||||
else:
|
||||
queries_list = sys.argv[1]
|
||||
client_number = int(sys.argv[2])
|
||||
print(
|
||||
"queries_file_path: %s, client_number: %d" % (queries_list, client_number)
|
||||
)
|
||||
if not os.path.isfile(queries_list) or not os.access(queries_list, os.R_OK):
|
||||
print("please check the right path for queries file")
|
||||
sys.exit()
|
||||
if (
|
||||
not checkInt(sys.argv[2])
|
||||
or int(sys.argv[2]) > max_instances_number
|
||||
or int(sys.argv[2]) < 1
|
||||
):
|
||||
print("client_number should be in [1~%d]" % max_instances_number)
|
||||
sys.exit()
|
||||
|
||||
client_list = {}
|
||||
queries_id, queries = read_queries(queries_list)
|
||||
|
||||
for c_idx in range(client_number):
|
||||
client_list[c_idx] = setup_client(c_idx)
|
||||
# clear cache
|
||||
os.system("sync; echo 3 > /proc/sys/vm/drop_caches")
|
||||
|
||||
print("###Polit Run Begin")
|
||||
for i in queries:
|
||||
warm_client(client_number, client_list, i, 1)
|
||||
print("###Polit Run End -> Start stressing....")
|
||||
|
||||
query_index = 0
|
||||
for q in queries:
|
||||
print(
|
||||
"\n###START -> Index: %d, ID: %s, Query: %s"
|
||||
% (query_index, queries_id[query_index], q)
|
||||
)
|
||||
warm_client(client_number, client_list, q, warmup_runs)
|
||||
print("###Warm Done!")
|
||||
for j in range(0, retest_number):
|
||||
totalT = run_multi_clients_caculated(
|
||||
client_number, client_list, q, calculated_runs
|
||||
)
|
||||
curr_loop = int(seconds * calculated_runs / totalT) + 1
|
||||
print(
|
||||
"###Calculation Done! -> loopN: %d, expected seconds:%d"
|
||||
% (curr_loop, seconds)
|
||||
)
|
||||
|
||||
print("###Stress Running! -> %d iterations......" % curr_loop)
|
||||
|
||||
totalT, totalP95 = run_multi_clients(
|
||||
client_number, client_list, q, curr_loop
|
||||
)
|
||||
|
||||
if totalT > (seconds - retest_tolerance) and totalT < (
|
||||
seconds + retest_tolerance
|
||||
):
|
||||
break
|
||||
else:
|
||||
print(
|
||||
"###totalT:%d is far way from expected seconds:%d. Run again ->j:%d!"
|
||||
% (totalT, seconds, j)
|
||||
)
|
||||
|
||||
print(
|
||||
"###Completed! -> ID: %s, clientN: %d, totalT: %.2f s, latencyAVG: %.2f ms, P95: %.2f ms, QPS_Final: %.2f"
|
||||
% (
|
||||
queries_id[query_index],
|
||||
client_number,
|
||||
totalT,
|
||||
totalT * 1000 / (curr_loop * client_number),
|
||||
totalP95,
|
||||
((curr_loop * client_number) / totalT),
|
||||
)
|
||||
)
|
||||
query_index += 1
|
||||
print("###Finished!")
|
||||
@ -0,0 +1,10 @@
|
||||
Q1.1$SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE toYear(LO_ORDERDATE) = 1993 AND LO_DISCOUNT BETWEEN 1 AND 3 AND LO_QUANTITY < 25;
|
||||
Q2.1$SELECT sum(LO_REVENUE),toYear(LO_ORDERDATE) AS year,P_BRAND FROM lineorder_flat WHERE P_CATEGORY = 'MFGR#12' AND S_REGION = 'AMERICA' GROUP BY year,P_BRAND ORDER BY year,P_BRAND;
|
||||
Q2.2$SELECT sum(LO_REVENUE),toYear(LO_ORDERDATE) AS year,P_BRAND FROM lineorder_flat WHERE P_BRAND >= 'MFGR#2221' AND P_BRAND <= 'MFGR#2228' AND S_REGION = 'ASIA' GROUP BY year,P_BRAND ORDER BY year,P_BRAND;
|
||||
Q2.3$SELECT sum(LO_REVENUE),toYear(LO_ORDERDATE) AS year,P_BRAND FROM lineorder_flat WHERE P_BRAND = 'MFGR#2239' AND S_REGION = 'EUROPE' GROUP BY year,P_BRAND ORDER BY year,P_BRAND;
|
||||
Q3.1$SELECT C_NATION,S_NATION,toYear(LO_ORDERDATE) AS year,sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE C_REGION = 'ASIA' AND S_REGION = 'ASIA' AND year >= 1992 AND year <= 1997 GROUP BY C_NATION,S_NATION,year ORDER BY year ASC,revenue DESC;
|
||||
Q3.2$SELECT C_CITY,S_CITY,toYear(LO_ORDERDATE) AS year,sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE C_NATION = 'UNITED STATES' AND S_NATION = 'UNITED STATES' AND year >= 1992 AND year <= 1997 GROUP BY C_CITY,S_CITY,year ORDER BY year ASC,revenue DESC;
|
||||
Q3.3$SELECT C_CITY,S_CITY,toYear(LO_ORDERDATE) AS year,sum(LO_REVENUE) AS revenue FROM lineorder_flat WHERE (C_CITY = 'UNITED KI1' OR C_CITY = 'UNITED KI5') AND (S_CITY = 'UNITED KI1' OR S_CITY = 'UNITED KI5') AND year >= 1992 AND year <= 1997 GROUP BY C_CITY,S_CITY,year ORDER BY year ASC,revenue DESC;
|
||||
Q4.1$SELECT toYear(LO_ORDERDATE) AS year,C_NATION,sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') GROUP BY year,C_NATION ORDER BY year ASC,C_NATION ASC;
|
||||
Q4.2$SELECT toYear(LO_ORDERDATE) AS year,S_NATION,P_CATEGORY,sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (year = 1997 OR year = 1998) AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') GROUP BY year,S_NATION,P_CATEGORY ORDER BY year ASC,S_NATION ASC,P_CATEGORY ASC;
|
||||
Q4.3$SELECT toYear(LO_ORDERDATE) AS year,S_CITY,P_BRAND,sum(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat WHERE S_NATION = 'UNITED STATES' AND (year = 1997 OR year = 1998) AND P_CATEGORY = 'MFGR#14' GROUP BY year,S_CITY,P_BRAND ORDER BY year ASC,S_CITY ASC,P_BRAND ASC;
|
||||
@ -0,0 +1,6 @@
|
||||
WORKING_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.."
|
||||
if [ ! -d "${WORKING_DIR}/output" ]; then
|
||||
mkdir ${WORKING_DIR}/output
|
||||
fi
|
||||
bash allin1_ssb.sh 2 > ${WORKING_DIR}/output/run.log
|
||||
echo "Please check log in: ${WORKING_DIR}/output/run.log"
|
||||
@ -0,0 +1,49 @@
|
||||
<!-- This file was generated automatically.
|
||||
Do not edit it: it is likely to be discarded and generated again before it's read next time.
|
||||
Files used to generate this file:
|
||||
config.xml -->
|
||||
|
||||
<!-- Config that is used when server is run without config file. --><clickhouse>
|
||||
<logger>
|
||||
<level>trace</level>
|
||||
<console>true</console>
|
||||
</logger>
|
||||
|
||||
<http_port>8123</http_port>
|
||||
<tcp_port>9000</tcp_port>
|
||||
<mysql_port>9004</mysql_port>
|
||||
|
||||
<path>./</path>
|
||||
|
||||
<uncompressed_cache_size>8589934592</uncompressed_cache_size>
|
||||
<mark_cache_size>5368709120</mark_cache_size>
|
||||
<mlock_executable>true</mlock_executable>
|
||||
|
||||
<compression>
|
||||
<case>
|
||||
<method>deflate_qpl</method>
|
||||
</case>
|
||||
</compression>
|
||||
|
||||
<users>
|
||||
<default>
|
||||
<password/>
|
||||
|
||||
<networks>
|
||||
<ip>::/0</ip>
|
||||
</networks>
|
||||
|
||||
<profile>default</profile>
|
||||
<quota>default</quota>
|
||||
<access_management>1</access_management>
|
||||
</default>
|
||||
</users>
|
||||
|
||||
<profiles>
|
||||
<default/>
|
||||
</profiles>
|
||||
|
||||
<quotas>
|
||||
<default/>
|
||||
</quotas>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,49 @@
|
||||
<!-- This file was generated automatically.
|
||||
Do not edit it: it is likely to be discarded and generated again before it's read next time.
|
||||
Files used to generate this file:
|
||||
config.xml -->
|
||||
|
||||
<!-- Config that is used when server is run without config file. --><clickhouse>
|
||||
<logger>
|
||||
<level>trace</level>
|
||||
<console>true</console>
|
||||
</logger>
|
||||
|
||||
<http_port>8124</http_port>
|
||||
<tcp_port>9001</tcp_port>
|
||||
<mysql_port>9005</mysql_port>
|
||||
|
||||
<path>./</path>
|
||||
|
||||
<uncompressed_cache_size>8589934592</uncompressed_cache_size>
|
||||
<mark_cache_size>5368709120</mark_cache_size>
|
||||
<mlock_executable>true</mlock_executable>
|
||||
|
||||
<compression>
|
||||
<case>
|
||||
<method>deflate_qpl</method>
|
||||
</case>
|
||||
</compression>
|
||||
|
||||
<users>
|
||||
<default>
|
||||
<password/>
|
||||
|
||||
<networks>
|
||||
<ip>::/0</ip>
|
||||
</networks>
|
||||
|
||||
<profile>default</profile>
|
||||
<quota>default</quota>
|
||||
<access_management>1</access_management>
|
||||
</default>
|
||||
</users>
|
||||
|
||||
<profiles>
|
||||
<default/>
|
||||
</profiles>
|
||||
|
||||
<quotas>
|
||||
<default/>
|
||||
</quotas>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,49 @@
|
||||
<!-- This file was generated automatically.
|
||||
Do not edit it: it is likely to be discarded and generated again before it's read next time.
|
||||
Files used to generate this file:
|
||||
config.xml -->
|
||||
|
||||
<!-- Config that is used when server is run without config file. --><clickhouse>
|
||||
<logger>
|
||||
<level>trace</level>
|
||||
<console>true</console>
|
||||
</logger>
|
||||
|
||||
<http_port>8123</http_port>
|
||||
<tcp_port>9000</tcp_port>
|
||||
<mysql_port>9004</mysql_port>
|
||||
|
||||
<path>./</path>
|
||||
|
||||
<uncompressed_cache_size>8589934592</uncompressed_cache_size>
|
||||
<mark_cache_size>5368709120</mark_cache_size>
|
||||
<mlock_executable>true</mlock_executable>
|
||||
|
||||
<compression>
|
||||
<case>
|
||||
<method>lz4</method>
|
||||
</case>
|
||||
</compression>
|
||||
|
||||
<users>
|
||||
<default>
|
||||
<password/>
|
||||
|
||||
<networks>
|
||||
<ip>::/0</ip>
|
||||
</networks>
|
||||
|
||||
<profile>default</profile>
|
||||
<quota>default</quota>
|
||||
<access_management>1</access_management>
|
||||
</default>
|
||||
</users>
|
||||
|
||||
<profiles>
|
||||
<default/>
|
||||
</profiles>
|
||||
|
||||
<quotas>
|
||||
<default/>
|
||||
</quotas>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,49 @@
|
||||
<!-- This file was generated automatically.
|
||||
Do not edit it: it is likely to be discarded and generated again before it's read next time.
|
||||
Files used to generate this file:
|
||||
config.xml -->
|
||||
|
||||
<!-- Config that is used when server is run without config file. --><clickhouse>
|
||||
<logger>
|
||||
<level>trace</level>
|
||||
<console>true</console>
|
||||
</logger>
|
||||
|
||||
<http_port>8124</http_port>
|
||||
<tcp_port>9001</tcp_port>
|
||||
<mysql_port>9005</mysql_port>
|
||||
|
||||
<path>./</path>
|
||||
|
||||
<uncompressed_cache_size>8589934592</uncompressed_cache_size>
|
||||
<mark_cache_size>5368709120</mark_cache_size>
|
||||
<mlock_executable>true</mlock_executable>
|
||||
|
||||
<compression>
|
||||
<case>
|
||||
<method>lz4</method>
|
||||
</case>
|
||||
</compression>
|
||||
|
||||
<users>
|
||||
<default>
|
||||
<password/>
|
||||
|
||||
<networks>
|
||||
<ip>::/0</ip>
|
||||
</networks>
|
||||
|
||||
<profile>default</profile>
|
||||
<quota>default</quota>
|
||||
<access_management>1</access_management>
|
||||
</default>
|
||||
</users>
|
||||
|
||||
<profiles>
|
||||
<default/>
|
||||
</profiles>
|
||||
|
||||
<quotas>
|
||||
<default/>
|
||||
</quotas>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,49 @@
|
||||
<!-- This file was generated automatically.
|
||||
Do not edit it: it is likely to be discarded and generated again before it's read next time.
|
||||
Files used to generate this file:
|
||||
config.xml -->
|
||||
|
||||
<!-- Config that is used when server is run without config file. --><clickhouse>
|
||||
<logger>
|
||||
<level>trace</level>
|
||||
<console>true</console>
|
||||
</logger>
|
||||
|
||||
<http_port>8123</http_port>
|
||||
<tcp_port>9000</tcp_port>
|
||||
<mysql_port>9004</mysql_port>
|
||||
|
||||
<path>./</path>
|
||||
|
||||
<uncompressed_cache_size>8589934592</uncompressed_cache_size>
|
||||
<mark_cache_size>5368709120</mark_cache_size>
|
||||
<mlock_executable>true</mlock_executable>
|
||||
|
||||
<compression>
|
||||
<case>
|
||||
<method>zstd</method>
|
||||
</case>
|
||||
</compression>
|
||||
|
||||
<users>
|
||||
<default>
|
||||
<password/>
|
||||
|
||||
<networks>
|
||||
<ip>::/0</ip>
|
||||
</networks>
|
||||
|
||||
<profile>default</profile>
|
||||
<quota>default</quota>
|
||||
<access_management>1</access_management>
|
||||
</default>
|
||||
</users>
|
||||
|
||||
<profiles>
|
||||
<default/>
|
||||
</profiles>
|
||||
|
||||
<quotas>
|
||||
<default/>
|
||||
</quotas>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,49 @@
|
||||
<!-- This file was generated automatically.
|
||||
Do not edit it: it is likely to be discarded and generated again before it's read next time.
|
||||
Files used to generate this file:
|
||||
config.xml -->
|
||||
|
||||
<!-- Config that is used when server is run without config file. --><clickhouse>
|
||||
<logger>
|
||||
<level>trace</level>
|
||||
<console>true</console>
|
||||
</logger>
|
||||
|
||||
<http_port>8124</http_port>
|
||||
<tcp_port>9001</tcp_port>
|
||||
<mysql_port>9005</mysql_port>
|
||||
|
||||
<path>./</path>
|
||||
|
||||
<uncompressed_cache_size>8589934592</uncompressed_cache_size>
|
||||
<mark_cache_size>5368709120</mark_cache_size>
|
||||
<mlock_executable>true</mlock_executable>
|
||||
|
||||
<compression>
|
||||
<case>
|
||||
<method>zstd</method>
|
||||
</case>
|
||||
</compression>
|
||||
|
||||
<users>
|
||||
<default>
|
||||
<password/>
|
||||
|
||||
<networks>
|
||||
<ip>::/0</ip>
|
||||
</networks>
|
||||
|
||||
<profile>default</profile>
|
||||
<quota>default</quota>
|
||||
<access_management>1</access_management>
|
||||
</default>
|
||||
</users>
|
||||
|
||||
<profiles>
|
||||
<default/>
|
||||
</profiles>
|
||||
|
||||
<quotas>
|
||||
<default/>
|
||||
</quotas>
|
||||
</clickhouse>
|
||||
2
contrib/sqlite-amalgamation
vendored
2
contrib/sqlite-amalgamation
vendored
@ -1 +1 @@
|
||||
Subproject commit 400ad7152a0c7ee07756d96ab4f6a8f6d1080916
|
||||
Subproject commit 20598079891d27ef1a3ad3f66bbfa3f983c25268
|
||||
2
contrib/vectorscan
vendored
2
contrib/vectorscan
vendored
@ -1 +1 @@
|
||||
Subproject commit f6250ae3e5a3085000239313ad0689cc1e00cdc2
|
||||
Subproject commit b4bba94b1a250603b0b198e0394946e32f6c3f30
|
||||
@ -3,7 +3,9 @@ set -ex
|
||||
set -o pipefail
|
||||
trap "exit" INT TERM
|
||||
trap 'kill $(jobs -pr) ||:' EXIT
|
||||
S3_URL=${S3_URL:="https://clickhouse-builds.s3.amazonaws.com"}
|
||||
BUILD_NAME=${BUILD_NAME:-package_release}
|
||||
export S3_URL BUILD_NAME
|
||||
|
||||
mkdir db0 ||:
|
||||
mkdir left ||:
|
||||
@ -28,8 +30,9 @@ function download
|
||||
# Historically there were various paths for the performance test package.
|
||||
# Test all of them.
|
||||
declare -a urls_to_try=(
|
||||
"https://s3.amazonaws.com/clickhouse-builds/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst"
|
||||
"https://s3.amazonaws.com/clickhouse-builds/$left_pr/$left_sha/$BUILD_NAME/performance.tgz"
|
||||
"$S3_URL/PRs/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst"
|
||||
"$S3_URL/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst"
|
||||
"$S3_URL/$left_pr/$left_sha/$BUILD_NAME/performance.tgz"
|
||||
)
|
||||
|
||||
for path in "${urls_to_try[@]}"
|
||||
|
||||
@ -6,11 +6,7 @@ export CHPC_CHECK_START_TIMESTAMP
|
||||
|
||||
S3_URL=${S3_URL:="https://clickhouse-builds.s3.amazonaws.com"}
|
||||
BUILD_NAME=${BUILD_NAME:-package_release}
|
||||
|
||||
COMMON_BUILD_PREFIX="/clickhouse_build_check"
|
||||
if [[ $S3_URL == *"s3.amazonaws.com"* ]]; then
|
||||
COMMON_BUILD_PREFIX=""
|
||||
fi
|
||||
export S3_URL BUILD_NAME
|
||||
|
||||
# Sometimes AWS responde with DNS error and it's impossible to retry it with
|
||||
# current curl version options.
|
||||
@ -66,8 +62,9 @@ function find_reference_sha
|
||||
# test all of them.
|
||||
unset found
|
||||
declare -a urls_to_try=(
|
||||
"https://s3.amazonaws.com/clickhouse-builds/0/$REF_SHA/$BUILD_NAME/performance.tar.zst"
|
||||
"https://s3.amazonaws.com/clickhouse-builds/0/$REF_SHA/$BUILD_NAME/performance.tgz"
|
||||
"$S3_URL/PRs/0/$REF_SHA/$BUILD_NAME/performance.tar.zst"
|
||||
"$S3_URL/0/$REF_SHA/$BUILD_NAME/performance.tar.zst"
|
||||
"$S3_URL/0/$REF_SHA/$BUILD_NAME/performance.tgz"
|
||||
)
|
||||
for path in "${urls_to_try[@]}"
|
||||
do
|
||||
@ -92,10 +89,15 @@ chmod 777 workspace output
|
||||
cd workspace
|
||||
|
||||
# Download the package for the version we are going to test.
|
||||
if curl_with_retry "$S3_URL/$PR_TO_TEST/$SHA_TO_TEST$COMMON_BUILD_PREFIX/$BUILD_NAME/performance.tar.zst"
|
||||
then
|
||||
right_path="$S3_URL/$PR_TO_TEST/$SHA_TO_TEST$COMMON_BUILD_PREFIX/$BUILD_NAME/performance.tar.zst"
|
||||
fi
|
||||
# A temporary solution for migrating into PRs directory
|
||||
for prefix in "$S3_URL/PRs" "$S3_URL";
|
||||
do
|
||||
if curl_with_retry "$prefix/$PR_TO_TEST/$SHA_TO_TEST/$BUILD_NAME/performance.tar.zst"
|
||||
then
|
||||
right_path="$prefix/$PR_TO_TEST/$SHA_TO_TEST/$BUILD_NAME/performance.tar.zst"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
mkdir right
|
||||
wget -nv -nd -c "$right_path" -O- | tar -C right --no-same-owner --strip-components=1 --zstd --extract --verbose
|
||||
|
||||
@ -26,6 +26,7 @@ logging.basicConfig(
|
||||
total_start_seconds = time.perf_counter()
|
||||
stage_start_seconds = total_start_seconds
|
||||
|
||||
|
||||
# Thread executor that does not hides exception that happens during function
|
||||
# execution, and rethrows it after join()
|
||||
class SafeThread(Thread):
|
||||
@ -158,6 +159,7 @@ for e in subst_elems:
|
||||
|
||||
available_parameters[name] = values
|
||||
|
||||
|
||||
# Takes parallel lists of templates, substitutes them with all combos of
|
||||
# parameters. The set of parameters is determined based on the first list.
|
||||
# Note: keep the order of queries -- sometimes we have DROP IF EXISTS
|
||||
|
||||
@ -670,7 +670,6 @@ if args.report == "main":
|
||||
)
|
||||
|
||||
elif args.report == "all-queries":
|
||||
|
||||
print((header_template.format()))
|
||||
|
||||
add_tested_commits()
|
||||
|
||||
@ -128,7 +128,7 @@ function run_tests()
|
||||
set +e
|
||||
|
||||
if [[ -n "$USE_PARALLEL_REPLICAS" ]] && [[ "$USE_PARALLEL_REPLICAS" -eq 1 ]]; then
|
||||
clickhouse-test --client="clickhouse-client --use_hedged_requests=0 --allow_experimental_parallel_reading_from_replicas=1 \
|
||||
clickhouse-test --client="clickhouse-client --use_hedged_requests=0 --allow_experimental_parallel_reading_from_replicas=1 --parallel_replicas_for_non_replicated_merge_tree=1 \
|
||||
--max_parallel_replicas=100 --cluster_for_parallel_replicas='parallel_replicas'" \
|
||||
-j 2 --testname --shard --zookeeper --check-zookeeper-session --no-stateless --no-parallel-replicas --hung-check --print-time "${ADDITIONAL_OPTIONS[@]}" \
|
||||
"$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
|
||||
@ -10,31 +10,38 @@ import requests
|
||||
import tempfile
|
||||
|
||||
|
||||
DEFAULT_URL = 'https://clickhouse-datasets.s3.amazonaws.com'
|
||||
DEFAULT_URL = "https://clickhouse-datasets.s3.amazonaws.com"
|
||||
|
||||
AVAILABLE_DATASETS = {
|
||||
'hits': 'hits_v1.tar',
|
||||
'visits': 'visits_v1.tar',
|
||||
"hits": "hits_v1.tar",
|
||||
"visits": "visits_v1.tar",
|
||||
}
|
||||
|
||||
RETRIES_COUNT = 5
|
||||
|
||||
|
||||
def _get_temp_file_name():
|
||||
return os.path.join(tempfile._get_default_tempdir(), next(tempfile._get_candidate_names()))
|
||||
return os.path.join(
|
||||
tempfile._get_default_tempdir(), next(tempfile._get_candidate_names())
|
||||
)
|
||||
|
||||
|
||||
def build_url(base_url, dataset):
|
||||
return os.path.join(base_url, dataset, 'partitions', AVAILABLE_DATASETS[dataset])
|
||||
return os.path.join(base_url, dataset, "partitions", AVAILABLE_DATASETS[dataset])
|
||||
|
||||
|
||||
def dowload_with_progress(url, path):
|
||||
logging.info("Downloading from %s to temp path %s", url, path)
|
||||
for i in range(RETRIES_COUNT):
|
||||
try:
|
||||
with open(path, 'wb') as f:
|
||||
with open(path, "wb") as f:
|
||||
response = requests.get(url, stream=True)
|
||||
response.raise_for_status()
|
||||
total_length = response.headers.get('content-length')
|
||||
total_length = response.headers.get("content-length")
|
||||
if total_length is None or int(total_length) == 0:
|
||||
logging.info("No content-length, will download file without progress")
|
||||
logging.info(
|
||||
"No content-length, will download file without progress"
|
||||
)
|
||||
f.write(response.content)
|
||||
else:
|
||||
dl = 0
|
||||
@ -46,7 +53,11 @@ def dowload_with_progress(url, path):
|
||||
if sys.stdout.isatty():
|
||||
done = int(50 * dl / total_length)
|
||||
percent = int(100 * float(dl) / total_length)
|
||||
sys.stdout.write("\r[{}{}] {}%".format('=' * done, ' ' * (50-done), percent))
|
||||
sys.stdout.write(
|
||||
"\r[{}{}] {}%".format(
|
||||
"=" * done, " " * (50 - done), percent
|
||||
)
|
||||
)
|
||||
sys.stdout.flush()
|
||||
break
|
||||
except Exception as ex:
|
||||
@ -56,14 +67,21 @@ def dowload_with_progress(url, path):
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
else:
|
||||
raise Exception("Cannot download dataset from {}, all retries exceeded".format(url))
|
||||
raise Exception(
|
||||
"Cannot download dataset from {}, all retries exceeded".format(url)
|
||||
)
|
||||
|
||||
sys.stdout.write("\n")
|
||||
logging.info("Downloading finished")
|
||||
|
||||
|
||||
def unpack_to_clickhouse_directory(tar_path, clickhouse_path):
|
||||
logging.info("Will unpack data from temp path %s to clickhouse db %s", tar_path, clickhouse_path)
|
||||
with tarfile.open(tar_path, 'r') as comp_file:
|
||||
logging.info(
|
||||
"Will unpack data from temp path %s to clickhouse db %s",
|
||||
tar_path,
|
||||
clickhouse_path,
|
||||
)
|
||||
with tarfile.open(tar_path, "r") as comp_file:
|
||||
comp_file.extractall(path=clickhouse_path)
|
||||
logging.info("Unpack finished")
|
||||
|
||||
@ -72,15 +90,21 @@ if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Simple tool for dowloading datasets for clickhouse from S3")
|
||||
description="Simple tool for dowloading datasets for clickhouse from S3"
|
||||
)
|
||||
|
||||
parser.add_argument('--dataset-names', required=True, nargs='+', choices=list(AVAILABLE_DATASETS.keys()))
|
||||
parser.add_argument('--url-prefix', default=DEFAULT_URL)
|
||||
parser.add_argument('--clickhouse-data-path', default='/var/lib/clickhouse/')
|
||||
parser.add_argument(
|
||||
"--dataset-names",
|
||||
required=True,
|
||||
nargs="+",
|
||||
choices=list(AVAILABLE_DATASETS.keys()),
|
||||
)
|
||||
parser.add_argument("--url-prefix", default=DEFAULT_URL)
|
||||
parser.add_argument("--clickhouse-data-path", default="/var/lib/clickhouse/")
|
||||
|
||||
args = parser.parse_args()
|
||||
datasets = args.dataset_names
|
||||
logging.info("Will fetch following datasets: %s", ', '.join(datasets))
|
||||
logging.info("Will fetch following datasets: %s", ", ".join(datasets))
|
||||
for dataset in datasets:
|
||||
logging.info("Processing %s", dataset)
|
||||
temp_archive_path = _get_temp_file_name()
|
||||
@ -92,10 +116,11 @@ if __name__ == "__main__":
|
||||
logging.info("Some exception occured %s", str(ex))
|
||||
raise
|
||||
finally:
|
||||
logging.info("Will remove downloaded file %s from filesystem if it exists", temp_archive_path)
|
||||
logging.info(
|
||||
"Will remove downloaded file %s from filesystem if it exists",
|
||||
temp_archive_path,
|
||||
)
|
||||
if os.path.exists(temp_archive_path):
|
||||
os.remove(temp_archive_path)
|
||||
logging.info("Processing of %s finished", dataset)
|
||||
logging.info("Fetch finished, enjoy your tables!")
|
||||
|
||||
|
||||
|
||||
@ -170,6 +170,7 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
|
||||
fi
|
||||
|
||||
rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server.log ||:
|
||||
rg -A50 -Fa "============" /var/log/clickhouse-server/stderr.log ||:
|
||||
zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst &
|
||||
|
||||
# Compress tables.
|
||||
|
||||
@ -41,6 +41,9 @@ if [ "$is_tsan_build" -eq "0" ]; then
|
||||
export THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_BEFORE_SLEEP_TIME_US=10000
|
||||
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US=10000
|
||||
|
||||
export THREAD_FUZZER_EXPLICIT_SLEEP_PROBABILITY=0.01
|
||||
export THREAD_FUZZER_EXPLICIT_MEMORY_EXCEPTION_PROBABILITY=0.01
|
||||
fi
|
||||
|
||||
export ZOOKEEPER_FAULT_INJECTION=1
|
||||
|
||||
@ -11,13 +11,14 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
|
||||
aspell \
|
||||
curl \
|
||||
git \
|
||||
file \
|
||||
libxml2-utils \
|
||||
moreutils \
|
||||
python3-fuzzywuzzy \
|
||||
python3-pip \
|
||||
shellcheck \
|
||||
yamllint \
|
||||
&& pip3 install black==22.8.0 boto3 codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2 \
|
||||
&& pip3 install black==23.1.0 boto3 codespell==2.2.1 dohq-artifactory mypy PyGithub unidiff pylint==2.6.2 \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /root/.cache/pip
|
||||
|
||||
|
||||
@ -109,8 +109,7 @@ mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/c
|
||||
|
||||
# Install and start new server
|
||||
install_packages package_folder
|
||||
# Disable fault injections on start (we don't test them here, and it can lead to tons of requests in case of huge number of tables).
|
||||
export ZOOKEEPER_FAULT_INJECTION=0
|
||||
export ZOOKEEPER_FAULT_INJECTION=1
|
||||
configure
|
||||
start 500
|
||||
clickhouse-client --query "SELECT 'Server successfully started', 'OK', NULL, ''" >> /test_output/test_results.tsv \
|
||||
|
||||
1
docs/.gitignore
vendored
1
docs/.gitignore
vendored
@ -1 +1,2 @@
|
||||
build
|
||||
clickhouse-docs
|
||||
|
||||
@ -40,6 +40,8 @@ The documentation contains information about all the aspects of the ClickHouse l
|
||||
|
||||
At the moment, [documentation](https://clickhouse.com/docs) exists in English, Russian, and Chinese. We store the reference documentation besides the ClickHouse source code in the [GitHub repository](https://github.com/ClickHouse/ClickHouse/tree/master/docs), and user guides in a separate repo [Clickhouse/clickhouse-docs](https://github.com/ClickHouse/clickhouse-docs).
|
||||
|
||||
To get the latter launch the `get-clickhouse-docs.sh` script.
|
||||
|
||||
Each language lies in the corresponding folder. Files that are not translated from English are symbolic links to the English ones.
|
||||
|
||||
<a name="how-to-contribute"/>
|
||||
|
||||
@ -108,7 +108,7 @@ sidebar_label: 2022
|
||||
* Print out git status information at CMake configure stage. [#28047](https://github.com/ClickHouse/ClickHouse/pull/28047) ([Braulio Valdivielso Martínez](https://github.com/BraulioVM)).
|
||||
* Add new log level `<test>` for testing environments. [#28559](https://github.com/ClickHouse/ClickHouse/pull/28559) ([alesapin](https://github.com/alesapin)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Fix handling null value with type of Nullable(String) in function JSONExtract. This fixes [#27929](https://github.com/ClickHouse/ClickHouse/issues/27929) and [#27930](https://github.com/ClickHouse/ClickHouse/issues/27930) . This was introduced in https://github.com/ClickHouse/ClickHouse/pull/25452 . [#27939](https://github.com/ClickHouse/ClickHouse/pull/27939) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)).
|
||||
|
||||
@ -17,7 +17,7 @@ sidebar_label: 2022
|
||||
* Backported in [#29970](https://github.com/ClickHouse/ClickHouse/issues/29970): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Backported in [#30051](https://github.com/ClickHouse/ClickHouse/issues/30051): Fix releasing query ID and session ID at the end of query processing while handing gRPC call. This PR fixes flaky test [test_grpc_protocol/test.py::test_session](https://clickhouse-test-reports.s3.yandex.net/0/1ac03811a2df9717fa7c633d1af03def821d24b6/integration_tests_(memory).html). [#29954](https://github.com/ClickHouse/ClickHouse/pull/29954) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#29054](https://github.com/ClickHouse/ClickHouse/issues/29054): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#28795](https://github.com/ClickHouse/ClickHouse/issues/28795): - Fix the number of arguments required by s2RectAdd and s2RectContains functions. [#28663](https://github.com/ClickHouse/ClickHouse/pull/28663) ([Bharat Nallan](https://github.com/bharatnc)).
|
||||
|
||||
@ -18,11 +18,11 @@ sidebar_label: 2022
|
||||
* Backported in [#30620](https://github.com/ClickHouse/ClickHouse/issues/30620): Fix reading from empty file on encrypted disk. [#30494](https://github.com/ClickHouse/ClickHouse/pull/30494) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Backported in [#31369](https://github.com/ClickHouse/ClickHouse/issues/31369): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release
|
||||
|
||||
* Backported in [#30915](https://github.com/ClickHouse/ClickHouse/issues/30915): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#30824](https://github.com/ClickHouse/ClickHouse/issues/30824): Fix "Column is not under aggregate function and not in GROUP BY" with PREWHERE (Fixes: [#28461](https://github.com/ClickHouse/ClickHouse/issues/28461)). [#28502](https://github.com/ClickHouse/ClickHouse/pull/28502) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Backported in [#30766](https://github.com/ClickHouse/ClickHouse/issues/30766): Fix hanging DDL queries on Replicated database while adding a new replica. [#29328](https://github.com/ClickHouse/ClickHouse/pull/29328) ([Kevin Michel](https://github.com/kmichel-aiven)).
|
||||
|
||||
@ -13,7 +13,7 @@ sidebar_label: 2022
|
||||
#### Bug Fix
|
||||
* Backported in [#31573](https://github.com/ClickHouse/ClickHouse/issues/31573): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#31518](https://github.com/ClickHouse/ClickHouse/issues/31518): Remove not like function into RPNElement. [#31169](https://github.com/ClickHouse/ClickHouse/pull/31169) ([sundyli](https://github.com/sundy-li)).
|
||||
* Backported in [#31554](https://github.com/ClickHouse/ClickHouse/issues/31554): Resolve `nullptr` in STS credentials provider for S3. [#31409](https://github.com/ClickHouse/ClickHouse/pull/31409) ([Vladimir Chebotarev](https://github.com/excitoon)).
|
||||
|
||||
@ -11,7 +11,7 @@ sidebar_label: 2022
|
||||
* Backported in [#32252](https://github.com/ClickHouse/ClickHouse/issues/32252): Fix skipping columns while writing protobuf. This PR fixes [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160), see the comment [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160)#issuecomment-980595318. [#31988](https://github.com/ClickHouse/ClickHouse/pull/31988) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Backported in [#32346](https://github.com/ClickHouse/ClickHouse/issues/32346): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#32151](https://github.com/ClickHouse/ClickHouse/issues/32151): Fix crash when function `dictGet` with type is used for dictionary attribute when type is `Nullable`. Fixes [#30980](https://github.com/ClickHouse/ClickHouse/issues/30980). [#31800](https://github.com/ClickHouse/ClickHouse/pull/31800) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Backported in [#32093](https://github.com/ClickHouse/ClickHouse/issues/32093): Some `GET_PART` entry might hang in replication queue if part is lost on all replicas and there are no other parts in the same partition. It's fixed in cases when partition key contains only columns of integer types or `Date[Time]`. Fixes [#31485](https://github.com/ClickHouse/ClickHouse/issues/31485). [#31887](https://github.com/ClickHouse/ClickHouse/pull/31887) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
|
||||
@ -10,7 +10,7 @@ sidebar_label: 2022
|
||||
#### Bug Fix
|
||||
* Backported in [#32692](https://github.com/ClickHouse/ClickHouse/issues/32692): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#32680](https://github.com/ClickHouse/ClickHouse/issues/32680): Fix unexpected projection removal when detaching parts. [#32067](https://github.com/ClickHouse/ClickHouse/pull/32067) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#32285](https://github.com/ClickHouse/ClickHouse/issues/32285): Dictionaries fix cases when `{condition}` does not work for custom database queries. [#32117](https://github.com/ClickHouse/ClickHouse/pull/32117) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
@ -23,7 +23,7 @@ sidebar_label: 2022
|
||||
* Backported in [#33182](https://github.com/ClickHouse/ClickHouse/issues/33182): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Backported in [#33655](https://github.com/ClickHouse/ClickHouse/issues/33655): Fix hdfs url check that didn't allow using HA namenode address. Bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/31042. [#32976](https://github.com/ClickHouse/ClickHouse/pull/32976) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release):
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release):
|
||||
|
||||
* Backported in [#32657](https://github.com/ClickHouse/ClickHouse/issues/32657): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
|
||||
|
||||
@ -124,11 +124,11 @@ sidebar_label: 2022
|
||||
* Recursive submodules are no longer needed for ClickHouse. [#30315](https://github.com/ClickHouse/ClickHouse/pull/30315) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Added docker image to build docs. [#30499](https://github.com/ClickHouse/ClickHouse/pull/30499) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release
|
||||
|
||||
* Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Fix hanging DDL queries on Replicated database while adding a new replica. [#29328](https://github.com/ClickHouse/ClickHouse/pull/29328) ([Kevin Michel](https://github.com/kmichel-aiven)).
|
||||
* Fix vertical merges of projection parts. This fixes [#29253](https://github.com/ClickHouse/ClickHouse/issues/29253) . This PR also fixes several projection merge/mutation issues introduced in https://github.com/ClickHouse/ClickHouse/pull/25165. [#29337](https://github.com/ClickHouse/ClickHouse/pull/29337) ([Amos Bird](https://github.com/amosbird)).
|
||||
|
||||
@ -7,6 +7,6 @@ sidebar_label: 2022
|
||||
|
||||
### ClickHouse release v21.11.11.1-stable FIXME as compared to v21.11.10.1-stable
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#33656](https://github.com/ClickHouse/ClickHouse/issues/33656): Fix hdfs url check that didn't allow using HA namenode address. Bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/31042. [#32976](https://github.com/ClickHouse/ClickHouse/pull/32976) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
@ -7,7 +7,7 @@ sidebar_label: 2022
|
||||
|
||||
### ClickHouse release v21.11.2.2-stable FIXME as compared to v21.11.1.8636-prestable
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#31154](https://github.com/ClickHouse/ClickHouse/issues/31154): Skip max_partition_size_to_drop check in case of ATTACH PARTITION ... FROM and MOVE PARTITION ... [#30995](https://github.com/ClickHouse/ClickHouse/pull/30995) ([Amr Alaa](https://github.com/amralaa-MSFT)).
|
||||
* Backported in [#31027](https://github.com/ClickHouse/ClickHouse/issues/31027): Using `formatRow` function with not row formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
@ -10,7 +10,7 @@ sidebar_label: 2022
|
||||
#### Bug Fix
|
||||
* Backported in [#31246](https://github.com/ClickHouse/ClickHouse/issues/31246): Memory amount was incorrectly estimated when ClickHouse is run in containers with cgroup limits. [#31157](https://github.com/ClickHouse/ClickHouse/pull/31157) ([Pavel Medvedev](https://github.com/pmed)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#31206](https://github.com/ClickHouse/ClickHouse/issues/31206): Fix possible assert in `hdfs` table function/engine, add test. [#31036](https://github.com/ClickHouse/ClickHouse/pull/31036) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Backported in [#31202](https://github.com/ClickHouse/ClickHouse/issues/31202): Fix abort in debug server and `DB::Exception: std::out_of_range: basic_string` error in release server in case of bad hdfs url by adding additional check of hdfs url structure. [#31042](https://github.com/ClickHouse/ClickHouse/pull/31042) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
@ -10,7 +10,7 @@ sidebar_label: 2022
|
||||
#### Bug Fix
|
||||
* Backported in [#31370](https://github.com/ClickHouse/ClickHouse/issues/31370): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#31282](https://github.com/ClickHouse/ClickHouse/issues/31282): Fix some corner cases with intersect/except. Closes [#30803](https://github.com/ClickHouse/ClickHouse/issues/30803). [#30965](https://github.com/ClickHouse/ClickHouse/pull/30965) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#31237](https://github.com/ClickHouse/ClickHouse/issues/31237): Fix bug which broke select queries if they happened after dropping materialized view. Found in [#30691](https://github.com/ClickHouse/ClickHouse/issues/30691). [#30997](https://github.com/ClickHouse/ClickHouse/pull/30997) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
|
||||
@ -13,7 +13,7 @@ sidebar_label: 2022
|
||||
#### Bug Fix
|
||||
* Backported in [#31572](https://github.com/ClickHouse/ClickHouse/issues/31572): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#31517](https://github.com/ClickHouse/ClickHouse/issues/31517): Remove not like function into RPNElement. [#31169](https://github.com/ClickHouse/ClickHouse/pull/31169) ([sundyli](https://github.com/sundy-li)).
|
||||
* Backported in [#31551](https://github.com/ClickHouse/ClickHouse/issues/31551): Resolve `nullptr` in STS credentials provider for S3. [#31409](https://github.com/ClickHouse/ClickHouse/pull/31409) ([Vladimir Chebotarev](https://github.com/excitoon)).
|
||||
|
||||
@ -11,7 +11,7 @@ sidebar_label: 2022
|
||||
* Backported in [#32254](https://github.com/ClickHouse/ClickHouse/issues/32254): Fix skipping columns while writing protobuf. This PR fixes [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160), see the comment [#31160](https://github.com/ClickHouse/ClickHouse/issues/31160)#issuecomment-980595318. [#31988](https://github.com/ClickHouse/ClickHouse/pull/31988) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Backported in [#32345](https://github.com/ClickHouse/ClickHouse/issues/32345): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#32152](https://github.com/ClickHouse/ClickHouse/issues/32152): Fix crash when function `dictGet` with type is used for dictionary attribute when type is `Nullable`. Fixes [#30980](https://github.com/ClickHouse/ClickHouse/issues/30980). [#31800](https://github.com/ClickHouse/ClickHouse/pull/31800) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Backported in [#32298](https://github.com/ClickHouse/ClickHouse/issues/32298): Fix recursive user defined functions crash. Closes [#30856](https://github.com/ClickHouse/ClickHouse/issues/30856). [#31820](https://github.com/ClickHouse/ClickHouse/pull/31820) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
|
||||
@ -10,13 +10,13 @@ sidebar_label: 2022
|
||||
#### Bug Fix
|
||||
* Backported in [#32691](https://github.com/ClickHouse/ClickHouse/issues/32691): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#32711](https://github.com/ClickHouse/ClickHouse/issues/32711): Fix failures in queries that are trying to use skipping indices, which are not materialized yet. Fixes [#32292](https://github.com/ClickHouse/ClickHouse/issues/32292) and [#30343](https://github.com/ClickHouse/ClickHouse/issues/30343). [#32359](https://github.com/ClickHouse/ClickHouse/pull/32359) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Backported in [#32568](https://github.com/ClickHouse/ClickHouse/issues/32568): Fix crash in `JoinCommon::removeColumnNullability`, close [#32458](https://github.com/ClickHouse/ClickHouse/issues/32458). [#32508](https://github.com/ClickHouse/ClickHouse/pull/32508) ([Vladimir C](https://github.com/vdimir)).
|
||||
* Backported in [#32732](https://github.com/ClickHouse/ClickHouse/issues/32732): Fix surprisingly bad code in function `file`. [#32640](https://github.com/ClickHouse/ClickHouse/pull/32640) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release):
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release):
|
||||
|
||||
* Backported in [#32617](https://github.com/ClickHouse/ClickHouse/issues/32617): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ sidebar_label: 2022
|
||||
|
||||
### ClickHouse release v21.11.8.4-stable FIXME as compared to v21.11.7.9-stable
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#32679](https://github.com/ClickHouse/ClickHouse/issues/32679): Fix unexpected projection removal when detaching parts. [#32067](https://github.com/ClickHouse/ClickHouse/pull/32067) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#32543](https://github.com/ClickHouse/ClickHouse/issues/32543): Some replication queue entries might hang for `temporary_directories_lifetime` (1 day by default) with `Directory tmp_merge_<part_name>` or `Part ... (state Deleting) already exists, but it will be deleted soon` or similar error. It's fixed. Fixes [#29616](https://github.com/ClickHouse/ClickHouse/issues/29616). [#32201](https://github.com/ClickHouse/ClickHouse/pull/32201) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
|
||||
@ -7,7 +7,7 @@ sidebar_label: 2022
|
||||
|
||||
### ClickHouse release v21.11.9.1-stable FIXME as compared to v21.11.8.4-stable
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#33181](https://github.com/ClickHouse/ClickHouse/issues/33181): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
|
||||
|
||||
@ -132,7 +132,7 @@ sidebar_label: 2022
|
||||
* Build rpm and tgz packages in master and release branches workfolw. [#32048](https://github.com/ClickHouse/ClickHouse/pull/32048) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Fix broken symlink for sysroot/linux-riscv64/usr/lib. [#32071](https://github.com/ClickHouse/ClickHouse/pull/32071) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Fix some corner cases with intersect/except. Closes [#30803](https://github.com/ClickHouse/ClickHouse/issues/30803). [#30965](https://github.com/ClickHouse/ClickHouse/pull/30965) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Skip max_partition_size_to_drop check in case of ATTACH PARTITION ... FROM and MOVE PARTITION ... [#30995](https://github.com/ClickHouse/ClickHouse/pull/30995) ([Amr Alaa](https://github.com/amralaa-MSFT)).
|
||||
|
||||
@ -10,7 +10,7 @@ sidebar_label: 2022
|
||||
#### Bug Fix
|
||||
* Backported in [#32693](https://github.com/ClickHouse/ClickHouse/issues/32693): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#32681](https://github.com/ClickHouse/ClickHouse/issues/32681): Fix unexpected projection removal when detaching parts. [#32067](https://github.com/ClickHouse/ClickHouse/pull/32067) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#32483](https://github.com/ClickHouse/ClickHouse/issues/32483): Fix 'APPLY lambda' parsing which could lead to client/server crash. [#32138](https://github.com/ClickHouse/ClickHouse/pull/32138) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
@ -23,7 +23,7 @@ sidebar_label: 2022
|
||||
* Backported in [#32733](https://github.com/ClickHouse/ClickHouse/issues/32733): Fix surprisingly bad code in function `file`. [#32640](https://github.com/ClickHouse/ClickHouse/pull/32640) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Backported in [#32793](https://github.com/ClickHouse/ClickHouse/issues/32793): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release):
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release):
|
||||
|
||||
* Backported in [#32616](https://github.com/ClickHouse/ClickHouse/issues/32616): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
|
||||
|
||||
@ -10,7 +10,7 @@ sidebar_label: 2022
|
||||
#### Bug Fix
|
||||
* Backported in [#33018](https://github.com/ClickHouse/ClickHouse/issues/33018): - ClickHouse Keeper handler should remove operation when response sent. [#32988](https://github.com/ClickHouse/ClickHouse/pull/32988) ([JackyWoo](https://github.com/JackyWoo)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#32890](https://github.com/ClickHouse/ClickHouse/issues/32890): Fix LOGICAL_ERROR when the target of a materialized view is a JOIN or a SET table. [#32669](https://github.com/ClickHouse/ClickHouse/pull/32669) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#33183](https://github.com/ClickHouse/ClickHouse/issues/33183): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
|
||||
@ -13,7 +13,7 @@ sidebar_label: 2022
|
||||
#### Bug Fix
|
||||
* Backported in [#33551](https://github.com/ClickHouse/ClickHouse/issues/33551): Fix null pointer dereference in low cardinality data when deserializing LowCardinality data in the Native format. [#33021](https://github.com/ClickHouse/ClickHouse/pull/33021) ([Harry Lee](https://github.com/HarryLeeIBM)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#33537](https://github.com/ClickHouse/ClickHouse/issues/33537): Fix ORC stripe reading. [#32929](https://github.com/ClickHouse/ClickHouse/pull/32929) ([Ernest Zaslavsky](https://github.com/kreuzerkrieg)).
|
||||
* Backported in [#33654](https://github.com/ClickHouse/ClickHouse/issues/33654): Fix segfault in Avro that appears after the second insert into file. [#33566](https://github.com/ClickHouse/ClickHouse/pull/33566) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
@ -25,7 +25,7 @@ sidebar_label: 2022
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Backported in [#28075](https://github.com/ClickHouse/ClickHouse/issues/28075): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#28181](https://github.com/ClickHouse/ClickHouse/issues/28181): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)).
|
||||
* Backported in [#28293](https://github.com/ClickHouse/ClickHouse/issues/28293): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
@ -10,7 +10,7 @@ sidebar_label: 2022
|
||||
#### Bug Fix
|
||||
* Backported in [#28647](https://github.com/ClickHouse/ClickHouse/issues/28647): Fix a rare bug in `DROP PART` which can lead to the error `Unexpected merged part intersects drop range`. [#27807](https://github.com/ClickHouse/ClickHouse/pull/27807) ([alesapin](https://github.com/alesapin)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#28569](https://github.com/ClickHouse/ClickHouse/issues/28569): Fix bug which can lead to error `Existing table metadata in ZooKeeper differs in sorting key expression.` after alter of `ReplicatedVersionedCollapsingMergeTree`. Fixes [#28515](https://github.com/ClickHouse/ClickHouse/issues/28515). [#28528](https://github.com/ClickHouse/ClickHouse/pull/28528) ([alesapin](https://github.com/alesapin)).
|
||||
* Backported in [#28857](https://github.com/ClickHouse/ClickHouse/issues/28857): Fix benign race condition in ReplicatedMergeTreeQueue. Shouldn't be visible for user, but can lead to subtle bugs. [#28734](https://github.com/ClickHouse/ClickHouse/pull/28734) ([alesapin](https://github.com/alesapin)).
|
||||
|
||||
@ -14,7 +14,7 @@ sidebar_label: 2022
|
||||
#### Bug Fix
|
||||
* Backported in [#30041](https://github.com/ClickHouse/ClickHouse/issues/30041): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#29260](https://github.com/ClickHouse/ClickHouse/issues/29260): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#29026](https://github.com/ClickHouse/ClickHouse/issues/29026): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
|
||||
@ -14,11 +14,11 @@ sidebar_label: 2022
|
||||
* Backported in [#31577](https://github.com/ClickHouse/ClickHouse/issues/31577): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)).
|
||||
* Backported in [#32347](https://github.com/ClickHouse/ClickHouse/issues/32347): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release
|
||||
|
||||
* Backported in [#30913](https://github.com/ClickHouse/ClickHouse/issues/30913): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#30750](https://github.com/ClickHouse/ClickHouse/issues/30750): Functions for case-insensitive search in UTF8 strings like `positionCaseInsensitiveUTF8` and `countSubstringsCaseInsensitiveUTF8` might find substrings that actually does not match, it's fixed. [#30663](https://github.com/ClickHouse/ClickHouse/pull/30663) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Backported in [#31038](https://github.com/ClickHouse/ClickHouse/issues/31038): Using `formatRow` function with not row formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
@ -11,7 +11,7 @@ sidebar_label: 2022
|
||||
* Backported in [#32690](https://github.com/ClickHouse/ClickHouse/issues/32690): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)).
|
||||
* Backported in [#33727](https://github.com/ClickHouse/ClickHouse/issues/33727): Fix null pointer dereference in low cardinality data when deserializing LowCardinality data in the Native format. [#33021](https://github.com/ClickHouse/ClickHouse/pull/33021) ([Harry Lee](https://github.com/HarryLeeIBM)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#32791](https://github.com/ClickHouse/ClickHouse/issues/32791): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)).
|
||||
|
||||
|
||||
@ -40,7 +40,7 @@ sidebar_label: 2022
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Backported in [#28030](https://github.com/ClickHouse/ClickHouse/issues/28030): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#28119](https://github.com/ClickHouse/ClickHouse/issues/28119): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)).
|
||||
* Backported in [#28179](https://github.com/ClickHouse/ClickHouse/issues/28179): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)).
|
||||
|
||||
@ -14,7 +14,7 @@ sidebar_label: 2022
|
||||
* Backported in [#27925](https://github.com/ClickHouse/ClickHouse/issues/27925): Fix PostgreSQL-style cast (`::` operator) with negative numbers. [#27876](https://github.com/ClickHouse/ClickHouse/pull/27876) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Backported in [#28752](https://github.com/ClickHouse/ClickHouse/issues/28752): Fix transformation of disjunctions chain to `IN` (controlled by settings `optimize_min_equality_disjunction_chain_length`) in distributed queries with settings `legacy_column_name_of_tuple_literal = 0`. [#28658](https://github.com/ClickHouse/ClickHouse/pull/28658) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#28509](https://github.com/ClickHouse/ClickHouse/issues/28509): Fixed possible ZooKeeper watches leak on background processing of distributed DDL queue. Closes [#26036](https://github.com/ClickHouse/ClickHouse/issues/26036). [#28446](https://github.com/ClickHouse/ClickHouse/pull/28446) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Backported in [#28570](https://github.com/ClickHouse/ClickHouse/issues/28570): Fix bug which can lead to error `Existing table metadata in ZooKeeper differs in sorting key expression.` after alter of `ReplicatedVersionedCollapsingMergeTree`. Fixes [#28515](https://github.com/ClickHouse/ClickHouse/issues/28515). [#28528](https://github.com/ClickHouse/ClickHouse/pull/28528) ([alesapin](https://github.com/alesapin)).
|
||||
|
||||
@ -7,7 +7,7 @@ sidebar_label: 2022
|
||||
|
||||
### ClickHouse release v21.7.11.3-stable FIXME as compared to v21.7.10.4-stable
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#29024](https://github.com/ClickHouse/ClickHouse/issues/29024): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#29195](https://github.com/ClickHouse/ClickHouse/issues/29195): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
@ -23,7 +23,7 @@ sidebar_label: 2022
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Backported in [#28032](https://github.com/ClickHouse/ClickHouse/issues/28032): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#28116](https://github.com/ClickHouse/ClickHouse/issues/28116): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)).
|
||||
* Backported in [#28183](https://github.com/ClickHouse/ClickHouse/issues/28183): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)).
|
||||
|
||||
@ -10,7 +10,7 @@ sidebar_label: 2022
|
||||
#### Improvement
|
||||
* Backported in [#30452](https://github.com/ClickHouse/ClickHouse/issues/30452): Allow symlinks to files in user_files directory for file table function. [#30309](https://github.com/ClickHouse/ClickHouse/pull/30309) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#29724](https://github.com/ClickHouse/ClickHouse/issues/29724): Fix null deference for `GROUP BY WITH TOTALS HAVING` (when the column from `HAVING` wasn't selected). [#29553](https://github.com/ClickHouse/ClickHouse/pull/29553) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Backported in [#30233](https://github.com/ClickHouse/ClickHouse/issues/30233): Fix INSERT SELECT incorrectly fills MATERIALIZED column based of Nullable column. [#30189](https://github.com/ClickHouse/ClickHouse/pull/30189) ([Azat Khuzhin](https://github.com/azat)).
|
||||
|
||||
@ -16,11 +16,11 @@ sidebar_label: 2022
|
||||
#### Bug Fix
|
||||
* Backported in [#31368](https://github.com/ClickHouse/ClickHouse/issues/31368): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release
|
||||
|
||||
* Backported in [#30914](https://github.com/ClickHouse/ClickHouse/issues/30914): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#28756](https://github.com/ClickHouse/ClickHouse/issues/28756): Fix NOT-IN index optimization when not all key columns are used. This fixes [#28120](https://github.com/ClickHouse/ClickHouse/issues/28120). [#28315](https://github.com/ClickHouse/ClickHouse/pull/28315) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#30825](https://github.com/ClickHouse/ClickHouse/issues/30825): Fix "Column is not under aggregate function and not in GROUP BY" with PREWHERE (Fixes: [#28461](https://github.com/ClickHouse/ClickHouse/issues/28461)). [#28502](https://github.com/ClickHouse/ClickHouse/pull/28502) ([Azat Khuzhin](https://github.com/azat)).
|
||||
|
||||
@ -13,7 +13,7 @@ sidebar_label: 2022
|
||||
#### Bug Fix
|
||||
* Backported in [#31575](https://github.com/ClickHouse/ClickHouse/issues/31575): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#31204](https://github.com/ClickHouse/ClickHouse/issues/31204): Fix abort in debug server and `DB::Exception: std::out_of_range: basic_string` error in release server in case of bad hdfs url by adding additional check of hdfs url structure. [#31042](https://github.com/ClickHouse/ClickHouse/pull/31042) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Backported in [#31253](https://github.com/ClickHouse/ClickHouse/issues/31253): Fix bug in Keeper which can lead to inability to start when some coordination logs was lost and we have more fresh snapshot than our latest log. [#31150](https://github.com/ClickHouse/ClickHouse/pull/31150) ([alesapin](https://github.com/alesapin)).
|
||||
|
||||
@ -11,7 +11,7 @@ sidebar_label: 2022
|
||||
* Backported in [#32688](https://github.com/ClickHouse/ClickHouse/issues/32688): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31656](https://github.com/ClickHouse/ClickHouse/pull/31656) ([sunny](https://github.com/sunny19930321)).
|
||||
* Backported in [#32343](https://github.com/ClickHouse/ClickHouse/issues/32343): Fix bug when remove unneeded columns in subquery. If there is an aggregation function in query without group by, do not remove if it is unneeded. [#32289](https://github.com/ClickHouse/ClickHouse/pull/32289) ([dongyifeng](https://github.com/dyf6372)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#32108](https://github.com/ClickHouse/ClickHouse/issues/32108): Fix crash with empty result on odbc query. Closes [#31465](https://github.com/ClickHouse/ClickHouse/issues/31465). [#31766](https://github.com/ClickHouse/ClickHouse/pull/31766) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#32150](https://github.com/ClickHouse/ClickHouse/issues/32150): Fix crash when function `dictGet` with type is used for dictionary attribute when type is `Nullable`. Fixes [#30980](https://github.com/ClickHouse/ClickHouse/issues/30980). [#31800](https://github.com/ClickHouse/ClickHouse/pull/31800) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
@ -26,7 +26,7 @@ sidebar_label: 2022
|
||||
* Backported in [#33048](https://github.com/ClickHouse/ClickHouse/issues/33048): Fix possible exception at RabbitMQ storage startup by delaying channel creation. [#32584](https://github.com/ClickHouse/ClickHouse/pull/32584) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#32795](https://github.com/ClickHouse/ClickHouse/issues/32795): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release):
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release):
|
||||
|
||||
* Backported in [#32659](https://github.com/ClickHouse/ClickHouse/issues/32659): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ sidebar_label: 2022
|
||||
|
||||
### ClickHouse release v21.8.14.5-lts FIXME as compared to v21.8.13.6-lts
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#33184](https://github.com/ClickHouse/ClickHouse/issues/33184): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Backported in [#33659](https://github.com/ClickHouse/ClickHouse/issues/33659): Fix hdfs url check that didn't allow using HA namenode address. Bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/31042. [#32976](https://github.com/ClickHouse/ClickHouse/pull/32976) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
@ -7,7 +7,7 @@ sidebar_label: 2022
|
||||
|
||||
### ClickHouse release v21.8.15.7-lts FIXME as compared to v21.8.14.5-lts
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#34121](https://github.com/ClickHouse/ClickHouse/issues/34121): Fix usage of functions `array` and `tuple` with literal arguments in distributed queries. Previously it could lead to `Not found columns` exception. [#33938](https://github.com/ClickHouse/ClickHouse/pull/33938) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Backported in [#34097](https://github.com/ClickHouse/ClickHouse/issues/34097): Fix segfault while parsing ORC file with corrupted footer. Closes [#33797](https://github.com/ClickHouse/ClickHouse/issues/33797). [#33984](https://github.com/ClickHouse/ClickHouse/pull/33984) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
@ -25,7 +25,7 @@ sidebar_label: 2022
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Backported in [#28031](https://github.com/ClickHouse/ClickHouse/issues/28031): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#27974](https://github.com/ClickHouse/ClickHouse/issues/27974): Fix handling null value with type of Nullable(String) in function JSONExtract. This fixes [#27929](https://github.com/ClickHouse/ClickHouse/issues/27929) and [#27930](https://github.com/ClickHouse/ClickHouse/issues/27930) . This was introduced in https://github.com/ClickHouse/ClickHouse/pull/25452 . [#27939](https://github.com/ClickHouse/ClickHouse/pull/27939) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#28117](https://github.com/ClickHouse/ClickHouse/issues/28117): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)).
|
||||
|
||||
@ -16,7 +16,7 @@ sidebar_label: 2022
|
||||
* Backported in [#27923](https://github.com/ClickHouse/ClickHouse/issues/27923): Fix PostgreSQL-style cast (`::` operator) with negative numbers. [#27876](https://github.com/ClickHouse/ClickHouse/pull/27876) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Backported in [#28753](https://github.com/ClickHouse/ClickHouse/issues/28753): Fix transformation of disjunctions chain to `IN` (controlled by settings `optimize_min_equality_disjunction_chain_length`) in distributed queries with settings `legacy_column_name_of_tuple_literal = 0`. [#28658](https://github.com/ClickHouse/ClickHouse/pull/28658) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#28644](https://github.com/ClickHouse/ClickHouse/issues/28644): Fix rare case when changes of `clickhouse-keeper` settings may lead to lost logs and server hung. [#28360](https://github.com/ClickHouse/ClickHouse/pull/28360) ([alesapin](https://github.com/alesapin)).
|
||||
* Backported in [#28508](https://github.com/ClickHouse/ClickHouse/issues/28508): Fix lack of quotes for table names in MaterializedPostgreSQL engine. Closes [#28316](https://github.com/ClickHouse/ClickHouse/issues/28316). [#28433](https://github.com/ClickHouse/ClickHouse/pull/28433) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
|
||||
@ -7,7 +7,7 @@ sidebar_label: 2022
|
||||
|
||||
### ClickHouse release v21.8.7.22-lts FIXME as compared to v21.8.6.15-lts
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#29121](https://github.com/ClickHouse/ClickHouse/issues/29121): Better check for connection usability and also catch any exception in RabbitMQ shutdown just in case. [#28797](https://github.com/ClickHouse/ClickHouse/pull/28797) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#29027](https://github.com/ClickHouse/ClickHouse/issues/29027): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
|
||||
@ -10,7 +10,7 @@ sidebar_label: 2022
|
||||
#### Bug Fix
|
||||
* Backported in [#29128](https://github.com/ClickHouse/ClickHouse/issues/29128): Fix bug in `clickhouse-keeper-converter` which can lead to incorrect ZooKeeper log deserialization. [#29071](https://github.com/ClickHouse/ClickHouse/pull/29071) ([小路](https://github.com/nicelulu)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#29262](https://github.com/ClickHouse/ClickHouse/issues/29262): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#29106](https://github.com/ClickHouse/ClickHouse/issues/29106): Fix waiting for mutation with `mutations_sync=2`. [#28889](https://github.com/ClickHouse/ClickHouse/pull/28889) ([Azat Khuzhin](https://github.com/azat)).
|
||||
|
||||
@ -14,7 +14,7 @@ sidebar_label: 2022
|
||||
* Backported in [#29817](https://github.com/ClickHouse/ClickHouse/issues/29817): Allow using a materialized column as the sharding key in a distributed table even if `insert_allow_materialized_columns=0`:. [#28637](https://github.com/ClickHouse/ClickHouse/pull/28637) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Backported in [#29973](https://github.com/ClickHouse/ClickHouse/issues/29973): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#29676](https://github.com/ClickHouse/ClickHouse/issues/29676): Fix vertical merges of projection parts. This fixes [#29253](https://github.com/ClickHouse/ClickHouse/issues/29253) . This PR also fixes several projection merge/mutation issues introduced in https://github.com/ClickHouse/ClickHouse/pull/25165. [#29337](https://github.com/ClickHouse/ClickHouse/pull/29337) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#29538](https://github.com/ClickHouse/ClickHouse/issues/29538): Fix possible `Block structure mismatch` for subqueries with pushed-down `HAVING` predicate. Fixes [#29010](https://github.com/ClickHouse/ClickHouse/issues/29010). [#29475](https://github.com/ClickHouse/ClickHouse/pull/29475) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
|
||||
@ -25,7 +25,7 @@ sidebar_label: 2022
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Backported in [#28029](https://github.com/ClickHouse/ClickHouse/issues/28029): Temporarily switched ubuntu apt repository to mirror ru.archive.ubuntu.com as default one(archive.ubuntu.com) is not responding from our CI. [#28016](https://github.com/ClickHouse/ClickHouse/pull/28016) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#27973](https://github.com/ClickHouse/ClickHouse/issues/27973): Fix handling null value with type of Nullable(String) in function JSONExtract. This fixes [#27929](https://github.com/ClickHouse/ClickHouse/issues/27929) and [#27930](https://github.com/ClickHouse/ClickHouse/issues/27930) . This was introduced in https://github.com/ClickHouse/ClickHouse/pull/25452 . [#27939](https://github.com/ClickHouse/ClickHouse/pull/27939) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#28118](https://github.com/ClickHouse/ClickHouse/issues/28118): Fix extremely rare segfaults on shutdown due to incorrect order of context/config reloader shutdown. [#28088](https://github.com/ClickHouse/ClickHouse/pull/28088) ([nvartolomei](https://github.com/nvartolomei)).
|
||||
|
||||
@ -10,7 +10,7 @@ sidebar_label: 2022
|
||||
#### Improvement
|
||||
* Backported in [#28897](https://github.com/ClickHouse/ClickHouse/issues/28897): Use real tmp file instead of predefined "rows_sources" for vertical merges. This avoids generating garbage directories in tmp disks. [#28299](https://github.com/ClickHouse/ClickHouse/pull/28299) ([Amos Bird](https://github.com/amosbird)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#28815](https://github.com/ClickHouse/ClickHouse/issues/28815): Fix possible crash for `SELECT` with partially created aggregate projection in case of exception. [#28700](https://github.com/ClickHouse/ClickHouse/pull/28700) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#28789](https://github.com/ClickHouse/ClickHouse/issues/28789): Fix benign race condition in ReplicatedMergeTreeQueue. Shouldn't be visible for user, but can lead to subtle bugs. [#28734](https://github.com/ClickHouse/ClickHouse/pull/28734) ([alesapin](https://github.com/alesapin)).
|
||||
|
||||
@ -7,7 +7,7 @@ sidebar_label: 2022
|
||||
|
||||
### ClickHouse release v21.9.4.35-stable FIXME as compared to v21.9.3.30-stable
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#29191](https://github.com/ClickHouse/ClickHouse/issues/29191): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
|
||||
@ -17,7 +17,7 @@ sidebar_label: 2022
|
||||
* Backported in [#29972](https://github.com/ClickHouse/ClickHouse/issues/29972): Fix shutdown of `AccessControlManager`. Now there can't be reloading of the configuration after AccessControlManager has been destroyed. This PR fixes the flaky test [test_user_directories/test.py::test_relative_path](https://clickhouse-test-reports.s3.yandex.net/0/f0e3122507ed8bea3f177495531c7d56bcb32466/integration_tests_(thread).html). [#29951](https://github.com/ClickHouse/ClickHouse/pull/29951) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Backported in [#30052](https://github.com/ClickHouse/ClickHouse/issues/30052): Fix releasing query ID and session ID at the end of query processing while handing gRPC call. This PR fixes flaky test [test_grpc_protocol/test.py::test_session](https://clickhouse-test-reports.s3.yandex.net/0/1ac03811a2df9717fa7c633d1af03def821d24b6/integration_tests_(memory).html). [#29954](https://github.com/ClickHouse/ClickHouse/pull/29954) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#29055](https://github.com/ClickHouse/ClickHouse/issues/29055): Fix invalid constant type conversion when nullable or lowcardinality primary key is used. [#28636](https://github.com/ClickHouse/ClickHouse/pull/28636) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#29107](https://github.com/ClickHouse/ClickHouse/issues/29107): Fix waiting for mutation with `mutations_sync=2`. [#28889](https://github.com/ClickHouse/ClickHouse/pull/28889) ([Azat Khuzhin](https://github.com/azat)).
|
||||
|
||||
@ -21,11 +21,11 @@ sidebar_label: 2022
|
||||
* Backported in [#31371](https://github.com/ClickHouse/ClickHouse/issues/31371): Fix SHOW GRANTS when partial revokes are used. This PR fixes [#31138](https://github.com/ClickHouse/ClickHouse/issues/31138). [#31249](https://github.com/ClickHouse/ClickHouse/pull/31249) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Backported in [#31576](https://github.com/ClickHouse/ClickHouse/issues/31576): Quota limit was not reached, but the limit was exceeded. This PR fixes [#31174](https://github.com/ClickHouse/ClickHouse/issues/31174). [#31337](https://github.com/ClickHouse/ClickHouse/pull/31337) ([sunny](https://github.com/sunny19930321)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release
|
||||
|
||||
* Backported in [#30916](https://github.com/ClickHouse/ClickHouse/issues/30916): Fix `ORDER BY ... WITH FILL` with set `TO` and `FROM` and no rows in result set. [#30888](https://github.com/ClickHouse/ClickHouse/pull/30888) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#30823](https://github.com/ClickHouse/ClickHouse/issues/30823): Fix "Column is not under aggregate function and not in GROUP BY" with PREWHERE (Fixes: [#28461](https://github.com/ClickHouse/ClickHouse/issues/28461)). [#28502](https://github.com/ClickHouse/ClickHouse/pull/28502) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Backported in [#30609](https://github.com/ClickHouse/ClickHouse/issues/30609): Fix bad optimizations of ORDER BY if it contains WITH FILL. This closes [#28908](https://github.com/ClickHouse/ClickHouse/issues/28908). This closes [#26049](https://github.com/ClickHouse/ClickHouse/issues/26049). [#28910](https://github.com/ClickHouse/ClickHouse/pull/28910) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
|
||||
@ -118,7 +118,7 @@ sidebar_label: 2022
|
||||
* Remove editing /etc/hosts from Dockerfile. [#33635](https://github.com/ClickHouse/ClickHouse/pull/33635) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Properly separate thrift-cmake from arrow-cmake after https://github.com/ClickHouse/ClickHouse/pull/31104 . cc @taiyang-li. [#33661](https://github.com/ClickHouse/ClickHouse/pull/33661) ([Amos Bird](https://github.com/amosbird)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Fixed CAST from String to IPv4 or IPv6 and back. Fixed error message in case of failed conversion. [#29224](https://github.com/ClickHouse/ClickHouse/pull/29224) ([Dmitry Novik](https://github.com/novikd)).
|
||||
* Fix base64Encode adding trailing bytes on small strings. [#31797](https://github.com/ClickHouse/ClickHouse/pull/31797) ([Kevin Michel](https://github.com/kmichel-aiven)).
|
||||
@ -185,7 +185,7 @@ sidebar_label: 2022
|
||||
* Fix segfault in Avro that appears after the second insert into file. [#33566](https://github.com/ClickHouse/ClickHouse/pull/33566) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fix wrong database for JOIN w/o explicit database in distributed queries (Fixes: [#10471](https://github.com/ClickHouse/ClickHouse/issues/10471)). [#33611](https://github.com/ClickHouse/ClickHouse/pull/33611) ([Azat Khuzhin](https://github.com/azat)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release):
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release):
|
||||
|
||||
* Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
|
||||
|
||||
@ -10,6 +10,6 @@ sidebar_label: 2022
|
||||
#### Improvement
|
||||
* Backported in [#33793](https://github.com/ClickHouse/ClickHouse/issues/33793): Create parent directories in DiskS3::restoreFileOperations method. [#33730](https://github.com/ClickHouse/ClickHouse/pull/33730) ([ianton-ru](https://github.com/ianton-ru)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#33898](https://github.com/ClickHouse/ClickHouse/issues/33898): Fix usage of sparse columns (which can be enabled by experimental setting `ratio_of_defaults_for_sparse_serialization`). [#33849](https://github.com/ClickHouse/ClickHouse/pull/33849) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
@ -10,7 +10,7 @@ sidebar_label: 2022
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Backport CI checks to 22.1 release branch. [#34897](https://github.com/ClickHouse/ClickHouse/pull/34897) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Backported in [#34119](https://github.com/ClickHouse/ClickHouse/issues/34119): Fix usage of functions `array` and `tuple` with literal arguments in distributed queries. Previously it could lead to `Not found columns` exception. [#33938](https://github.com/ClickHouse/ClickHouse/pull/33938) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Backported in [#34124](https://github.com/ClickHouse/ClickHouse/issues/34124): Fix crash while reading of nested tuples. Fixes [#33838](https://github.com/ClickHouse/ClickHouse/issues/33838). [#33956](https://github.com/ClickHouse/ClickHouse/pull/33956) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
@ -105,7 +105,7 @@ sidebar_label: 2022
|
||||
* Update tzdata to 2022e to support the new timezone changes. Palestine transitions are now Saturdays at 02:00. Simplify three Ukraine zones into one. Jordan and Syria switch from +02/+03 with DST to year-round +03. (https://data.iana.org/time-zones/tzdb/NEWS). This closes [#42252](https://github.com/ClickHouse/ClickHouse/issues/42252). [#42327](https://github.com/ClickHouse/ClickHouse/pull/42327) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix power8 support. [#42462](https://github.com/ClickHouse/ClickHouse/pull/42462) ([Boris Kuschel](https://github.com/bkuschel)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehavior in official stable release)
|
||||
|
||||
* Several fixes for DiskWeb. [#41652](https://github.com/ClickHouse/ClickHouse/pull/41652) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fixes issue when docker run will fail if "https_port" is not present in config. [#41693](https://github.com/ClickHouse/ClickHouse/pull/41693) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
@ -147,7 +147,7 @@ sidebar_label: 2022
|
||||
* This closes [#42453](https://github.com/ClickHouse/ClickHouse/issues/42453). [#42573](https://github.com/ClickHouse/ClickHouse/pull/42573) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix function `arrayElement` with type `Map` with `Nullable` values and `Nullable` index. [#42623](https://github.com/ClickHouse/ClickHouse/pull/42623) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
#### Bug Fix (user-visible misbehaviour in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehaviour in official stable release)
|
||||
|
||||
* Fix unexpected table loading error when partition key contains alias function names during server upgrade. [#36379](https://github.com/ClickHouse/ClickHouse/pull/36379) ([Amos Bird](https://github.com/amosbird)).
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ sidebar_label: 2022
|
||||
|
||||
### ClickHouse release v22.10.2.11-stable (d2bfcaba002) FIXME as compared to v22.10.1.1877-stable (98ab5a3c189)
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in official stable or prestable release)
|
||||
#### Bug Fix (user-visible misbehavior in official stable release)
|
||||
|
||||
* Backported in [#42750](https://github.com/ClickHouse/ClickHouse/issues/42750): A segmentation fault related to DNS & c-ares has been reported. The below error ocurred in multiple threads: ``` 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008088 [ 356 ] {} <Fatal> BaseDaemon: ######################################## 2022-09-28 15:41:19.008,"2022.09.28 15:41:19.008147 [ 356 ] {} <Fatal> BaseDaemon: (version 22.8.5.29 (official build), build id: 92504ACA0B8E2267) (from thread 353) (no query) Received signal Segmentation fault (11)" 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008196 [ 356 ] {} <Fatal> BaseDaemon: Address: 0xf Access: write. Address not mapped to object. 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008216 [ 356 ] {} <Fatal> BaseDaemon: Stack trace: 0x188f8212 0x1626851b 0x1626a69e 0x16269b3f 0x16267eab 0x13cf8284 0x13d24afc 0x13c5217e 0x14ec2495 0x15ba440f 0x15b9d13b 0x15bb2699 0x1891ccb3 0x1891e00d 0x18ae0769 0x18ade022 0x7f76aa985609 0x7f76aa8aa133 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008274 [ 356 ] {} <Fatal> BaseDaemon: 2. Poco::Net::IPAddress::family() const @ 0x188f8212 in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008297 [ 356 ] {} <Fatal> BaseDaemon: 3. ? @ 0x1626851b in /usr/bin/clickhouse 2022-09-28 15:41:19.008,2022.09.28 15:41:19.008309 [ 356 ] {} <Fatal> BaseDaemon: 4. ? @ 0x1626a69e in /usr/bin/clickhouse ```. [#42234](https://github.com/ClickHouse/ClickHouse/pull/42234) ([Arthur Passos](https://github.com/arthurpassos)).
|
||||
* Backported in [#42793](https://github.com/ClickHouse/ClickHouse/issues/42793): Fix a bug in ParserFunction that could have led to a segmentation fault. [#42724](https://github.com/ClickHouse/ClickHouse/pull/42724) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user