Merge branch 'master' into pretty-print-number-nullable-low-cardinality

This commit is contained in:
Alexey Milovidov 2024-05-20 05:14:29 +02:00
commit 64645e4010
17 changed files with 197 additions and 39 deletions

View File

@ -28,3 +28,10 @@ runs:
run: |
# to remove every leftovers
sudo rm -fr "$TEMP_PATH" && mkdir -p "$TEMP_PATH"
- name: Tune vm.mmap_rnd_bits for sanitizers
shell: bash
run: |
sudo sysctl vm.mmap_rnd_bits
# https://github.com/google/sanitizers/issues/856
echo "Tune vm.mmap_rnd_bits for sanitizers"
sudo sysctl vm.mmap_rnd_bits=28

View File

@ -1,11 +1,23 @@
set (DEFAULT_LIBS "-nodefaultlibs")
if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "amd64")
execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-x86_64.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
set(system_processor "x86_64")
else ()
execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
set(system_processor "${CMAKE_SYSTEM_PROCESSOR}")
endif ()
file(GLOB bprefix "/usr/local/llvm${COMPILER_VERSION_MAJOR}/lib/clang/${COMPILER_VERSION_MAJOR}/lib/${system_processor}-portbld-freebsd*/")
message(STATUS "-Bprefix: ${bprefix}")
execute_process(COMMAND ${CMAKE_CXX_COMPILER} -Bprefix=${bprefix} --print-file-name=libclang_rt.builtins-${system_processor}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
# --print-file-name simply prints what you passed in case of nothing was resolved, so let's try one other possible option
if (BUILTINS_LIBRARY STREQUAL "libclang_rt.builtins-${system_processor}.a")
execute_process(COMMAND ${CMAKE_CXX_COMPILER} -Bprefix=${bprefix} --print-file-name=libclang_rt.builtins.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
endif()
if (BUILTINS_LIBRARY STREQUAL "libclang_rt.builtins.a")
message(FATAL_ERROR "libclang_rt.builtins had not been found")
endif()
set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread")
message(STATUS "Default libraries: ${DEFAULT_LIBS}")

View File

@ -41,10 +41,7 @@ if (CMAKE_CROSSCOMPILING)
set (ENABLE_ICU OFF CACHE INTERNAL "")
set (ENABLE_FASTOPS OFF CACHE INTERNAL "")
elseif (OS_LINUX OR OS_ANDROID)
if (ARCH_AARCH64)
# FIXME: broken dependencies
set (ENABLE_GRPC OFF CACHE INTERNAL "")
elseif (ARCH_PPC64LE)
if (ARCH_PPC64LE)
set (ENABLE_GRPC OFF CACHE INTERNAL "")
elseif (ARCH_RISCV64)
# RISC-V support is preliminary

View File

@ -31,3 +31,123 @@ add_library(_ch_contrib_grpc INTERFACE)
target_link_libraries(_ch_contrib_grpc INTERFACE ${gRPC_LIBRARIES})
target_include_directories(_ch_contrib_grpc SYSTEM INTERFACE ${gRPC_INCLUDE_DIRS})
add_library(ch_contrib::grpc ALIAS _ch_contrib_grpc)
# Here we are trying to build a binary tool grpc_cpp_plugin in case of cross-compilation.
# We need this file only during compilation process itself so we need it for our "host"
# platform, not "target" platform.
# If we are doing normal compilation this executable will be produced in grpc.cmake.
#
# All code inside this block looks so weird because cmake fundametally doesn't
# support different toolchains for different targets. So we just running it
# in "bash script" fashion with different (actually without, i.e. default) toolchain.
#
# FIXME Sorry, I don't know cmake.
if (NOT CMAKE_HOST_SYSTEM_NAME STREQUAL CMAKE_SYSTEM_NAME
OR NOT CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR)
# First we need to build openssl for host plaform
set(OPENSSL_BUILD_DIR "${_gRPC_BINARY_DIR}/build_openssl")
set(OPENSSL_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/openssl-cmake")
execute_process(
COMMAND mkdir -p ${OPENSSL_BUILD_DIR}
COMMAND_ECHO STDOUT
)
if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64")
set (HOST_ARCH_AMD64 1)
elseif (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)")
set (HOST_ARCH_AARCH64 1)
elseif (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(powerpc64le.*|ppc64le.*|PPC64LE.*)")
set (HOST_ARCH_PPC64LE 1)
elseif (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(s390x.*|S390X.*)")
set (HOST_ARCH_S390X 1)
elseif (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "riscv64")
set (HOST_ARCH_RISCV64 1)
endif ()
if (CMAKE_HOST_SYSTEM_NAME MATCHES "Linux")
set (HOST_OS_LINUX 1)
elseif (CMAKE_HOST_SYSTEM_NAME MATCHES "Darwin")
set (HOST_OS_DARWIN 1)
endif ()
execute_process(
COMMAND ${CMAKE_COMMAND}
"-G${CMAKE_GENERATOR}"
"-DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}"
"-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
"-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
"-DARCH_AMD64=${HOST_ARCH_AMD64}"
"-DARCH_AARCH64=${HOST_ARCH_AARCH64}"
"-DARCH_PPC64LE=${HOST_ARCH_PPC64LE}"
"-DARCH_S390X=${HOST_ARCH_S390X}"
"-DARCH_RISCV64=${HOST_ARCH_RISCV64}"
"-DOS_DARWIN=${HOST_OS_DARWIN}"
"-DOPENSSL_AUX_BUILD_FOR_CROSS_COMPILATION=1"
"-DClickHouse_BINARY_DIR=${ClickHouse_BINARY_DIR}"
"-DClickHouse_SOURCE_DIR=${ClickHouse_SOURCE_DIR}"
"${OPENSSL_SOURCE_DIR}"
WORKING_DIRECTORY "${OPENSSL_BUILD_DIR}"
COMMAND_ECHO STDOUT)
execute_process(
COMMAND ${CMAKE_COMMAND} --build "${OPENSSL_BUILD_DIR}"
COMMAND_ECHO STDOUT)
execute_process(
COMMAND ${CMAKE_COMMAND} --install "${OPENSSL_BUILD_DIR}"
COMMAND_ECHO STDOUT)
# It's not important on which file we depend, we just want to specify right order
add_library(openssl_for_grpc STATIC IMPORTED GLOBAL)
set_target_properties (openssl_for_grpc PROPERTIES IMPORTED_LOCATION "${OPENSSL_BUILD_DIR}/libssl.a")
add_dependencies(openssl_for_grpc "${OPENSSL_BUILD_DIR}/libssl.a")
# Okay, openssl ready, let's build grpc_cpp_plugin
set (GRPC_CPP_PLUGIN_BUILD_DIR "${_gRPC_BINARY_DIR}/build")
execute_process(
COMMAND mkdir -p ${GRPC_CPP_PLUGIN_BUILD_DIR}
COMMAND_ECHO STDOUT
)
set(abseil_source_dir "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp")
set(protobuf_source_dir "${ClickHouse_SOURCE_DIR}/contrib/google-protobuf")
set(re2_source_dir "${ClickHouse_SOURCE_DIR}/contrib/re2")
set(ssl_source_dir "${ClickHouse_SOURCE_DIR}/contrib/openssl-cmake")
set(zlib_source_dir "${ClickHouse_SOURCE_DIR}/contrib/zlib-ng")
# For some reason config exists only in this directory
set(zlib_config_source_dir "${ClickHouse_BINARY_DIR}/contrib/zlib-ng-cmake")
set(cares_source_dir "${ClickHouse_SOURCE_DIR}/contrib/c-ares")
execute_process(
COMMAND ${CMAKE_COMMAND}
"-G${CMAKE_GENERATOR}"
"-DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}"
"-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
"-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
"-DABSL_ROOT_DIR=${abseil_source_dir}"
"-DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES=${zlib_config_source_dir}"
"-DgRPC_INSTALL=0"
"-DABSL_ENABLE_INSTALL=1"
"-DPROTOBUF_ROOT_DIR=${protobuf_source_dir}"
"-DRE2_ROOT_DIR=${re2_source_dir}"
"-DCARES_ROOT_DIR=${cares_source_dir}"
"-DOPENSSL_ROOT_DIR=${OPENSSL_BUILD_DIR}"
"-DOPENSSL_INCLUDE_DIR=${OPENSSL_BUILD_DIR}/include"
"-DZLIB_ROOT_DIR=${zlib_source_dir}"
"-DgRPC_SSL_PROVIDER=package"
"${_gRPC_SOURCE_DIR}"
WORKING_DIRECTORY "${GRPC_CPP_PLUGIN_BUILD_DIR}"
COMMAND_ECHO STDOUT)
execute_process(
COMMAND ${CMAKE_COMMAND} --build "${GRPC_CPP_PLUGIN_BUILD_DIR}"
COMMAND_ECHO STDOUT)
add_executable(grpc_cpp_plugin IMPORTED GLOBAL)
set_target_properties (grpc_cpp_plugin PROPERTIES IMPORTED_LOCATION "${GRPC_CPP_PLUGIN_BUILD_DIR}/grpc_cpp_plugin")
add_dependencies(grpc_cpp_plugin "${GRPC_CPP_PLUGIN_BUILD_DIR}/grpc_cpp_plugin")
add_dependencies(grpc_cpp_plugin openssl_for_grpc)
endif()

View File

@ -1829,6 +1829,8 @@ target_link_libraries(grpc_plugin_support
${_gRPC_PROTOBUF_PROTOC_LIBRARIES}
)
if (CMAKE_HOST_SYSTEM_NAME STREQUAL CMAKE_SYSTEM_NAME
AND CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR)
add_executable(grpc_cpp_plugin
${_gRPC_SOURCE_DIR}/src/compiler/cpp_plugin.cc
@ -1852,3 +1854,5 @@ target_link_libraries(grpc_cpp_plugin
${_gRPC_ALLTARGETS_LIBRARIES}
grpc_plugin_support
)
endif()

View File

@ -32,9 +32,16 @@ set(OPENSSLDIR "/etc/ssl" CACHE PATH "Set the default openssl directory")
set(OPENSSL_ENGINESDIR "/usr/local/lib/engines-3" CACHE PATH "Set the default openssl directory for engines")
set(OPENSSL_MODULESDIR "/usr/local/lib/ossl-modules" CACHE PATH "Set the default openssl directory for modules")
add_definitions(-DOPENSSL_NO_KTLS -DOPENSSLDIR="${OPENSSLDIR}" -DENGINESDIR="${OPENSSL_ENGINESDIR}" -DMODULESDIR="${OPENSSL_MODULESDIR}" -DOPENSSL_USE_NODELETE -DOPENSSL_PIC)
target_compile_options(global-group INTERFACE "-Wno-deprecated-declarations")
target_compile_options(global-group INTERFACE "-Wno-poison-system-directories")
# special type of build during cross-compilation
if(OPENSSL_AUX_BUILD_FOR_CROSS_COMPILATION)
add_definitions(-DOPENSSL_NO_KTLS -DOPENSSLDIR="\\\"${OPENSSLDIR}\\\"" -DENGINESDIR="\\\"${OPENSSL_ENGINESDIR}\\\"" -DMODULESDIR="\\\"${OPENSSL_MODULESDIR}\\\"" -DOPENSSL_USE_NODELETE -DOPENSSL_PIC)
add_compile_options("-Wno-deprecated-declarations")
add_compile_options("-Wno-poison-system-directories")
else()
add_definitions(-DOPENSSL_NO_KTLS -DOPENSSLDIR="${OPENSSLDIR}" -DENGINESDIR="${OPENSSL_ENGINESDIR}" -DMODULESDIR="${OPENSSL_MODULESDIR}" -DOPENSSL_USE_NODELETE -DOPENSSL_PIC)
target_compile_options(global-group INTERFACE "-Wno-deprecated-declarations")
target_compile_options(global-group INTERFACE "-Wno-poison-system-directories")
endif()
if(ARCH_AMD64)
if(OS_DARWIN)
@ -1473,4 +1480,9 @@ target_link_libraries(ssl crypto)
add_library(OpenSSL::Crypto ALIAS crypto)
add_library(OpenSSL::SSL ALIAS ssl)
install(FILES openssl.conf fipsmodule.conf DESTINATION "${CLICKHOUSE_ETC_DIR}/clickhouse-server" COMPONENT clickhouse)
if(OPENSSL_AUX_BUILD_FOR_CROSS_COMPILATION)
install(DIRECTORY "${PLATFORM_DIRECTORY}/include" DESTINATION "${CMAKE_BINARY_DIR}")
install(DIRECTORY "${OPENSSL_SOURCE_DIR}/include" DESTINATION "${CMAKE_BINARY_DIR}")
else()
install(FILES openssl.conf fipsmodule.conf DESTINATION "${CLICKHOUSE_ETC_DIR}/clickhouse-server" COMPONENT clickhouse)
endif()

View File

@ -1,5 +1,4 @@
# rebuild in #33610
# docker build -t clickhouse/fasttest .
# docker build -t clickhouse/fasttest .
ARG FROM_TAG=latest
FROM clickhouse/test-util:$FROM_TAG

View File

@ -5,6 +5,14 @@ FROM ubuntu:22.04
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
# FIXME: rebuild for clang 18.1.3, that contains a workaround [1] for
# sanitizers issue [2]:
#
# $ git tag --contains c2a57034eff048cd36c563c8e0051db3a70991b3 | tail -1
# llvmorg-18.1.3
#
# [1]: https://github.com/llvm/llvm-project/commit/c2a57034eff048cd36c563c8e0051db3a70991b3
# [2]: https://github.com/ClickHouse/ClickHouse/issues/64086
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=18
RUN apt-get update \

View File

@ -45,7 +45,7 @@ When merging, `ReplacingMergeTree` from all the rows with the same sorting key l
- The last in the selection, if `ver` not set. A selection is a set of rows in a set of parts participating in the merge. The most recently created part (the last insert) will be the last one in the selection. Thus, after deduplication, the very last row from the most recent insert will remain for each unique sorting key.
- With the maximum version, if `ver` specified. If `ver` is the same for several rows, then it will use "if `ver` is not specified" rule for them, i.e. the most recent inserted row will remain.
Example:
Example:
```sql
-- without ver - the last inserted 'wins'
@ -90,14 +90,14 @@ SELECT * FROM mySecondReplacingMT FINAL;
### is_deleted
`is_deleted` — Name of a column used during a merge to determine whether the data in this row represents the state or is to be deleted; `1` is a “deleted“ row, `0` is a “state“ row.
`is_deleted` — Name of a column used during a merge to determine whether the data in this row represents the state or is to be deleted; `1` is a "deleted" row, `0` is a "state" row.
Column data type — `UInt8`.
:::note
`is_deleted` can only be enabled when `ver` is used.
The row is deleted when `OPTIMIZE ... FINAL CLEANUP` or `OPTIMIZE ... FINAL` is used.
The row is deleted only when `OPTIMIZE ... FINAL CLEANUP`. This `CLEANUP` special keyword is not allowed by default unless `allow_experimental_replacing_merge_with_cleanup` MergeTree setting is enabled.
No matter the operation on the data, the version must be increased. If two inserted rows have the same version number, the last inserted row is the one kept.
@ -114,21 +114,22 @@ CREATE OR REPLACE TABLE myThirdReplacingMT
`is_deleted` UInt8
)
ENGINE = ReplacingMergeTree(eventTime, is_deleted)
ORDER BY key;
ORDER BY key
SETTINGS allow_experimental_replacing_merge_with_cleanup = 1;
INSERT INTO myThirdReplacingMT Values (1, 'first', '2020-01-01 01:01:01', 0);
INSERT INTO myThirdReplacingMT Values (1, 'first', '2020-01-01 01:01:01', 1);
INSERT INTO myThirdReplacingMT Values (1, 'first', '2020-01-01 01:01:01', 1);
select * from myThirdReplacingMT final;
0 rows in set. Elapsed: 0.003 sec.
-- delete rows with is_deleted
OPTIMIZE TABLE myThirdReplacingMT FINAL CLEANUP;
OPTIMIZE TABLE myThirdReplacingMT FINAL CLEANUP;
INSERT INTO myThirdReplacingMT Values (1, 'first', '2020-01-01 00:00:00', 0);
select * from myThirdReplacingMT final;
select * from myThirdReplacingMT final;
┌─key─┬─someCol─┬───────────eventTime─┬─is_deleted─┐
│ 1 │ first │ 2020-01-01 00:00:00 │ 0 │

View File

@ -23,6 +23,13 @@ thread_local ThreadStatus constinit * current_thread = nullptr;
namespace
{
#if defined(__aarch64__)
/// For aarch64 16K is not enough (likely due to tons of registers)
static constexpr size_t UNWIND_MINSIGSTKSZ = 32 << 10;
#else
static constexpr size_t UNWIND_MINSIGSTKSZ = 16 << 10;
#endif
/// Alternative stack for signal handling.
///
/// This stack should not be located in the TLS (thread local storage), since:
@ -50,7 +57,7 @@ struct ThreadStack
free(data);
}
static size_t getSize() { return std::max<size_t>(16 << 10, MINSIGSTKSZ); }
static size_t getSize() { return std::max<size_t>(UNWIND_MINSIGSTKSZ, MINSIGSTKSZ); }
void * getData() const { return data; }
private:

View File

@ -20,6 +20,7 @@ namespace ErrorCodes
{
extern const int CANNOT_RESTORE_FROM_FIELD_DUMP;
extern const int DECIMAL_OVERFLOW;
extern const int INCORRECT_DATA;
}
template <is_decimal T>
@ -28,7 +29,7 @@ T DecimalField<T>::getScaleMultiplier() const
return DecimalUtils::scaleMultiplier<T>(scale);
}
inline Field getBinaryValue(UInt8 type, ReadBuffer & buf)
Field getBinaryValue(UInt8 type, ReadBuffer & buf)
{
switch (static_cast<Field::Types::Which>(type))
{
@ -146,7 +147,7 @@ inline Field getBinaryValue(UInt8 type, ReadBuffer & buf)
case Field::Types::CustomType:
return Field();
}
UNREACHABLE();
throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown field type {}", std::to_string(type));
}
void readBinary(Array & x, ReadBuffer & buf)
@ -575,7 +576,7 @@ template bool decimalLessOrEqual<Decimal256>(Decimal256 x, Decimal256 y, UInt32
template bool decimalLessOrEqual<DateTime64>(DateTime64 x, DateTime64 y, UInt32 x_scale, UInt32 y_scale);
inline void writeText(const Null & x, WriteBuffer & buf)
void writeText(const Null & x, WriteBuffer & buf)
{
if (x.isNegativeInfinity())
writeText("-Inf", buf);

View File

@ -49,7 +49,7 @@ public:
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be Float64",
arg->getName(), i, getName());
arg->getName(), i + 1, getName());
}
return std::make_shared<DataTypeFloat64>();
}

View File

@ -846,7 +846,7 @@ LockedKey::~LockedKey()
/// See comment near cleanupThreadFunc() for more details.
key_metadata->key_state = KeyMetadata::KeyState::REMOVING;
LOG_DEBUG(key_metadata->logger(), "Submitting key {} for removal", getKey());
LOG_TRACE(key_metadata->logger(), "Submitting key {} for removal", getKey());
key_metadata->addToCleanupQueue();
}

View File

@ -91,6 +91,8 @@ apt-get install --yes --no-install-recommends azure-cli
# Increase the limit on number of virtual memory mappings to aviod 'Cannot mmap' error
echo "vm.max_map_count = 2097152" > /etc/sysctl.d/01-increase-map-counts.conf
# Workarond for sanitizers uncompatibility with some kernels, see https://github.com/google/sanitizers/issues/856
echo "vm.mmap_rnd_bits=28" > /etc/sysctl.d/02-vm-mmap_rnd_bits.conf
systemctl restart docker

View File

@ -39,10 +39,6 @@ def wait_for_clickhouse_stop(started_node):
assert result == "OK", "ClickHouse process is still running"
@pytest.mark.skipif(
helpers.cluster.is_arm(),
reason="Fails on ARM, issue https://github.com/ClickHouse/ClickHouse/issues/63855",
)
def test_pkill(started_node):
if (
started_node.is_built_with_thread_sanitizer()
@ -63,10 +59,6 @@ def test_pkill(started_node):
)
@pytest.mark.skipif(
helpers.cluster.is_arm(),
reason="Fails on ARM, issue https://github.com/ClickHouse/ClickHouse/issues/63855",
)
def test_pkill_query_log(started_node):
for signal in ["SEGV", "4"]:
# force create query_log if it was not created

View File

@ -35,10 +35,6 @@ def started_node():
pass
@pytest.mark.skipif(
helpers.cluster.is_arm(),
reason="Fails on ARM, issue https://github.com/ClickHouse/ClickHouse/issues/63855",
)
def test_send_segfault(started_node):
# NOTE: another option is to increase waiting time.
if (

View File

@ -18,5 +18,5 @@ CREATE table table_tar2star Engine S3(s3_conn, filename='03036_archive2.tar :: e
SELECT id, data, _file, _path FROM table_tar2star ORDER BY (id, _file, _path);
CREATE table table_tarstarglobs Engine S3(s3_conn, filename='03036_archive*.tar* :: example{2..3}.csv');
SELECT id, data, _file, _path FROM table_tarstarglobs ORDER BY (id, _file, _path);
CREATE table table_noexist Engine s3(s3_conn, filename='03036_archive2.zip :: nonexistent.csv'); -- { serverError INCORRECT_QUERY }
SELECT id, data, _file, _path FROM s3(s3_conn, filename='03036_compressed_file_archive.zip :: example7.csv', format='CSV', structure='auto', compression_method='gz') ORDER BY (id, _file, _path)
CREATE table table_noexist Engine s3(s3_conn, filename='03036_archive2.zip :: nonexistent.csv'); -- { serverError UNKNOWN_STORAGE }
SELECT id, data, _file, _path FROM s3(s3_conn, filename='03036_compressed_file_archive.zip :: example7.csv', format='CSV', structure='auto', compression_method='gz') ORDER BY (id, _file, _path)