thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-25 19:20:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into make_mysqlxx_pool_backward_compatible

Browse Source
This commit is contained in:
mergify[bot] 2021-09-08 14:55:57 +00:00 committed by GitHub
commit e9feb9f880
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 243 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.clang-tidy
View File

@ -203,3 +203,5 @@ CheckOptions:
value: CamelCase
- key: readability-identifier-naming.UsingCase
value: CamelCase
- key: modernize-loop-convert.UseCxx20ReverseRanges
value: false

2
cmake/analysis.cmake
View File

@ -6,7 +6,7 @@ if (ENABLE_CLANG_TIDY)
message(FATAL_ERROR "clang-tidy requires CMake version at least 3.6.")
endif()
find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-11" "clang-tidy-10" "clang-tidy-9" "clang-tidy-8")
find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-12" "clang-tidy-11" "clang-tidy-10" "clang-tidy-9" "clang-tidy-8")
if (CLANG_TIDY_PATH)
message(STATUS

2
cmake/freebsd/toolchain-x86_64.cmake
View File

@ -10,7 +10,7 @@ set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it
set (CMAKE_AR "/usr/bin/ar" CACHE FILEPATH "" FORCE)
set (CMAKE_RANLIB "/usr/bin/ranlib" CACHE FILEPATH "" FORCE)
set (LINKER_NAME "lld" CACHE STRING "" FORCE)
set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")

2
cmake/linux/toolchain-aarch64.cmake
View File

@ -13,7 +13,7 @@ set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_D
set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64")
set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64")
set (LINKER_NAME "lld" CACHE STRING "" FORCE)
set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE)
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")

5
cmake/tools.cmake
View File

@ -79,8 +79,9 @@ endif ()
if (LINKER_NAME)
if (COMPILER_CLANG AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 12.0.0 OR CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 12.0.0))
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --ld-path=${LINKER_NAME}")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LINKER_NAME}")
find_program (LLD_PATH NAMES ${LINKER_NAME})
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --ld-path=${LLD_PATH}")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LLD_PATH}")
else ()
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")

2
docker/builder/Dockerfile
View File

@ -1,6 +1,6 @@
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list

2
docker/builder/build.sh
View File

@ -4,7 +4,7 @@ set -e
#ccache -s # uncomment to display CCache statistics
mkdir -p /server/build_docker
cd /server/build_docker
cmake -G Ninja /server "-DCMAKE_C_COMPILER=$(command -v clang-11)" "-DCMAKE_CXX_COMPILER=$(command -v clang++-11)"
cmake -G Ninja /server "-DCMAKE_C_COMPILER=$(command -v clang-12)" "-DCMAKE_CXX_COMPILER=$(command -v clang++-12)"
# Set the number of build jobs to the half of number of virtual CPU cores (rounded up).
# By default, ninja use all virtual CPU cores, that leads to very high memory consumption without much improvement in build time.

14
docker/packager/binary/Dockerfile
View File

@ -1,7 +1,7 @@
# docker build -t yandex/clickhouse-binary-builder .
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
@ -39,8 +39,6 @@ RUN apt-get update \
bash \
build-essential \
ccache \
clang-11 \
clang-tidy-11 \
cmake \
curl \
g++-10 \
@ -50,9 +48,13 @@ RUN apt-get update \
gperf \
libicu-dev \
libreadline-dev \
lld-11 \
llvm-11 \
llvm-11-dev \
clang-12 \
clang-tidy-12 \
lld-12 \
llvm-12 \
llvm-12-dev \
libicu-dev \
libreadline-dev \
moreutils \
ninja-build \
pigz \

2
docker/packager/binary/build.sh
View File

@ -4,7 +4,6 @@ set -x -e
mkdir -p build/cmake/toolchain/darwin-x86_64
tar xJf MacOSX11.0.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1
ln -sf darwin-x86_64 build/cmake/toolchain/darwin-aarch64
mkdir -p build/cmake/toolchain/linux-aarch64
@ -23,6 +22,7 @@ cd build/build_docker
rm -f CMakeCache.txt
# Read cmake arguments into array (possibly empty)
read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}"
env
cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
ccache --show-config ||:

12
docker/packager/deb/Dockerfile
View File

@ -1,7 +1,7 @@
# docker build -t yandex/clickhouse-deb-builder .
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list
@ -37,17 +37,17 @@ RUN curl -O https://clickhouse-datasets.s3.yandex.net/utils/1/dpkg-deb \
RUN apt-get update \
&& apt-get install \
alien \
clang-11 \
clang-tidy-11 \
clang-12 \
clang-tidy-12 \
cmake \
debhelper \
devscripts \
gdb \
git \
gperf \
lld-11 \
llvm-11 \
llvm-11-dev \
lld-12 \
llvm-12 \
llvm-12-dev \
moreutils \
ninja-build \
perl \

5
docker/packager/packager
View File

@ -75,7 +75,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
# Explicitly use LLD with Clang by default.
# Don't force linker for cross-compilation.
if is_clang and not is_cross_compile:
cmake_flags.append("-DLINKER_NAME=lld")
cmake_flags.append("-DLINKER_NAME=ld.lld")
if is_cross_darwin:
cc = compiler[:-len(DARWIN_SUFFIX)]
@ -204,7 +204,8 @@ if __name__ == "__main__":
parser.add_argument("--output-dir", required=True)
parser.add_argument("--build-type", choices=("debug", ""), default="")
parser.add_argument("--compiler", choices=("clang-11", "clang-11-darwin", "clang-11-darwin-aarch64", "clang-11-aarch64",
"clang-11-freebsd", "gcc-10"), default="clang-11")
"clang-12", "clang-12-darwin", "clang-12-darwin-aarch64", "clang-12-aarch64",
"clang-11-freebsd", "clang-12-freebsd", "gcc-10"), default="clang-12")
parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="")
parser.add_argument("--unbundled", action="store_true")
parser.add_argument("--split-binary", action="store_true")

2
docker/test/base/Dockerfile
View File

@ -1,7 +1,7 @@
# docker build -t yandex/clickhouse-test-base .
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list

4
docker/test/codebrowser/Dockerfile
View File

@ -11,7 +11,7 @@ RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-9 libl
# https://github.com/ClickHouse-Extras/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b
RUN git clone https://github.com/ClickHouse-Extras/woboq_codebrowser
RUN cd woboq_codebrowser && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-9 -DCMAKE_C_COMPILER=clang-9 && make -j
RUN cd woboq_codebrowser && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-12 -DCMAKE_C_COMPILER=clang-12 && make -j
ENV CODEGEN=/woboq_codebrowser/generator/codebrowser_generator
ENV CODEINDEX=/woboq_codebrowser/indexgenerator/codebrowser_indexgenerator
@ -24,7 +24,7 @@ ENV SHA=nosha
ENV DATA="data"
CMD mkdir -p $BUILD_DIRECTORY && cd $BUILD_DIRECTORY && \
cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-11 -DCMAKE_C_COMPILER=/usr/bin/clang-11 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \
cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-12 -DCMAKE_C_COMPILER=/usr/bin/clang-12 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \
mkdir -p $HTML_RESULT_DIRECTORY && \
$CODEGEN -b $BUILD_DIRECTORY -a -o $HTML_RESULT_DIRECTORY -p ClickHouse:$SOURCE_DIRECTORY:$SHA -d $DATA | ts '%Y-%m-%d %H:%M:%S' && \
cp -r $STATIC_DATA $HTML_RESULT_DIRECTORY/ &&\

2
docker/test/fasttest/Dockerfile
View File

@ -1,7 +1,7 @@
# docker build -t yandex/clickhouse-fasttest .
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=12
RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list

2
docker/test/fasttest/run.sh
View File

@ -9,7 +9,7 @@ trap 'kill $(jobs -pr) ||:' EXIT
stage=${stage:-}
# Compiler version, normally set by Dockerfile
export LLVM_VERSION=${LLVM_VERSION:-11}
export LLVM_VERSION=${LLVM_VERSION:-12}
# A variable to pass additional flags to CMake.
# Here we explicitly default it to nothing so that bash doesn't complain about

2
docker/test/fuzzer/run-fuzzer.sh
View File

@ -12,7 +12,7 @@ stage=${stage:-}
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
echo "$script_dir"
repo_dir=ch
BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-11_debug_none_bundled_unsplitted_disable_False_binary"}
BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-12_debug_none_bundled_unsplitted_disable_False_binary"}
function clone
{

2
docker/test/keeper-jepsen/run.sh
View File

@ -2,7 +2,7 @@
set -euo pipefail
CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-11_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"}
CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-12_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"}
CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""}

4
docker/test/pvs/Dockerfile
View File

@ -28,7 +28,7 @@ RUN apt-get update --yes \
ENV PKG_VERSION="pvs-studio-latest"
RUN set -x \
&& export PUBKEY_HASHSUM="686e5eb8b3c543a5c54442c39ec876b6c2d912fe8a729099e600017ae53c877dda3368fe38ed7a66024fe26df6b5892a" \
&& export PUBKEY_HASHSUM="ad369a2e9d8b8c30f5a9f2eb131121739b79c78e03fef0f016ea51871a5f78cd4e6257b270dca0ac3be3d1f19d885516" \
&& wget -nv https://files.viva64.com/etc/pubkey.txt -O /tmp/pubkey.txt \
&& echo "${PUBKEY_HASHSUM} /tmp/pubkey.txt" | sha384sum -c \
&& apt-key add /tmp/pubkey.txt \
@ -38,7 +38,7 @@ RUN set -x \
&& dpkg -i "${PKG_VERSION}.deb"
CMD echo "Running PVS version $PKG_VERSION" && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic \
&& cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"USE_INTERNAL_PROTOBUF_LIBRARY"=OFF -D"USE_INTERNAL_GRPC_LIBRARY"=OFF \
&& cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"USE_INTERNAL_PROTOBUF_LIBRARY"=OFF -D"USE_INTERNAL_GRPC_LIBRARY"=OFF -DCMAKE_C_COMPILER=clang-12 -DCMAKE_CXX_COMPILER=clang\+\+-12 \
&& ninja re2_st clickhouse_grpc_protos \
&& pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 -l ./licence.lic; \
cp /repo_folder/pvs-studio.log /test_output; \

8
docs/en/development/build.md
View File

@ -23,7 +23,7 @@ $ sudo apt-get install git cmake python ninja-build
Or cmake3 instead of cmake on older systems.
### Install clang-11 (recommended) {#install-clang-11}
### Install clang-12 (recommended) {#install-clang-12}
On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/))
@ -33,11 +33,11 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
For other Linux distribution - check the availability of the [prebuild packages](https://releases.llvm.org/download.html) or build clang [from sources](https://clang.llvm.org/get_started.html).
#### Use clang-11 for Builds
#### Use clang-12 for Builds
``` bash
$ export CC=clang-11
$ export CXX=clang++-11
$ export CC=clang-12
$ export CXX=clang++-12
```
Gcc can also be used though it is discouraged.

9
src/Common/FiberStack.h
View File

@ -27,7 +27,12 @@ private:
size_t stack_size;
size_t page_size = 0;
public:
static constexpr size_t default_stack_size = 128 * 1024; /// 64KB was not enough for tests
/// NOTE: If you see random segfaults in CI and stack starts from boost::context::...fiber...
/// probably it worth to try to increase stack size for coroutines.
///
/// Current value is just enough for all tests in our CI. It's not selected in some special
/// way. We will have 36 pages with 4KB page size.
static constexpr size_t default_stack_size = 144 * 1024; /// 64KB was not enough for tests
explicit FiberStack(size_t stack_size_ = default_stack_size) : stack_size(stack_size_)
{
@ -43,6 +48,8 @@ public:
if (MAP_FAILED == vp)
DB::throwFromErrno(fmt::format("FiberStack: Cannot mmap {}.", ReadableSize(num_bytes)), DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
/// TODO: make reports on illegal guard page access more clear.
/// Currently we will see segfault and almost random stacktrace.
if (-1 == ::mprotect(vp, page_size, PROT_NONE))
{
::munmap(vp, num_bytes);

3
src/Compression/CompressionCodecEncrypted.cpp
View File

@ -113,7 +113,8 @@ namespace DB
std::string CompressionCodecEncrypted::deriveKey(const std::string_view & master_key)
{
std::string_view salt(""); // No salt: derive keys in a deterministic manner.
/// No salt: derive keys in a deterministic manner.
std::string_view salt(""); // NOLINT
std::string_view info("Codec Encrypted('AES-128-GCM-SIV') key generation key");
std::array<char, 32> result;

6
src/IO/WriteBufferFromFile.cpp
View File

@ -32,7 +32,7 @@ WriteBufferFromFile::WriteBufferFromFile(
mode_t mode,
char * existing_memory,
size_t alignment)
: WriteBufferFromFileDescriptor(-1, buf_size, existing_memory, alignment), file_name(file_name_)
: WriteBufferFromFileDescriptor(-1, buf_size, existing_memory, alignment, file_name_)
{
ProfileEvents::increment(ProfileEvents::FileOpen);
@ -65,9 +65,7 @@ WriteBufferFromFile::WriteBufferFromFile(
size_t buf_size,
char * existing_memory,
size_t alignment)
:
WriteBufferFromFileDescriptor(fd_, buf_size, existing_memory, alignment),
file_name(original_file_name.empty() ? "(fd = " + toString(fd_) + ")" : original_file_name)
: WriteBufferFromFileDescriptor(fd_, buf_size, existing_memory, alignment, original_file_name)
{
fd_ = -1;
}

1
src/IO/WriteBufferFromFile.h
View File

@ -25,7 +25,6 @@ namespace DB
class WriteBufferFromFile : public WriteBufferFromFileDescriptor
{
protected:
std::string file_name;
CurrentMetrics::Increment metric_increment{CurrentMetrics::OpenFileForWrite};
public:

42
src/IO/WriteBufferFromFileDescriptor.cpp
View File

@ -61,7 +61,12 @@ void WriteBufferFromFileDescriptor::nextImpl()
if ((-1 == res || 0 == res) && errno != EINTR)
{
ProfileEvents::increment(ProfileEvents::WriteBufferFromFileDescriptorWriteFailed);
throwFromErrnoWithPath("Cannot write to file " + getFileName(), getFileName(),
/// Don't use getFileName() here because this method can be called from destructor
String error_file_name = file_name;
if (error_file_name.empty())
error_file_name = "(fd = " + toString(fd) + ")";
throwFromErrnoWithPath("Cannot write to file " + error_file_name, error_file_name,
ErrorCodes::CANNOT_WRITE_TO_FILE_DESCRIPTOR);
}
@ -73,20 +78,20 @@ void WriteBufferFromFileDescriptor::nextImpl()
ProfileEvents::increment(ProfileEvents::WriteBufferFromFileDescriptorWriteBytes, bytes_written);
}
/// Name or some description of file.
std::string WriteBufferFromFileDescriptor::getFileName() const
{
return "(fd = " + toString(fd) + ")";
}
/// NOTE: This class can be used as a very low-level building block, for example
/// in trace collector. In such places allocations of memory can be dangerous,
/// so don't allocate anything in this constructor.
WriteBufferFromFileDescriptor::WriteBufferFromFileDescriptor(
int fd_,
size_t buf_size,
char * existing_memory,
size_t alignment)
: WriteBufferFromFileBase(buf_size, existing_memory, alignment), fd(fd_) {}
size_t alignment,
std::string file_name_)
: WriteBufferFromFileBase(buf_size, existing_memory, alignment)
, fd(fd_)
, file_name(std::move(file_name_))
{
}
WriteBufferFromFileDescriptor::~WriteBufferFromFileDescriptor()
@ -115,7 +120,7 @@ void WriteBufferFromFileDescriptor::sync()
}
off_t WriteBufferFromFileDescriptor::seek(off_t offset, int whence)
off_t WriteBufferFromFileDescriptor::seek(off_t offset, int whence) // NOLINT
{
off_t res = lseek(fd, offset, whence);
if (-1 == res)
@ -125,7 +130,7 @@ off_t WriteBufferFromFileDescriptor::seek(off_t offset, int whence)
}
void WriteBufferFromFileDescriptor::truncate(off_t length)
void WriteBufferFromFileDescriptor::truncate(off_t length) // NOLINT
{
int res = ftruncate(fd, length);
if (-1 == res)
@ -133,7 +138,7 @@ void WriteBufferFromFileDescriptor::truncate(off_t length)
}
off_t WriteBufferFromFileDescriptor::size()
off_t WriteBufferFromFileDescriptor::size() const
{
struct stat buf;
int res = fstat(fd, &buf);
@ -142,4 +147,13 @@ off_t WriteBufferFromFileDescriptor::size()
return buf.st_size;
}
std::string WriteBufferFromFileDescriptor::getFileName() const
{
if (file_name.empty())
return "(fd = " + toString(fd) + ")";
return file_name;
}
}

21
src/IO/WriteBufferFromFileDescriptor.h
View File

@ -13,17 +13,17 @@ class WriteBufferFromFileDescriptor : public WriteBufferFromFileBase
protected:
int fd;
/// If file has name contains filename, otherwise contains string "(fd=...)"
std::string file_name;
void nextImpl() override;
/// Name or some description of file.
std::string getFileName() const override;
public:
WriteBufferFromFileDescriptor(
int fd_ = -1,
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
char * existing_memory = nullptr,
size_t alignment = 0);
size_t alignment = 0,
std::string file_name_ = "");
/** Could be used before initialization if needed 'fd' was not passed to constructor.
* It's not possible to change 'fd' during work.
@ -42,10 +42,15 @@ public:
void sync() override;
off_t seek(off_t offset, int whence);
void truncate(off_t length);
/// clang-tidy wants these methods to be const, but
/// they are not const semantically
off_t seek(off_t offset, int whence); // NOLINT
void truncate(off_t length); // NOLINT
off_t size();
/// Name or some description of file.
std::string getFileName() const override;
off_t size() const;
};
}

24
src/Interpreters/LogicalExpressionsOptimizer.cpp
View File

@ -225,22 +225,19 @@ void LogicalExpressionsOptimizer::addInExpression(const DisjunctiveEqualityChain
/// 1. Create a new IN expression based on information from the OR-chain.
/// Construct a list of literals `x1, ..., xN` from the string `expr = x1 OR ... OR expr = xN`
ASTPtr value_list = std::make_shared<ASTExpressionList>();
/// Construct a tuple of literals `x1, ..., xN` from the string `expr = x1 OR ... OR expr = xN`
Tuple tuple;
tuple.reserve(equality_functions.size());
for (const auto * function : equality_functions)
{
const auto & operands = getFunctionOperands(function);
value_list->children.push_back(operands[1]);
tuple.push_back(operands[1]->as<ASTLiteral>()->value);
}
/// Sort the literals so that they are specified in the same order in the IN expression.
/// Otherwise, they would be specified in the order of the ASTLiteral addresses, which is nondeterministic.
std::sort(value_list->children.begin(), value_list->children.end(), [](const DB::ASTPtr & lhs, const DB::ASTPtr & rhs)
{
const auto * val_lhs = lhs->as<ASTLiteral>();
const auto * val_rhs = rhs->as<ASTLiteral>();
return val_lhs->value < val_rhs->value;
});
std::sort(tuple.begin(), tuple.end());
/// Get the expression `expr` from the chain `expr = x1 OR ... OR expr = xN`
ASTPtr equals_expr_lhs;
@ -250,14 +247,11 @@ void LogicalExpressionsOptimizer::addInExpression(const DisjunctiveEqualityChain
equals_expr_lhs = operands[0];
}
auto tuple_function = std::make_shared<ASTFunction>();
tuple_function->name = "tuple";
tuple_function->arguments = value_list;
tuple_function->children.push_back(tuple_function->arguments);
auto tuple_literal = std::make_shared<ASTLiteral>(std::move(tuple));
ASTPtr expression_list = std::make_shared<ASTExpressionList>();
expression_list->children.push_back(equals_expr_lhs);
expression_list->children.push_back(tuple_function);
expression_list->children.push_back(tuple_literal);
/// Construct the expression `expr IN (x1, ..., xN)`
auto in_function = std::make_shared<ASTFunction>();

7
src/Storages/StorageDistributed.cpp
View File

@ -1332,7 +1332,12 @@ void registerStorageDistributed(StorageFactory & factory)
String remote_table = engine_args[2]->as<ASTLiteral &>().value.safeGet<String>();
const auto & sharding_key = engine_args.size() >= 4 ? engine_args[3] : nullptr;
const auto & storage_policy = engine_args.size() >= 5 ? engine_args[4]->as<ASTLiteral &>().value.safeGet<String>() : "default";
String storage_policy = "default";
if (engine_args.size() >= 5)
{
engine_args[4] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[4], local_context);
storage_policy = engine_args[4]->as<ASTLiteral &>().value.safeGet<String>();
}
/// Check that sharding_key exists in the table and has numeric type.
if (sharding_key)

1
src/Storages/ya.make
View File

@ -214,7 +214,6 @@ SRCS(
System/StorageSystemTables.cpp
System/StorageSystemUserDirectories.cpp
System/StorageSystemUsers.cpp
System/StorageSystemViews.cpp
System/StorageSystemWarnings.cpp
System/StorageSystemZeros.cpp
System/StorageSystemZooKeeper.cpp

114
tests/ci/ci_config.json
View File

@ -1,7 +1,7 @@
{
"build_config": [
{
"compiler": "clang-11",
"compiler": "clang-12",
"build-type": "",
"sanitizer": "",
"package-type": "deb",
@ -12,7 +12,7 @@
"with_coverage": false
},
{
"compiler": "clang-11",
"compiler": "clang-12",
"build-type": "",
"sanitizer": "",
"package-type": "performance",
@ -32,7 +32,7 @@
"with_coverage": false
},
{
"compiler": "clang-11",
"compiler": "clang-12",
"build-type": "",
"sanitizer": "address",
"package-type": "deb",
@ -42,7 +42,7 @@
"with_coverage": false
},
{
"compiler": "clang-11",
"compiler": "clang-12",
"build-type": "",
"sanitizer": "undefined",
"package-type": "deb",
@ -52,7 +52,7 @@
"with_coverage": false
},
{
"compiler": "clang-11",
"compiler": "clang-12",
"build-type": "",
"sanitizer": "thread",
"package-type": "deb",
@ -62,7 +62,7 @@
"with_coverage": false
},
{
"compiler": "clang-11",
"compiler": "clang-12",
"build-type": "",
"sanitizer": "memory",
"package-type": "deb",
@ -72,7 +72,7 @@
"with_coverage": false
},
{
"compiler": "clang-11",
"compiler": "clang-12",
"build-type": "debug",
"sanitizer": "",
"package-type": "deb",
@ -92,7 +92,7 @@
"with_coverage": false
},
{
"compiler": "clang-11",
"compiler": "clang-12",
"build-type": "",
"sanitizer": "",
"package-type": "binary",
@ -104,7 +104,7 @@
],
"special_build_config": [
{
"compiler": "clang-11",
"compiler": "clang-12",
"build-type": "debug",
"sanitizer": "",
"package-type": "deb",
@ -114,7 +114,7 @@
"with_coverage": false
},
{
"compiler": "clang-11",
"compiler": "clang-12",
"build-type": "",
"sanitizer": "",
"package-type": "binary",
@ -124,7 +124,7 @@
"with_coverage": false
},
{
"compiler": "clang-11-darwin",
"compiler": "clang-12-darwin",
"build-type": "",
"sanitizer": "",
"package-type": "binary",
@ -134,7 +134,7 @@
"with_coverage": false
},
{
"compiler": "clang-11-aarch64",
"compiler": "clang-12-aarch64",
"build-type": "",
"sanitizer": "",
"package-type": "binary",
@ -144,7 +144,7 @@
"with_coverage": false
},
{
"compiler": "clang-11-freebsd",
"compiler": "clang-12-freebsd",
"build-type": "",
"sanitizer": "",
"package-type": "binary",
@ -154,7 +154,7 @@
"with_coverage": false
},
{
"compiler": "clang-11-darwin-aarch64",
"compiler": "clang-12-darwin-aarch64",
"build-type": "",
"sanitizer": "",
"package-type": "binary",
@ -167,7 +167,7 @@
"tests_config": {
"Functional stateful tests (address)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
@ -179,7 +179,7 @@
},
"Functional stateful tests (thread)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
@ -191,7 +191,7 @@
},
"Functional stateful tests (memory)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
@ -203,7 +203,7 @@
},
"Functional stateful tests (ubsan)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "undefined",
@ -215,7 +215,7 @@
},
"Functional stateful tests (debug)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "debug",
"sanitizer": "none",
@ -227,7 +227,7 @@
},
"Functional stateful tests (release)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
@ -239,7 +239,7 @@
},
"Functional stateful tests (release, DatabaseOrdinary)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
@ -251,7 +251,7 @@
},
"Functional stateful tests (release, DatabaseReplicated)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
@ -263,7 +263,7 @@
},
"Functional stateless tests (address)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
@ -275,7 +275,7 @@
},
"Functional stateless tests (thread)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
@ -287,7 +287,7 @@
},
"Functional stateless tests (memory)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
@ -299,7 +299,7 @@
},
"Functional stateless tests (ubsan)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "undefined",
@ -311,7 +311,7 @@
},
"Functional stateless tests (debug)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "debug",
"sanitizer": "none",
@ -323,7 +323,7 @@
},
"Functional stateless tests (release)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
@ -335,7 +335,7 @@
},
"Functional stateless tests (pytest)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
@ -359,7 +359,7 @@
},
"Functional stateless tests (release, wide parts enabled)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
@ -371,7 +371,7 @@
},
"Functional stateless tests (release, DatabaseOrdinary)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
@ -383,7 +383,7 @@
},
"Functional stateless tests (release, DatabaseReplicated)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
@ -395,7 +395,7 @@
},
"Stress test (address)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
@ -407,7 +407,7 @@
},
"Stress test (thread)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
@ -419,7 +419,7 @@
},
"Stress test (undefined)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "undefined",
@ -431,7 +431,7 @@
},
"Stress test (memory)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
@ -443,7 +443,7 @@
},
"Stress test (debug)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "debug",
"sanitizer": "none",
@ -455,7 +455,7 @@
},
"Integration tests (asan)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
@ -467,7 +467,7 @@
},
"Integration tests (thread)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
@ -479,7 +479,7 @@
},
"Integration tests (release)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
@ -491,7 +491,7 @@
},
"Integration tests (memory)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
@ -503,7 +503,7 @@
},
"Integration tests flaky check (asan)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
@ -515,7 +515,7 @@
},
"Compatibility check": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
@ -527,7 +527,7 @@
},
"Split build smoke test": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "binary",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
@ -539,7 +539,7 @@
},
"Testflows check": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
@ -563,7 +563,7 @@
},
"Unit tests release clang": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "binary",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
@ -575,7 +575,7 @@
},
"Unit tests ASAN": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "binary",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
@ -587,7 +587,7 @@
},
"Unit tests MSAN": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "binary",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
@ -599,7 +599,7 @@
},
"Unit tests TSAN": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "binary",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
@ -611,7 +611,7 @@
},
"Unit tests UBSAN": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "binary",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
@ -623,7 +623,7 @@
},
"AST fuzzer (debug)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "binary",
"build_type": "debug",
"sanitizer": "none",
@ -635,7 +635,7 @@
},
"AST fuzzer (ASan)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "binary",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
@ -647,7 +647,7 @@
},
"AST fuzzer (MSan)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "binary",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
@ -659,7 +659,7 @@
},
"AST fuzzer (TSan)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "binary",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
@ -671,7 +671,7 @@
},
"AST fuzzer (UBSan)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "binary",
"build_type": "relwithdebuginfo",
"sanitizer": "undefined",
@ -683,7 +683,7 @@
},
"Release": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
@ -695,7 +695,7 @@
},
"Functional stateless tests flaky check (address)": {
"required_build_properties": {
"compiler": "clang-11",
"compiler": "clang-12",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",

1
tests/queries/0_stateless/02015_shard_crash_clang_12_build.reference Normal file
View File

@ -0,0 +1 @@
1

47
tests/queries/0_stateless/02015_shard_crash_clang_12_build.sh Executable file
View File

@ -0,0 +1,47 @@
#!/usr/bin/env bash
# This test reproduces crash in case of insufficient coroutines stack size
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS local"
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS distributed"
$CLICKHOUSE_CLIENT --query "CREATE TABLE local (x UInt8) ENGINE = Memory;"
$CLICKHOUSE_CLIENT --query "CREATE TABLE distributed AS local ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), local, x);"
$CLICKHOUSE_CLIENT --insert_distributed_sync=0 --network_compression_method='zstd' --query "INSERT INTO distributed SELECT number FROM numbers(256);"
$CLICKHOUSE_CLIENT --insert_distributed_sync=0 --network_compression_method='zstd' --query "SYSTEM FLUSH DISTRIBUTED distributed;"
function select_thread()
{
while true; do
$CLICKHOUSE_CLIENT --insert_distributed_sync=0 --network_compression_method='zstd' --query "SELECT count() FROM local" >/dev/null
$CLICKHOUSE_CLIENT --insert_distributed_sync=0 --network_compression_method='zstd' --query "SELECT count() FROM distributed" >/dev/null
done
}
export -f select_thread;
TIMEOUT=30
timeout $TIMEOUT bash -c select_thread 2> /dev/null &
timeout $TIMEOUT bash -c select_thread 2> /dev/null &
timeout $TIMEOUT bash -c select_thread 2> /dev/null &
timeout $TIMEOUT bash -c select_thread 2> /dev/null &
timeout $TIMEOUT bash -c select_thread 2> /dev/null &
timeout $TIMEOUT bash -c select_thread 2> /dev/null &
timeout $TIMEOUT bash -c select_thread 2> /dev/null &
timeout $TIMEOUT bash -c select_thread 2> /dev/null &
timeout $TIMEOUT bash -c select_thread 2> /dev/null &
timeout $TIMEOUT bash -c select_thread 2> /dev/null &
wait
$CLICKHOUSE_CLIENT --query "SELECT 1"
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS local"
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS distributed"

0
tests/queries/0_stateless/02017_create_distributed_table_coredump.reference Normal file
View File

12
tests/queries/0_stateless/02017_create_distributed_table_coredump.sql Normal file
View File

@ -0,0 +1,12 @@
drop table if exists t;
drop table if exists td1;
drop table if exists td2;
drop table if exists td3;
create table t (val UInt32) engine = MergeTree order by val;
create table td1 engine = Distributed(test_shard_localhost, currentDatabase(), 't') as t;
create table td2 engine = Distributed(test_shard_localhost, currentDatabase(), 't', xxHash32(val), default) as t;
create table td3 engine = Distributed(test_shard_localhost, currentDatabase(), 't', xxHash32(val), 'default') as t;
drop table if exists t;
drop table if exists td1;
drop table if exists td2;
drop table if exists td3;

2
tests/queries/0_stateless/02023_transform_or_to_in.reference Normal file
View File

@ -0,0 +1,2 @@
0
0

15
tests/queries/0_stateless/02023_transform_or_to_in.sql Normal file
View File

@ -0,0 +1,15 @@
DROP TABLE IF EXISTS t_transform_or;
CREATE TABLE t_transform_or(B AggregateFunction(uniq, String), A String) Engine=MergeTree ORDER BY (A);
INSERT INTO t_transform_or SELECT uniqState(''), '0';
SELECT uniqMergeIf(B, (A = '1') OR (A = '2') OR (A = '3'))
FROM cluster(test_cluster_two_shards, currentDatabase(), t_transform_or)
SETTINGS legacy_column_name_of_tuple_literal = 0;
SELECT uniqMergeIf(B, (A = '1') OR (A = '2') OR (A = '3'))
FROM cluster(test_cluster_two_shards, currentDatabase(), t_transform_or)
SETTINGS legacy_column_name_of_tuple_literal = 1;
DROP TABLE t_transform_or;