thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'ClickHouse:master' into master

Browse Source
This commit is contained in:
michael1589 2021-11-27 23:30:25 +08:00 committed by GitHub
commit dbfe637f7b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
419 changed files with 4200 additions and 1125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
.github/workflows/main.yml vendored
View File

@ -152,7 +152,7 @@ jobs:
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 compatibility_check.py 0
cd $REPO_COPY/tests/ci && python3 compatibility_check.py
- name: Cleanup
if: always()
run: |
@ -518,6 +518,7 @@ jobs:
- BuilderDebDebug
- BuilderDebSplitted
runs-on: [self-hosted, style-checker]
if: always()
steps:
- name: Download json reports
uses: actions/download-artifact@v2
@ -604,7 +605,9 @@ jobs:
sudo rm -fr $TEMP_PATH
FunctionalStatelessTestTsan:
needs: [BuilderDebTsan]
runs-on: [self-hosted, func-tester]
# tests can consume more than 60GB of memory,
# so use bigger server
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
@ -1297,6 +1300,34 @@ jobs:
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
IntegrationTestsFlakyCheck:
needs: [BuilderDebAsan]
runs-on: [self-hosted, stress-tester]
steps:
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{runner.temp}}/reports_dir
- name: Check out repository code
uses: actions/checkout@v2
- name: Integration test
env:
TEMP_PATH: ${{runner.temp}}/integration_tests_asan_flaky_check
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Integration tests flaky check (asan, actions)'
REPO_COPY: ${{runner.temp}}/integration_tests_asan_flaky_check/ClickHouse
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 integration_test_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH
#############################################################################################
#################################### UNIT TESTS #############################################
#############################################################################################
@ -1481,6 +1512,7 @@ jobs:
- UnitTestsReleaseClang
- SplitBuildSmokeTest
- CompatibilityCheck
- IntegrationTestsFlakyCheck
runs-on: [self-hosted, style-checker]
steps:
- name: Check out repository code

8
CMakeLists.txt
View File

@ -169,9 +169,7 @@ endif ()
include (cmake/check_flags.cmake)
include (cmake/add_warning.cmake)
if (NOT MSVC)
set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror and many more is also added inside cmake/warnings.cmake
endif ()
set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror and many more is also added inside cmake/warnings.cmake
if (COMPILER_CLANG)
# clang: warning: argument unused during compilation: '-specs=/usr/share/dpkg/no-pie-compile.specs' [-Wunused-command-line-argument]
@ -312,6 +310,10 @@ include(cmake/cpu_features.cmake)
# Enable it explicitly.
set (COMPILER_FLAGS "${COMPILER_FLAGS} -fasynchronous-unwind-tables")
# Reproducible builds
set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -ffile-prefix-map=${CMAKE_SOURCE_DIR}=.")
if (${CMAKE_VERSION} VERSION_LESS "3.12.4")
# CMake < 3.12 doesn't support setting 20 as a C++ standard version.
# We will add C++ standard controlling flag in CMAKE_CXX_FLAGS manually for now.

23
PreLoad.cmake
View File

@ -18,3 +18,26 @@ if (NOT DEFINED ENV{CLION_IDE} AND NOT DEFINED ENV{XCODE_IDE})
set(CMAKE_GENERATOR "Ninja" CACHE INTERNAL "" FORCE)
endif ()
endif()
# Default toolchain - this is needed to avoid dependency on OS files.
execute_process(COMMAND uname -s OUTPUT_VARIABLE OS)
execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH)
if (OS MATCHES "Linux"
AND NOT DEFINED CMAKE_TOOLCHAIN_FILE
AND NOT DISABLE_HERMETIC_BUILD
AND ($ENV{CC} MATCHES ".*clang.*" OR CMAKE_C_COMPILER MATCHES ".*clang.*")
AND (USE_STATIC_LIBRARIES OR NOT DEFINED USE_STATIC_LIBRARIES))
if (ARCH MATCHES "amd64|x86_64")
set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-x86_64.cmake" CACHE INTERNAL "" FORCE)
elseif (ARCH MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)")
set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "" FORCE)
elseif (ARCH MATCHES "^(ppc64le.*|PPC64LE.*)")
set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-ppc64le.cmake" CACHE INTERNAL "" FORCE)
else ()
message (FATAL_ERROR "Unsupported architecture: ${ARCH}")
endif ()
endif()

4
SECURITY.md
View File

@ -19,9 +19,11 @@ The following versions of ClickHouse server are currently being supported with s
| 21.4 | :x: |
| 21.5 | :x: |
| 21.6 | :x: |
| 21.7 | ✅ |
| 21.7 | :x: |
| 21.8 | ✅ |
| 21.9 | ✅ |
| 21.10 | ✅ |
| 21.11 | ✅ |
## Reporting a Vulnerability

4
base/base/CachedFn.h
View File

@ -18,8 +18,8 @@ struct CachedFn
{
private:
using Traits = FnTraits<decltype(Func)>;
using DecayedArgs = TLMap<std::decay_t, typename Traits::Args>;
using Key = TLChangeRoot<std::tuple, DecayedArgs>;
using DecayedArgs = TypeListMap<std::decay_t, typename Traits::Args>;
using Key = TypeListChangeRoot<std::tuple, DecayedArgs>;
using Result = typename Traits::Ret;
std::map<Key, Result> cache; // Can't use hashmap as tuples are unhashable by default

4
base/base/FnTraits.h
View File

@ -1,6 +1,6 @@
#pragma once
#include "Typelist.h"
#include "TypeList.h"
namespace detail
{
@ -14,7 +14,7 @@ struct FnTraits<R(A...)>
static constexpr bool value = std::is_invocable_r_v<R, F, A...>;
using Ret = R;
using Args = Typelist<A...>;
using Args = TypeList<A...>;
};
template <class R, class ...A>

44
base/base/TypeList.h Normal file
View File

@ -0,0 +1,44 @@
#pragma once
#include <cstddef>
#include <type_traits>
#include <utility>
#include "defines.h"
#include "TypePair.h"
/// General-purpose typelist. Easy on compilation times as it does not use recursion.
template <typename ...Args>
struct TypeList { static constexpr size_t size = sizeof...(Args); };
namespace TypeListUtils /// In some contexts it's more handy to use functions instead of aliases
{
template <typename ...LArgs, typename ...RArgs>
constexpr TypeList<LArgs..., RArgs...> concat(TypeList<LArgs...>, TypeList<RArgs...>) { return {}; }
template <typename T, typename ...Args>
constexpr TypeList<T, Args...> prepend(TypeList<Args...>) { return {}; }
template <typename T, typename ...Args>
constexpr TypeList<Args..., T> append(TypeList<Args...>) { return {}; }
template <template <typename> typename F, typename ...Args>
constexpr TypeList<F<Args>...> map(TypeList<Args...>) { return {}; }
template <template <typename...> typename Root, typename ...Args>
constexpr Root<Args...> changeRoot(TypeList<Args...>) { return {}; }
template <typename F, typename ...Args>
constexpr void forEach(TypeList<Args...>, F && f) { (std::forward<F>(f)(Id<Args>{}), ...); }
}
template <typename TypeListLeft, typename TypeListRight>
using TypeListConcat = decltype(TypeListUtils::concat(TypeListLeft{}, TypeListRight{}));
template <typename T, typename List> using TypeListPrepend = decltype(TypeListUtils::prepend<T>(List{}));
template <typename T, typename List> using TypeListAppend = decltype(TypeListUtils::append<T>(List{}));
template <template <typename> typename F, typename List>
using TypeListMap = decltype(TypeListUtils::map<F>(List{}));
template <template <typename...> typename Root, typename List>
using TypeListChangeRoot = decltype(TypeListUtils::changeRoot<Root>(List{}));

21
base/base/TypeLists.h Normal file
View File

@ -0,0 +1,21 @@
#pragma once
#include "TypeList.h"
#include "extended_types.h"
#include "Decimal.h"
#include "UUID.h"
namespace DB
{
using TypeListNativeInt = TypeList<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64>;
using TypeListFloat = TypeList<Float32, Float64>;
using TypeListNativeNumber = TypeListConcat<TypeListNativeInt, TypeListFloat>;
using TypeListWideInt = TypeList<UInt128, Int128, UInt256, Int256>;
using TypeListInt = TypeListConcat<TypeListNativeInt, TypeListWideInt>;
using TypeListIntAndFloat = TypeListConcat<TypeListInt, TypeListFloat>;
using TypeListDecimal = TypeList<Decimal32, Decimal64, Decimal128, Decimal256>;
using TypeListNumber = TypeListConcat<TypeListIntAndFloat, TypeListDecimal>;
using TypeListNumberWithUUID = TypeListAppend<UUID, TypeListNumber>;
}

4
base/base/TypePair.h
View File

@ -1,4 +1,4 @@
#pragma once
template <class T, class V> struct TypePair { };
template <class T> struct Id { };
template <typename T, typename V> struct TypePair {};
template <typename T> struct Id {};

44
base/base/Typelist.h
View File

@ -1,44 +0,0 @@
#pragma once
#include <cstddef>
#include <type_traits>
#include <utility>
#include "defines.h"
#include "TypePair.h"
/// General-purpose typelist. Easy on compilation times as it does not use recursion.
template <class ...Args>
struct Typelist { static constexpr size_t size = sizeof...(Args); };
namespace TLUtils /// In some contexts it's more handy to use functions instead of aliases
{
template <class ...LArgs, class ...RArgs>
constexpr Typelist<LArgs..., RArgs...> concat(Typelist<LArgs...>, Typelist<RArgs...>) { return {}; }
template <class T, class ...Args>
constexpr Typelist<T, Args...> prepend(Typelist<Args...>) { return {}; }
template <class T, class ...Args>
constexpr Typelist<Args..., T> append(Typelist<Args...>) { return {}; }
template <template<class> class F, class ...Args>
constexpr Typelist<F<Args>...> map(Typelist<Args...>) { return {}; }
template <template<class...> class Root, class ...Args>
constexpr Root<Args...> changeRoot(Typelist<Args...>) { return {}; }
template <class F, class ...Args>
constexpr void forEach(Typelist<Args...>, F && f) { (std::forward<F>(f)(Id<Args>{}), ...); }
}
template <class TLLeft, class TLRight>
using TLConcat = decltype(TLUtils::concat(TLLeft{}, TLRight{}));
template <class T, class Typelist> using TLPrepend = decltype(TLUtils::prepend<T>(Typelist{}));
template <class T, class Typelist> using TLAppend = decltype(TLUtils::append<T>(Typelist{}));
template <template<class> class F, class Typelist>
using TLMap = decltype(TLUtils::map<F>(Typelist{}));
template <template<class...> class Root, class Typelist>
using TLChangeRoot = decltype(TLUtils::changeRoot<Root>(Typelist{}));

18
base/base/Typelists.h
View File

@ -1,18 +0,0 @@
#pragma once
#include "Typelist.h"
#include "extended_types.h"
#include "Decimal.h"
#include "UUID.h"
namespace DB
{
using TLIntegral = Typelist<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64>;
using TLExtendedIntegral = Typelist<UInt128, Int128, UInt256, Int256>;
using TLDecimals = Typelist<Decimal32, Decimal64, Decimal128, Decimal256>;
using TLIntegralWithExtended = TLConcat<TLIntegral, TLExtendedIntegral>;
using TLNumbers = TLConcat<TLIntegralWithExtended, TLDecimals>;
using TLNumbersWithUUID = TLAppend<UUID, TLNumbers>;
}

2
base/base/UUID.h
View File

@ -5,5 +5,5 @@
namespace DB
{
using UUID = StrongTypedef<UInt128, struct UUIDTag>;
using UUID = StrongTypedef<UInt128, struct UUIDTag>;
}

23
base/base/iostream_debug_helpers.h
View File

@ -6,6 +6,7 @@
#include <tuple>
#include <iomanip>
#include <iostream>
#include <magic_enum.hpp>
/** Usage:
*
@ -61,6 +62,13 @@ std::enable_if_t<priority == 2, Out> & dumpImpl(Out & out, T && x, std::decay_t<
}
template <int priority, typename Out, typename T>
std::enable_if_t<priority == 3 && std::is_enum_v<std::decay_t<T>>, Out> &
dumpImpl(Out & out, T && x)
{
return out << magic_enum::enum_name(x);
}
/// string and const char * - output not as container or pointer.
template <int priority, typename Out, typename T>
@ -131,15 +139,26 @@ Out & dumpValue(Out & out, T && x)
template <typename Out, typename T>
Out & dump(Out & out, const char * name, T && x)
{
// Dumping string literal, printing name and demangled type is irrelevant.
if constexpr (std::is_same_v<const char *, std::decay_t<std::remove_reference_t<T>>>)
{
const auto name_len = strlen(name);
const auto value_len = strlen(x);
// `name` is the same as quoted `x`
if (name_len > 2 && value_len > 0 && name[0] == '"' && name[name_len - 1] == '"'
&& strncmp(name + 1, x, std::min(value_len, name_len) - 1) == 0)
return out << x;
}
out << demangle(typeid(x).name()) << " " << name << " = ";
return dumpValue(out, x);
return dumpValue(out, x) << "; ";
}
#ifdef __clang__
#pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments"
#endif
#define DUMPVAR(VAR) ::dump(std::cerr, #VAR, (VAR)); std::cerr << "; ";
#define DUMPVAR(VAR) ::dump(std::cerr, #VAR, (VAR));
#define DUMPHEAD std::cerr << __FILE__ << ':' << __LINE__ << " [ " << getThreadId() << " ] ";
#define DUMPTAIL std::cerr << '\n';

6
cmake/add_warning.cmake
View File

@ -12,13 +12,13 @@ macro (add_warning flag)
if (SUPPORTS_CXXFLAG_${underscored_flag})
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W${flag}")
else ()
message (WARNING "Flag -W${flag} is unsupported")
message (STATUS "Flag -W${flag} is unsupported")
endif ()
if (SUPPORTS_CFLAG_${underscored_flag})
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -W${flag}")
else ()
message (WARNING "Flag -W${flag} is unsupported")
message (STATUS "Flag -W${flag} is unsupported")
endif ()
endmacro ()
@ -39,7 +39,7 @@ macro (target_add_warning target flag)
if (SUPPORTS_CXXFLAG_${underscored_flag})
target_compile_options (${target} PRIVATE "-W${flag}")
else ()
message (WARNING "Flag -W${flag} is unsupported")
message (STATUS "Flag -W${flag} is unsupported")
endif ()
endmacro ()

2
cmake/find/parquet.cmake
View File

@ -1,5 +1,5 @@
if (Protobuf_PROTOC_EXECUTABLE)
option (ENABLE_PARQUET "Enable parquet" ON)
option (ENABLE_PARQUET "Enable parquet" ${ENABLE_LIBRARIES})
elseif(ENABLE_PARQUET OR USE_INTERNAL_PARQUET_LIBRARY)
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use parquet without protoc executable")
endif()

2
cmake/freebsd/toolchain-aarch64.cmake
View File

@ -3,7 +3,7 @@ set (CMAKE_SYSTEM_PROCESSOR "aarch64")
set (CMAKE_C_COMPILER_TARGET "aarch64-unknown-freebsd12")
set (CMAKE_CXX_COMPILER_TARGET "aarch64-unknown-freebsd12")
set (CMAKE_ASM_COMPILER_TARGET "aarch64-unknown-freebsd12")
set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/freebsd-aarch64")
set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-aarch64")
set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake

2
cmake/freebsd/toolchain-x86_64.cmake
View File

@ -3,7 +3,7 @@ set (CMAKE_SYSTEM_PROCESSOR "x86_64")
set (CMAKE_C_COMPILER_TARGET "x86_64-pc-freebsd11")
set (CMAKE_CXX_COMPILER_TARGET "x86_64-pc-freebsd11")
set (CMAKE_ASM_COMPILER_TARGET "x86_64-pc-freebsd11")
set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/freebsd-x86_64")
set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-x86_64")
set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake

8
cmake/linux/default_libs.cmake
View File

@ -5,8 +5,12 @@ set (DEFAULT_LIBS "-nodefaultlibs")
# We need builtins from Clang's RT even without libcxx - for ubsan+int128.
# See https://bugs.llvm.org/show_bug.cgi?id=16404
if (COMPILER_CLANG AND NOT CMAKE_CROSSCOMPILING)
execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
if (COMPILER_CLANG)
execute_process (COMMAND ${CMAKE_CXX_COMPILER} --target=${CMAKE_CXX_COMPILER_TARGET} --print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
if (NOT EXISTS "${BUILTINS_LIBRARY}")
set (BUILTINS_LIBRARY "-lgcc")
endif ()
else ()
set (BUILTINS_LIBRARY "-lgcc")
endif ()

6
cmake/print_flags.cmake
View File

@ -1,6 +1,12 @@
set (FULL_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
set (FULL_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
set (FULL_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CMAKE_EXE_LINKER_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
message (STATUS "compiler C = ${CMAKE_C_COMPILER} ${FULL_C_FLAGS}")
message (STATUS "compiler CXX = ${CMAKE_CXX_COMPILER} ${FULL_CXX_FLAGS}")
message (STATUS "LINKER_FLAGS = ${FULL_EXE_LINKER_FLAGS}")
# Reproducible builds
string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_C_FLAGS_NORMALIZED "${FULL_C_FLAGS}")
string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_CXX_FLAGS_NORMALIZED "${FULL_CXX_FLAGS}")
string (REPLACE "${CMAKE_SOURCE_DIR}" "." FULL_EXE_LINKER_FLAGS_NORMALIZED "${FULL_EXE_LINKER_FLAGS}")

9
cmake/sanitize.cmake
View File

@ -78,6 +78,15 @@ if (SANITIZE)
elseif (SANITIZE STREQUAL "undefined")
set (UBSAN_FLAGS "-fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero")
if (ENABLE_FUZZING)
# Unsigned integer overflow is well defined behaviour from a perspective of C++ standard,
# compilers or CPU. We use in hash functions like SipHash and many other places in our codebase.
# This flag is needed only because fuzzers are run inside oss-fuzz infrastructure
# and they have a bunch of flags not halt the program if UIO happend and even to silence that warnings.
# But for unknown reason that flags don't work with ClickHouse or we don't understand how to properly use them,
# that's why we often receive reports about UIO. The simplest way to avoid this is just set this flag here.
set(UBSAN_FLAGS "${SAN_FLAGS} -fno-sanitize=unsigned-integer-overflow")
endif()
if (COMPILER_CLANG)
set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/ubsan_suppressions.txt")
else()

2
contrib/cctz-cmake/CMakeLists.txt
View File

@ -82,7 +82,7 @@ if (NOT EXTERNAL_CCTZ_LIBRARY_FOUND OR NOT EXTERNAL_CCTZ_LIBRARY_WORKS)
# each file in that dir (except of tab and localtime) store the info about timezone
execute_process(COMMAND
bash -c "cd ${TZDIR} && find * -type f -and ! -name '*.tab' -and ! -name 'localtime' | sort | paste -sd ';' -"
bash -c "cd ${TZDIR} && find * -type f -and ! -name '*.tab' -and ! -name 'localtime' | LC_ALL=C sort | paste -sd ';' -"
OUTPUT_STRIP_TRAILING_WHITESPACE
OUTPUT_VARIABLE TIMEZONES)

4
contrib/krb5-cmake/CMakeLists.txt
View File

@ -500,7 +500,6 @@ function(preprocess_et out_var)
COMMAND perl "${KRB5_SOURCE_DIR}/util/et/compile_et" -d "${KRB5_SOURCE_DIR}/util/et" ${in_f}
DEPENDS ${in_f} "${KRB5_SOURCE_DIR}/util/et/compile_et"
WORKING_DIRECTORY ${ET_PATH}
COMMENT "Creating preprocessed file ${F_C}"
VERBATIM
)
list(APPEND result ${F_C})
@ -526,7 +525,6 @@ add_custom_command(
add_custom_target(
ERROR_MAP_H
DEPENDS "${KRB5_SOURCE_DIR}/lib/gssapi/krb5/error_map.h"
COMMENT "generating error_map.h"
VERBATIM
)
@ -539,14 +537,12 @@ add_custom_command(
add_custom_target(
ERRMAP_H
DEPENDS "${KRB5_SOURCE_DIR}/lib/gssapi/generic/errmap.h"
COMMENT "generating errmap.h"
VERBATIM
)
add_custom_target(
KRB_5_H
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/include/krb5/krb5.h"
COMMENT "generating krb5.h"
VERBATIM
)

2
contrib/libhdfs3 vendored

@ -1 +1 @@
Subproject commit a8c37ee001af1ae88e5dfa637ae5b31b087c96d3
Subproject commit 9194af44588633c1b2dae44bf945804401ff883e

7
contrib/protobuf-cmake/CMakeLists.txt
View File

@ -110,6 +110,9 @@ set(libprotobuf_files
)
add_library(libprotobuf ${libprotobuf_lite_files} ${libprotobuf_files})
if (ENABLE_FUZZING)
target_compile_options(libprotobuf PRIVATE "-fsanitize-recover=all")
endif()
target_link_libraries(libprotobuf pthread)
target_link_libraries(libprotobuf ${ZLIB_LIBRARIES})
if(${CMAKE_SYSTEM_NAME} STREQUAL "Android")
@ -211,7 +214,9 @@ add_library(protobuf::libprotoc ALIAS libprotoc)
set(protoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/main.cc)
if (NOT CMAKE_CROSSCOMPILING)
if (CMAKE_HOST_SYSTEM_NAME STREQUAL CMAKE_SYSTEM_NAME
AND CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR)
add_executable(protoc ${protoc_files})
target_link_libraries(protoc libprotoc libprotobuf pthread)
add_executable(protobuf::protoc ALIAS protoc)

2
contrib/sysroot vendored

@ -1 +1 @@
Subproject commit 1a64956aa7c280448be6526251bb2b8e6d380ab1
Subproject commit 4ef348b7f30f2ad5b02b266268b3c948e51ad457

3
docker/packager/binary/Dockerfile
View File

@ -93,9 +93,6 @@ RUN git clone https://github.com/tpoechtrager/cctools-port.git \
# Download toolchain and SDK for Darwin
RUN wget -nv https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz
# Download toolchain for FreeBSD 11.3
RUN wget -nv https://clickhouse-datasets.s3.yandex.net/toolchains/toolchains/freebsd-11.3-toolchain.tar.xz
# NOTE: Seems like gcc-11 is too new for ubuntu20 repository
RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
&& apt-get update \

3
docker/packager/binary/build.sh
View File

@ -6,9 +6,6 @@ mkdir -p build/cmake/toolchain/darwin-x86_64
tar xJf MacOSX11.0.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1
ln -sf darwin-x86_64 build/cmake/toolchain/darwin-aarch64
mkdir -p build/cmake/toolchain/freebsd-x86_64
tar xJf freebsd-11.3-toolchain.tar.xz -C build/cmake/toolchain/freebsd-x86_64 --strip-components=1
# Uncomment to debug ccache. Don't put ccache log in /output right away, or it
# will be confusingly packed into the "performance" package.
# export CCACHE_LOGFILE=/build/ccache.log

8
docker/test/base/Dockerfile
View File

@ -52,14 +52,18 @@ RUN apt-get update \
--yes --no-install-recommends
# Sanitizer options for services (clickhouse-server)
RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment; \
# Set resident memory limit for TSAN to 45GiB (46080MiB) to avoid OOMs in Stress tests
# and MEMORY_LIMIT_EXCEEDED exceptions in Functional tests (total memory limit in Functional tests is ~55.24 GiB).
# TSAN will flush shadow memory when reaching this limit.
# It may cause false-negatives, but it's better than OOM.
RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7 memory_limit_mb=46080'" >> /etc/environment; \
echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \
echo "MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1'" >> /etc/environment; \
echo "LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt'" >> /etc/environment; \
ln -s /usr/lib/llvm-${LLVM_VERSION}/bin/llvm-symbolizer /usr/bin/llvm-symbolizer;
# Sanitizer options for current shell (not current, but the one that will be spawned on "docker run")
# (but w/o verbosity for TSAN, otherwise test.reference will not match)
ENV TSAN_OPTIONS='halt_on_error=1 history_size=7'
ENV TSAN_OPTIONS='halt_on_error=1 history_size=7 memory_limit_mb=46080'
ENV UBSAN_OPTIONS='print_stacktrace=1'
ENV MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1'

1
docker/test/fasttest/run.sh
View File

@ -159,6 +159,7 @@ function clone_submodules
cd "$FASTTEST_SOURCE"
SUBMODULES_TO_UPDATE=(
contrib/sysroot
contrib/magic_enum
contrib/abseil-cpp
contrib/boost

2
docker/test/fuzzer/run-fuzzer.sh
View File

@ -77,7 +77,7 @@ function configure
function watchdog
{
sleep 3600
sleep 1800
echo "Fuzzing run has timed out"
for _ in {1..10}

14
docker/test/performance-comparison/report.py
View File

@ -227,10 +227,20 @@ def tableEnd():
return '</table>'
def tsvRows(n):
result = []
try:
with open(n, encoding='utf-8') as fd:
return [row for row in csv.reader(fd, delimiter="\t", quotechar='"')]
result = []
for row in csv.reader(fd, delimiter="\t", quoting=csv.QUOTE_NONE):
new_row = []
for e in row:
# The first one .encode('utf-8').decode('unicode-escape') decodes the escape characters from the strings.
# The second one (encode('latin1').decode('utf-8')) fixes the changes with unicode vs utf-8 chars, so
# 'Чем зÐ<C2B7>нимаеÑ<C2B5>ЬÑ<C2AC>Ñ<EFBFBD>' is transformed back into 'Чем зАнимаешЬся'.
new_row.append(e.encode('utf-8').decode('unicode-escape').encode('latin1').decode('utf-8'))
result.append(new_row)
return result
except:
report_errors.append(
traceback.format_exception_only(

2
docker/test/pvs/Dockerfile
View File

@ -40,7 +40,7 @@ RUN set -x \
ENV CCACHE_DIR=/test_output/ccache
CMD echo "Running PVS version $PKG_VERSION" && mkdir -p $CCACHE_DIR && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic \
&& cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"USE_INTERNAL_PROTOBUF_LIBRARY"=OFF -D"USE_INTERNAL_GRPC_LIBRARY"=OFF -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \
&& cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"DISABLE_HERMETIC_BUILD"=ON -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \
&& ninja re2_st clickhouse_grpc_protos \
&& pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 -l ./licence.lic; \
cp /repo_folder/pvs-studio.log /test_output; \

7
docker/test/stateful/run.sh
View File

@ -120,8 +120,12 @@ timeout "$MAX_RUN_TIME" bash -c run_tests ||:
./process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server.log ||:
pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz ||:
# FIXME: remove once only github actions will be left
rm /var/log/clickhouse-server/clickhouse-server.log
mv /var/log/clickhouse-server/stderr.log /test_output/ ||:
if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
tar -chf /test_output/clickhouse_coverage.tar.gz /profraw ||:
fi
@ -130,6 +134,9 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server2.log ||:
pigz < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.gz ||:
pigz < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.gz ||:
# FIXME: remove once only github actions will be left
rm /var/log/clickhouse-server/clickhouse-server1.log
rm /var/log/clickhouse-server/clickhouse-server2.log
mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:
mv /var/log/clickhouse-server/stderr2.log /test_output/ ||:
fi

5
docker/test/stateless/run.sh
View File

@ -135,6 +135,8 @@ done
wait ||:
# Compressed (FIXME: remove once only github actions will be left)
rm /var/log/clickhouse-server/clickhouse-server.log
mv /var/log/clickhouse-server/stderr.log /test_output/ ||:
if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
tar -chf /test_output/clickhouse_coverage.tar.gz /profraw ||:
@ -155,6 +157,9 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
grep -Fa "Fatal" /var/log/clickhouse-server/clickhouse-server2.log ||:
pigz < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.gz ||:
pigz < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.gz ||:
# FIXME: remove once only github actions will be left
rm /var/log/clickhouse-server/clickhouse-server1.log
rm /var/log/clickhouse-server/clickhouse-server2.log
mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:
mv /var/log/clickhouse-server/stderr2.log /test_output/ ||:
tar -chf /test_output/zookeeper_log_dump1.tar /var/lib/clickhouse1/data/system/zookeeper_log ||:

2
docker/test/stress/run.sh
View File

@ -185,6 +185,8 @@ zgrep -Fa "########################################" /test_output/* > /dev/null
for log_file in /var/log/clickhouse-server/clickhouse-server.log*
do
pigz < "${log_file}" > /test_output/"$(basename ${log_file})".gz
# FIXME: remove once only github actions will be left
rm "${log_file}"
done
tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:

2
docker/test/stress/stress
View File

@ -138,7 +138,7 @@ if __name__ == "__main__":
parser.add_argument("--client-cmd", default='clickhouse-client')
parser.add_argument("--server-log-folder", default='/var/log/clickhouse-server')
parser.add_argument("--output-folder")
parser.add_argument("--global-time-limit", type=int, default=3600)
parser.add_argument("--global-time-limit", type=int, default=1800)
parser.add_argument("--num-parallel", type=int, default=cpu_count())
parser.add_argument('--hung-check', action='store_true', default=False)
# make sense only for hung check

2
docker/test/style/Dockerfile
View File

@ -10,7 +10,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
python3-pip \
pylint \
yamllint \
&& pip3 install codespell PyGithub boto3 unidiff
&& pip3 install codespell PyGithub boto3 unidiff dohq-artifactory
COPY run.sh /
COPY process_style_check_result.py /

2
docs/en/development/build.md
View File

@ -74,7 +74,7 @@ The build requires the following components:
- Git (is used only to checkout the sources, it’s not needed for the build)
- CMake 3.10 or newer
- Ninja
- C++ compiler: clang-11 or newer
- C++ compiler: clang-13 or newer
- Linker: lld
- Python (is only used inside LLVM build and it is optional)

2
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -588,7 +588,7 @@ For more information, see the section [Creating replicated tables](../../engines
Approximate size (in bytes) of the cache of marks used by table engines of the [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) family.
The cache is shared for the server and memory is allocated as needed. The cache size must be at least 5368709120.
The cache is shared for the server and memory is allocated as needed.
**Example**

2
docs/en/sql-reference/functions/json-functions.md
View File

@ -275,7 +275,7 @@ If the part does not exist or isn’t array, an empty array will be returned.
Example:
``` sql
SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ['-100', '200.0', '"hello"']';
SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ['-100', '200.0', '"hello"'];
```
## JSONExtractKeysAndValuesRaw {#json-extract-keys-and-values-raw}

2
docs/en/sql-reference/functions/splitting-merging-functions.md
View File

@ -290,7 +290,7 @@ ngrams(string, ngramsize)
- Array with n-grams.
Type: [Array](../../sql-reference/data-types/array.md)([FixedString](../../sql-reference/data-types/fixedstring.md)).
Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
**Example**

2
docs/ru/sql-reference/functions/splitting-merging-functions.md
View File

@ -252,7 +252,7 @@ ngrams(string, ngramsize)
- Массив с n-граммами.
Тип: [Array](../../sql-reference/data-types/array.md)([FixedString](../../sql-reference/data-types/fixedstring.md)).
Тип: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
**Пример**

1
programs/copier/ClusterCopier.cpp
View File

@ -8,6 +8,7 @@
#include <Common/setThreadName.h>
#include <IO/ConnectionTimeoutsContext.h>
#include <Interpreters/InterpreterInsertQuery.h>
#include <Parsers/ASTFunction.h>
#include <Processors/Transforms/ExpressionTransform.h>
#include <QueryPipeline/QueryPipelineBuilder.h>
#include <QueryPipeline/Chain.h>

6
programs/copier/Internals.cpp
View File

@ -1,8 +1,10 @@
#include "Internals.h"
#include <Storages/MergeTree/MergeTreeData.h>
#include <Storages/extractKeyExpressionList.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Processors/Executors/PullingPipelineExecutor.h>
#include <Processors/Transforms/SquashingChunksTransform.h>
#include <Storages/MergeTree/MergeTreeData.h>
#include <Storages/extractKeyExpressionList.h>
namespace DB
{

1
programs/copier/TaskTableAndShard.h
View File

@ -5,6 +5,7 @@
#include "ClusterPartition.h"
#include <Core/Defines.h>
#include <Parsers/ASTFunction.h>
#include <base/map.h>
#include <boost/algorithm/string/join.hpp>

14
programs/odbc-bridge/ODBCBlockInputStream.cpp
View File

@ -2,6 +2,7 @@
#include <vector>
#include <IO/ReadBufferFromString.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeDateTime64.h>
#include <Columns/ColumnNullable.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnsNumber.h>
@ -141,7 +142,18 @@ void ODBCSource::insertValue(
assert_cast<ColumnUInt32 &>(column).insertValue(time);
break;
}
case ValueType::vtDateTime64:[[fallthrough]];
case ValueType::vtDateTime64:
{
auto value = row.get<std::string>(idx);
ReadBufferFromString in(value);
DateTime64 time = 0;
const auto * datetime_type = assert_cast<const DataTypeDateTime64 *>(data_type.get());
readDateTime64Text(time, datetime_type->getScale(), in, datetime_type->getTimeZone());
if (time < 0)
time = 0;
assert_cast<DataTypeDateTime64::ColumnType &>(column).insertValue(time);
break;
}
case ValueType::vtDecimal32: [[fallthrough]];
case ValueType::vtDecimal64: [[fallthrough]];
case ValueType::vtDecimal128: [[fallthrough]];

3
programs/server/Server.cpp
View File

@ -85,9 +85,6 @@
#if !defined(ARCADIA_BUILD)
# include "config_core.h"
# include "Common/config_version.h"
# if USE_OPENCL
# include "Common/BitonicSort.h"
# endif
#endif
#if defined(OS_LINUX)

1
src/Access/AccessEntityIO.cpp
View File

@ -7,6 +7,7 @@
#include <Access/SettingsProfile.h>
#include <Access/User.h>
#include <Core/Defines.h>
#include <IO/WriteHelpers.h>
#include <Interpreters/Access/InterpreterCreateQuotaQuery.h>
#include <Interpreters/Access/InterpreterCreateRoleQuery.h>
#include <Interpreters/Access/InterpreterCreateRowPolicyQuery.h>

42
src/AggregateFunctions/AggregateFunctionSparkbar.h
View File

@ -102,11 +102,16 @@ private:
size_t width;
X min_x;
X max_x;
bool specified_min_max_x;
String getBar(const UInt8 value) const
template <class T>
String getBar(const T value) const
{
if (isNaN(value) || value > 8 || value < 1)
return " ";
// ▁▂▃▄▅▆▇█
switch (value)
switch (static_cast<UInt8>(value))
{
case 1: return "▁";
case 2: return "▂";
@ -136,9 +141,20 @@ private:
String value;
if (data.points.empty() || !width)
return value;
X local_min_x = data.min_x;
X local_max_x = data.max_x;
size_t diff_x = local_max_x - local_min_x;
size_t diff_x;
X min_x_local;
if (specified_min_max_x)
{
diff_x = max_x - min_x;
min_x_local = min_x;
}
else
{
diff_x = data.max_x - data.min_x;
min_x_local = data.min_x;
}
if ((diff_x + 1) <= width)
{
Y min_y = data.min_y;
@ -149,15 +165,15 @@ private:
{
for (size_t i = 0; i <= diff_x; ++i)
{
auto it = data.points.find(local_min_x + i);
auto it = data.points.find(min_x_local + i);
bool found = it != data.points.end();
value += getBar(found ? static_cast<UInt8>(std::round(((it->getMapped() - min_y) / diff_y) * 7) + 1) : 0);
value += getBar(found ? std::round(((it->getMapped() - min_y) / diff_y) * 7) + 1 : 0.0);
}
}
else
{
for (size_t i = 0; i <= diff_x; ++i)
value += getBar(data.points.has(local_min_x + i) ? 1 : 0);
value += getBar(data.points.has(min_x_local + i) ? 1 : 0);
}
}
else
@ -186,9 +202,9 @@ private:
if (i == bound.first) // is bound
{
Float64 proportion = bound.second - bound.first;
auto it = data.points.find(local_min_x + i);
auto it = data.points.find(min_x_local + i);
bool found = (it != data.points.end());
if (found)
if (found && proportion > 0)
new_y = new_y.value_or(0) + it->getMapped() * proportion;
if (new_y)
@ -213,7 +229,7 @@ private:
}
else
{
auto it = data.points.find(local_min_x + i);
auto it = data.points.find(min_x_local + i);
if (it != data.points.end())
new_y = new_y.value_or(0) + it->getMapped();
}
@ -226,7 +242,7 @@ private:
auto getBars = [&] (const std::optional<Float64> & point_y)
{
value += getBar(point_y ? static_cast<UInt8>(std::round(((point_y.value() - min_y.value()) / diff_y) * 7) + 1) : 0);
value += getBar(point_y ? std::round(((point_y.value() - min_y.value()) / diff_y) * 7) + 1 : 0);
};
auto getBarsForConstant = [&] (const std::optional<Float64> & point_y)
{
@ -250,11 +266,13 @@ public:
width = params.at(0).safeGet<UInt64>();
if (params.size() == 3)
{
specified_min_max_x = true;
min_x = params.at(1).safeGet<X>();
max_x = params.at(2).safeGet<X>();
}
else
{
specified_min_max_x = false;
min_x = std::numeric_limits<X>::min();
max_x = std::numeric_limits<X>::max();
}

2
src/AggregateFunctions/parseAggregateFunctionParameters.cpp
View File

@ -1,6 +1,6 @@
#include <AggregateFunctions/parseAggregateFunctionParameters.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ExpressionListParsers.h>
#include <Parsers/parseQuery.h>

4
src/AggregateFunctions/parseAggregateFunctionParameters.h
View File

@ -1,8 +1,8 @@
#pragma once
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTExpressionList.h>
#include <Interpreters/Context_fwd.h>
#include <Parsers/ASTExpressionList.h>
#include <Parsers/IAST_fwd.h>
namespace DB

7
src/Backups/renameInCreateQuery.cpp
View File

@ -1,12 +1,13 @@
#include <Backups/renameInCreateQuery.h>
#include <Backups/BackupRenamingConfig.h>
#include <Backups/renameInCreateQuery.h>
#include <Interpreters/InDepthNodeVisitor.h>
#include <Interpreters/evaluateConstantExpression.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Interpreters/InDepthNodeVisitor.h>
#include <Interpreters/evaluateConstantExpression.h>
namespace DB

1
src/Client/ClientBase.cpp
View File

@ -40,7 +40,6 @@
#include <Parsers/ASTDropQuery.h>
#include <Parsers/ASTSetQuery.h>
#include <Parsers/ASTUseQuery.h>
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ASTQueryWithOutput.h>
#include <Parsers/ASTLiteral.h>

2
src/Columns/ColumnMap.cpp
View File

@ -263,7 +263,7 @@ void ColumnMap::getExtremes(Field & min, Field & max) const
void ColumnMap::forEachSubcolumn(ColumnCallback callback)
{
nested->forEachSubcolumn(callback);
callback(nested);
}
bool ColumnMap::structureEquals(const IColumn & rhs) const

2
src/Common/ErrorCodes.cpp
View File

@ -598,6 +598,8 @@
M(628, OFFSET_FETCH_WITHOUT_ORDER_BY) \
M(629, HTTP_RANGE_NOT_SATISFIABLE) \
M(630, HAVE_DEPENDENT_OBJECTS) \
M(631, UNKNOWN_FILE_SIZE) \
M(632, UNEXPECTED_DATA_AFTER_PARSED_VALUE) \
\
M(999, KEEPER_EXCEPTION) \
M(1000, POCO_EXCEPTION) \

2
src/Common/ProfileEvents.cpp
View File

@ -261,6 +261,8 @@
M(RemoteFSUnprefetchedReads, "Number of reads from unprefetched buffer") \
M(RemoteFSBuffers, "Number of buffers created for asynchronous reading from remote filesystem") \
\
M(ReadBufferSeekCancelConnection, "Number of seeks which lead to new connection (s3, http)") \
\
M(SleepFunctionCalls, "Number of times a sleep function (sleep, sleepEachRow) has been called.") \
M(SleepFunctionMicroseconds, "Time spent sleeping due to a sleep function call.") \
\

14
src/Common/Visitor.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <base/demangle.h>
#include <base/Typelist.h>
#include <base/TypeList.h>
#include <Common/Exception.h>
/* Generic utils which are intended for visitor pattern implementation.
@ -64,7 +64,7 @@ template <>
class Visitor<>
{
public:
using List = Typelist<>;
using List = TypeList<>;
protected:
~Visitor() = default;
@ -74,7 +74,7 @@ template <typename Type>
class Visitor<Type> : public Visitor<>
{
public:
using List = Typelist<Type>;
using List = TypeList<Type>;
virtual void visit(Type &) = 0;
@ -86,7 +86,7 @@ template <typename Type, typename ... Types>
class Visitor<Type, Types ...> : public Visitor<Types ...>
{
public:
using List = Typelist<Type, Types ...>;
using List = TypeList<Type, Types ...>;
using Visitor<Types ...>::visit;
virtual void visit(Type &) = 0;
@ -145,10 +145,10 @@ protected:
template <typename Derived, typename VisitorBase>
class VisitorImpl : public
TLChangeRoot<
TypeListChangeRoot<
VisitorImplHelper,
TLConcat<
Typelist<Derived, VisitorBase>,
TypeListConcat<
TypeList<Derived, VisitorBase>,
typename VisitorBase::List
>
>

89
src/Common/renameat2.cpp
View File

@ -4,13 +4,6 @@
#include <Poco/Environment.h>
#include <filesystem>
#if defined(linux) || defined(__linux) || defined(__linux__)
#include <unistd.h>
#include <fcntl.h>
#include <sys/syscall.h>
#include <linux/fs.h>
#endif
namespace fs = std::filesystem;
namespace DB
@ -25,25 +18,56 @@ namespace ErrorCodes
extern const int FILE_ALREADY_EXISTS;
}
}
#if defined(__linux__)
#include <unistd.h>
#include <fcntl.h>
#include <sys/syscall.h>
#include <linux/fs.h>
/// For old versions of libc.
#if !defined(RENAME_NOREPLACE)
#define RENAME_NOREPLACE 1
#endif
#if !defined(RENAME_EXCHANGE)
#define RENAME_EXCHANGE 2
#endif
#if !defined(__NR_renameat2)
#if defined(__x86_64__)
#define __NR_renameat2 316
#elif defined(__aarch64__)
#define __NR_renameat2 276
#elif defined(__ppc64__)
#define __NR_renameat2 357
#elif defined(__riscv)
#define __NR_renameat2 276
#else
#error "Unsupported architecture"
#endif
#endif
namespace DB
{
static bool supportsRenameat2Impl()
{
#if defined(__NR_renameat2)
VersionNumber renameat2_minimal_version(3, 15, 0);
VersionNumber linux_version(Poco::Environment::osVersion());
return linux_version >= renameat2_minimal_version;
#else
return false;
#endif
}
#if defined(__NR_renameat2)
static bool renameat2(const std::string & old_path, const std::string & new_path, int flags)
{
if (!supportsRenameat2())
return false;
if (old_path.empty() || new_path.empty())
throw Exception("Cannot rename " + old_path + " to " + new_path + ": path is empty", ErrorCodes::LOGICAL_ERROR);
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot rename {} to {}: path is empty", old_path, new_path);
/// int olddirfd (ignored for absolute oldpath), const char *oldpath,
/// int newdirfd (ignored for absolute newpath), const char *newpath,
@ -63,28 +87,50 @@ static bool renameat2(const std::string & old_path, const std::string & new_path
return false;
if (errno == EEXIST)
throwFromErrno("Cannot rename " + old_path + " to " + new_path + " because the second path already exists", ErrorCodes::ATOMIC_RENAME_FAIL);
throwFromErrno(fmt::format("Cannot rename {} to {} because the second path already exists", old_path, new_path), ErrorCodes::ATOMIC_RENAME_FAIL);
if (errno == ENOENT)
throwFromErrno("Paths cannot be exchanged because " + old_path + " or " + new_path + " does not exist", ErrorCodes::ATOMIC_RENAME_FAIL);
throwFromErrnoWithPath("Cannot rename " + old_path + " to " + new_path, new_path, ErrorCodes::SYSTEM_ERROR);
throwFromErrno(fmt::format("Paths cannot be exchanged because {} or {} does not exist", old_path, new_path), ErrorCodes::ATOMIC_RENAME_FAIL);
throwFromErrnoWithPath(fmt::format("Cannot rename {} to {}", old_path, new_path), new_path, ErrorCodes::SYSTEM_ERROR);
}
bool supportsRenameat2()
{
static bool supports = supportsRenameat2Impl();
return supports;
}
}
#else
#define RENAME_NOREPLACE -1
#define RENAME_EXCHANGE -1
namespace DB
{
static bool renameat2(const std::string &, const std::string &, int)
{
return false;
}
bool supportsRenameat2()
{
return false;
}
}
#endif
namespace DB
{
static void renameNoReplaceFallback(const std::string & old_path, const std::string & new_path)
{
/// NOTE it's unsafe
if (fs::exists(new_path))
throw Exception("File " + new_path + " exists", ErrorCodes::FILE_ALREADY_EXISTS);
throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "File {} exists", new_path);
fs::rename(old_path, new_path);
}
@ -97,13 +143,6 @@ static void renameExchangeFallback(const std::string &, const std::string &)
}
#pragma GCC diagnostic pop
bool supportsRenameat2()
{
static bool supports = supportsRenameat2Impl();
return supports;
}
void renameNoReplace(const std::string & old_path, const std::string & new_path)
{
if (!renameat2(old_path, new_path, RENAME_NOREPLACE))

4
src/Compression/CachedCompressedReadBuffer.cpp
View File

@ -87,7 +87,7 @@ CachedCompressedReadBuffer::CachedCompressedReadBuffer(
void CachedCompressedReadBuffer::seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block)
{
/// Nothing to do if we already at required position
if (file_pos == offset_in_compressed_file
if (!owned_cell && file_pos == offset_in_compressed_file
&& (offset() == offset_in_decompressed_block ||
nextimpl_working_buffer_offset == offset_in_decompressed_block))
return;
@ -106,6 +106,8 @@ void CachedCompressedReadBuffer::seek(size_t offset_in_compressed_file, size_t o
bytes += offset();
/// No data, everything discarded
pos = working_buffer.end();
owned_cell.reset();
/// Remember required offset in decompressed block which will be set in
/// the next ReadBuffer::next() call
nextimpl_working_buffer_offset = offset_in_decompressed_block;

2
src/Compression/CompressedReadBufferFromFile.h
View File

@ -49,7 +49,7 @@ public:
explicit CompressedReadBufferFromFile(std::unique_ptr<ReadBufferFromFileBase> buf, bool allow_different_codecs_ = false);
/// Seek is lazy in some sense. We move position in compressed file_in to offset_in_compressed_file, but don't
/// read data into working_buffer and don't shit our position to offset_in_decompressed_block. Instead
/// read data into working_buffer and don't shift our position to offset_in_decompressed_block. Instead
/// we store this offset inside nextimpl_working_buffer_offset.
void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block);

2
src/Compression/CompressionCodecEncrypted.cpp
View File

@ -7,11 +7,9 @@
#include <Compression/CompressionCodecEncrypted.h>
#include <Poco/Logger.h>
#include <base/logger_useful.h>
#include <Common/ErrorCodes.h>
// This depends on BoringSSL-specific API, notably <openssl/aead.h>.
#if USE_SSL && USE_INTERNAL_SSL_LIBRARY
#include <Parsers/ASTLiteral.h>
#include <openssl/digest.h>
#include <openssl/err.h>
#include <boost/algorithm/hex.hpp>

1
src/Compression/CompressionCodecGorilla.cpp
View File

@ -7,7 +7,6 @@
#include <Compression/CompressionFactory.h>
#include <base/unaligned.h>
#include <Parsers/IAST_fwd.h>
#include <Parsers/ASTIdentifier.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadBufferFromMemory.h>
#include <IO/BitHelpers.h>

1
src/Compression/CompressionCodecNone.cpp
View File

@ -1,7 +1,6 @@
#include <Compression/CompressionCodecNone.h>
#include <Compression/CompressionInfo.h>
#include <Compression/CompressionFactory.h>
#include <Parsers/ASTIdentifier.h>
namespace DB

1
src/Compression/CompressionCodecT64.cpp
View File

@ -5,7 +5,6 @@
#include <base/unaligned.h>
#include <Parsers/IAST.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTFunction.h>
#include <IO/WriteHelpers.h>
#include <Core/Types.h>

2
src/Core/Settings.h
View File

@ -514,6 +514,7 @@ class IColumn;
M(Int64, read_priority, 0, "Priority to read data from local filesystem. Only supported for 'pread_threadpool' method.", 0) \
M(UInt64, merge_tree_min_rows_for_concurrent_read_for_remote_filesystem, (20 * 8192), "If at least as many lines are read from one file, the reading can be parallelized, when reading from remote filesystem.", 0) \
M(UInt64, merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem, (24 * 10 * 1024 * 1024), "If at least as many bytes are read from one file, the reading can be parallelized, when reading from remote filesystem.", 0) \
M(UInt64, remote_read_min_bytes_for_seek, DBMS_DEFAULT_BUFFER_SIZE, "Min bytes required for remote read (url, s3) to do seek, instead for read with ignore.", 0) \
\
M(UInt64, async_insert_threads, 16, "Maximum number of threads to actually parse and insert data in background. Zero means asynchronous mode is disabled", 0) \
M(Bool, async_insert, false, "If true, data from INSERT query is stored in queue and later flushed to table in background. Makes sense only for inserts via HTTP protocol. If wait_for_async_insert is false, INSERT query is processed almost instantly, otherwise client will wait until data will be flushed to table", 0) \
@ -582,6 +583,7 @@ class IColumn;
M(Bool, input_format_arrow_import_nested, false, "Allow to insert array of structs into Nested table in Arrow input format.", 0) \
M(Bool, input_format_orc_import_nested, false, "Allow to insert array of structs into Nested table in ORC input format.", 0) \
M(Bool, input_format_parquet_import_nested, false, "Allow to insert array of structs into Nested table in Parquet input format.", 0) \
M(Bool, input_format_allow_seeks, true, "Allow seeks while reading in ORC/Parquet/Arrow input formats", 0) \
\
M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic' and 'best_effort'.", 0) \
M(DateTimeOutputFormat, date_time_output_format, FormatSettings::DateTimeOutputFormat::Simple, "Method to write DateTime to text output. Possible values: 'simple', 'iso', 'unix_timestamp'.", 0) \

2
src/DataTypes/DataTypeAggregateFunction.cpp
View File

@ -15,8 +15,8 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier_fwd.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTIdentifier.h>
namespace DB

3
src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp
View File

@ -8,8 +8,9 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <boost/algorithm/string/join.hpp>

6
src/DataTypes/DataTypeLowCardinality.cpp
View File

@ -6,7 +6,7 @@
#include <Common/assert_cast.h>
#include <Common/typeid_cast.h>
#include <Core/Field.h>
#include <base/Typelists.h>
#include <base/TypeLists.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeFactory.h>
@ -53,7 +53,7 @@ namespace
{
}
template <class T>
template <typename T>
void operator()(Id<T>)
{
if (typeid_cast<const DataTypeNumber<T> *>(&keys_type))
@ -89,7 +89,7 @@ MutableColumnUniquePtr DataTypeLowCardinality::createColumnUniqueImpl(const IDat
else if (which.isInt() || which.isUInt() || which.isFloat())
{
MutableColumnUniquePtr column;
TLUtils::forEach(TLIntegral{}, CreateColumnVector(column, *type, creator));
TypeListUtils::forEach(TypeListIntAndFloat{}, CreateColumnVector(column, *type, creator));
if (!column)
throw Exception("Unexpected numeric type: " + type->getName(), ErrorCodes::LOGICAL_ERROR);

13
src/DataTypes/Serializations/ISerialization.cpp
View File

@ -15,6 +15,7 @@ namespace DB
namespace ErrorCodes
{
extern const int MULTIPLE_STREAMS_REQUIRED;
extern const int UNEXPECTED_DATA_AFTER_PARSED_VALUE;
}
String ISerialization::Substream::toString() const
@ -256,5 +257,17 @@ ISerialization::SubstreamData ISerialization::createFromPath(const SubstreamPath
return res;
}
void ISerialization::throwUnexpectedDataAfterParsedValue(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const String & type_name) const
{
WriteBufferFromOwnString ostr;
serializeText(column, column.size() - 1, ostr, settings);
throw Exception(
ErrorCodes::UNEXPECTED_DATA_AFTER_PARSED_VALUE,
"Unexpected data '{}' after parsed {} value '{}'",
std::string(istr.position(), std::min(size_t(10), istr.available())),
type_name,
ostr.str());
}
}

3
src/DataTypes/Serializations/ISerialization.h
View File

@ -304,6 +304,9 @@ public:
static bool hasSubcolumnForPath(const SubstreamPath & path, size_t prefix_len);
static SubstreamData createFromPath(const SubstreamPath & path, size_t prefix_len);
protected:
[[noreturn]] void throwUnexpectedDataAfterParsedValue(IColumn & column, ReadBuffer & istr, const FormatSettings &, const String & type_name) const;
};
using SerializationPtr = std::shared_ptr<const ISerialization>;

5
src/DataTypes/Serializations/SerializationArray.cpp
View File

@ -450,13 +450,16 @@ void SerializationArray::serializeText(const IColumn & column, size_t row_num, W
}
void SerializationArray::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
void SerializationArray::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
{
deserializeTextImpl(column, istr,
[&](IColumn & nested_column)
{
nested->deserializeTextQuoted(nested_column, istr, settings);
}, false);
if (whole && !istr.eof())
throwUnexpectedDataAfterParsedValue(column, istr, settings, "Array");
}
void SerializationArray::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const

2
src/DataTypes/Serializations/SerializationArray.h
View File

@ -19,7 +19,7 @@ public:
void deserializeBinary(IColumn & column, ReadBuffer & istr) const override;
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &, bool whole) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;

4
src/DataTypes/Serializations/SerializationCustomSimpleText.cpp
View File

@ -21,7 +21,7 @@ String serializeToString(const SerializationCustomSimpleText & domain, const ICo
void deserializeFromString(const SerializationCustomSimpleText & domain, IColumn & column, const String & s, const FormatSettings & settings)
{
ReadBufferFromString istr(s);
domain.deserializeText(column, istr, settings);
domain.deserializeText(column, istr, settings, true);
}
}
@ -37,7 +37,7 @@ SerializationCustomSimpleText::SerializationCustomSimpleText(const Serialization
void SerializationCustomSimpleText::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
String str;
readString(str, istr);
readStringUntilEOF(str, istr);
deserializeFromString(*this, column, str, settings);
}

6
src/DataTypes/Serializations/SerializationCustomSimpleText.h
View File

@ -19,9 +19,11 @@ public:
// Methods that subclasses must override in order to get full serialization/deserialization support.
virtual void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override = 0;
virtual void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0;
/// whole = true means that buffer contains only one value, so we should read until EOF.
/// It's needed to check if there is garbage after parsed field.
virtual void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &, bool whole) const = 0;
/** Text deserialization without quoting or escaping.
/** Text deserialization in case when buffer contains only one value, without any escaping and delimiters.
*/
void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;

3
src/DataTypes/Serializations/SerializationDate.cpp
View File

@ -5,7 +5,6 @@
#include <Columns/ColumnsNumber.h>
#include <Formats/ProtobufReader.h>
#include <Formats/ProtobufWriter.h>
#include <Common/assert_cast.h>
@ -20,6 +19,8 @@ void SerializationDate::serializeText(const IColumn & column, size_t row_num, Wr
void SerializationDate::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
deserializeTextEscaped(column, istr, settings);
if (!istr.eof())
throwUnexpectedDataAfterParsedValue(column, istr, settings, "Date");
}
void SerializationDate::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const

2
src/DataTypes/Serializations/SerializationDate32.cpp
View File

@ -17,6 +17,8 @@ void SerializationDate32::serializeText(const IColumn & column, size_t row_num,
void SerializationDate32::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
deserializeTextEscaped(column, istr, settings);
if (!istr.eof())
throwUnexpectedDataAfterParsedValue(column, istr, settings, "Date32");
}
void SerializationDate32::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const

2
src/DataTypes/Serializations/SerializationDateTime.cpp
View File

@ -62,6 +62,8 @@ void SerializationDateTime::serializeTextEscaped(const IColumn & column, size_t
void SerializationDateTime::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
deserializeTextEscaped(column, istr, settings);
if (!istr.eof())
throwUnexpectedDataAfterParsedValue(column, istr, settings, "DateTime");
}
void SerializationDateTime::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const

10
src/DataTypes/Serializations/SerializationDateTime64.cpp
View File

@ -2,12 +2,9 @@
#include <Columns/ColumnVector.h>
#include <Common/assert_cast.h>
#include <Common/typeid_cast.h>
#include <base/DateLUT.h>
#include <Formats/FormatSettings.h>
#include <Formats/ProtobufReader.h>
#include <Formats/ProtobufWriter.h>
#include <IO/Operators.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteBufferFromString.h>
#include <IO/WriteHelpers.h>
@ -40,16 +37,21 @@ void SerializationDateTime64::serializeText(const IColumn & column, size_t row_n
}
}
void SerializationDateTime64::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
void SerializationDateTime64::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
{
DateTime64 result = 0;
readDateTime64Text(result, scale, istr, time_zone);
assert_cast<ColumnType &>(column).getData().push_back(result);
if (whole && !istr.eof())
throwUnexpectedDataAfterParsedValue(column, istr, settings, "DateTime64");
}
void SerializationDateTime64::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
deserializeTextEscaped(column, istr, settings);
if (!istr.eof())
throwUnexpectedDataAfterParsedValue(column, istr, settings, "DateTime64");
}
void SerializationDateTime64::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const

2
src/DataTypes/Serializations/SerializationDateTime64.h
View File

@ -14,7 +14,7 @@ public:
SerializationDateTime64(UInt32 scale_, const TimezoneMixin & time_zone_);
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const override;
void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;

5
src/DataTypes/Serializations/SerializationDecimal.cpp
View File

@ -51,11 +51,14 @@ void SerializationDecimal<T>::serializeText(const IColumn & column, size_t row_n
}
template <typename T>
void SerializationDecimal<T>::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
void SerializationDecimal<T>::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
{
T x;
readText(x, istr);
assert_cast<ColumnType &>(column).getData().push_back(x);
if (whole && !istr.eof())
ISerialization::throwUnexpectedDataAfterParsedValue(column, istr, settings, "Decimal");
}
template <typename T>

2
src/DataTypes/Serializations/SerializationDecimal.h
View File

@ -15,7 +15,7 @@ public:
: SerializationDecimalBase<T>(precision_, scale_) {}
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &, bool whole) const override;
void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void readText(T & x, ReadBuffer & istr, bool csv = false) const { readText(x, istr, this->precision, this->scale, csv); }

6
src/DataTypes/Serializations/SerializationEnum.cpp
View File

@ -53,11 +53,15 @@ template <typename Type>
void SerializationEnum<Type>::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
if (settings.tsv.input_format_enum_as_number)
{
assert_cast<ColumnType &>(column).getData().push_back(readValue(istr));
if (!istr.eof())
ISerialization::throwUnexpectedDataAfterParsedValue(column, istr, settings, "Enum");
}
else
{
std::string field_name;
readString(field_name, istr);
readStringUntilEOF(field_name, istr);
assert_cast<ColumnType &>(column).getData().push_back(this->getValue(StringRef(field_name), true));
}
}

10
src/DataTypes/Serializations/SerializationIP.cpp
View File

@ -36,7 +36,7 @@ void SerializationIPv4::serializeText(const IColumn & column, size_t row_num, Wr
ostr.write(buffer, strlen(buffer));
}
void SerializationIPv4::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
void SerializationIPv4::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
{
ColumnUInt32 * col = typeid_cast<ColumnUInt32 *>(&column);
if (!col)
@ -53,6 +53,9 @@ void SerializationIPv4::deserializeText(IColumn & column, ReadBuffer & istr, con
}
col->insert(ipv4_value);
if (whole && !istr.eof())
throwUnexpectedDataAfterParsedValue(column, istr, settings, "IPv4");
}
SerializationIPv6::SerializationIPv6(const SerializationPtr & nested_)
@ -74,7 +77,7 @@ void SerializationIPv6::serializeText(const IColumn & column, size_t row_num, Wr
ostr.write(buffer, strlen(buffer));
}
void SerializationIPv6::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
void SerializationIPv6::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
{
ColumnFixedString * col = typeid_cast<ColumnFixedString *>(&column);
if (!col)
@ -92,6 +95,9 @@ void SerializationIPv6::deserializeText(IColumn & column, ReadBuffer & istr, con
}
col->insertString(ipv6_value);
if (whole && !istr.eof())
throwUnexpectedDataAfterParsedValue(column, istr, settings, "IPv6");
}
}

4
src/DataTypes/Serializations/SerializationIP.h
View File

@ -11,7 +11,7 @@ public:
SerializationIPv4(const SerializationPtr & nested_);
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const override;
};
class SerializationIPv6 : public SerializationCustomSimpleText
@ -20,7 +20,7 @@ public:
SerializationIPv6(const SerializationPtr & nested_);
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const override;
};
}

14
src/DataTypes/Serializations/SerializationMap.cpp
View File

@ -1,23 +1,16 @@
#include <base/map.h>
#include <DataTypes/Serializations/SerializationMap.h>
#include <DataTypes/Serializations/SerializationArray.h>
#include <DataTypes/Serializations/SerializationTuple.h>
#include <DataTypes/DataTypeMap.h>
#include <Common/StringUtils/StringUtils.h>
#include <Columns/ColumnMap.h>
#include <Columns/ColumnArray.h>
#include <Core/Field.h>
#include <Formats/FormatSettings.h>
#include <Common/typeid_cast.h>
#include <Common/assert_cast.h>
#include <Common/quoteString.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteBufferFromString.h>
#include <IO/ReadBufferFromString.h>
#include <IO/Operators.h>
namespace DB
@ -176,13 +169,16 @@ void SerializationMap::serializeText(const IColumn & column, size_t row_num, Wri
serializeTextImpl(column, row_num, ostr, writer, writer);
}
void SerializationMap::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
void SerializationMap::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
{
deserializeTextImpl(column, istr,
[&settings](ReadBuffer & buf, const SerializationPtr & subcolumn_serialization, IColumn & subcolumn)
{
subcolumn_serialization->deserializeTextQuoted(subcolumn, buf, settings);
});
if (whole && !istr.eof())
throwUnexpectedDataAfterParsedValue(column, istr, settings, "Map");
}
void SerializationMap::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
@ -248,7 +244,7 @@ void SerializationMap::deserializeTextCSV(IColumn & column, ReadBuffer & istr, c
String s;
readCSV(s, istr, settings.csv);
ReadBufferFromString rb(s);
deserializeText(column, rb, settings);
deserializeText(column, rb, settings, true);
}
void SerializationMap::enumerateStreams(

2
src/DataTypes/Serializations/SerializationMap.h
View File

@ -23,7 +23,7 @@ public:
void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void deserializeBinary(IColumn & column, ReadBuffer & istr) const override;
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &, bool whole) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;

2
src/DataTypes/Serializations/SerializationNothing.h
View File

@ -24,7 +24,7 @@ public:
void serializeBinary(const IColumn &, size_t, WriteBuffer &) const override { throwNoSerialization(); }
void deserializeBinary(IColumn &, ReadBuffer &) const override { throwNoSerialization(); }
void serializeText(const IColumn &, size_t, WriteBuffer &, const FormatSettings &) const override { throwNoSerialization(); }
void deserializeText(IColumn &, ReadBuffer &, const FormatSettings &) const override { throwNoSerialization(); }
void deserializeText(IColumn &, ReadBuffer &, const FormatSettings &, bool) const override { throwNoSerialization(); }
/// These methods read and write zero bytes just to allow to figure out size of column.
void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override;

5
src/DataTypes/Serializations/SerializationNumber.cpp
View File

@ -21,7 +21,7 @@ void SerializationNumber<T>::serializeText(const IColumn & column, size_t row_nu
}
template <typename T>
void SerializationNumber<T>::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
void SerializationNumber<T>::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
{
T x;
@ -31,6 +31,9 @@ void SerializationNumber<T>::deserializeText(IColumn & column, ReadBuffer & istr
readText(x, istr);
assert_cast<ColumnVector<T> &>(column).getData().push_back(x);
if (whole && !istr.eof())
throwUnexpectedDataAfterParsedValue(column, istr, settings, "Number");
}
template <typename T>

2
src/DataTypes/Serializations/SerializationNumber.h
View File

@ -16,7 +16,7 @@ public:
using ColumnType = ColumnVector<T>;
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override;
void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;

7
src/DataTypes/Serializations/SerializationTuple.cpp
View File

@ -1,4 +1,3 @@
#include <base/map.h>
#include <base/range.h>
#include <DataTypes/Serializations/SerializationTuple.h>
#include <DataTypes/DataTypeTuple.h>
@ -8,7 +7,6 @@
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteBufferFromString.h>
#include <IO/Operators.h>
namespace DB
@ -121,7 +119,7 @@ void SerializationTuple::serializeText(const IColumn & column, size_t row_num, W
writeChar(')', ostr);
}
void SerializationTuple::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
void SerializationTuple::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
{
const size_t size = elems.size();
assertChar('(', istr);
@ -149,6 +147,9 @@ void SerializationTuple::deserializeText(IColumn & column, ReadBuffer & istr, co
}
skipWhitespaceIfAny(istr);
assertChar(')', istr);
if (whole && !istr.eof())
throwUnexpectedDataAfterParsedValue(column, istr, settings, "Tuple");
}
void SerializationTuple::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const

2
src/DataTypes/Serializations/SerializationTuple.h
View File

@ -20,7 +20,7 @@ public:
void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void deserializeBinary(IColumn & column, ReadBuffer & istr) const override;
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &, bool whole) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;

7
src/DataTypes/Serializations/SerializationUUID.cpp
View File

@ -15,16 +15,19 @@ void SerializationUUID::serializeText(const IColumn & column, size_t row_num, Wr
writeText(assert_cast<const ColumnUUID &>(column).getData()[row_num], ostr);
}
void SerializationUUID::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
void SerializationUUID::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const
{
UUID x;
readText(x, istr);
assert_cast<ColumnUUID &>(column).getData().push_back(x);
if (whole && !istr.eof())
throwUnexpectedDataAfterParsedValue(column, istr, settings, "UUID");
}
void SerializationUUID::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
deserializeText(column, istr, settings);
deserializeText(column, istr, settings, false);
}
void SerializationUUID::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const

2
src/DataTypes/Serializations/SerializationUUID.h
View File

@ -9,7 +9,7 @@ class SerializationUUID : public SimpleTextSerialization
{
public:
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, bool whole) const override;
void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;

14
src/DataTypes/Serializations/SimpleTextSerialization.h
View File

@ -33,30 +33,32 @@ protected:
void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override
{
deserializeText(column, istr, settings);
deserializeText(column, istr, settings, true);
}
void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override
{
deserializeText(column, istr, settings);
deserializeText(column, istr, settings, false);
}
void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override
{
deserializeText(column, istr, settings);
deserializeText(column, istr, settings, false);
}
void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override
{
deserializeText(column, istr, settings);
deserializeText(column, istr, settings, false);
}
void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override
{
deserializeText(column, istr, settings);
deserializeText(column, istr, settings, false);
}
virtual void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0;
/// whole = true means that buffer contains only one value, so we should read until EOF.
/// It's needed to check if there is garbage after parsed field.
virtual void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &, bool whole) const = 0;
};
}

1
src/DataTypes/convertMySQLDataType.cpp
View File

@ -5,7 +5,6 @@
#include <Core/MultiEnum.h>
#include <Core/SettingsEnums.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/IAST.h>
#include "DataTypeDate.h"
#include "DataTypeDateTime.h"

9
src/Databases/DDLDependencyVisitor.cpp
View File

@ -1,10 +1,11 @@
#include <Databases/DDLDependencyVisitor.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTIdentifier.h>
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
#include <Interpreters/Context.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Poco/String.h>
namespace DB

13
src/Databases/DatabaseAtomic.cpp
View File

@ -229,15 +229,8 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_
StoragePtr table = getTableUnlocked(table_name, db_lock);
if (table->isDictionary() && !dictionary)
{
if (exchange)
throw Exception(ErrorCodes::INCORRECT_QUERY,
"Use EXCHANGE DICTIONARIES for dictionaries and EXCHANGE TABLES for tables.");
else
throw Exception(ErrorCodes::INCORRECT_QUERY,
"Use RENAME DICTIONARY for dictionaries and RENAME TABLE for tables.");
}
if (dictionary && !table->isDictionary())
throw Exception(ErrorCodes::INCORRECT_QUERY, "Use RENAME/EXCHANGE TABLE (instead of RENAME/EXCHANGE DICTIONARY) for tables");
table->checkTableCanBeRenamed();
assert_can_move_mat_view(table);
@ -245,6 +238,8 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_
if (exchange)
{
other_table = other_db.getTableUnlocked(to_table_name, other_db_lock);
if (dictionary && !other_table->isDictionary())
throw Exception(ErrorCodes::INCORRECT_QUERY, "Use RENAME/EXCHANGE TABLE (instead of RENAME/EXCHANGE DICTIONARY) for tables");
other_table->checkTableCanBeRenamed();
assert_can_move_mat_view(other_table);
}

2
src/Databases/DatabaseFactory.cpp
View File

@ -11,9 +11,7 @@
#include <Interpreters/evaluateConstantExpression.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/formatAST.h>
#include <Parsers/queryToString.h>
#include <Storages/ExternalDataSourceConfiguration.h>
#include <Common/Macros.h>

1
src/Databases/DatabaseMemory.cpp
View File

@ -5,6 +5,7 @@
#include <Databases/DDLDependencyVisitor.h>
#include <Interpreters/Context.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTFunction.h>
#include <Storages/IStorage.h>
#include <filesystem>

8
src/Databases/DatabaseOnDisk.cpp
View File

@ -7,6 +7,7 @@
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/formatAST.h>
#include <Parsers/parseQuery.h>
@ -40,6 +41,7 @@ namespace ErrorCodes
extern const int TABLE_ALREADY_EXISTS;
extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
extern const int DATABASE_NOT_EMPTY;
extern const int INCORRECT_QUERY;
}
@ -349,8 +351,6 @@ void DatabaseOnDisk::renameTable(
{
if (exchange)
throw Exception("Tables can be exchanged only in Atomic databases", ErrorCodes::NOT_IMPLEMENTED);
if (dictionary)
throw Exception("Dictionaries can be renamed only in Atomic databases", ErrorCodes::NOT_IMPLEMENTED);
bool from_ordinary_to_atomic = false;
bool from_atomic_to_ordinary = false;
@ -372,7 +372,11 @@ void DatabaseOnDisk::renameTable(
ASTPtr attach_query;
/// DatabaseLazy::detachTable may return nullptr even if table exists, so we need tryGetTable for this case.
StoragePtr table = tryGetTable(table_name, local_context);
if (dictionary && table && !table->isDictionary())
throw Exception("Use RENAME/EXCHANGE TABLE (instead of RENAME/EXCHANGE DICTIONARY) for tables", ErrorCodes::INCORRECT_QUERY);
detachTable(local_context, table_name);
UUID prev_uuid = UUIDHelpers::Nil;
try
{

Some files were not shown because too many files have changed in this diff Show More