thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-10-09 01:50:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:ClickHouse/ClickHouse into feature/randomASKII-function

Browse Source
This commit is contained in:
Sergey Shtykov 2019-10-30 14:13:04 +03:00
commit 789f5d5550
30 changed files with 244 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -245,3 +245,6 @@ website/package-lock.json
/.ccls-cache /.ccls-cache
/compile_commands.json /compile_commands.json
# Toolchains
/cmake/toolchain/*

6
CMakeLists.txt
View File

@ -13,7 +13,10 @@ foreach(policy
endforeach() endforeach()
project(ClickHouse) project(ClickHouse)
include (cmake/arch.cmake)
include (cmake/target.cmake) include (cmake/target.cmake)
include (cmake/tools.cmake)
# Ignore export() since we don't use it, # Ignore export() since we don't use it,
# but it gets broken with a global targets via link_libraries() # but it gets broken with a global targets via link_libraries()
@ -26,8 +29,6 @@ set(CMAKE_LINK_DEPENDS_NO_SHARED 1) # Do not relink all depended targets on .so
set(CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel" CACHE STRING "" FORCE) set(CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel" CACHE STRING "" FORCE)
set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a postfix.") # To be consistent with CMakeLists from contrib libs. set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a postfix.") # To be consistent with CMakeLists from contrib libs.
include (cmake/arch.cmake)
option(ENABLE_IPO "Enable inter-procedural optimization (aka LTO)" OFF) # need cmake 3.9+ option(ENABLE_IPO "Enable inter-procedural optimization (aka LTO)" OFF) # need cmake 3.9+
if(ENABLE_IPO) if(ENABLE_IPO)
cmake_policy(SET CMP0069 NEW) cmake_policy(SET CMP0069 NEW)
@ -230,7 +231,6 @@ include(cmake/dbms_glob_sources.cmake)
if (OS_LINUX) if (OS_LINUX)
include(cmake/linux/default_libs.cmake) include(cmake/linux/default_libs.cmake)
elseif (OS_DARWIN) elseif (OS_DARWIN)
include(cmake/darwin/sdk.cmake)
include(cmake/darwin/default_libs.cmake) include(cmake/darwin/default_libs.cmake)
endif () endif ()

1
cmake/arch.cmake
View File

@ -17,6 +17,7 @@ endif ()
if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le.*|PPC64LE.*)") if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le.*|PPC64LE.*)")
set (ARCH_PPC64LE 1) set (ARCH_PPC64LE 1)
# FIXME: move this check into tools.cmake
if (COMPILER_CLANG OR (COMPILER_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)) if (COMPILER_CLANG OR (COMPILER_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8))
message(FATAL_ERROR "Only gcc-8 is supported for powerpc architecture") message(FATAL_ERROR "Only gcc-8 is supported for powerpc architecture")
endif () endif ()

8
cmake/darwin/default_libs.cmake
View File

@ -11,6 +11,14 @@ message(STATUS "Default libraries: ${DEFAULT_LIBS}")
set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS}) set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})
set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS}) set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
# Minimal supported SDK version
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mmacosx-version-min=10.14")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mmacosx-version-min=10.14")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -mmacosx-version-min=10.14")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -mmacosx-version-min=10.14")
# Global libraries # Global libraries
add_library(global-libs INTERFACE) add_library(global-libs INTERFACE)

11
cmake/darwin/sdk.cmake
View File

@ -1,11 +0,0 @@
option (SDK_PATH "Path to the SDK to build with" "")
if (NOT EXISTS "${SDK_PATH}/SDKSettings.plist")
message (FATAL_ERROR "Wrong SDK path provided: ${SDK_PATH}")
endif ()
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14")

13
cmake/darwin/toolchain-x86_64.cmake Normal file
View File

@ -0,0 +1,13 @@
set (CMAKE_SYSTEM_NAME "Darwin")
set (CMAKE_SYSTEM_PROCESSOR "x86_64")
set (CMAKE_C_COMPILER_TARGET "x86_64-apple-darwin")
set (CMAKE_CXX_COMPILER_TARGET "x86_64-apple-darwin")
set (CMAKE_OSX_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/darwin-x86_64")
set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

2
cmake/linux/default_libs.cmake
View File

@ -5,7 +5,7 @@ set (DEFAULT_LIBS "-nodefaultlibs")
# We need builtins from Clang's RT even without libcxx - for ubsan+int128. # We need builtins from Clang's RT even without libcxx - for ubsan+int128.
# See https://bugs.llvm.org/show_bug.cgi?id=16404 # See https://bugs.llvm.org/show_bug.cgi?id=16404
if (COMPILER_CLANG) if (COMPILER_CLANG AND NOT (CMAKE_CROSSCOMPILING AND ARCH_AARCH64))
execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE) execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
else () else ()
set (BUILTINS_LIBRARY "-lgcc") set (BUILTINS_LIBRARY "-lgcc")

20
cmake/linux/toolchain-aarch64.cmake Normal file
View File

@ -0,0 +1,20 @@
set (CMAKE_SYSTEM_NAME "Linux")
set (CMAKE_SYSTEM_PROCESSOR "aarch64")
set (CMAKE_C_COMPILER_TARGET "aarch64-linux-gnu")
set (CMAKE_CXX_COMPILER_TARGET "aarch64-linux-gnu")
set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/aarch64-linux-gnu/libc")
# We don't use compiler from toolchain because it's gcc-8, and we provide support only for gcc-9.
set (CMAKE_AR "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/bin/aarch64-linux-gnu-ar")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64")
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

67
cmake/target.cmake
View File

@ -9,62 +9,8 @@ elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
add_definitions(-D OS_DARWIN) add_definitions(-D OS_DARWIN)
endif () endif ()
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set (COMPILER_GCC 1)
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set (COMPILER_CLANG 1)
endif ()
if (COMPILER_GCC)
# Require minimum version of gcc
set (GCC_MINIMUM_VERSION 8)
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION} AND NOT CMAKE_VERSION VERSION_LESS 2.8.9)
message (FATAL_ERROR "GCC version must be at least ${GCC_MINIMUM_VERSION}. For example, if GCC ${GCC_MINIMUM_VERSION} is available under gcc-${GCC_MINIMUM_VERSION}, g++-${GCC_MINIMUM_VERSION} names, do the following: export CC=gcc-${GCC_MINIMUM_VERSION} CXX=g++-${GCC_MINIMUM_VERSION}; rm -rf CMakeCache.txt CMakeFiles; and re run cmake or ./release.")
endif ()
elseif (COMPILER_CLANG)
# Require minimum version of clang
set (CLANG_MINIMUM_VERSION 7)
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.")
endif ()
else ()
message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang 6+ and GCC 7+.")
endif ()
string(REGEX MATCH "-?[0-9]+(.[0-9]+)?$" COMPILER_POSTFIX ${CMAKE_CXX_COMPILER})
if (OS_LINUX)
find_program (LLD_PATH NAMES "lld${COMPILER_POSTFIX}" "lld")
find_program (GOLD_PATH NAMES "ld.gold" "gold")
endif()
option (LINKER_NAME "Linker name or full path")
if (NOT LINKER_NAME)
if (COMPILER_CLANG AND LLD_PATH)
set (LINKER_NAME "lld")
elseif (GOLD_PATH)
set (LINKER_NAME "gold")
endif ()
endif ()
if (LINKER_NAME)
message(STATUS "Using linker: ${LINKER_NAME} (selected from: LLD_PATH=${LLD_PATH}; GOLD_PATH=${GOLD_PATH}; COMPILER_POSTFIX=${COMPILER_POSTFIX})")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
endif ()
if (CMAKE_CROSSCOMPILING) if (CMAKE_CROSSCOMPILING)
if (OS_DARWIN) if (OS_DARWIN)
set (CMAKE_SYSTEM_PROCESSOR x86_64)
set (CMAKE_C_COMPILER_TARGET x86_64-apple-darwin)
set (CMAKE_CXX_COMPILER_TARGET x86_64-apple-darwin)
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
# FIXME: broken dependencies # FIXME: broken dependencies
set (USE_SNAPPY OFF CACHE INTERNAL "") set (USE_SNAPPY OFF CACHE INTERNAL "")
set (ENABLE_SSL OFF CACHE INTERNAL "") set (ENABLE_SSL OFF CACHE INTERNAL "")
@ -73,12 +19,19 @@ if (CMAKE_CROSSCOMPILING)
set (ENABLE_READLINE OFF CACHE INTERNAL "") set (ENABLE_READLINE OFF CACHE INTERNAL "")
set (ENABLE_ICU OFF CACHE INTERNAL "") set (ENABLE_ICU OFF CACHE INTERNAL "")
set (ENABLE_FASTOPS OFF CACHE INTERNAL "") set (ENABLE_FASTOPS OFF CACHE INTERNAL "")
elseif (OS_LINUX)
message (STATUS "Cross-compiling for Darwin") if (ARCH_AARCH64)
# FIXME: broken dependencies
set (ENABLE_PROTOBUF OFF CACHE INTERNAL "")
set (ENABLE_PARQUET OFF CACHE INTERNAL "")
set (ENABLE_MYSQL OFF CACHE INTERNAL "")
endif ()
else () else ()
message (FATAL_ERROR "Trying to cross-compile to unsupported target: ${CMAKE_SYSTEM_NAME}!") message (FATAL_ERROR "Trying to cross-compile to unsupported system: ${CMAKE_SYSTEM_NAME}!")
endif () endif ()
# Don't know why but CXX_STANDARD doesn't work for cross-compilation # Don't know why but CXX_STANDARD doesn't work for cross-compilation
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
message (STATUS "Cross-compiling for target: ${CMAKE_CXX_COMPILE_TARGET}")
endif () endif ()

2
cmake/toolchain/darwin-x86_64/README.txt Normal file
View File

@ -0,0 +1,2 @@
wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz
tar --strip-components=1 xJf MacOSX10.14.sdk.tar.xz

2
cmake/toolchain/linux-aarch64/README.txt Normal file
View File

@ -0,0 +1,2 @@
wget https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz
tar --strip-components=1 xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz

41
cmake/tools.cmake Normal file
View File

@ -0,0 +1,41 @@
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set (COMPILER_GCC 1)
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set (COMPILER_CLANG 1)
endif ()
if (COMPILER_GCC)
# Require minimum version of gcc
set (GCC_MINIMUM_VERSION 8)
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION} AND NOT CMAKE_VERSION VERSION_LESS 2.8.9)
message (FATAL_ERROR "GCC version must be at least ${GCC_MINIMUM_VERSION}. For example, if GCC ${GCC_MINIMUM_VERSION} is available under gcc-${GCC_MINIMUM_VERSION}, g++-${GCC_MINIMUM_VERSION} names, do the following: export CC=gcc-${GCC_MINIMUM_VERSION} CXX=g++-${GCC_MINIMUM_VERSION}; rm -rf CMakeCache.txt CMakeFiles; and re run cmake or ./release.")
endif ()
elseif (COMPILER_CLANG)
# Require minimum version of clang
set (CLANG_MINIMUM_VERSION 7)
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.")
endif ()
else ()
message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang 6+ and GCC 7+.")
endif ()
option (LINKER_NAME "Linker name or full path")
find_program (LLD_PATH NAMES "ld.lld" "lld")
find_program (GOLD_PATH NAMES "ld.gold" "gold")
if (NOT LINKER_NAME)
if (LLD_PATH)
set (LINKER_NAME "lld")
elseif (GOLD_PATH)
set (LINKER_NAME "gold")
endif ()
endif ()
if (LINKER_NAME)
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
message(STATUS "Using custom linker by name: ${LINKER_NAME}")
endif ()

2
dbms/src/Common/StackTrace.cpp
View File

@ -30,7 +30,7 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext
else else
error << "Address: " << info.si_addr; error << "Address: " << info.si_addr;
#if defined(__x86_64__) && !defined(__FreeBSD__) && !defined(__APPLE__) #if defined(__x86_64__) && !defined(__FreeBSD__) && !defined(__APPLE__) && !defined(__arm__)
auto err_mask = context.uc_mcontext.gregs[REG_ERR]; auto err_mask = context.uc_mcontext.gregs[REG_ERR];
if ((err_mask & 0x02)) if ((err_mask & 0x02))
error << " Access: write."; error << " Access: write.";

6
dbms/src/Common/tests/int_hashes_perf.cpp
View File

@ -31,7 +31,7 @@ void setAffinity()
static inline ALWAYS_INLINE UInt64 rdtsc() static inline ALWAYS_INLINE UInt64 rdtsc()
{ {
#if __x86_64__ #if defined(__x86_64__)
UInt32 a, d; UInt32 a, d;
__asm__ volatile ("rdtsc" : "=a" (a), "=d" (d)); __asm__ volatile ("rdtsc" : "=a" (a), "=d" (d));
return static_cast<UInt64>(a) | (static_cast<UInt64>(d) << 32); return static_cast<UInt64>(a) | (static_cast<UInt64>(d) << 32);
@ -109,7 +109,7 @@ static inline size_t murmurMix(UInt64 x)
} }
#if __x86_64__ #if defined(__x86_64__)
static inline size_t crc32Hash(UInt64 x) static inline size_t crc32Hash(UInt64 x)
{ {
UInt64 crc = -1ULL; UInt64 crc = -1ULL;
@ -309,7 +309,7 @@ int main(int argc, char ** argv)
if (!method || method == 8) test<mulShift> (n, data.data(), "7: mulShift"); if (!method || method == 8) test<mulShift> (n, data.data(), "7: mulShift");
if (!method || method == 9) test<tabulation>(n, data.data(), "8: tabulation"); if (!method || method == 9) test<tabulation>(n, data.data(), "8: tabulation");
#if __x86_64__ #if defined(__x86_64__)
if (!method || method == 10) test<crc32Hash> (n, data.data(), "9: crc32"); if (!method || method == 10) test<crc32Hash> (n, data.data(), "9: crc32");
#endif #endif

18
dbms/src/Core/iostream_debug_helpers.cpp
View File

@ -18,15 +18,17 @@
namespace DB namespace DB
{ {
std::ostream & operator<<(std::ostream & stream, const IBlockInputStream & what)
template <>
std::ostream & operator<< <Field>(std::ostream & stream, const Field & what)
{ {
stream << "IBlockInputStream(name = " << what.getName() << ")"; stream << applyVisitor(FieldVisitorDump(), what);
return stream; return stream;
} }
std::ostream & operator<<(std::ostream & stream, const Field & what) std::ostream & operator<<(std::ostream & stream, const IBlockInputStream & what)
{ {
stream << applyVisitor(FieldVisitorDump(), what); stream << "IBlockInputStream(name = " << what.getName() << ")";
return stream; return stream;
} }
@ -102,14 +104,6 @@ std::ostream & operator<<(std::ostream & stream, const Connection::Packet & what
return stream; return stream;
} }
std::ostream & operator<<(std::ostream & stream, const IAST & what)
{
stream << "IAST{";
what.dumpTree(stream);
stream << "}";
return stream;
}
std::ostream & operator<<(std::ostream & stream, const ExpressionAction & what) std::ostream & operator<<(std::ostream & stream, const ExpressionAction & what)
{ {
stream << "ExpressionAction(" << what.toString() << ")"; stream << "ExpressionAction(" << what.toString() << ")";

16
dbms/src/Core/iostream_debug_helpers.h
View File

@ -7,18 +7,15 @@
namespace DB namespace DB
{ {
// Used to disable implicit casting for certain overloaded types such as Field, which leads to // Use template to disable implicit casting for certain overloaded types such as Field, which leads
// overload resolution ambiguity. // to overload resolution ambiguity.
template <typename T> struct Dumpable; class Field;
template <typename T> template <typename T, typename U = std::enable_if_t<std::is_same_v<T, Field>>>
std::ostream & operator<<(std::ostream & stream, const typename Dumpable<T>::Type & what); std::ostream & operator<<(std::ostream & stream, const T & what);
class IBlockInputStream; class IBlockInputStream;
std::ostream & operator<<(std::ostream & stream, const IBlockInputStream & what); std::ostream & operator<<(std::ostream & stream, const IBlockInputStream & what);
class Field;
template <> struct Dumpable<Field> { using Type = Field; };
struct NameAndTypePair; struct NameAndTypePair;
std::ostream & operator<<(std::ostream & stream, const NameAndTypePair & what); std::ostream & operator<<(std::ostream & stream, const NameAndTypePair & what);
@ -43,9 +40,6 @@ std::ostream & operator<<(std::ostream & stream, const ColumnWithTypeAndName & w
class IColumn; class IColumn;
std::ostream & operator<<(std::ostream & stream, const IColumn & what); std::ostream & operator<<(std::ostream & stream, const IColumn & what);
class IAST;
std::ostream & operator<<(std::ostream & stream, const IAST & what);
std::ostream & operator<<(std::ostream & stream, const Connection::Packet & what); std::ostream & operator<<(std::ostream & stream, const Connection::Packet & what);
struct ExpressionAction; struct ExpressionAction;

2
dbms/src/IO/tests/parse_int_perf.cpp
View File

@ -17,7 +17,7 @@
UInt64 rdtsc() UInt64 rdtsc()
{ {
#if __x86_64__ #if defined(__x86_64__)
UInt64 val; UInt64 val;
__asm__ __volatile__("rdtsc" : "=A" (val) :); __asm__ __volatile__("rdtsc" : "=A" (val) :);
return val; return val;

15
dbms/src/Interpreters/AnalyzedJoin.cpp
View File

@ -204,21 +204,6 @@ void AnalyzedJoin::addJoinedColumnsAndCorrectNullability(Block & sample_block) c
bool make_nullable = join_use_nulls && left_or_full_join; bool make_nullable = join_use_nulls && left_or_full_join;
if (!make_nullable)
{
/// Keys from right table are usually not stored in Join, but copied from the left one.
/// So, if left key is nullable, let's make right key nullable too.
/// Note: for some join types it's not needed and, probably, may be removed.
/// Note: changing this code, take into account the implementation in Join.cpp.
auto it = std::find(key_names_right.begin(), key_names_right.end(), col.name);
if (it != key_names_right.end())
{
auto pos = it - key_names_right.begin();
const auto & left_key_name = key_names_left[pos];
make_nullable = sample_block.getByName(left_key_name).type->isNullable();
}
}
if (make_nullable && res_type->canBeInsideNullable()) if (make_nullable && res_type->canBeInsideNullable())
res_type = makeNullable(res_type); res_type = makeNullable(res_type);

2
dbms/src/Interpreters/AnalyzedJoin.h
View File

@ -94,7 +94,7 @@ public:
void addOnKeys(ASTPtr & left_table_ast, ASTPtr & right_table_ast); void addOnKeys(ASTPtr & left_table_ast, ASTPtr & right_table_ast);
bool hasUsing() const { return table_join.using_expression_list != nullptr; } bool hasUsing() const { return table_join.using_expression_list != nullptr; }
bool hasOn() const { return !hasUsing(); } bool hasOn() const { return table_join.on_expression != nullptr; }
NameSet getQualifiedColumnsSet() const; NameSet getQualifiedColumnsSet() const;
NameSet getOriginalColumnsSet() const; NameSet getOriginalColumnsSet() const;

66
dbms/src/Interpreters/Join.cpp
View File

@ -37,11 +37,52 @@ namespace ErrorCodes
} }
/// Converts column to nullable if needed. No backward convertion. static ColumnPtr filterWithBlanks(ColumnPtr src_column, const IColumn::Filter & filter, bool inverse_filter = false)
{
ColumnPtr column = src_column->convertToFullColumnIfConst();
MutableColumnPtr mut_column = column->cloneEmpty();
mut_column->reserve(column->size());
if (inverse_filter)
{
for (size_t row = 0; row < filter.size(); ++row)
{
if (filter[row])
mut_column->insertDefault();
else
mut_column->insertFrom(*column, row);
}
}
else
{
for (size_t row = 0; row < filter.size(); ++row)
{
if (filter[row])
mut_column->insertFrom(*column, row);
else
mut_column->insertDefault();
}
}
return mut_column;
}
static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column, bool nullable) static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column, bool nullable)
{ {
if (nullable) if (nullable)
{
JoinCommon::convertColumnToNullable(column); JoinCommon::convertColumnToNullable(column);
}
else
{
/// We have to replace values masked by NULLs with defaults.
if (column.column)
if (auto * nullable_column = checkAndGetColumn<ColumnNullable>(*column.column))
column.column = filterWithBlanks(column.column, nullable_column->getNullMapColumn().getData(), true);
JoinCommon::removeColumnNullability(column);
}
return std::move(column); return std::move(column);
} }
@ -57,6 +98,9 @@ static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column,
column.column = std::move(mutable_column); column.column = std::move(mutable_column);
} }
} }
else
JoinCommon::removeColumnNullability(column);
return std::move(column); return std::move(column);
} }
@ -769,7 +813,7 @@ void Join::joinBlockImpl(
for (size_t i = 0; i < existing_columns; ++i) for (size_t i = 0; i < existing_columns; ++i)
block.safeGetByPosition(i).column = block.safeGetByPosition(i).column->filter(row_filter, -1); block.safeGetByPosition(i).column = block.safeGetByPosition(i).column->filter(row_filter, -1);
/// Add join key columns from right block if they has different name. /// Add join key columns from right block if needed.
for (size_t i = 0; i < right_table_keys.columns(); ++i) for (size_t i = 0; i < right_table_keys.columns(); ++i)
{ {
const auto & right_key = right_table_keys.getByPosition(i); const auto & right_key = right_table_keys.getByPosition(i);
@ -791,7 +835,7 @@ void Join::joinBlockImpl(
null_map_filter.getData().swap(row_filter); null_map_filter.getData().swap(row_filter);
const IColumn::Filter & filter = null_map_filter.getData(); const IColumn::Filter & filter = null_map_filter.getData();
/// Add join key columns from right block if they has different name. /// Add join key columns from right block if needed.
for (size_t i = 0; i < right_table_keys.columns(); ++i) for (size_t i = 0; i < right_table_keys.columns(); ++i)
{ {
const auto & right_key = right_table_keys.getByPosition(i); const auto & right_key = right_table_keys.getByPosition(i);
@ -800,20 +844,10 @@ void Join::joinBlockImpl(
if (required_right_keys.count(right_key.name) && !block.has(right_key.name)) if (required_right_keys.count(right_key.name) && !block.has(right_key.name))
{ {
const auto & col = block.getByName(left_name); const auto & col = block.getByName(left_name);
ColumnPtr column = col.column->convertToFullColumnIfConst();
MutableColumnPtr mut_column = column->cloneEmpty();
mut_column->reserve(column->size());
for (size_t row = 0; row < filter.size(); ++row)
{
if (filter[row])
mut_column->insertFrom(*column, row);
else
mut_column->insertDefault();
}
bool is_nullable = nullable_right_side || right_key.type->isNullable(); bool is_nullable = nullable_right_side || right_key.type->isNullable();
block.insert(correctNullability({std::move(mut_column), col.type, right_key.name}, is_nullable, null_map_filter));
ColumnPtr thin_column = filterWithBlanks(col.column, filter);
block.insert(correctNullability({thin_column, col.type, right_key.name}, is_nullable, null_map_filter));
if constexpr (is_all_join) if constexpr (is_all_join)
right_keys_to_replicate.push_back(block.getPositionByName(right_key.name)); right_keys_to_replicate.push_back(block.getPositionByName(right_key.name));

16
dbms/src/Interpreters/join_common.cpp
View File

@ -32,6 +32,22 @@ void convertColumnsToNullable(Block & block, size_t starting_pos)
convertColumnToNullable(block.getByPosition(i)); convertColumnToNullable(block.getByPosition(i));
} }
/// @warning It assumes that every NULL has default value in nested column (or it does not matter)
void removeColumnNullability(ColumnWithTypeAndName & column)
{
if (!column.type->isNullable())
return;
column.type = static_cast<const DataTypeNullable &>(*column.type).getNestedType();
if (column.column)
{
auto * nullable_column = checkAndGetColumn<ColumnNullable>(*column.column);
ColumnPtr nested_column = nullable_column->getNestedColumnPtr();
MutableColumnPtr mutable_column = (*std::move(nested_column)).mutate();
column.column = std::move(mutable_column);
}
}
ColumnRawPtrs temporaryMaterializeColumns(const Block & block, const Names & names, Columns & materialized) ColumnRawPtrs temporaryMaterializeColumns(const Block & block, const Names & names, Columns & materialized)
{ {
ColumnRawPtrs ptrs; ColumnRawPtrs ptrs;

1
dbms/src/Interpreters/join_common.h
View File

@ -15,6 +15,7 @@ namespace JoinCommon
void convertColumnToNullable(ColumnWithTypeAndName & column); void convertColumnToNullable(ColumnWithTypeAndName & column);
void convertColumnsToNullable(Block & block, size_t starting_pos = 0); void convertColumnsToNullable(Block & block, size_t starting_pos = 0);
void removeColumnNullability(ColumnWithTypeAndName & column);
ColumnRawPtrs temporaryMaterializeColumns(const Block & block, const Names & names, Columns & materialized); ColumnRawPtrs temporaryMaterializeColumns(const Block & block, const Names & names, Columns & materialized);
void removeLowCardinalityInplace(Block & block); void removeLowCardinalityInplace(Block & block);

4
dbms/src/Interpreters/tests/hash_map_string.cpp
View File

@ -131,7 +131,7 @@ struct FastHash64
}; };
#if __x86_64__ #if defined(__x86_64__)
struct CrapWow struct CrapWow
{ {
size_t operator() (CompactStringRef x) const size_t operator() (CompactStringRef x) const
@ -381,7 +381,7 @@ int main(int argc, char ** argv)
<< std::endl; << std::endl;
} }
#if __x86_64__ #if defined(__x86_64__)
if (!m || m == 3) if (!m || m == 3)
{ {
Stopwatch watch; Stopwatch watch;

9
dbms/src/Parsers/iostream_debug_helpers.cpp
View File

@ -1,4 +1,5 @@
#include "iostream_debug_helpers.h" #include "iostream_debug_helpers.h"
#include <Parsers/IAST.h>
#include <Parsers/IParser.h> #include <Parsers/IParser.h>
#include <Parsers/Lexer.h> #include <Parsers/Lexer.h>
#include <Parsers/TokenIterator.h> #include <Parsers/TokenIterator.h>
@ -20,4 +21,12 @@ std::ostream & operator<<(std::ostream & stream, const Expected & what)
return stream; return stream;
} }
std::ostream & operator<<(std::ostream & stream, const IAST & what)
{
stream << "IAST{";
what.dumpTree(stream);
stream << "}";
return stream;
}
} }

3
dbms/src/Parsers/iostream_debug_helpers.h
View File

@ -9,6 +9,9 @@ std::ostream & operator<<(std::ostream & stream, const Token & what);
struct Expected; struct Expected;
std::ostream & operator<<(std::ostream & stream, const Expected & what); std::ostream & operator<<(std::ostream & stream, const Expected & what);
class IAST;
std::ostream & operator<<(std::ostream & stream, const IAST & what);
} }
#include <Core/iostream_debug_helpers.h> #include <Core/iostream_debug_helpers.h>

15
dbms/tests/performance/grear_circle_dist.xml Normal file
View File

@ -0,0 +1,15 @@
<test>
<type>once</type>
<stop_conditions>
<any_of>
<average_speed_not_changing_for_ms>1000</average_speed_not_changing_for_ms>
<total_time_ms>10000</total_time_ms>
</any_of>
</stop_conditions>
<!-- lon [-180; 180], lat [-90; 90] -->
<query>SELECT count() FROM system.numbers WHERE NOT ignore(greatCircleDistance((rand() % 360) * 1. - 180, (number % 150) * 1.2 - 90, (number % 360) + toFloat64(rand()) / 4294967296 - 180, (rand() % 180) * 1. - 90))</query>
<!-- 55.755830, 37.617780 is center of Moscow -->
<query>SELECT count() FROM system.numbers WHERE NOT ignore(greatCircleDistance(55. + toFloat64(rand()) / 4294967296, 37. + toFloat64(rand()) / 4294967296, 55. + toFloat64(rand()) / 4294967296, 37. + toFloat64(rand()) / 4294967296))</query>
</test>

16
dbms/tests/queries/0_stateless/00853_join_with_nulls_crash.reference
View File

@ -1,11 +1,11 @@
\N test 0 1 Nullable(String) Nullable(String) \N test 0 1 Nullable(String) String
bar bar 1 2 Nullable(String) Nullable(String) bar bar 1 2 Nullable(String) String
\N 0 1 Nullable(String) Nullable(String) \N 0 1 Nullable(String) String
foo \N 2 0 Nullable(String) Nullable(String) foo 2 0 Nullable(String) String
\N test 0 1 Nullable(String) Nullable(String) \N test 0 1 Nullable(String) String
bar bar 1 2 Nullable(String) Nullable(String) bar bar 1 2 Nullable(String) String
\N 0 1 Nullable(String) Nullable(String) \N 0 1 Nullable(String) String
foo \N 2 0 Nullable(String) Nullable(String) foo 2 0 Nullable(String) String
foo \N 2 0 String Nullable(String) foo \N 2 0 String Nullable(String)
bar bar 1 2 String Nullable(String) bar bar 1 2 String Nullable(String)
test 0 1 String Nullable(String) test 0 1 String Nullable(String)

4
docker/packager/binary/Dockerfile
View File

@ -71,14 +71,10 @@ RUN cd cctools-port/cctools && ./configure --prefix=/cctools --with-libtapi=/cct
RUN rm -rf cctools-port RUN rm -rf cctools-port
# Download toolchain for Darwin # Download toolchain for Darwin
RUN mkdir -p /build/cmake/toolchain/darwin-x86_64
RUN wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz RUN wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz
RUN tar xJf MacOSX10.14.sdk.tar.xz -C /build/cmake/toolchain/darwin-x86_64 --strip-components=1
# Download toolchain for ARM # Download toolchain for ARM
RUN mkdir -p /build/cmake/toolchain/linux-aarch64
RUN wget "https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en" -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz RUN wget "https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en" -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz
RUN tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C /build/cmake/toolchain/linux-aarch64 --strip-components=1
COPY build.sh / COPY build.sh /
CMD ["/bin/bash", "/build.sh"] CMD ["/bin/bash", "/build.sh"]

6
docker/packager/binary/build.sh
View File

@ -2,6 +2,12 @@
set -x -e set -x -e
mkdir -p build/cmake/toolchain/darwin-x86_64
tar xJf MacOSX10.14.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1
mkdir -p build/cmake/toolchain/linux-aarch64
tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build/cmake/toolchain/linux-aarch64 --strip-components=1
mkdir -p build/build_docker mkdir -p build/build_docker
cd build/build_docker cd build/build_docker
ccache --show-stats ||: ccache --show-stats ||:

16
docker/packager/packager
View File

@ -107,6 +107,14 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, di
result = [] result = []
cmake_flags = ['$CMAKE_FLAGS', '-DADD_GDB_INDEX_FOR_GOLD=1'] cmake_flags = ['$CMAKE_FLAGS', '-DADD_GDB_INDEX_FOR_GOLD=1']
is_clang = "clang" in compiler
is_cross_compile = ("darwin" in compiler) or ("aarch64" in compiler)
# Explicitly use LLD with Clang by default.
# Don't force linker for cross-compilation.
if is_clang and not is_cross_compile:
cmake_flags = ['$CMAKE_FLAGS', '-DLINKER_NAME=lld']
if compiler.endswith("-darwin"): if compiler.endswith("-darwin"):
cc = compiler[:-len("-darwin")] cc = compiler[:-len("-darwin")]
else: else:
@ -124,9 +132,11 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, di
if "darwin" in compiler: if "darwin" in compiler:
cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/x86_64-apple-darwin-ar") cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/x86_64-apple-darwin-ar")
cmake_flags.append("-DCMAKE_RANLIB:FILEPATH=/cctools/bin/x86_64-apple-darwin-ranlib") cmake_flags.append("-DCMAKE_RANLIB:FILEPATH=/cctools/bin/x86_64-apple-darwin-ranlib")
cmake_flags.append("-DCMAKE_SYSTEM_NAME=Darwin")
cmake_flags.append("-DSDK_PATH=/cctools/MacOSX10.14.sdk")
cmake_flags.append("-DLINKER_NAME=/cctools/bin/x86_64-apple-darwin-ld") cmake_flags.append("-DLINKER_NAME=/cctools/bin/x86_64-apple-darwin-ld")
cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-x86_64.cmake")
if "aarch64" in compiler:
cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake")
if sanitizer: if sanitizer:
result.append("SANITIZER={}".format(sanitizer)) result.append("SANITIZER={}".format(sanitizer))
@ -181,7 +191,7 @@ if __name__ == "__main__":
parser.add_argument("--clickhouse-repo-path", default="../../") parser.add_argument("--clickhouse-repo-path", default="../../")
parser.add_argument("--output-dir", required=True) parser.add_argument("--output-dir", required=True)
parser.add_argument("--build-type", choices=("debug", ""), default="") parser.add_argument("--build-type", choices=("debug", ""), default="")
parser.add_argument("--compiler", choices=("clang-6.0", "clang-7", "gcc-7", "clang-8", "clang-8-darwin", "gcc-8", "gcc-9"), default="gcc-7") parser.add_argument("--compiler", choices=("clang-6.0", "clang-7", "gcc-7", "clang-8", "clang-8-darwin", "clang-8-aarch64", "gcc-8", "gcc-9"), default="gcc-7")
parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="") parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="")
parser.add_argument("--unbundled", action="store_true") parser.add_argument("--unbundled", action="store_true")
parser.add_argument("--split-binary", action="store_true") parser.add_argument("--split-binary", action="store_true")