thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into base64_functions

Browse Source
This commit is contained in:
alexey-milovidov 2018-11-23 21:33:56 +03:00 committed by GitHub
commit af013285e9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
335 changed files with 7912 additions and 2967 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.clang-format
View File

@ -1,6 +1,6 @@
BasedOnStyle: WebKit
Language: Cpp
AlignAfterOpenBracket: false
AlignAfterOpenBracket: AlwaysBreak
BreakBeforeBraces: Custom
BraceWrapping:
AfterClass: true
@ -25,7 +25,7 @@ Standard: Cpp11
PointerAlignment: Middle
MaxEmptyLinesToKeep: 2
KeepEmptyLinesAtTheStartOfBlocks: false
AllowShortFunctionsOnASingleLine: Empty
AllowShortFunctionsOnASingleLine: InlineOnly
AlwaysBreakTemplateDeclarations: true
IndentCaseLabels: true
SpaceAfterTemplateKeyword: true

1
.gitignore vendored
View File

@ -11,6 +11,7 @@
/build
/build_*
/build-*
/docs/build
/docs/edit
/docs/tools/venv/

0
.travis.yml → .travis.yml.bak
View File

2237
CHANGELOG.md
View File

File diff suppressed because it is too large Load Diff

13
CHANGELOG_RU.md
View File

@ -1,3 +1,16 @@
## ClickHouse release 18.14.15, 2018-11-21
### Исправления ошибок:
* При чтении столбцов типа `Array(String)`, размер требуемого куска памяти оценивался слишком большим, что приводило к исключению "Memory limit exceeded" при выполнении запроса. Ошибка появилась в версии 18.12.13. [#3589](https://github.com/yandex/ClickHouse/issues/3589)
## ClickHouse release 18.14.14, 2018-11-20
### Исправления ошибок:
* Исправлена работа запросов `ON CLUSTER` в случае, когда в конфигурации кластера включено шифрование (флаг `<secure>`). [#3599](https://github.com/yandex/ClickHouse/pull/3599)
### Улучшения процесса сборки ClickHouse:
* Испрпавлены проблемы сборки (llvm-7 из системы, macos) [#3582](https://github.com/yandex/ClickHouse/pull/3582)
## ClickHouse release 18.14.13, 2018-11-08
### Исправления ошибок:

2
CMakeLists.txt
View File

@ -115,7 +115,7 @@ endif ()
include (cmake/test_cpu.cmake)
option (ARCH_NATIVE "Enable -march=native compiler flag" OFF)
option (ARCH_NATIVE "Enable -march=native compiler flag" ${ARCH_ARM})
if (ARCH_NATIVE)
set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
endif ()

3
README.md
View File

@ -2,11 +2,10 @@
ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time.
🎤🥂 **ClickHouse Meetup in [Amsterdam on November 15](https://events.yandex.com/events/meetings/15-11-2018/)** 🍰🔥🐻
## Useful Links
* [Official website](https://clickhouse.yandex/) has quick high-level overview of ClickHouse on main page.
* [Tutorial](https://clickhouse.yandex/tutorial.html) shows how to set up and query small ClickHouse cluster.
* [Documentation](https://clickhouse.yandex/docs/en/) provides more in-depth information.
* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
* [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any.

6
cmake/find_llvm.cmake
View File

@ -18,12 +18,12 @@ if (ENABLE_EMBEDDED_COMPILER)
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
find_package(LLVM ${CMAKE_CXX_COMPILER_VERSION} CONFIG PATHS ${LLVM_PATHS})
else ()
find_package (LLVM 6 CONFIG PATHS ${LLVM_PATHS})
find_package (LLVM 7 CONFIG PATHS ${LLVM_PATHS})
if (NOT LLVM_FOUND)
find_package (LLVM 5 CONFIG PATHS ${LLVM_PATHS})
find_package (LLVM 6 CONFIG PATHS ${LLVM_PATHS})
endif ()
if (NOT LLVM_FOUND)
find_package (LLVM 7 CONFIG PATHS ${LLVM_PATHS})
find_package (LLVM 5 CONFIG PATHS ${LLVM_PATHS})
endif ()
endif ()

8
cmake/find_odbc.cmake
View File

@ -71,10 +71,10 @@ if (ENABLE_ODBC)
)
# MinGW find usually fails
if(MINGW)
if (MINGW)
set(ODBC_INCLUDE_DIRECTORIES ".")
set(ODBC_LIBRARIES odbc32)
endif()
endif ()
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(ODBC
@ -82,6 +82,10 @@ if (ENABLE_ODBC)
ODBC_INCLUDE_DIRECTORIES
ODBC_LIBRARIES)
if (USE_STATIC_LIBRARIES)
list(APPEND ODBC_LIBRARIES ${LTDL_LIBRARY})
endif ()
mark_as_advanced(ODBC_FOUND ODBC_LIBRARIES ODBC_INCLUDE_DIRECTORIES)
endif ()
endif ()

4
cmake/find_poco.cmake
View File

@ -93,8 +93,8 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
endif ()
if (OPENSSL_FOUND AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL))
set (Poco_NetSSL_LIBRARY PocoNetSSL)
set (Poco_Crypto_LIBRARY PocoCrypto)
set (Poco_NetSSL_LIBRARY PocoNetSSL ${OPENSSL_LIBRARIES})
set (Poco_Crypto_LIBRARY PocoCrypto ${OPENSSL_LIBRARIES})
endif ()
if (USE_STATIC_LIBRARIES AND USE_INTERNAL_ZLIB_LIBRARY)

1
cmake/lib_name.cmake
View File

@ -1,5 +1,4 @@
set(DIVIDE_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libdivide)
set(CITYHASH_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/cityhash102/include)
set(COMMON_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/libs/libcommon/include ${ClickHouse_BINARY_DIR}/libs/libcommon/include)
set(DBMS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/dbms/src ${ClickHouse_BINARY_DIR}/dbms/src)
set(DOUBLE_CONVERSION_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/double-conversion)

3
cmake/print_include_directories.cmake
View File

@ -10,6 +10,9 @@ list(APPEND dirs ${dirs1})
get_property (dirs1 TARGET common PROPERTY INCLUDE_DIRECTORIES)
list(APPEND dirs ${dirs1})
get_property (dirs1 TARGET cityhash PROPERTY INCLUDE_DIRECTORIES)
list(APPEND dirs ${dirs1})
if (USE_INTERNAL_BOOST_LIBRARY)
get_property (dirs1 TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY} PROPERTY INCLUDE_DIRECTORIES)
list(APPEND dirs ${dirs1})

35
contrib/CMakeLists.txt vendored
View File

@ -159,16 +159,6 @@ if (USE_INTERNAL_POCO_LIBRARY)
endif ()
endif ()
if (USE_INTERNAL_LLVM_LIBRARY)
# ld: unknown option: --color-diagnostics
if (APPLE)
set (LINKER_SUPPORTS_COLOR_DIAGNOSTICS 0 CACHE INTERNAL "")
endif ()
set (LLVM_ENABLE_EH 1 CACHE INTERNAL "")
set (LLVM_ENABLE_RTTI 1 CACHE INTERNAL "")
add_subdirectory (llvm/llvm)
endif ()
if (USE_INTERNAL_GTEST_LIBRARY)
# Google Test from sources
add_subdirectory(${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest ${CMAKE_CURRENT_BINARY_DIR}/googletest)
@ -177,7 +167,30 @@ if (USE_INTERNAL_GTEST_LIBRARY)
target_include_directories (gtest SYSTEM INTERFACE ${ClickHouse_SOURCE_DIR}/contrib/googletest/include)
endif ()
if (USE_INTERNAL_LLVM_LIBRARY)
file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/empty.cpp CONTENT " ")
add_library(LLVM0 ${CMAKE_CURRENT_BINARY_DIR}/empty.cpp) # silly cmake bug fix
# ld: unknown option: --color-diagnostics
if (APPLE)
set (LINKER_SUPPORTS_COLOR_DIAGNOSTICS 0 CACHE INTERNAL "")
endif ()
set (LLVM_ENABLE_EH 1 CACHE INTERNAL "")
set (LLVM_ENABLE_RTTI 1 CACHE INTERNAL "")
set (LLVM_INCLUDE_TESTS 0 CACHE INTERNAL "")
set (LLVM_INCLUDE_EXAMPLES 0 CACHE INTERNAL "")
set (LLVM_INCLUDE_TOOLS 0 CACHE INTERNAL "")
set (LLVM_INSTALL_TOOLCHAIN_ONLY 0 CACHE INTERNAL "")
set (CLANG_BUILT_STANDALONE 0 CACHE INTERNAL "")
set (LLDB_BUILT_STANDALONE 0 CACHE INTERNAL "")
set (CLANG_ENABLE_STATIC_ANALYZER 0 CACHE INTERNAL "")
set (CLANG_ENABLE_ARCMT 0 CACHE INTERNAL "")
set (CLANG_BUILD_TOOLS 0 CACHE INTERNAL "")
set (BENCHMARK_ENABLE_GTEST_TESTS 0 CACHE INTERNAL "")
set (BENCHMARK_ENABLE_ASSEMBLY_TESTS 0 CACHE INTERNAL "")
set (LLVM_TARGETS_TO_BUILD "X86;AArch64" CACHE INTERNAL "")
add_subdirectory (llvm/llvm)
endif ()
if (USE_BASE64)
add_subdirectory (base64-cmake)
endif()
endif()

5
contrib/cityhash102/CMakeLists.txt
View File

@ -1,9 +1,8 @@
add_library(cityhash
src/city.cc
include/citycrc.h
include/city.h
src/config.h)
target_include_directories (cityhash BEFORE PUBLIC include)
target_include_directories (cityhash PRIVATE src)
target_include_directories(cityhash BEFORE PUBLIC include)
target_include_directories(cityhash PRIVATE src)

2
contrib/poco vendored

@ -1 +1 @@
Subproject commit d7a4383c4d85b51938b62ed5812bc0935245edb3
Subproject commit 20c1d877773b6a672f1bbfe3290dfea42a117ed5

2
contrib/ssl vendored

@ -1 +1 @@
Subproject commit 919f6f1331d500bfdd26f8bbbf88e92c0119879b
Subproject commit dbbbcdbbd17785566f8f9c107b714f9e213d7293

52
dbms/CMakeLists.txt
View File

@ -86,7 +86,6 @@ list (APPEND dbms_sources
src/AggregateFunctions/AggregateFunctionFactory.cpp
src/AggregateFunctions/AggregateFunctionCombinatorFactory.cpp
src/AggregateFunctions/AggregateFunctionState.cpp
src/AggregateFunctions/FactoryHelpers.cpp
src/AggregateFunctions/parseAggregateFunctionParameters.cpp)
list (APPEND dbms_headers
@ -120,7 +119,7 @@ endif ()
if (USE_EMBEDDED_COMPILER)
llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
target_link_libraries (dbms ${REQUIRED_LLVM_LIBRARIES})
target_link_libraries (dbms PRIVATE ${REQUIRED_LLVM_LIBRARIES})
target_include_directories (dbms SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
endif ()
@ -151,33 +150,48 @@ if (NOT ARCH_ARM AND CPUID_LIBRARY)
endif()
target_link_libraries (clickhouse_common_io
PUBLIC
common
PRIVATE
string_utils
widechar_width
${LINK_LIBRARIES_ONLY_ON_X86_64}
${LZ4_LIBRARY}
${ZSTD_LIBRARY}
${DOUBLE_CONVERSION_LIBRARIES}
pocoext
PUBLIC
${Poco_Net_LIBRARY}
${Poco_Util_LIBRARY}
${Poco_Foundation_LIBRARY}
${RE2_LIBRARY}
${RE2_ST_LIBRARY}
${CITYHASH_LIBRARIES}
PRIVATE
${ZLIB_LIBRARIES}
${EXECINFO_LIBRARY}
${ELF_LIBRARY}
PUBLIC
${Boost_SYSTEM_LIBRARY}
PRIVATE
apple_rt
${CMAKE_DL_LIBS}
)
target_link_libraries (dbms
PRIVATE
clickhouse_parsers
clickhouse_common_config
PUBLIC
clickhouse_common_io
pocoext
PUBLIC
${MYSQLXX_LIBRARY}
${RE2_LIBRARY}
${RE2_ST_LIBRARY}
PRIVATE
${BTRIE_LIBRARIES}
${Boost_PROGRAM_OPTIONS_LIBRARY}
PUBLIC
${Boost_SYSTEM_LIBRARY}
)
if (NOT USE_INTERNAL_RE2_LIBRARY)
@ -194,8 +208,8 @@ if (Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
endif()
if (USE_POCO_SQLODBC)
target_link_libraries (clickhouse_common_io ${Poco_SQL_LIBRARY})
target_link_libraries (dbms ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
target_link_libraries (clickhouse_common_io PRIVATE ${Poco_SQL_LIBRARY})
target_link_libraries (dbms PRIVATE ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
if (NOT USE_INTERNAL_POCO_LIBRARY)
target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIR})
target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR} PUBLIC ${Poco_SQL_INCLUDE_DIR})
@ -209,48 +223,44 @@ if (Poco_Data_FOUND)
endif()
if (USE_POCO_DATAODBC)
target_link_libraries (clickhouse_common_io ${Poco_Data_LIBRARY})
target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
target_link_libraries (clickhouse_common_io PRIVATE ${Poco_Data_LIBRARY})
target_link_libraries (dbms PRIVATE ${Poco_DataODBC_LIBRARY})
if (NOT USE_INTERNAL_POCO_LIBRARY)
target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
endif()
endif()
if (USE_POCO_MONGODB)
target_link_libraries (dbms ${Poco_MongoDB_LIBRARY})
target_link_libraries (dbms PRIVATE ${Poco_MongoDB_LIBRARY})
endif()
if (USE_POCO_NETSSL)
target_link_libraries (clickhouse_common_io ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
target_link_libraries (dbms ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
target_link_libraries (clickhouse_common_io PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
target_link_libraries (dbms PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
endif()
target_link_libraries (dbms ${Poco_Foundation_LIBRARY})
target_link_libraries (dbms PRIVATE ${Poco_Foundation_LIBRARY})
if (USE_ICU)
target_link_libraries (dbms ${ICU_LIBS})
target_link_libraries (dbms PRIVATE ${ICU_LIBS})
target_include_directories (dbms SYSTEM PRIVATE ${ICU_INCLUDE_DIR})
endif ()
if (USE_CAPNP)
target_link_libraries (dbms ${CAPNP_LIBRARY})
target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARY})
if (NOT USE_INTERNAL_CAPNP_LIBRARY)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR})
endif ()
endif ()
if (USE_RDKAFKA)
target_link_libraries (dbms ${RDKAFKA_LIBRARY})
target_link_libraries (dbms PRIVATE ${RDKAFKA_LIBRARY})
if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
endif ()
endif ()
target_link_libraries(dbms ${OPENSSL_CRYPTO_LIBRARY})
target_link_libraries (dbms
Threads::Threads
)
target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY} Threads::Threads)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
@ -286,6 +296,6 @@ if (ENABLE_TESTS)
# attach all dbms gtest sources
grep_gtest_sources(${ClickHouse_SOURCE_DIR}/dbms dbms_gtest_sources)
add_executable(unit_tests_dbms ${dbms_gtest_sources})
target_link_libraries(unit_tests_dbms gtest_main dbms)
target_link_libraries(unit_tests_dbms PRIVATE gtest_main dbms clickhouse_common_zookeeper)
add_check(unit_tests_dbms)
endif ()

2
dbms/cmake/version.cmake
View File

@ -22,3 +22,5 @@ endif ()
set (VERSION_NAME "${PROJECT_NAME}" CACHE STRING "")
set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}" CACHE STRING "")
set (VERSION_SO "${VERSION_STRING}" CACHE STRING "")
math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000")

26
dbms/programs/CMakeLists.txt
View File

@ -48,45 +48,45 @@ else ()
link_directories (${LLVM_LIBRARY_DIRS})
endif ()
add_executable (clickhouse main.cpp)
target_link_libraries (clickhouse clickhouse_common_io)
target_link_libraries (clickhouse PRIVATE clickhouse_common_io)
target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
if (ENABLE_CLICKHOUSE_SERVER)
target_link_libraries (clickhouse clickhouse-server-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-server-lib)
endif ()
if (ENABLE_CLICKHOUSE_CLIENT)
target_link_libraries (clickhouse clickhouse-client-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-client-lib)
endif ()
if (ENABLE_CLICKHOUSE_LOCAL)
target_link_libraries (clickhouse clickhouse-local-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-local-lib)
endif ()
if (ENABLE_CLICKHOUSE_BENCHMARK)
target_link_libraries (clickhouse clickhouse-benchmark-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-benchmark-lib)
endif ()
if (ENABLE_CLICKHOUSE_PERFORMANCE)
target_link_libraries (clickhouse clickhouse-performance-test-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-performance-test-lib)
endif ()
if (ENABLE_CLICKHOUSE_COPIER)
target_link_libraries (clickhouse clickhouse-copier-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-copier-lib)
endif ()
if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
target_link_libraries (clickhouse clickhouse-extract-from-config-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-extract-from-config-lib)
endif ()
if (ENABLE_CLICKHOUSE_COMPRESSOR)
target_link_libraries (clickhouse clickhouse-compressor-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-compressor-lib)
endif ()
if (ENABLE_CLICKHOUSE_FORMAT)
target_link_libraries (clickhouse clickhouse-format-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-format-lib)
endif ()
if (ENABLE_CLICKHOUSE_OBFUSCATOR)
target_link_libraries (clickhouse clickhouse-obfuscator-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-obfuscator-lib)
endif ()
if (USE_EMBEDDED_COMPILER)
target_link_libraries (clickhouse clickhouse-compiler-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-compiler-lib)
endif ()
if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
target_link_libraries (clickhouse clickhouse-odbc-bridge-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-odbc-bridge-lib)
endif()
set (CLICKHOUSE_BUNDLE)

6
dbms/programs/benchmark/Benchmark.cpp
View File

@ -42,10 +42,8 @@ namespace DB
namespace ErrorCodes
{
extern const int POCO_EXCEPTION;
extern const int STD_EXCEPTION;
extern const int UNKNOWN_EXCEPTION;
extern const int BAD_ARGUMENTS;
extern const int EMPTY_DATA_PASSED;
}
class Benchmark
@ -170,7 +168,7 @@ private:
}
if (queries.empty())
throw Exception("Empty list of queries.");
throw Exception("Empty list of queries.", ErrorCodes::EMPTY_DATA_PASSED);
std::cerr << "Loaded " << queries.size() << " queries.\n";
}

4
dbms/programs/benchmark/CMakeLists.txt
View File

@ -1,8 +1,8 @@
add_library (clickhouse-benchmark-lib ${LINK_MODE} Benchmark.cpp)
target_link_libraries (clickhouse-benchmark-lib clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-benchmark-lib PRIVATE clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_include_directories (clickhouse-benchmark-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-benchmark clickhouse-benchmark.cpp)
target_link_libraries (clickhouse-benchmark clickhouse-benchmark-lib clickhouse_aggregate_functions)
target_link_libraries (clickhouse-benchmark PRIVATE clickhouse-benchmark-lib clickhouse_aggregate_functions)
endif ()

4
dbms/programs/clang/CMakeLists.txt
View File

@ -6,9 +6,9 @@ if (CLICKHOUSE_SPLIT_BINARY)
if (USE_EMBEDDED_COMPILER)
link_directories (${LLVM_LIBRARY_DIRS})
add_executable (clickhouse-clang clickhouse-clang.cpp)
target_link_libraries (clickhouse-clang clickhouse-compiler-lib)
target_link_libraries (clickhouse-clang PRIVATE clickhouse-compiler-lib)
add_executable (clickhouse-lld clickhouse-lld.cpp)
target_link_libraries (clickhouse-lld clickhouse-compiler-lib)
target_link_libraries (clickhouse-lld PRIVATE clickhouse-compiler-lib)
install (TARGETS clickhouse-clang clickhouse-lld RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
endif ()
endif ()

2
dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt
View File

@ -1,5 +1,7 @@
add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG)
link_directories(${LLVM_LIBRARY_DIRS})
add_library(clickhouse-compiler-lib ${LINK_MODE}
driver.cpp
cc1_main.cpp

2
dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt
View File

@ -1,5 +1,7 @@
add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG)
link_directories(${LLVM_LIBRARY_DIRS})
add_library(clickhouse-compiler-lib ${LINK_MODE}
driver.cpp
cc1_main.cpp

2
dbms/programs/clang/Compiler-6.0.0/cc1as_main.cpp
View File

@ -441,7 +441,7 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
auto Pair = StringRef(S).split('=');
auto Sym = Pair.first;
auto Val = Pair.second;
int64_t Value;
int64_t Value = 0;
// We have already error checked this in the driver.
Val.getAsInteger(0, Value);
Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value);

4
dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt
View File

@ -1,8 +1,11 @@
add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG)
link_directories(${LLVM_LIBRARY_DIRS})
add_library(clickhouse-compiler-lib ${LINK_MODE}
driver.cpp
cc1_main.cpp
cc1gen_reproducer_main.cpp
cc1as_main.cpp
lld.cpp)
@ -19,7 +22,6 @@ target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE
# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.
target_link_libraries(clickhouse-compiler-lib PRIVATE
clangBasic clangCodeGen clangDriver
clangFrontend
clangFrontendTool

24
dbms/programs/clang/Compiler-7.0.0/cc1_main.cpp
View File

@ -16,6 +16,7 @@
#include "llvm/Option/Arg.h"
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
#include "clang/Config/config.h"
#include "clang/Basic/Stack.h"
#include "clang/Driver/DriverDiagnostic.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/CompilerInstance.h"
@ -76,13 +77,6 @@ void initializePollyPasses(llvm::PassRegistry &Registry);
#endif
#ifdef CLANG_HAVE_RLIMITS
// The amount of stack we think is "sufficient". If less than this much is
// available, we may be unable to reach our template instantiation depth
// limit and other similar limits.
// FIXME: Unify this with the stack we request when spawning a thread to build
// a module.
static const int kSufficientStack = 8 << 20;
#if defined(__linux__) && defined(__PIE__)
static size_t getCurrentStackAllocation() {
// If we can't compute the current stack usage, allow for 512K of command
@ -120,7 +114,7 @@ static size_t getCurrentStackAllocation() {
#include <alloca.h>
LLVM_ATTRIBUTE_NOINLINE
static void ensureStackAddressSpace(int ExtraChunks = 0) {
static void ensureStackAddressSpace() {
// Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary
// relatively close to the stack (they are only guaranteed to be 128MiB
// apart). This results in crashes if we happen to heap-allocate more than
@ -129,7 +123,7 @@ static void ensureStackAddressSpace(int ExtraChunks = 0) {
// To avoid these crashes, ensure that we have sufficient virtual memory
// pages allocated before we start running.
size_t Curr = getCurrentStackAllocation();
const int kTargetStack = kSufficientStack - 256 * 1024;
const int kTargetStack = DesiredStackSize - 256 * 1024;
if (Curr < kTargetStack) {
volatile char *volatile Alloc =
static_cast<volatile char *>(alloca(kTargetStack - Curr));
@ -149,21 +143,23 @@ static void ensureSufficientStack() {
// Increase the soft stack limit to our desired level, if necessary and
// possible.
if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) {
if (rlim.rlim_cur != RLIM_INFINITY &&
rlim.rlim_cur < rlim_t(DesiredStackSize)) {
// Try to allocate sufficient stack.
if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack)
rlim.rlim_cur = kSufficientStack;
if (rlim.rlim_max == RLIM_INFINITY ||
rlim.rlim_max >= rlim_t(DesiredStackSize))
rlim.rlim_cur = DesiredStackSize;
else if (rlim.rlim_cur == rlim.rlim_max)
return;
else
rlim.rlim_cur = rlim.rlim_max;
if (setrlimit(RLIMIT_STACK, &rlim) != 0 ||
rlim.rlim_cur != kSufficientStack)
rlim.rlim_cur != DesiredStackSize)
return;
}
// We should now have a stack of size at least kSufficientStack. Ensure
// We should now have a stack of size at least DesiredStackSize. Ensure
// that we can actually use that much, if necessary.
ensureStackAddressSpace();
}

57
dbms/programs/clang/Compiler-7.0.0/cc1as_main.cpp
View File

@ -29,6 +29,7 @@
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCInstrInfo.h"
#include "llvm/MC/MCObjectFileInfo.h"
#include "llvm/MC/MCObjectWriter.h"
#include "llvm/MC/MCParser/MCAsmParser.h"
#include "llvm/MC/MCParser/MCTargetAsmParser.h"
#include "llvm/MC/MCRegisterInfo.h"
@ -59,10 +60,9 @@ using namespace clang::driver::options;
using namespace llvm;
using namespace llvm::opt;
namespace {
/// \brief Helper class for representing a single invocation of the assembler.
/// Helper class for representing a single invocation of the assembler.
struct AssemblerInvocation {
/// @name Target Options
/// @{
@ -94,9 +94,11 @@ struct AssemblerInvocation {
std::string DwarfDebugFlags;
std::string DwarfDebugProducer;
std::string DebugCompilationDir;
std::map<const std::string, const std::string> DebugPrefixMap;
llvm::DebugCompressionType CompressDebugSections =
llvm::DebugCompressionType::None;
std::string MainFileName;
std::string SplitDwarfFile;
/// @}
/// @name Frontend Options
@ -232,6 +234,9 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir);
Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name);
for (const auto &Arg : Args.getAllArgValues(OPT_fdebug_prefix_map_EQ))
Opts.DebugPrefixMap.insert(StringRef(Arg).split('='));
// Frontend Options
if (Args.hasArg(OPT_INPUT)) {
bool First = true;
@ -247,6 +252,7 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
}
Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm);
Opts.OutputPath = Args.getLastArgValue(OPT_o);
Opts.SplitDwarfFile = Args.getLastArgValue(OPT_split_dwarf_file);
if (Arg *A = Args.getLastArg(OPT_filetype)) {
StringRef Name = A->getValue();
unsigned OutputType = StringSwitch<unsigned>(Name)
@ -282,22 +288,17 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
}
static std::unique_ptr<raw_fd_ostream>
getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags,
bool Binary) {
if (Opts.OutputPath.empty())
Opts.OutputPath = "-";
getOutputStream(StringRef Path, DiagnosticsEngine &Diags, bool Binary) {
// Make sure that the Out file gets unlinked from the disk if we get a
// SIGINT.
if (Opts.OutputPath != "-")
sys::RemoveFileOnSignal(Opts.OutputPath);
if (Path != "-")
sys::RemoveFileOnSignal(Path);
std::error_code EC;
auto Out = llvm::make_unique<raw_fd_ostream>(
Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text));
Path, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text));
if (EC) {
Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath
<< EC.message();
Diags.Report(diag::err_fe_unable_to_open_output) << Path << EC.message();
return nullptr;
}
@ -342,9 +343,15 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations);
bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj;
std::unique_ptr<raw_fd_ostream> FDOS = getOutputStream(Opts, Diags, IsBinary);
if (Opts.OutputPath.empty())
Opts.OutputPath = "-";
std::unique_ptr<raw_fd_ostream> FDOS =
getOutputStream(Opts.OutputPath, Diags, IsBinary);
if (!FDOS)
return true;
std::unique_ptr<raw_fd_ostream> DwoOS;
if (!Opts.SplitDwarfFile.empty())
DwoOS = getOutputStream(Opts.SplitDwarfFile, Diags, IsBinary);
// FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and
// MCObjectFileInfo needs a MCContext reference in order to initialize itself.
@ -374,6 +381,9 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer));
if (!Opts.DebugCompilationDir.empty())
Ctx.setCompilationDir(Opts.DebugCompilationDir);
if (!Opts.DebugPrefixMap.empty())
for (const auto &KV : Opts.DebugPrefixMap)
Ctx.addDebugPrefixMapEntry(KV.first, KV.second);
if (!Opts.MainFileName.empty())
Ctx.setMainFileName(StringRef(Opts.MainFileName));
Ctx.setDwarfVersion(Opts.DwarfVersion);
@ -427,11 +437,14 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
MCTargetOptions MCOptions;
std::unique_ptr<MCAsmBackend> MAB(
TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
std::unique_ptr<MCObjectWriter> OW =
DwoOS ? MAB->createDwoObjectWriter(*Out, *DwoOS)
: MAB->createObjectWriter(*Out);
Triple T(Opts.Triple);
Str.reset(TheTarget->createMCObjectStreamer(
T, Ctx, std::move(MAB), *Out, std::move(CE), *STI, Opts.RelaxAll,
Opts.IncrementalLinkerCompatible,
T, Ctx, std::move(MAB), std::move(OW), std::move(CE), *STI,
Opts.RelaxAll, Opts.IncrementalLinkerCompatible,
/*DWARFMustBeAtTheEnd*/ true));
Str.get()->InitSections(Opts.NoExecStack);
}
@ -456,7 +469,7 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
auto Pair = StringRef(S).split('=');
auto Sym = Pair.first;
auto Val = Pair.second;
int64_t Value;
int64_t Value = 1;
// We have already error checked this in the driver.
Val.getAsInteger(0, Value);
Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value);
@ -475,14 +488,18 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
FDOS.reset();
// Delete output file if there were errors.
if (Failed && Opts.OutputPath != "-")
sys::fs::remove(Opts.OutputPath);
if (Failed) {
if (Opts.OutputPath != "-")
sys::fs::remove(Opts.OutputPath);
if (!Opts.SplitDwarfFile.empty() && Opts.SplitDwarfFile != "-")
sys::fs::remove(Opts.SplitDwarfFile);
}
return Failed;
}
static void LLVMErrorHandler(void *UserData, const std::string &Message,
bool /*GenCrashDiag*/) {
bool GenCrashDiag) {
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
Diags.Report(diag::err_fe_error_backend) << Message;
@ -491,7 +508,7 @@ static void LLVMErrorHandler(void *UserData, const std::string &Message,
exit(1);
}
int cc1as_main(ArrayRef<const char *> Argv, const char */*Argv0*/, void */*MainAddr*/) {
int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
// Initialize targets and assembly printers/parsers.
InitializeAllTargetInfos();
InitializeAllTargetMCs();

196
dbms/programs/clang/Compiler-7.0.0/cc1gen_reproducer_main.cpp Normal file
View File

@ -0,0 +1,196 @@
//===-- cc1gen_reproducer_main.cpp - Clang reproducer generator ----------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the clang -cc1gen-reproducer functionality, which
// generates reproducers for invocations for clang-based tools.
//
//===----------------------------------------------------------------------===//
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/LLVM.h"
#include "clang/Basic/VirtualFileSystem.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/Driver.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/YAMLTraits.h"
#include "llvm/Support/raw_ostream.h"
using namespace clang;
namespace {
struct UnsavedFileHash {
std::string Name;
std::string MD5;
};
struct ClangInvocationInfo {
std::string Toolchain;
std::string LibclangOperation;
std::string LibclangOptions;
std::vector<std::string> Arguments;
std::vector<std::string> InvocationArguments;
std::vector<UnsavedFileHash> UnsavedFileHashes;
bool Dump = false;
};
} // end anonymous namespace
LLVM_YAML_IS_SEQUENCE_VECTOR(UnsavedFileHash)
namespace llvm {
namespace yaml {
template <> struct MappingTraits<UnsavedFileHash> {
static void mapping(IO &IO, UnsavedFileHash &Info) {
IO.mapRequired("name", Info.Name);
IO.mapRequired("md5", Info.MD5);
}
};
template <> struct MappingTraits<ClangInvocationInfo> {
static void mapping(IO &IO, ClangInvocationInfo &Info) {
IO.mapRequired("toolchain", Info.Toolchain);
IO.mapOptional("libclang.operation", Info.LibclangOperation);
IO.mapOptional("libclang.opts", Info.LibclangOptions);
IO.mapRequired("args", Info.Arguments);
IO.mapOptional("invocation-args", Info.InvocationArguments);
IO.mapOptional("unsaved_file_hashes", Info.UnsavedFileHashes);
}
};
} // end namespace yaml
} // end namespace llvm
static std::string generateReproducerMetaInfo(const ClangInvocationInfo &Info) {
std::string Result;
llvm::raw_string_ostream OS(Result);
OS << '{';
bool NeedComma = false;
auto EmitKey = [&](StringRef Key) {
if (NeedComma)
OS << ", ";
NeedComma = true;
OS << '"' << Key << "\": ";
};
auto EmitStringKey = [&](StringRef Key, StringRef Value) {
if (Value.empty())
return;
EmitKey(Key);
OS << '"' << Value << '"';
};
EmitStringKey("libclang.operation", Info.LibclangOperation);
EmitStringKey("libclang.opts", Info.LibclangOptions);
if (!Info.InvocationArguments.empty()) {
EmitKey("invocation-args");
OS << '[';
for (const auto &Arg : llvm::enumerate(Info.InvocationArguments)) {
if (Arg.index())
OS << ',';
OS << '"' << Arg.value() << '"';
}
OS << ']';
}
OS << '}';
// FIXME: Compare unsaved file hashes and report mismatch in the reproducer.
if (Info.Dump)
llvm::outs() << "REPRODUCER METAINFO: " << OS.str() << "\n";
return std::move(OS.str());
}
/// Generates a reproducer for a set of arguments from a specific invocation.
static llvm::Optional<driver::Driver::CompilationDiagnosticReport>
generateReproducerForInvocationArguments(ArrayRef<const char *> Argv,
const ClangInvocationInfo &Info) {
using namespace driver;
auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(Argv[0]);
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions;
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
DiagnosticsEngine Diags(DiagID, &*DiagOpts, new IgnoringDiagConsumer());
ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false);
Driver TheDriver(Argv[0], llvm::sys::getDefaultTargetTriple(), Diags);
TheDriver.setTargetAndMode(TargetAndMode);
std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(Argv));
if (C && !C->containsError()) {
for (const auto &J : C->getJobs()) {
if (const Command *Cmd = dyn_cast<Command>(&J)) {
Driver::CompilationDiagnosticReport Report;
TheDriver.generateCompilationDiagnostics(
*C, *Cmd, generateReproducerMetaInfo(Info), &Report);
return Report;
}
}
}
return None;
}
std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes);
static void printReproducerInformation(
llvm::raw_ostream &OS, const ClangInvocationInfo &Info,
const driver::Driver::CompilationDiagnosticReport &Report) {
OS << "REPRODUCER:\n";
OS << "{\n";
OS << R"("files":[)";
for (const auto &File : llvm::enumerate(Report.TemporaryFiles)) {
if (File.index())
OS << ',';
OS << '"' << File.value() << '"';
}
OS << "]\n}\n";
}
int cc1gen_reproducer_main(ArrayRef<const char *> Argv, const char *Argv0,
void *MainAddr) {
if (Argv.size() < 1) {
llvm::errs() << "error: missing invocation file\n";
return 1;
}
// Parse the invocation descriptor.
StringRef Input = Argv[0];
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
llvm::MemoryBuffer::getFile(Input);
if (!Buffer) {
llvm::errs() << "error: failed to read " << Input << ": "
<< Buffer.getError().message() << "\n";
return 1;
}
llvm::yaml::Input YAML(Buffer.get()->getBuffer());
ClangInvocationInfo InvocationInfo;
YAML >> InvocationInfo;
if (Argv.size() > 1 && Argv[1] == StringRef("-v"))
InvocationInfo.Dump = true;
// Create an invocation that will produce the reproducer.
std::vector<const char *> DriverArgs;
for (const auto &Arg : InvocationInfo.Arguments)
DriverArgs.push_back(Arg.c_str());
std::string Path = GetExecutablePath(Argv0, /*CanonicalPrefixes=*/true);
DriverArgs[0] = Path.c_str();
llvm::Optional<driver::Driver::CompilationDiagnosticReport> Report =
generateReproducerForInvocationArguments(DriverArgs, InvocationInfo);
// Emit the information about the reproduce files to stdout.
int Result = 1;
if (Report) {
printReproducerInformation(llvm::outs(), InvocationInfo, *Report);
Result = 0;
}
// Remove the input file.
llvm::sys::fs::remove(Input);
return Result;
}

4
dbms/programs/clang/Compiler-7.0.0/driver.cpp
View File

@ -310,6 +310,8 @@ static int ExecuteCC1Tool(ArrayRef<const char *> argv, StringRef Tool) {
return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP);
if (Tool == "as")
return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP);
if (Tool == "gen-reproducer")
return cc1gen_reproducer_main(argv.slice(2), argv[0], GetExecutablePathVP);
// Reject unknown tools.
llvm::errs() << "error: unknown integrated tool '" << Tool << "'. "
@ -317,7 +319,7 @@ static int ExecuteCC1Tool(ArrayRef<const char *> argv, StringRef Tool) {
return 1;
}
int mainEntryClickHouseClang(int argc_, char **argv_) {
int mainEntryClickHouseClang(int argc_, /* const */ char **argv_) {
llvm::InitLLVM X(argc_, argv_);
SmallVector<const char *, 256> argv(argv_, argv_ + argc_);

156
dbms/programs/clang/Compiler-7.0.0/lld.cpp
View File

@ -1,10 +1,150 @@
#include "lld/Common/Driver.h"
#include "llvm/Support/InitLLVM.h"
#include <vector>
//===- tools/lld/lld.cpp - Linker Driver Dispatcher -----------------------===//
//
// The LLVM Linker
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file contains the main function of the lld executable. The main
// function is a thin wrapper which dispatches to the platform specific
// driver.
//
// lld is a single executable that contains four different linkers for ELF,
// COFF, WebAssembly and Mach-O. The main function dispatches according to
// argv[0] (i.e. command name). The most common name for each target is shown
// below:
//
// - ld.lld: ELF (Unix)
// - ld64: Mach-O (macOS)
// - lld-link: COFF (Windows)
// - ld-wasm: WebAssembly
//
// lld can be invoked as "lld" along with "-flavor" option. This is for
// backward compatibility and not recommended.
//
//===----------------------------------------------------------------------===//
int mainEntryClickHouseLLD(int argc, char ** argv)
{
llvm::InitLLVM X(argc, argv);
std::vector<const char *> args(argv, argv + argc);
return !lld::elf::link(args, false);
#include "lld/Common/Driver.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Support/InitLLVM.h"
#include "llvm/Support/Path.h"
#include <cstdlib>
using namespace lld;
using namespace llvm;
using namespace llvm::sys;
/*
enum Flavor {
Invalid,
Gnu, // -flavor gnu
WinLink, // -flavor link
Darwin, // -flavor darwin
Wasm, // -flavor wasm
};
LLVM_ATTRIBUTE_NORETURN static void die(const Twine &S) {
errs() << S << "\n";
exit(1);
}
static Flavor getFlavor(StringRef S) {
return StringSwitch<Flavor>(S)
.CasesLower("ld", "ld.lld", "gnu", Gnu)
.CasesLower("wasm", "ld-wasm", Wasm)
.CaseLower("link", WinLink)
.CasesLower("ld64", "ld64.lld", "darwin", Darwin)
.Default(Invalid);
}
static bool isPETarget(const std::vector<const char *> &V) {
for (auto It = V.begin(); It + 1 != V.end(); ++It) {
if (StringRef(*It) != "-m")
continue;
StringRef S = *(It + 1);
return S == "i386pe" || S == "i386pep" || S == "thumb2pe" || S == "arm64pe";
}
return false;
}
static Flavor parseProgname(StringRef Progname) {
#if __APPLE__
// Use Darwin driver for "ld" on Darwin.
if (Progname == "ld")
return Darwin;
#endif
#if LLVM_ON_UNIX
// Use GNU driver for "ld" on other Unix-like system.
if (Progname == "ld")
return Gnu;
#endif
// Progname may be something like "lld-gnu". Parse it.
SmallVector<StringRef, 3> V;
Progname.split(V, "-");
for (StringRef S : V)
if (Flavor F = getFlavor(S))
return F;
return Invalid;
}
static Flavor parseFlavor(std::vector<const char *> &V) {
// Parse -flavor option.
if (V.size() > 1 && V[1] == StringRef("-flavor")) {
if (V.size() <= 2)
die("missing arg value for '-flavor'");
Flavor F = getFlavor(V[2]);
if (F == Invalid)
die("Unknown flavor: " + StringRef(V[2]));
V.erase(V.begin() + 1, V.begin() + 3);
return F;
}
// Deduct the flavor from argv[0].
StringRef Arg0 = path::filename(V[0]);
if (Arg0.endswith_lower(".exe"))
Arg0 = Arg0.drop_back(4);
return parseProgname(Arg0);
}
*/
// If this function returns true, lld calls _exit() so that it quickly
// exits without invoking destructors of globally allocated objects.
//
// We don't want to do that if we are running tests though, because
// doing that breaks leak sanitizer. So, lit sets this environment variable,
// and we use it to detect whether we are running tests or not.
static bool canExitEarly() { return StringRef(getenv("LLD_IN_TEST")) != "1"; }
/// Universal linker main(). This linker emulates the gnu, darwin, or
/// windows linker based on the argv[0] or -flavor option.
int mainEntryClickHouseLLD(int Argc, /* const */ char **Argv) {
InitLLVM X(Argc, Argv);
std::vector<const char *> Args(Argv, Argv + Argc);
/*
switch (parseFlavor(Args)) {
case Gnu:
if (isPETarget(Args))
return !mingw::link(Args);
*/
return !elf::link(Args, canExitEarly());
/*
case WinLink:
return !coff::link(Args, canExitEarly());
case Darwin:
return !mach_o::link(Args, canExitEarly());
case Wasm:
return !wasm::link(Args, canExitEarly());
default:
die("lld is a generic driver.\n"
"Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-lld"
" (WebAssembly) instead");
}
*/
}

1
dbms/programs/clang/Compiler-7.0.0bundled
View File

@ -1 +0,0 @@
Compiler-7.0.0

49
dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt Normal file
View File

@ -0,0 +1,49 @@
add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG)
link_directories(${LLVM_LIBRARY_DIRS})
add_library(clickhouse-compiler-lib ${LINK_MODE}
driver.cpp
cc1_main.cpp
cc1as_main.cpp
lld.cpp)
target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0)
string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti
llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS})
# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.
target_link_libraries(clickhouse-compiler-lib PRIVATE
clangBasic clangCodeGen clangDriver
clangFrontend
clangFrontendTool
clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend
clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers
clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic
clangCrossTU clangIndex
lldCOFF
lldDriver
lldELF
lldMinGW
lldMachO
lldReaderWriter
lldYAML
lldCommon
lldCore
${REQUIRED_LLVM_LIBRARIES}
PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads
${MALLOC_LIBRARIES}
${GLIBC_COMPATIBILITY_LIBRARIES}
${MEMCPY_LIBRARIES}
)

243
dbms/programs/clang/Compiler-7.0.0bundled/cc1_main.cpp Normal file
View File

@ -0,0 +1,243 @@
//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the clang -cc1 functionality, which implements the
// core compiler functionality along with a number of additional tools for
// demonstration and testing purposes.
//
//===----------------------------------------------------------------------===//
#include "llvm/Option/Arg.h"
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
#include "clang/Config/config.h"
#include "clang/Driver/DriverDiagnostic.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "clang/Frontend/TextDiagnosticBuffer.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "clang/FrontendTool/Utils.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/LinkAllPasses.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/raw_ostream.h"
#include <cstdio>
#ifdef CLANG_HAVE_RLIMITS
#include <sys/resource.h>
#endif
// have no .a version in packages
#undef LINK_POLLY_INTO_TOOLS
using namespace clang;
using namespace llvm::opt;
//===----------------------------------------------------------------------===//
// Main driver
//===----------------------------------------------------------------------===//
static void LLVMErrorHandler(void *UserData, const std::string &Message,
bool GenCrashDiag) {
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
Diags.Report(diag::err_fe_error_backend) << Message;
// Run the interrupt handlers to make sure any special cleanups get done, in
// particular that we remove files registered with RemoveFileOnSignal.
llvm::sys::RunInterruptHandlers();
// We cannot recover from llvm errors. When reporting a fatal error, exit
// with status 70 to generate crash diagnostics. For BSD systems this is
// defined as an internal software error. Otherwise, exit with status 1.
exit(GenCrashDiag ? 70 : 1);
}
#ifdef LINK_POLLY_INTO_TOOLS
namespace polly {
void initializePollyPasses(llvm::PassRegistry &Registry);
}
#endif
#ifdef CLANG_HAVE_RLIMITS
// The amount of stack we think is "sufficient". If less than this much is
// available, we may be unable to reach our template instantiation depth
// limit and other similar limits.
// FIXME: Unify this with the stack we request when spawning a thread to build
// a module.
static const int kSufficientStack = 8 << 20;
#if defined(__linux__) && defined(__PIE__)
static size_t getCurrentStackAllocation() {
// If we can't compute the current stack usage, allow for 512K of command
// line arguments and environment.
size_t Usage = 512 * 1024;
if (FILE *StatFile = fopen("/proc/self/stat", "r")) {
// We assume that the stack extends from its current address to the end of
// the environment space. In reality, there is another string literal (the
// program name) after the environment, but this is close enough (we only
// need to be within 100K or so).
unsigned long StackPtr, EnvEnd;
// Disable silly GCC -Wformat warning that complains about length
// modifiers on ignored format specifiers. We want to retain these
// for documentation purposes even though they have no effect.
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wformat"
#endif
if (fscanf(StatFile,
"%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu "
"%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu "
"%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d "
"%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d",
&StackPtr, &EnvEnd) == 2) {
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic pop
#endif
Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd;
}
fclose(StatFile);
}
return Usage;
}
#include <alloca.h>
LLVM_ATTRIBUTE_NOINLINE
static void ensureStackAddressSpace(int ExtraChunks = 0) {
// Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary
// relatively close to the stack (they are only guaranteed to be 128MiB
// apart). This results in crashes if we happen to heap-allocate more than
// 128MiB before we reach our stack high-water mark.
//
// To avoid these crashes, ensure that we have sufficient virtual memory
// pages allocated before we start running.
size_t Curr = getCurrentStackAllocation();
const int kTargetStack = kSufficientStack - 256 * 1024;
if (Curr < kTargetStack) {
volatile char *volatile Alloc =
static_cast<volatile char *>(alloca(kTargetStack - Curr));
Alloc[0] = 0;
Alloc[kTargetStack - Curr - 1] = 0;
}
}
#else
static void ensureStackAddressSpace() {}
#endif
/// Attempt to ensure that we have at least 8MiB of usable stack space.
static void ensureSufficientStack() {
struct rlimit rlim;
if (getrlimit(RLIMIT_STACK, &rlim) != 0)
return;
// Increase the soft stack limit to our desired level, if necessary and
// possible.
if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) {
// Try to allocate sufficient stack.
if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack)
rlim.rlim_cur = kSufficientStack;
else if (rlim.rlim_cur == rlim.rlim_max)
return;
else
rlim.rlim_cur = rlim.rlim_max;
if (setrlimit(RLIMIT_STACK, &rlim) != 0 ||
rlim.rlim_cur != kSufficientStack)
return;
}
// We should now have a stack of size at least kSufficientStack. Ensure
// that we can actually use that much, if necessary.
ensureStackAddressSpace();
}
#else
static void ensureSufficientStack() {}
#endif
int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
ensureSufficientStack();
std::unique_ptr<CompilerInstance> Clang(new CompilerInstance());
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
// Register the support for object-file-wrapped Clang modules.
auto PCHOps = Clang->getPCHContainerOperations();
PCHOps->registerWriter(llvm::make_unique<ObjectFilePCHContainerWriter>());
PCHOps->registerReader(llvm::make_unique<ObjectFilePCHContainerReader>());
// Initialize targets first, so that --version shows registered targets.
llvm::InitializeAllTargets();
llvm::InitializeAllTargetMCs();
llvm::InitializeAllAsmPrinters();
llvm::InitializeAllAsmParsers();
#ifdef LINK_POLLY_INTO_TOOLS
llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry();
polly::initializePollyPasses(Registry);
#endif
// Buffer diagnostics from argument parsing so that we can output them using a
// well formed diagnostic object.
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer;
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer);
bool Success = CompilerInvocation::CreateFromArgs(
Clang->getInvocation(), Argv.begin(), Argv.end(), Diags);
// Infer the builtin include path if unspecified.
if (Clang->getHeaderSearchOpts().UseBuiltinIncludes &&
Clang->getHeaderSearchOpts().ResourceDir.empty())
Clang->getHeaderSearchOpts().ResourceDir =
CompilerInvocation::GetResourcesPath(Argv0, MainAddr);
// Create the actual diagnostics engine.
Clang->createDiagnostics();
if (!Clang->hasDiagnostics())
return 1;
// Set an error handler, so that any LLVM backend diagnostics go through our
// error handler.
llvm::install_fatal_error_handler(LLVMErrorHandler,
static_cast<void*>(&Clang->getDiagnostics()));
DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics());
if (!Success)
return 1;
// Execute the frontend actions.
Success = ExecuteCompilerInvocation(Clang.get());
// If any timers were active but haven't been destroyed yet, print their
// results now. This happens in -disable-free mode.
llvm::TimerGroup::printAll(llvm::errs());
// Our error handler depends on the Diagnostics object, which we're
// potentially about to delete. Uninstall the handler now so that any
// later errors use the default handling behavior instead.
llvm::remove_fatal_error_handler();
// When running with -disable-free, don't do any destruction or shutdown.
if (Clang->getFrontendOpts().DisableFree) {
BuryPointer(std::move(Clang));
return !Success;
}
return !Success;
}

555
dbms/programs/clang/Compiler-7.0.0bundled/cc1as_main.cpp Normal file
View File

@ -0,0 +1,555 @@
//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the clang -cc1as functionality, which implements
// the direct interface to the LLVM MC based assembler.
//
//===----------------------------------------------------------------------===//
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Driver/DriverDiagnostic.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Triple.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/MC/MCAsmBackend.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCCodeEmitter.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCInstrInfo.h"
#include "llvm/MC/MCObjectFileInfo.h"
#include "llvm/MC/MCParser/MCAsmParser.h"
#include "llvm/MC/MCParser/MCTargetAsmParser.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/MCSubtargetInfo.h"
#include "llvm/MC/MCTargetOptions.h"
#include "llvm/Option/Arg.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/FormattedStream.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/TargetRegistry.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/raw_ostream.h"
#include <memory>
#include <system_error>
using namespace clang;
using namespace clang::driver;
using namespace clang::driver::options;
using namespace llvm;
using namespace llvm::opt;
namespace {
/// \brief Helper class for representing a single invocation of the assembler.
struct AssemblerInvocation {
/// @name Target Options
/// @{
/// The name of the target triple to assemble for.
std::string Triple;
/// If given, the name of the target CPU to determine which instructions
/// are legal.
std::string CPU;
/// The list of target specific features to enable or disable -- this should
/// be a list of strings starting with '+' or '-'.
std::vector<std::string> Features;
/// The list of symbol definitions.
std::vector<std::string> SymbolDefs;
/// @}
/// @name Language Options
/// @{
std::vector<std::string> IncludePaths;
unsigned NoInitialTextSection : 1;
unsigned SaveTemporaryLabels : 1;
unsigned GenDwarfForAssembly : 1;
unsigned RelaxELFRelocations : 1;
unsigned DwarfVersion;
std::string DwarfDebugFlags;
std::string DwarfDebugProducer;
std::string DebugCompilationDir;
llvm::DebugCompressionType CompressDebugSections =
llvm::DebugCompressionType::None;
std::string MainFileName;
/// @}
/// @name Frontend Options
/// @{
std::string InputFile;
std::vector<std::string> LLVMArgs;
std::string OutputPath;
enum FileType {
FT_Asm, ///< Assembly (.s) output, transliterate mode.
FT_Null, ///< No output, for timing purposes.
FT_Obj ///< Object file output.
};
FileType OutputType;
unsigned ShowHelp : 1;
unsigned ShowVersion : 1;
/// @}
/// @name Transliterate Options
/// @{
unsigned OutputAsmVariant;
unsigned ShowEncoding : 1;
unsigned ShowInst : 1;
/// @}
/// @name Assembler Options
/// @{
unsigned RelaxAll : 1;
unsigned NoExecStack : 1;
unsigned FatalWarnings : 1;
unsigned IncrementalLinkerCompatible : 1;
/// The name of the relocation model to use.
std::string RelocationModel;
/// @}
public:
AssemblerInvocation() {
Triple = "";
NoInitialTextSection = 0;
InputFile = "-";
OutputPath = "-";
OutputType = FT_Asm;
OutputAsmVariant = 0;
ShowInst = 0;
ShowEncoding = 0;
RelaxAll = 0;
NoExecStack = 0;
FatalWarnings = 0;
IncrementalLinkerCompatible = 0;
DwarfVersion = 0;
}
static bool CreateFromArgs(AssemblerInvocation &Res,
ArrayRef<const char *> Argv,
DiagnosticsEngine &Diags);
};
}
bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
ArrayRef<const char *> Argv,
DiagnosticsEngine &Diags) {
bool Success = true;
// Parse the arguments.
std::unique_ptr<OptTable> OptTbl(createDriverOptTable());
const unsigned IncludedFlagsBitmask = options::CC1AsOption;
unsigned MissingArgIndex, MissingArgCount;
InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount,
IncludedFlagsBitmask);
// Check for missing argument error.
if (MissingArgCount) {
Diags.Report(diag::err_drv_missing_argument)
<< Args.getArgString(MissingArgIndex) << MissingArgCount;
Success = false;
}
// Issue errors on unknown arguments.
for (const Arg *A : Args.filtered(OPT_UNKNOWN)) {
auto ArgString = A->getAsString(Args);
std::string Nearest;
if (OptTbl->findNearest(ArgString, Nearest, IncludedFlagsBitmask) > 1)
Diags.Report(diag::err_drv_unknown_argument) << ArgString;
else
Diags.Report(diag::err_drv_unknown_argument_with_suggestion)
<< ArgString << Nearest;
Success = false;
}
// Construct the invocation.
// Target Options
Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple));
Opts.CPU = Args.getLastArgValue(OPT_target_cpu);
Opts.Features = Args.getAllArgValues(OPT_target_feature);
// Use the default target triple if unspecified.
if (Opts.Triple.empty())
Opts.Triple = llvm::sys::getDefaultTargetTriple();
// Language Options
Opts.IncludePaths = Args.getAllArgValues(OPT_I);
Opts.NoInitialTextSection = Args.hasArg(OPT_n);
Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels);
// Any DebugInfoKind implies GenDwarfForAssembly.
Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ);
if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections,
OPT_compress_debug_sections_EQ)) {
if (A->getOption().getID() == OPT_compress_debug_sections) {
// TODO: be more clever about the compression type auto-detection
Opts.CompressDebugSections = llvm::DebugCompressionType::GNU;
} else {
Opts.CompressDebugSections =
llvm::StringSwitch<llvm::DebugCompressionType>(A->getValue())
.Case("none", llvm::DebugCompressionType::None)
.Case("zlib", llvm::DebugCompressionType::Z)
.Case("zlib-gnu", llvm::DebugCompressionType::GNU)
.Default(llvm::DebugCompressionType::None);
}
}
Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations);
Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags);
Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags);
Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer);
Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir);
Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name);
// Frontend Options
if (Args.hasArg(OPT_INPUT)) {
bool First = true;
for (const Arg *A : Args.filtered(OPT_INPUT)) {
if (First) {
Opts.InputFile = A->getValue();
First = false;
} else {
Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args);
Success = false;
}
}
}
Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm);
Opts.OutputPath = Args.getLastArgValue(OPT_o);
if (Arg *A = Args.getLastArg(OPT_filetype)) {
StringRef Name = A->getValue();
unsigned OutputType = StringSwitch<unsigned>(Name)
.Case("asm", FT_Asm)
.Case("null", FT_Null)
.Case("obj", FT_Obj)
.Default(~0U);
if (OutputType == ~0U) {
Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name;
Success = false;
} else
Opts.OutputType = FileType(OutputType);
}
Opts.ShowHelp = Args.hasArg(OPT_help);
Opts.ShowVersion = Args.hasArg(OPT_version);
// Transliterate Options
Opts.OutputAsmVariant =
getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags);
Opts.ShowEncoding = Args.hasArg(OPT_show_encoding);
Opts.ShowInst = Args.hasArg(OPT_show_inst);
// Assemble Options
Opts.RelaxAll = Args.hasArg(OPT_mrelax_all);
Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack);
Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings);
Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic");
Opts.IncrementalLinkerCompatible =
Args.hasArg(OPT_mincremental_linker_compatible);
Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym);
return Success;
}
static std::unique_ptr<raw_fd_ostream>
getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags,
bool Binary) {
if (Opts.OutputPath.empty())
Opts.OutputPath = "-";
// Make sure that the Out file gets unlinked from the disk if we get a
// SIGINT.
if (Opts.OutputPath != "-")
sys::RemoveFileOnSignal(Opts.OutputPath);
std::error_code EC;
auto Out = llvm::make_unique<raw_fd_ostream>(
Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text));
if (EC) {
Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath
<< EC.message();
return nullptr;
}
return Out;
}
static bool ExecuteAssembler(AssemblerInvocation &Opts,
DiagnosticsEngine &Diags) {
// Get the target specific parser.
std::string Error;
const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error);
if (!TheTarget)
return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
ErrorOr<std::unique_ptr<MemoryBuffer>> Buffer =
MemoryBuffer::getFileOrSTDIN(Opts.InputFile);
if (std::error_code EC = Buffer.getError()) {
Error = EC.message();
return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile;
}
SourceMgr SrcMgr;
// Tell SrcMgr about this buffer, which is what the parser will pick up.
SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc());
// Record the location of the include directories so that the lexer can find
// it later.
SrcMgr.setIncludeDirs(Opts.IncludePaths);
std::unique_ptr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(Opts.Triple));
assert(MRI && "Unable to create target register info!");
std::unique_ptr<MCAsmInfo> MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple));
assert(MAI && "Unable to create target asm info!");
// Ensure MCAsmInfo initialization occurs before any use, otherwise sections
// may be created with a combination of default and explicit settings.
MAI->setCompressDebugSections(Opts.CompressDebugSections);
MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations);
bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj;
std::unique_ptr<raw_fd_ostream> FDOS = getOutputStream(Opts, Diags, IsBinary);
if (!FDOS)
return true;
// FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and
// MCObjectFileInfo needs a MCContext reference in order to initialize itself.
std::unique_ptr<MCObjectFileInfo> MOFI(new MCObjectFileInfo());
MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr);
bool PIC = false;
if (Opts.RelocationModel == "static") {
PIC = false;
} else if (Opts.RelocationModel == "pic") {
PIC = true;
} else {
assert(Opts.RelocationModel == "dynamic-no-pic" &&
"Invalid PIC model!");
PIC = false;
}
MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx);
if (Opts.SaveTemporaryLabels)
Ctx.setAllowTemporaryLabels(false);
if (Opts.GenDwarfForAssembly)
Ctx.setGenDwarfForAssembly(true);
if (!Opts.DwarfDebugFlags.empty())
Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags));
if (!Opts.DwarfDebugProducer.empty())
Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer));
if (!Opts.DebugCompilationDir.empty())
Ctx.setCompilationDir(Opts.DebugCompilationDir);
if (!Opts.MainFileName.empty())
Ctx.setMainFileName(StringRef(Opts.MainFileName));
Ctx.setDwarfVersion(Opts.DwarfVersion);
// Build up the feature string from the target feature list.
std::string FS;
if (!Opts.Features.empty()) {
FS = Opts.Features[0];
for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i)
FS += "," + Opts.Features[i];
}
std::unique_ptr<MCStreamer> Str;
std::unique_ptr<MCInstrInfo> MCII(TheTarget->createMCInstrInfo());
std::unique_ptr<MCSubtargetInfo> STI(
TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS));
raw_pwrite_stream *Out = FDOS.get();
std::unique_ptr<buffer_ostream> BOS;
// FIXME: There is a bit of code duplication with addPassesToEmitFile.
if (Opts.OutputType == AssemblerInvocation::FT_Asm) {
MCInstPrinter *IP = TheTarget->createMCInstPrinter(
llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI);
std::unique_ptr<MCCodeEmitter> CE;
if (Opts.ShowEncoding)
CE.reset(TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx));
MCTargetOptions MCOptions;
std::unique_ptr<MCAsmBackend> MAB(
TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
auto FOut = llvm::make_unique<formatted_raw_ostream>(*Out);
Str.reset(TheTarget->createAsmStreamer(
Ctx, std::move(FOut), /*asmverbose*/ true,
/*useDwarfDirectory*/ true, IP, std::move(CE), std::move(MAB),
Opts.ShowInst));
} else if (Opts.OutputType == AssemblerInvocation::FT_Null) {
Str.reset(createNullStreamer(Ctx));
} else {
assert(Opts.OutputType == AssemblerInvocation::FT_Obj &&
"Invalid file type!");
if (!FDOS->supportsSeeking()) {
BOS = make_unique<buffer_ostream>(*FDOS);
Out = BOS.get();
}
std::unique_ptr<MCCodeEmitter> CE(
TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx));
MCTargetOptions MCOptions;
std::unique_ptr<MCAsmBackend> MAB(
TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
Triple T(Opts.Triple);
Str.reset(TheTarget->createMCObjectStreamer(
T, Ctx, std::move(MAB), *Out, std::move(CE), *STI, Opts.RelaxAll,
Opts.IncrementalLinkerCompatible,
/*DWARFMustBeAtTheEnd*/ true));
Str.get()->InitSections(Opts.NoExecStack);
}
// Assembly to object compilation should leverage assembly info.
Str->setUseAssemblerInfoForParsing(true);
bool Failed = false;
std::unique_ptr<MCAsmParser> Parser(
createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI));
// FIXME: init MCTargetOptions from sanitizer flags here.
MCTargetOptions Options;
std::unique_ptr<MCTargetAsmParser> TAP(
TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options));
if (!TAP)
Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
// Set values for symbols, if any.
for (auto &S : Opts.SymbolDefs) {
auto Pair = StringRef(S).split('=');
auto Sym = Pair.first;
auto Val = Pair.second;
int64_t Value;
// We have already error checked this in the driver.
Val.getAsInteger(0, Value);
Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value);
}
if (!Failed) {
Parser->setTargetParser(*TAP.get());
Failed = Parser->Run(Opts.NoInitialTextSection);
}
// Close Streamer first.
// It might have a reference to the output stream.
Str.reset();
// Close the output stream early.
BOS.reset();
FDOS.reset();
// Delete output file if there were errors.
if (Failed && Opts.OutputPath != "-")
sys::fs::remove(Opts.OutputPath);
return Failed;
}
static void LLVMErrorHandler(void *UserData, const std::string &Message,
bool /*GenCrashDiag*/) {
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
Diags.Report(diag::err_fe_error_backend) << Message;
// We cannot recover from llvm errors.
exit(1);
}
int cc1as_main(ArrayRef<const char *> Argv, const char */*Argv0*/, void */*MainAddr*/) {
// Initialize targets and assembly printers/parsers.
InitializeAllTargetInfos();
InitializeAllTargetMCs();
InitializeAllAsmParsers();
// Construct our diagnostic client.
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
TextDiagnosticPrinter *DiagClient
= new TextDiagnosticPrinter(errs(), &*DiagOpts);
DiagClient->setPrefix("clang -cc1as");
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
// Set an error handler, so that any LLVM backend diagnostics go through our
// error handler.
ScopedFatalErrorHandler FatalErrorHandler
(LLVMErrorHandler, static_cast<void*>(&Diags));
// Parse the arguments.
AssemblerInvocation Asm;
if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags))
return 1;
if (Asm.ShowHelp) {
std::unique_ptr<OptTable> Opts(driver::createDriverOptTable());
Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler",
/*Include=*/driver::options::CC1AsOption, /*Exclude=*/0,
/*ShowAllAliases=*/false);
return 0;
}
// Honor -version.
//
// FIXME: Use a better -version message?
if (Asm.ShowVersion) {
llvm::cl::PrintVersionMessage();
return 0;
}
// Honor -mllvm.
//
// FIXME: Remove this, one day.
if (!Asm.LLVMArgs.empty()) {
unsigned NumArgs = Asm.LLVMArgs.size();
auto Args = llvm::make_unique<const char*[]>(NumArgs + 2);
Args[0] = "clang (LLVM option parsing)";
for (unsigned i = 0; i != NumArgs; ++i)
Args[i + 1] = Asm.LLVMArgs[i].c_str();
Args[NumArgs + 1] = nullptr;
llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get());
}
// Execute the invocation, unless there were parsing errors.
bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags);
// If any timers were active but haven't been destroyed yet, print their
// results now.
TimerGroup::printAll(errs());
return !!Failed;
}

512
dbms/programs/clang/Compiler-7.0.0bundled/driver.cpp Normal file
View File

@ -0,0 +1,512 @@
//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the clang driver; it is a thin wrapper
// for functionality in the Driver clang library.
//
//===----------------------------------------------------------------------===//
#include "clang/Driver/Driver.h"
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/DriverDiagnostic.h"
#include "clang/Driver/Options.h"
#include "clang/Driver/ToolChain.h"
#include "clang/Frontend/ChainedDiagnosticConsumer.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/SerializedDiagnosticPrinter.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Option/Option.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/InitLLVM.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/Program.h"
#include "llvm/Support/Regex.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/StringSaver.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/raw_ostream.h"
#include <memory>
#include <set>
#include <system_error>
using namespace clang;
using namespace clang::driver;
using namespace llvm::opt;
std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) {
if (!CanonicalPrefixes) {
SmallString<128> ExecutablePath(Argv0);
// Do a PATH lookup if Argv0 isn't a valid path.
if (!llvm::sys::fs::exists(ExecutablePath))
if (llvm::ErrorOr<std::string> P =
llvm::sys::findProgramByName(ExecutablePath))
ExecutablePath = *P;
return ExecutablePath.str();
}
// This just needs to be some symbol in the binary; C++ doesn't
// allow taking the address of ::main however.
void *P = (void*) (intptr_t) GetExecutablePath;
return llvm::sys::fs::getMainExecutable(Argv0, P);
}
static const char *GetStableCStr(std::set<std::string> &SavedStrings,
StringRef S) {
return SavedStrings.insert(S).first->c_str();
}
/// ApplyQAOverride - Apply a list of edits to the input argument lists.
///
/// The input string is a space separate list of edits to perform,
/// they are applied in order to the input argument lists. Edits
/// should be one of the following forms:
///
/// '#': Silence information about the changes to the command line arguments.
///
/// '^': Add FOO as a new argument at the beginning of the command line.
///
/// '+': Add FOO as a new argument at the end of the command line.
///
/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command
/// line.
///
/// 'xOPTION': Removes all instances of the literal argument OPTION.
///
/// 'XOPTION': Removes all instances of the literal argument OPTION,
/// and the following argument.
///
/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox'
/// at the end of the command line.
///
/// \param OS - The stream to write edit information to.
/// \param Args - The vector of command line arguments.
/// \param Edit - The override command to perform.
/// \param SavedStrings - Set to use for storing string representations.
static void ApplyOneQAOverride(raw_ostream &OS,
SmallVectorImpl<const char*> &Args,
StringRef Edit,
std::set<std::string> &SavedStrings) {
// This does not need to be efficient.
if (Edit[0] == '^') {
const char *Str =
GetStableCStr(SavedStrings, Edit.substr(1));
OS << "### Adding argument " << Str << " at beginning\n";
Args.insert(Args.begin() + 1, Str);
} else if (Edit[0] == '+') {
const char *Str =
GetStableCStr(SavedStrings, Edit.substr(1));
OS << "### Adding argument " << Str << " at end\n";
Args.push_back(Str);
} else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") &&
Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) {
StringRef MatchPattern = Edit.substr(2).split('/').first;
StringRef ReplPattern = Edit.substr(2).split('/').second;
ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1);
for (unsigned i = 1, e = Args.size(); i != e; ++i) {
// Ignore end-of-line response file markers
if (Args[i] == nullptr)
continue;
std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]);
if (Repl != Args[i]) {
OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n";
Args[i] = GetStableCStr(SavedStrings, Repl);
}
}
} else if (Edit[0] == 'x' || Edit[0] == 'X') {
auto Option = Edit.substr(1);
for (unsigned i = 1; i < Args.size();) {
if (Option == Args[i]) {
OS << "### Deleting argument " << Args[i] << '\n';
Args.erase(Args.begin() + i);
if (Edit[0] == 'X') {
if (i < Args.size()) {
OS << "### Deleting argument " << Args[i] << '\n';
Args.erase(Args.begin() + i);
} else
OS << "### Invalid X edit, end of command line!\n";
}
} else
++i;
}
} else if (Edit[0] == 'O') {
for (unsigned i = 1; i < Args.size();) {
const char *A = Args[i];
// Ignore end-of-line response file markers
if (A == nullptr)
continue;
if (A[0] == '-' && A[1] == 'O' &&
(A[2] == '\0' ||
(A[3] == '\0' && (A[2] == 's' || A[2] == 'z' ||
('0' <= A[2] && A[2] <= '9'))))) {
OS << "### Deleting argument " << Args[i] << '\n';
Args.erase(Args.begin() + i);
} else
++i;
}
OS << "### Adding argument " << Edit << " at end\n";
Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str()));
} else {
OS << "### Unrecognized edit: " << Edit << "\n";
}
}
/// ApplyQAOverride - Apply a comma separate list of edits to the
/// input argument lists. See ApplyOneQAOverride.
static void ApplyQAOverride(SmallVectorImpl<const char*> &Args,
const char *OverrideStr,
std::set<std::string> &SavedStrings) {
raw_ostream *OS = &llvm::errs();
if (OverrideStr[0] == '#') {
++OverrideStr;
OS = &llvm::nulls();
}
*OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n";
// This does not need to be efficient.
const char *S = OverrideStr;
while (*S) {
const char *End = ::strchr(S, ' ');
if (!End)
End = S + strlen(S);
if (End != S)
ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings);
S = End;
if (*S != '\0')
++S;
}
}
extern int cc1_main(ArrayRef<const char *> Argv, const char *Argv0,
void *MainAddr);
extern int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0,
void *MainAddr);
extern int cc1gen_reproducer_main(ArrayRef<const char *> Argv,
const char *Argv0, void *MainAddr);
static void insertTargetAndModeArgs(const ParsedClangName &NameParts,
SmallVectorImpl<const char *> &ArgVector,
std::set<std::string> &SavedStrings) {
// Put target and mode arguments at the start of argument list so that
// arguments specified in command line could override them. Avoid putting
// them at index 0, as an option like '-cc1' must remain the first.
int InsertionPoint = 0;
if (ArgVector.size() > 0)
++InsertionPoint;
if (NameParts.DriverMode) {
// Add the mode flag to the arguments.
ArgVector.insert(ArgVector.begin() + InsertionPoint,
GetStableCStr(SavedStrings, NameParts.DriverMode));
}
if (NameParts.TargetIsValid) {
const char *arr[] = {"-target", GetStableCStr(SavedStrings,
NameParts.TargetPrefix)};
ArgVector.insert(ArgVector.begin() + InsertionPoint,
std::begin(arr), std::end(arr));
}
}
static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver,
SmallVectorImpl<const char *> &Opts) {
llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts);
// The first instance of '#' should be replaced with '=' in each option.
for (const char *Opt : Opts)
if (char *NumberSignPtr = const_cast<char *>(::strchr(Opt, '#')))
*NumberSignPtr = '=';
}
static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) {
// Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE.
TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS");
if (TheDriver.CCPrintOptions)
TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE");
// Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE.
TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS");
if (TheDriver.CCPrintHeaders)
TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE");
// Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE.
TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS");
if (TheDriver.CCLogDiagnostics)
TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE");
}
static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient,
const std::string &Path) {
// If the clang binary happens to be named cl.exe for compatibility reasons,
// use clang-cl.exe as the prefix to avoid confusion between clang and MSVC.
StringRef ExeBasename(llvm::sys::path::filename(Path));
if (ExeBasename.equals_lower("cl.exe"))
ExeBasename = "clang-cl.exe";
DiagClient->setPrefix(ExeBasename);
}
// This lets us create the DiagnosticsEngine with a properly-filled-out
// DiagnosticOptions instance.
static DiagnosticOptions *
CreateAndPopulateDiagOpts(ArrayRef<const char *> argv) {
auto *DiagOpts = new DiagnosticOptions;
std::unique_ptr<OptTable> Opts(createDriverOptTable());
unsigned MissingArgIndex, MissingArgCount;
InputArgList Args =
Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount);
// We ignore MissingArgCount and the return value of ParseDiagnosticArgs.
// Any errors that would be diagnosed here will also be diagnosed later,
// when the DiagnosticsEngine actually exists.
(void)ParseDiagnosticArgs(*DiagOpts, Args);
return DiagOpts;
}
static void SetInstallDir(SmallVectorImpl<const char *> &argv,
Driver &TheDriver, bool CanonicalPrefixes) {
// Attempt to find the original path used to invoke the driver, to determine
// the installed path. We do this manually, because we want to support that
// path being a symlink.
SmallString<128> InstalledPath(argv[0]);
// Do a PATH lookup, if there are no directory components.
if (llvm::sys::path::filename(InstalledPath) == InstalledPath)
if (llvm::ErrorOr<std::string> Tmp = llvm::sys::findProgramByName(
llvm::sys::path::filename(InstalledPath.str())))
InstalledPath = *Tmp;
// FIXME: We don't actually canonicalize this, we just make it absolute.
if (CanonicalPrefixes)
llvm::sys::fs::make_absolute(InstalledPath);
StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath));
if (llvm::sys::fs::exists(InstalledPathParent))
TheDriver.setInstalledDir(InstalledPathParent);
}
static int ExecuteCC1Tool(ArrayRef<const char *> argv, StringRef Tool) {
void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath;
if (Tool == "")
return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP);
if (Tool == "as")
return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP);
// Reject unknown tools.
llvm::errs() << "error: unknown integrated tool '" << Tool << "'. "
<< "Valid tools include '-cc1' and '-cc1as'.\n";
return 1;
}
int mainEntryClickHouseClang(int argc_, char **argv_) {
llvm::InitLLVM X(argc_, argv_);
SmallVector<const char *, 256> argv(argv_, argv_ + argc_);
if (llvm::sys::Process::FixupStandardFileDescriptors())
return 1;
llvm::InitializeAllTargets();
auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]);
llvm::BumpPtrAllocator A;
llvm::StringSaver Saver(A);
// Parse response files using the GNU syntax, unless we're in CL mode. There
// are two ways to put clang in CL compatibility mode: argv[0] is either
// clang-cl or cl, or --driver-mode=cl is on the command line. The normal
// command line parsing can't happen until after response file parsing, so we
// have to manually search for a --driver-mode=cl argument the hard way.
// Finally, our -cc1 tools don't care which tokenization mode we use because
// response files written by clang will tokenize the same way in either mode.
bool ClangCLMode = false;
if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") ||
std::find_if(argv.begin(), argv.end(), [](const char *F) {
return F && strcmp(F, "--driver-mode=cl") == 0;
}) != argv.end()) {
ClangCLMode = true;
}
enum { Default, POSIX, Windows } RSPQuoting = Default;
for (const char *F : argv) {
if (strcmp(F, "--rsp-quoting=posix") == 0)
RSPQuoting = POSIX;
else if (strcmp(F, "--rsp-quoting=windows") == 0)
RSPQuoting = Windows;
}
// Determines whether we want nullptr markers in argv to indicate response
// files end-of-lines. We only use this for the /LINK driver argument with
// clang-cl.exe on Windows.
bool MarkEOLs = ClangCLMode;
llvm::cl::TokenizerCallback Tokenizer;
if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode))
Tokenizer = &llvm::cl::TokenizeWindowsCommandLine;
else
Tokenizer = &llvm::cl::TokenizeGNUCommandLine;
if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1"))
MarkEOLs = false;
llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs);
// Handle -cc1 integrated tools, even if -cc1 was expanded from a response
// file.
auto FirstArg = std::find_if(argv.begin() + 1, argv.end(),
[](const char *A) { return A != nullptr; });
if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) {
// If -cc1 came from a response file, remove the EOL sentinels.
if (MarkEOLs) {
auto newEnd = std::remove(argv.begin(), argv.end(), nullptr);
argv.resize(newEnd - argv.begin());
}
return ExecuteCC1Tool(argv, argv[1] + 4);
}
bool CanonicalPrefixes = true;
for (int i = 1, size = argv.size(); i < size; ++i) {
// Skip end-of-line response file markers
if (argv[i] == nullptr)
continue;
if (StringRef(argv[i]) == "-no-canonical-prefixes") {
CanonicalPrefixes = false;
break;
}
}
// Handle CL and _CL_ which permits additional command line options to be
// prepended or appended.
if (ClangCLMode) {
// Arguments in "CL" are prepended.
llvm::Optional<std::string> OptCL = llvm::sys::Process::GetEnv("CL");
if (OptCL.hasValue()) {
SmallVector<const char *, 8> PrependedOpts;
getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts);
// Insert right after the program name to prepend to the argument list.
argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end());
}
// Arguments in "_CL_" are appended.
llvm::Optional<std::string> Opt_CL_ = llvm::sys::Process::GetEnv("_CL_");
if (Opt_CL_.hasValue()) {
SmallVector<const char *, 8> AppendedOpts;
getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts);
// Insert at the end of the argument list to append.
argv.append(AppendedOpts.begin(), AppendedOpts.end());
}
}
std::set<std::string> SavedStrings;
// Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the
// scenes.
if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) {
// FIXME: Driver shouldn't take extra initial argument.
ApplyQAOverride(argv, OverrideStr, SavedStrings);
}
std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes);
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts =
CreateAndPopulateDiagOpts(argv);
TextDiagnosticPrinter *DiagClient
= new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
FixupDiagPrefixExeName(DiagClient, Path);
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
if (!DiagOpts->DiagnosticSerializationFile.empty()) {
auto SerializedConsumer =
clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile,
&*DiagOpts, /*MergeChildRecords=*/true);
Diags.setClient(new ChainedDiagnosticConsumer(
Diags.takeClient(), std::move(SerializedConsumer)));
}
ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false);
Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags);
SetInstallDir(argv, TheDriver, CanonicalPrefixes);
TheDriver.setTargetAndMode(TargetAndMode);
insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings);
SetBackdoorDriverOutputsFromEnvVars(TheDriver);
std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(argv));
int Res = 1;
if (C && !C->containsError()) {
SmallVector<std::pair<int, const Command *>, 4> FailingCommands;
Res = TheDriver.ExecuteCompilation(*C, FailingCommands);
// Force a crash to test the diagnostics.
if (TheDriver.GenReproducer) {
Diags.Report(diag::err_drv_force_crash)
<< !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH");
// Pretend that every command failed.
FailingCommands.clear();
for (const auto &J : C->getJobs())
if (const Command *C = dyn_cast<Command>(&J))
FailingCommands.push_back(std::make_pair(-1, C));
}
for (const auto &P : FailingCommands) {
int CommandRes = P.first;
const Command *FailingCommand = P.second;
if (!Res)
Res = CommandRes;
// If result status is < 0, then the driver command signalled an error.
// If result status is 70, then the driver command reported a fatal error.
// On Windows, abort will return an exit code of 3. In these cases,
// generate additional diagnostic information if possible.
bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70;
#ifdef _WIN32
DiagnoseCrash |= CommandRes == 3;
#endif
if (DiagnoseCrash) {
TheDriver.generateCompilationDiagnostics(*C, *FailingCommand);
break;
}
}
}
Diags.getClient()->finish();
// If any timers were active but haven't been destroyed yet, print their
// results now. This happens in -disable-free mode.
llvm::TimerGroup::printAll(llvm::errs());
#ifdef _WIN32
// Exit status should not be negative on Win32, unless abnormal termination.
// Once abnormal termiation was caught, negative status should not be
// propagated.
if (Res < 0)
Res = 1;
#endif
// If we have multiple failing commands, we return the result of the first
// failing command.
return Res;
}

10
dbms/programs/clang/Compiler-7.0.0bundled/lld.cpp Normal file
View File

@ -0,0 +1,10 @@
#include "lld/Common/Driver.h"
#include "llvm/Support/InitLLVM.h"
#include <vector>
int mainEntryClickHouseLLD(int argc, char ** argv)
{
llvm::InitLLVM X(argc, argv);
std::vector<const char *> args(argv, argv + argc);
return !lld::elf::link(args, false);
}

5
dbms/programs/client/CMakeLists.txt
View File

@ -1,13 +1,12 @@
add_library (clickhouse-client-lib ${LINK_MODE} Client.cpp)
target_link_libraries (clickhouse-client-lib clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-client-lib PRIVATE clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (READLINE_INCLUDE_DIR)
target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
endif ()
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-client clickhouse-client.cpp)
target_link_libraries (clickhouse-client clickhouse-client-lib)
target_link_libraries (clickhouse-client PRIVATE clickhouse-client-lib)
endif ()
install (FILES clickhouse-client.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-client COMPONENT clickhouse-client RENAME config.xml)

3
dbms/programs/client/Client.cpp
View File

@ -86,9 +86,6 @@ namespace DB
namespace ErrorCodes
{
extern const int POCO_EXCEPTION;
extern const int STD_EXCEPTION;
extern const int UNKNOWN_EXCEPTION;
extern const int NETWORK_ERROR;
extern const int NO_DATA_TO_INSERT;
extern const int BAD_ARGUMENTS;

2
dbms/programs/client/Suggest.h
View File

@ -184,7 +184,7 @@ public:
}
catch (...)
{
std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false) << "\n";
std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n";
}
/// Note that keyword suggestions are available even if we cannot load data from server.

4
dbms/programs/compressor/CMakeLists.txt
View File

@ -1,8 +1,8 @@
add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp)
target_link_libraries (clickhouse-compressor-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
# Also in utils
add_executable (clickhouse-compressor clickhouse-compressor.cpp)
target_link_libraries (clickhouse-compressor clickhouse-compressor-lib)
target_link_libraries (clickhouse-compressor PRIVATE clickhouse-compressor-lib)
endif ()

2
dbms/programs/copier/CMakeLists.txt
View File

@ -1,5 +1,5 @@
add_library (clickhouse-copier-lib ${LINK_MODE} ClusterCopier.cpp)
target_link_libraries (clickhouse-copier-lib clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions)
target_link_libraries (clickhouse-copier-lib PRIVATE clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions daemon)
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-copier clickhouse-copier.cpp)

4
dbms/programs/extract-from-config/CMakeLists.txt
View File

@ -1,7 +1,7 @@
add_library (clickhouse-extract-from-config-lib ${LINK_MODE} ExtractFromConfig.cpp)
target_link_libraries (clickhouse-extract-from-config-lib clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-extract-from-config-lib PRIVATE clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-extract-from-config clickhouse-extract-from-config.cpp)
target_link_libraries (clickhouse-extract-from-config clickhouse-extract-from-config-lib)
target_link_libraries (clickhouse-extract-from-config PRIVATE clickhouse-extract-from-config-lib)
endif ()

4
dbms/programs/format/CMakeLists.txt
View File

@ -1,6 +1,6 @@
add_library (clickhouse-format-lib ${LINK_MODE} Format.cpp)
target_link_libraries (clickhouse-format-lib dbms clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-format-lib PRIVATE dbms clickhouse_common_io clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-format clickhouse-format.cpp)
target_link_libraries (clickhouse-format clickhouse-format-lib)
target_link_libraries (clickhouse-format PRIVATE clickhouse-format-lib)
endif ()

4
dbms/programs/local/CMakeLists.txt
View File

@ -1,7 +1,7 @@
add_library (clickhouse-local-lib ${LINK_MODE} LocalServer.cpp)
target_link_libraries (clickhouse-local-lib clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-local-lib PRIVATE clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-local clickhouse-local.cpp)
target_link_libraries (clickhouse-local clickhouse-local-lib)
target_link_libraries (clickhouse-local PRIVATE clickhouse-local-lib)
endif ()

4
dbms/programs/obfuscator/CMakeLists.txt
View File

@ -1,8 +1,8 @@
add_library (clickhouse-obfuscator-lib ${LINK_MODE} Obfuscator.cpp)
target_link_libraries (clickhouse-obfuscator-lib dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-obfuscator-lib PRIVATE dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-obfuscator clickhouse-obfuscator.cpp)
set_target_properties(clickhouse-obfuscator PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
target_link_libraries (clickhouse-obfuscator clickhouse-obfuscator-lib)
target_link_libraries (clickhouse-obfuscator PRIVATE clickhouse-obfuscator-lib)
endif ()

12
dbms/programs/odbc-bridge/CMakeLists.txt
View File

@ -9,23 +9,23 @@ add_library (clickhouse-odbc-bridge-lib ${LINK_MODE}
validateODBCConnectionString.cpp
)
target_link_libraries (clickhouse-odbc-bridge-lib clickhouse_common_io daemon dbms)
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE clickhouse_common_io daemon dbms)
target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
if (USE_POCO_SQLODBC)
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQLODBC_LIBRARY})
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQLODBC_LIBRARY})
target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR})
endif ()
if (Poco_SQL_FOUND)
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQL_LIBRARY})
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQL_LIBRARY})
endif ()
if (USE_POCO_DATAODBC)
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_DataODBC_LIBRARY})
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_DataODBC_LIBRARY})
target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
endif()
if (Poco_Data_FOUND)
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_Data_LIBRARY})
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_Data_LIBRARY})
endif ()
@ -35,5 +35,5 @@ endif ()
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-odbc-bridge odbc-bridge.cpp)
target_link_libraries (clickhouse-odbc-bridge clickhouse-odbc-bridge-lib)
target_link_libraries (clickhouse-odbc-bridge PRIVATE clickhouse-odbc-bridge-lib)
endif ()

2
dbms/programs/odbc-bridge/tests/CMakeLists.txt
View File

@ -1,2 +1,2 @@
add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp)
target_link_libraries (validate-odbc-connection-string clickhouse-odbc-bridge-lib)
target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse-odbc-bridge-lib clickhouse_common_io)

4
dbms/programs/performance-test/CMakeLists.txt
View File

@ -1,8 +1,8 @@
add_library (clickhouse-performance-test-lib ${LINK_MODE} PerformanceTest.cpp)
target_link_libraries (clickhouse-performance-test-lib clickhouse_common_io dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-performance-test-lib PRIVATE dbms clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_include_directories (clickhouse-performance-test-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-performance-test clickhouse-performance-test.cpp)
target_link_libraries (clickhouse-performance-test clickhouse-performance-test-lib dbms)
target_link_libraries (clickhouse-performance-test PRIVATE clickhouse-performance-test-lib)
endif ()

46
dbms/programs/performance-test/PerformanceTest.cpp
View File

@ -49,10 +49,10 @@ namespace DB
{
namespace ErrorCodes
{
extern const int POCO_EXCEPTION;
extern const int STD_EXCEPTION;
extern const int UNKNOWN_EXCEPTION;
extern const int NOT_IMPLEMENTED;
extern const int LOGICAL_ERROR;
extern const int BAD_ARGUMENTS;
extern const int FILE_DOESNT_EXIST;
}
static String pad(size_t padding)
@ -156,7 +156,7 @@ struct StopConditionsSet
else if (key == "average_speed_not_changing_for_ms")
average_speed_not_changing_for_ms.value = stop_conditions_view->getUInt64(key);
else
throw DB::Exception("Met unkown stop condition: " + key);
throw DB::Exception("Met unkown stop condition: " + key, DB::ErrorCodes::LOGICAL_ERROR);
++initialized_count;
}
@ -521,7 +521,7 @@ public:
{
if (input_files.size() < 1)
{
throw DB::Exception("No tests were specified", 0);
throw DB::Exception("No tests were specified", DB::ErrorCodes::BAD_ARGUMENTS);
}
std::string name;
@ -694,7 +694,7 @@ private:
size_t ram_size_needed = config->getUInt64("preconditions.ram_size");
size_t actual_ram = getMemoryAmount();
if (!actual_ram)
throw DB::Exception("ram_size precondition not available on this platform", ErrorCodes::NOT_IMPLEMENTED);
throw DB::Exception("ram_size precondition not available on this platform", DB::ErrorCodes::NOT_IMPLEMENTED);
if (ram_size_needed > actual_ram)
{
@ -868,12 +868,12 @@ private:
if (!test_config->has("query") && !test_config->has("query_file"))
{
throw DB::Exception("Missing query fields in test's config: " + test_name);
throw DB::Exception("Missing query fields in test's config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
}
if (test_config->has("query") && test_config->has("query_file"))
{
throw DB::Exception("Found both query and query_file fields. Choose only one");
throw DB::Exception("Found both query and query_file fields. Choose only one", DB::ErrorCodes::BAD_ARGUMENTS);
}
if (test_config->has("query"))
@ -885,7 +885,7 @@ private:
{
const String filename = test_config->getString("query_file");
if (filename.empty())
throw DB::Exception("Empty file name");
throw DB::Exception("Empty file name", DB::ErrorCodes::BAD_ARGUMENTS);
bool tsv = fs::path(filename).extension().string() == ".tsv";
@ -909,7 +909,7 @@ private:
if (queries.empty())
{
throw DB::Exception("Did not find any query to execute: " + test_name);
throw DB::Exception("Did not find any query to execute: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
}
if (test_config->has("substitutions"))
@ -929,7 +929,7 @@ private:
if (!test_config->has("type"))
{
throw DB::Exception("Missing type property in config: " + test_name);
throw DB::Exception("Missing type property in config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
}
String config_exec_type = test_config->getString("type");
@ -938,7 +938,7 @@ private:
else if (config_exec_type == "once")
exec_type = ExecutionType::Once;
else
throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name);
throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
times_to_run = test_config->getUInt("times_to_run", 1);
@ -951,7 +951,7 @@ private:
}
if (stop_conditions_template.empty())
throw DB::Exception("No termination conditions were found in config");
throw DB::Exception("No termination conditions were found in config", DB::ErrorCodes::BAD_ARGUMENTS);
for (size_t i = 0; i < times_to_run * queries.size(); ++i)
stop_conditions_by_run.push_back(stop_conditions_template);
@ -978,7 +978,7 @@ private:
else
{
if (lite_output)
throw DB::Exception("Specify main_metric for lite output");
throw DB::Exception("Specify main_metric for lite output", DB::ErrorCodes::BAD_ARGUMENTS);
}
if (metrics.size() > 0)
@ -1023,22 +1023,14 @@ private:
if (exec_type == ExecutionType::Loop)
{
for (const String & metric : metrics)
{
if (std::find(non_loop_metrics.begin(), non_loop_metrics.end(), metric) != non_loop_metrics.end())
{
throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")");
}
}
throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS);
}
else
{
for (const String & metric : metrics)
{
if (std::find(loop_metrics.begin(), loop_metrics.end(), metric) != loop_metrics.end())
{
throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")");
}
}
throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS);
}
}
@ -1439,7 +1431,7 @@ try
if (input_files.empty())
{
std::cerr << std::endl;
throw DB::Exception("Did not find any xml files");
throw DB::Exception("Did not find any xml files", DB::ErrorCodes::BAD_ARGUMENTS);
}
else
std::cerr << " found " << input_files.size() << " files." << std::endl;
@ -1454,7 +1446,7 @@ try
fs::path file(filename);
if (!fs::exists(file))
throw DB::Exception("File '" + filename + "' does not exist");
throw DB::Exception("File '" + filename + "' does not exist", DB::ErrorCodes::FILE_DOESNT_EXIST);
if (fs::is_directory(file))
{
@ -1463,7 +1455,7 @@ try
else
{
if (file.extension().string() != ".xml")
throw DB::Exception("File '" + filename + "' does not have .xml extension");
throw DB::Exception("File '" + filename + "' does not have .xml extension", DB::ErrorCodes::BAD_ARGUMENTS);
collected_files.push_back(filename);
}
}

12
dbms/programs/server/CMakeLists.txt
View File

@ -10,17 +10,21 @@ add_library (clickhouse-server-lib ${LINK_MODE}
TCPHandler.cpp
)
target_link_libraries (clickhouse-server-lib clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions)
target_link_libraries (clickhouse-server-lib PRIVATE clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY})
if (USE_POCO_NETSSL)
target_link_libraries (clickhouse-server-lib PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
endif ()
target_include_directories (clickhouse-server-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-server clickhouse-server.cpp)
target_link_libraries (clickhouse-server clickhouse-server-lib)
target_link_libraries (clickhouse-server PRIVATE clickhouse-server-lib)
install (TARGETS clickhouse-server ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
endif ()
if (OS_LINUX)
set (GLIBC_MAX_REQUIRED 2.4)
if (OS_LINUX AND MAKE_STATIC_LIBRARIES)
set (GLIBC_MAX_REQUIRED 2.4 CACHE INTERNAL "")
add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-server | grep '@GLIBC' | grep -oP 'GLIBC_[\\d\\.]+' | sort | uniq | sort -r | perl -lnE 'exit 1 if $_ gt q{GLIBC_${GLIBC_MAX_REQUIRED}}'")
endif ()

1
dbms/programs/server/HTTPHandler.cpp
View File

@ -270,7 +270,6 @@ void HTTPHandler::processQuery(
std::string query_id = params.get("query_id", "");
context.setUser(user, password, request.clientAddress(), quota_key);
context.setCurrentQueryId(query_id);
CurrentThread::attachQueryContext(context);
/// The user could specify session identifier and session timeout.
/// It allows to modify settings, create temporary tables and reuse them in subsequent requests.

3
dbms/programs/server/InterserverIOHTTPHandler.cpp
View File

@ -19,9 +19,6 @@ namespace DB
namespace ErrorCodes
{
extern const int ABORTED;
extern const int POCO_EXCEPTION;
extern const int STD_EXCEPTION;
extern const int UNKNOWN_EXCEPTION;
extern const int TOO_MANY_SIMULTANEOUS_QUERIES;
}

8
dbms/programs/server/Server.cpp
View File

@ -55,6 +55,7 @@
namespace CurrentMetrics
{
extern const Metric Revision;
extern const Metric VersionInteger;
}
namespace DB
@ -66,6 +67,8 @@ namespace ErrorCodes
extern const int SUPPORT_IS_DISABLED;
extern const int ARGUMENT_OUT_OF_BOUND;
extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
extern const int INVALID_CONFIG_PARAMETER;
extern const int SYSTEM_ERROR;
}
@ -73,7 +76,7 @@ static std::string getCanonicalPath(std::string && path)
{
Poco::trimInPlace(path);
if (path.empty())
throw Exception("path configuration parameter is empty");
throw Exception("path configuration parameter is empty", ErrorCodes::INVALID_CONFIG_PARAMETER);
if (path.back() != '/')
path += '/';
return std::move(path);
@ -108,6 +111,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
registerStorages();
CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get());
CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
/** Context contains all that query execution is dependent:
* settings, available functions, data types, aggregate functions, databases...
@ -141,7 +145,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
{
LOG_TRACE(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds.");
if (0 != mlockall(MCL_CURRENT))
LOG_WARNING(log, "Failed mlockall: " + errnoToString());
LOG_WARNING(log, "Failed mlockall: " + errnoToString(ErrorCodes::SYSTEM_ERROR));
else
LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed");
}

2
dbms/programs/server/TCPHandler.cpp
View File

@ -718,7 +718,7 @@ bool TCPHandler::receiveData()
{
NamesAndTypesList columns = block.getNamesAndTypesList();
storage = StorageMemory::create(external_table_name,
ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}});
ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, ColumnComments{}});
storage->startup();
query_context.addExternalTable(external_table_name, storage);
}

14
dbms/programs/server/config.xml
View File

@ -164,6 +164,20 @@
</replica>
</shard>
</test_shard_localhost>
<test_cluster_two_shards_localhost>
<shard>
<replica>
<host>localhost</host>
<port>9000</port>
</replica>
</shard>
<shard>
<replica>
<host>localhost</host>
<port>9000</port>
</replica>
</shard>
</test_cluster_two_shards_localhost>
<test_shard_localhost_secure>
<shard>
<replica>

2
dbms/programs/server/users.xml
View File

@ -56,7 +56,7 @@
Each element of list has one of the following forms:
<ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
<host> Hostname. Example: server01.yandex.ru.
To check access, DNS query is performed, and all received addresses compared to peer address.
<host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$

2
dbms/scripts/gen-bias-data.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/python3.4
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import sys

2
dbms/src/AggregateFunctions/AggregateFunctionAvg.cpp
View File

@ -12,7 +12,7 @@ namespace
template <typename T>
struct Avg
{
using FieldType = std::conditional_t<IsDecimalNumber<T>, Decimal128, typename NearestFieldType<T>::Type>;
using FieldType = std::conditional_t<IsDecimalNumber<T>, Decimal128, NearestFieldType<T>>;
using Function = AggregateFunctionAvg<T, AggregateFunctionAvgData<FieldType>>;
};

48
dbms/src/AggregateFunctions/AggregateFunctionQuantile.cpp
View File

@ -1,11 +1,7 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/Helpers.h>
#include <AggregateFunctions/FactoryHelpers.h>
#include <AggregateFunctions/AggregateFunctionQuantile.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/Helpers.h>
namespace DB
@ -52,13 +48,11 @@ static constexpr bool SupportDecimal()
}
template <template <typename> class Function, bool have_second_arg>
template <template <typename> class Function>
AggregateFunctionPtr createAggregateFunctionQuantile(const std::string & name, const DataTypes & argument_types, const Array & params)
{
if constexpr (have_second_arg)
assertBinary(name, argument_types);
else
assertUnary(name, argument_types);
/// Second argument type check doesn't depend on the type of the first one.
Function<void>::assertSecondArg(argument_types);
const DataTypePtr & argument_type = argument_types[0];
WhichDataType which(argument_type);
@ -86,29 +80,29 @@ AggregateFunctionPtr createAggregateFunctionQuantile(const std::string & name, c
void registerAggregateFunctionsQuantile(AggregateFunctionFactory & factory)
{
factory.registerFunction(NameQuantile::name, createAggregateFunctionQuantile<FuncQuantile, false>);
factory.registerFunction(NameQuantiles::name, createAggregateFunctionQuantile<FuncQuantiles, false>);
factory.registerFunction(NameQuantile::name, createAggregateFunctionQuantile<FuncQuantile>);
factory.registerFunction(NameQuantiles::name, createAggregateFunctionQuantile<FuncQuantiles>);
factory.registerFunction(NameQuantileDeterministic::name, createAggregateFunctionQuantile<FuncQuantileDeterministic, true>);
factory.registerFunction(NameQuantilesDeterministic::name, createAggregateFunctionQuantile<FuncQuantilesDeterministic, true>);
factory.registerFunction(NameQuantileDeterministic::name, createAggregateFunctionQuantile<FuncQuantileDeterministic>);
factory.registerFunction(NameQuantilesDeterministic::name, createAggregateFunctionQuantile<FuncQuantilesDeterministic>);
factory.registerFunction(NameQuantileExact::name, createAggregateFunctionQuantile<FuncQuantileExact, false>);
factory.registerFunction(NameQuantilesExact::name, createAggregateFunctionQuantile<FuncQuantilesExact, false>);
factory.registerFunction(NameQuantileExact::name, createAggregateFunctionQuantile<FuncQuantileExact>);
factory.registerFunction(NameQuantilesExact::name, createAggregateFunctionQuantile<FuncQuantilesExact>);
factory.registerFunction(NameQuantileExactWeighted::name, createAggregateFunctionQuantile<FuncQuantileExactWeighted, true>);
factory.registerFunction(NameQuantilesExactWeighted::name, createAggregateFunctionQuantile<FuncQuantilesExactWeighted, true>);
factory.registerFunction(NameQuantileExactWeighted::name, createAggregateFunctionQuantile<FuncQuantileExactWeighted>);
factory.registerFunction(NameQuantilesExactWeighted::name, createAggregateFunctionQuantile<FuncQuantilesExactWeighted>);
factory.registerFunction(NameQuantileTiming::name, createAggregateFunctionQuantile<FuncQuantileTiming, false>);
factory.registerFunction(NameQuantilesTiming::name, createAggregateFunctionQuantile<FuncQuantilesTiming, false>);
factory.registerFunction(NameQuantileTiming::name, createAggregateFunctionQuantile<FuncQuantileTiming>);
factory.registerFunction(NameQuantilesTiming::name, createAggregateFunctionQuantile<FuncQuantilesTiming>);
factory.registerFunction(NameQuantileTimingWeighted::name, createAggregateFunctionQuantile<FuncQuantileTimingWeighted, true>);
factory.registerFunction(NameQuantilesTimingWeighted::name, createAggregateFunctionQuantile<FuncQuantilesTimingWeighted, true>);
factory.registerFunction(NameQuantileTimingWeighted::name, createAggregateFunctionQuantile<FuncQuantileTimingWeighted>);
factory.registerFunction(NameQuantilesTimingWeighted::name, createAggregateFunctionQuantile<FuncQuantilesTimingWeighted>);
factory.registerFunction(NameQuantileTDigest::name, createAggregateFunctionQuantile<FuncQuantileTDigest, false>);
factory.registerFunction(NameQuantilesTDigest::name, createAggregateFunctionQuantile<FuncQuantilesTDigest, false>);
factory.registerFunction(NameQuantileTDigest::name, createAggregateFunctionQuantile<FuncQuantileTDigest>);
factory.registerFunction(NameQuantilesTDigest::name, createAggregateFunctionQuantile<FuncQuantilesTDigest>);
factory.registerFunction(NameQuantileTDigestWeighted::name, createAggregateFunctionQuantile<FuncQuantileTDigestWeighted, true>);
factory.registerFunction(NameQuantilesTDigestWeighted::name, createAggregateFunctionQuantile<FuncQuantilesTDigestWeighted, true>);
factory.registerFunction(NameQuantileTDigestWeighted::name, createAggregateFunctionQuantile<FuncQuantileTDigestWeighted>);
factory.registerFunction(NameQuantilesTDigestWeighted::name, createAggregateFunctionQuantile<FuncQuantilesTDigestWeighted>);
/// 'median' is an alias for 'quantile'
factory.registerAlias("median", NameQuantile::name);

60
dbms/src/AggregateFunctions/AggregateFunctionQuantile.h
View File

@ -1,19 +1,28 @@
#pragma once
#include <type_traits>
#include <AggregateFunctions/FactoryHelpers.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeArray.h>
/// These must be exposed in header for the purpose of dynamic compilation.
#include <AggregateFunctions/QuantileReservoirSampler.h>
#include <AggregateFunctions/QuantileReservoirSamplerDeterministic.h>
#include <AggregateFunctions/QuantileExact.h>
#include <AggregateFunctions/QuantileExactWeighted.h>
#include <AggregateFunctions/QuantileTiming.h>
#include <AggregateFunctions/QuantileTDigest.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <AggregateFunctions/QuantilesCommon.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnDecimal.h>
#include <Columns/ColumnsNumber.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypesNumber.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <type_traits>
namespace DB
@ -36,10 +45,10 @@ template <
typename Data,
/// Structure with static member "name", containing the name of the aggregate function.
typename Name,
/// If true, the function accept second argument
/// If true, the function accepts the second argument
/// (in can be "weight" to calculate quantiles or "determinator" that is used instead of PRNG).
/// Second argument is always obtained through 'getUInt' method.
bool have_second_arg,
bool has_second_arg,
/// If non-void, the function will return float of specified type with possibly interpolated results and NaN if there was no values.
/// Otherwise it will return Value type and default value if there was no values.
/// As an example, the function cannot return floats, if the SQL type of argument is Date or DateTime.
@ -49,12 +58,14 @@ template <
bool returns_many
>
class AggregateFunctionQuantile final : public IAggregateFunctionDataHelper<Data,
AggregateFunctionQuantile<Value, Data, Name, have_second_arg, FloatReturnType, returns_many>>
AggregateFunctionQuantile<Value, Data, Name, has_second_arg, FloatReturnType, returns_many>>
{
private:
using ColVecType = std::conditional_t<IsDecimalNumber<Value>, ColumnDecimal<Value>, ColumnVector<Value>>;
static constexpr bool returns_float = !std::is_same_v<FloatReturnType, void>;
static constexpr bool returns_float = !(std::is_same_v<FloatReturnType, void>)
&& (!(std::is_same_v<Value, DataTypeDate::FieldType> || std::is_same_v<Value, DataTypeDateTime::FieldType>)
|| std::is_same_v<Data, QuantileTiming<Value>>);
static_assert(!IsDecimalNumber<Value> || !returns_float);
QuantileLevels<Float64> levels;
@ -92,7 +103,7 @@ public:
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
{
const auto & column = static_cast<const ColVecType &>(*columns[0]);
if constexpr (have_second_arg)
if constexpr (has_second_arg)
this->data(place).add(
column.getData()[row_num],
columns[1]->getUInt(row_num));
@ -159,22 +170,17 @@ public:
}
const char * getHeaderFilePath() const override { return __FILE__; }
static void assertSecondArg(const DataTypes & argument_types)
{
if constexpr (has_second_arg)
/// TODO: check that second argument is of numerical type.
assertBinary(Name::name, argument_types);
else
assertUnary(Name::name, argument_types);
}
};
}
/// These must be exposed in header for the purpose of dynamic compilation.
#include <AggregateFunctions/QuantileReservoirSampler.h>
#include <AggregateFunctions/QuantileReservoirSamplerDeterministic.h>
#include <AggregateFunctions/QuantileExact.h>
#include <AggregateFunctions/QuantileExactWeighted.h>
#include <AggregateFunctions/QuantileTiming.h>
#include <AggregateFunctions/QuantileTDigest.h>
namespace DB
{
struct NameQuantile { static constexpr auto name = "quantile"; };
struct NameQuantiles { static constexpr auto name = "quantiles"; };
struct NameQuantileDeterministic { static constexpr auto name = "quantileDeterministic"; };

2
dbms/src/AggregateFunctions/AggregateFunctionSum.cpp
View File

@ -14,7 +14,7 @@ template <typename T>
struct SumSimple
{
/// @note It uses slow Decimal128 (cause we need such a variant). sumWithOverflow is faster for Decimal32/64
using ResultType = std::conditional_t<IsDecimalNumber<T>, Decimal128, typename NearestFieldType<T>::Type>;
using ResultType = std::conditional_t<IsDecimalNumber<T>, Decimal128, NearestFieldType<T>>;
using AggregateDataType = AggregateFunctionSumData<ResultType>;
using Function = AggregateFunctionSum<T, ResultType, AggregateDataType>;
};

2
dbms/src/AggregateFunctions/AggregateFunctionSumMap.h
View File

@ -52,7 +52,7 @@ struct AggregateFunctionSumMapData
template <typename T>
class AggregateFunctionSumMap final : public IAggregateFunctionDataHelper<
AggregateFunctionSumMapData<typename NearestFieldType<T>::Type>, AggregateFunctionSumMap<T>>
AggregateFunctionSumMapData<NearestFieldType<T>>, AggregateFunctionSumMap<T>>
{
private:
using ColVecType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;

3
dbms/src/AggregateFunctions/AggregateFunctionUniq.cpp
View File

@ -130,9 +130,6 @@ void registerAggregateFunctionsUniq(AggregateFunctionFactory & factory)
factory.registerFunction("uniqExact",
createAggregateFunctionUniq<true, AggregateFunctionUniqExactData, AggregateFunctionUniqExactData<String>>);
factory.registerFunction("uniqCombined",
createAggregateFunctionUniq<false, AggregateFunctionUniqCombinedData, AggregateFunctionUniqCombinedData<UInt64>>);
}
}

88
dbms/src/AggregateFunctions/AggregateFunctionUniq.h
View File

@ -22,7 +22,6 @@
#include <Common/typeid_cast.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <AggregateFunctions/UniqCombinedBiasData.h>
#include <AggregateFunctions/UniqVariadicHash.h>
@ -124,46 +123,6 @@ struct AggregateFunctionUniqExactData<String>
static String getName() { return "uniqExact"; }
};
template <typename T>
struct AggregateFunctionUniqCombinedData
{
using Key = UInt32;
using Set = CombinedCardinalityEstimator<
Key,
HashSet<Key, TrivialHash, HashTableGrower<>>,
16,
14,
17,
TrivialHash,
UInt32,
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
HyperLogLogMode::FullFeatured>;
Set set;
static String getName() { return "uniqCombined"; }
};
template <>
struct AggregateFunctionUniqCombinedData<String>
{
using Key = UInt64;
using Set = CombinedCardinalityEstimator<
Key,
HashSet<Key, TrivialHash, HashTableGrower<>>,
16,
14,
17,
TrivialHash,
UInt64,
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
HyperLogLogMode::FullFeatured>;
Set set;
static String getName() { return "uniqCombined"; }
};
namespace detail
{
@ -199,39 +158,6 @@ template <> struct AggregateFunctionUniqTraits<Float64>
}
};
/** Hash function for uniqCombined.
*/
template <typename T> struct AggregateFunctionUniqCombinedTraits
{
static UInt32 hash(T x) { return static_cast<UInt32>(intHash64(x)); }
};
template <> struct AggregateFunctionUniqCombinedTraits<UInt128>
{
static UInt32 hash(UInt128 x)
{
return sipHash64(x);
}
};
template <> struct AggregateFunctionUniqCombinedTraits<Float32>
{
static UInt32 hash(Float32 x)
{
UInt64 res = ext::bit_cast<UInt64>(x);
return static_cast<UInt32>(intHash64(res));
}
};
template <> struct AggregateFunctionUniqCombinedTraits<Float64>
{
static UInt32 hash(Float64 x)
{
UInt64 res = ext::bit_cast<UInt64>(x);
return static_cast<UInt32>(intHash64(res));
}
};
/** The structure for the delegation work to add one element to the `uniq` aggregate functions.
* Used for partial specialization to add strings.
@ -255,19 +181,6 @@ struct OneAdder
data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
}
}
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqCombinedData<T>>)
{
if constexpr (!std::is_same_v<T, String>)
{
const auto & value = static_cast<const ColumnVector<T> &>(column).getData()[row_num];
data.set.insert(AggregateFunctionUniqCombinedTraits<T>::hash(value));
}
else
{
StringRef value = column.getDataAt(row_num);
data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
}
}
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqExactData<T>>)
{
if constexpr (!std::is_same_v<T, String>)
@ -387,5 +300,4 @@ public:
const char * getHeaderFilePath() const override { return __FILE__; }
};
}

127
dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp Normal file
View File

@ -0,0 +1,127 @@
#include <AggregateFunctions/AggregateFunctionUniqCombined.h>
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/Helpers.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int ARGUMENT_OUT_OF_BOUND;
}
namespace
{
template <UInt8 K>
struct WithK
{
template <typename T>
using AggregateFunction = AggregateFunctionUniqCombined<T, K>;
template <bool is_exact, bool argument_is_tuple>
using AggregateFunctionVariadic = AggregateFunctionUniqCombinedVariadic<is_exact, argument_is_tuple, K>;
};
template <UInt8 K>
AggregateFunctionPtr createAggregateFunctionWithK(const DataTypes & argument_types)
{
/// We use exact hash function if the arguments are not contiguous in memory, because only exact hash function has support for this case.
bool use_exact_hash_function = !isAllArgumentsContiguousInMemory(argument_types);
if (argument_types.size() == 1)
{
const IDataType & argument_type = *argument_types[0];
AggregateFunctionPtr res(createWithNumericType<WithK<K>::template AggregateFunction>(*argument_types[0]));
WhichDataType which(argument_type);
if (res)
return res;
else if (which.isDate())
return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeDate::FieldType>>();
else if (which.isDateTime())
return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeDateTime::FieldType>>();
else if (which.isStringOrFixedString())
return std::make_shared<typename WithK<K>::template AggregateFunction<String>>();
else if (which.isUUID())
return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeUUID::FieldType>>();
else if (which.isTuple())
{
if (use_exact_hash_function)
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<true, true>>(argument_types);
else
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<false, true>>(argument_types);
}
}
/// "Variadic" method also works as a fallback generic case for a single argument.
if (use_exact_hash_function)
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<true, false>>(argument_types);
else
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<false, false>>(argument_types);
}
AggregateFunctionPtr createAggregateFunctionUniqCombined(
const std::string & name, const DataTypes & argument_types, const Array & params)
{
/// log2 of the number of cells in HyperLogLog.
/// Reasonable default value, selected to be comparable in quality with "uniq" aggregate function.
UInt8 precision = 17;
if (!params.empty())
{
if (params.size() != 1)
throw Exception(
"Aggregate function " + name + " requires one parameter or less.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
UInt64 precision_param = applyVisitor(FieldVisitorConvertToNumber<UInt64>(), params[0]);
// This range is hardcoded below
if (precision_param > 20 || precision_param < 12)
throw Exception(
"Parameter for aggregate function " + name + "is out or range: [12, 20].", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
precision = precision_param;
}
if (argument_types.empty())
throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
switch (precision)
{
case 12:
return createAggregateFunctionWithK<12>(argument_types);
case 13:
return createAggregateFunctionWithK<13>(argument_types);
case 14:
return createAggregateFunctionWithK<14>(argument_types);
case 15:
return createAggregateFunctionWithK<15>(argument_types);
case 16:
return createAggregateFunctionWithK<16>(argument_types);
case 17:
return createAggregateFunctionWithK<17>(argument_types);
case 18:
return createAggregateFunctionWithK<18>(argument_types);
case 19:
return createAggregateFunctionWithK<19>(argument_types);
case 20:
return createAggregateFunctionWithK<20>(argument_types);
}
__builtin_unreachable();
}
} // namespace
void registerAggregateFunctionUniqCombined(AggregateFunctionFactory & factory)
{
factory.registerFunction("uniqCombined", createAggregateFunctionUniqCombined);
}
} // namespace DB

229
dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.h Normal file
View File

@ -0,0 +1,229 @@
#pragma once
#include <Common/CombinedCardinalityEstimator.h>
#include <DataTypes/DataTypeTuple.h>
#include <DataTypes/DataTypeUUID.h>
#include <DataTypes/DataTypesNumber.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <AggregateFunctions/UniqCombinedBiasData.h>
#include <AggregateFunctions/UniqVariadicHash.h>
#include <ext/bit_cast.h>
#include <Columns/ColumnVector.h>
#include <Columns/ColumnsNumber.h>
#include <Common/FieldVisitors.h>
#include <Common/SipHash.h>
#include <Common/typeid_cast.h>
namespace DB
{
namespace detail
{
/** Hash function for uniqCombined.
*/
template <typename T>
struct AggregateFunctionUniqCombinedTraits
{
static UInt32 hash(T x)
{
return static_cast<UInt32>(intHash64(x));
}
};
template <>
struct AggregateFunctionUniqCombinedTraits<UInt128>
{
static UInt32 hash(UInt128 x)
{
return sipHash64(x);
}
};
template <>
struct AggregateFunctionUniqCombinedTraits<Float32>
{
static UInt32 hash(Float32 x)
{
UInt64 res = ext::bit_cast<UInt64>(x);
return static_cast<UInt32>(intHash64(res));
}
};
template <>
struct AggregateFunctionUniqCombinedTraits<Float64>
{
static UInt32 hash(Float64 x)
{
UInt64 res = ext::bit_cast<UInt64>(x);
return static_cast<UInt32>(intHash64(res));
}
};
} // namespace detail
template <typename Key, UInt8 K>
struct AggregateFunctionUniqCombinedDataWithKey
{
// TODO(ilezhankin): pre-generate values for |UniqCombinedBiasData|,
// at the moment gen-bias-data.py script doesn't work.
// We want to migrate from |HashSet| to |HyperLogLogCounter| when the sizes in memory become almost equal.
// The size per element in |HashSet| is sizeof(Key)*2 bytes, and the overall size of |HyperLogLogCounter| is 2^K * 6 bits.
// For Key=UInt32 we can calculate: 2^X * 4 * 2 ≤ 2^(K-3) * 6 ⇒ X ≤ K-4.
using Set = CombinedCardinalityEstimator<Key, HashSet<Key, TrivialHash, HashTableGrower<>>, 16, K - 4, K, TrivialHash, Key>;
Set set;
};
template <typename Key>
struct AggregateFunctionUniqCombinedDataWithKey<Key, 17>
{
using Set = CombinedCardinalityEstimator<Key,
HashSet<Key, TrivialHash, HashTableGrower<>>,
16,
13,
17,
TrivialHash,
Key,
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
HyperLogLogMode::FullFeatured>;
Set set;
};
template <typename T, UInt8 K>
struct AggregateFunctionUniqCombinedData : public AggregateFunctionUniqCombinedDataWithKey<UInt32, K>
{
};
template <UInt8 K>
struct AggregateFunctionUniqCombinedData<String, K> : public AggregateFunctionUniqCombinedDataWithKey<UInt64, K>
{
};
template <typename T, UInt8 K>
class AggregateFunctionUniqCombined final
: public IAggregateFunctionDataHelper<AggregateFunctionUniqCombinedData<T, K>, AggregateFunctionUniqCombined<T, K>>
{
public:
String getName() const override
{
return "uniqCombined";
}
DataTypePtr getReturnType() const override
{
return std::make_shared<DataTypeUInt64>();
}
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
{
if constexpr (!std::is_same_v<T, String>)
{
const auto & value = static_cast<const ColumnVector<T> &>(*columns[0]).getData()[row_num];
this->data(place).set.insert(detail::AggregateFunctionUniqCombinedTraits<T>::hash(value));
}
else
{
StringRef value = columns[0]->getDataAt(row_num);
this->data(place).set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
}
}
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
{
this->data(place).set.merge(this->data(rhs).set);
}
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
{
this->data(place).set.write(buf);
}
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
{
this->data(place).set.read(buf);
}
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
{
static_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
}
const char * getHeaderFilePath() const override
{
return __FILE__;
}
};
/** For multiple arguments. To compute, hashes them.
* You can pass multiple arguments as is; You can also pass one argument - a tuple.
* But (for the possibility of efficient implementation), you can not pass several arguments, among which there are tuples.
*/
template <bool is_exact, bool argument_is_tuple, UInt8 K>
class AggregateFunctionUniqCombinedVariadic final : public IAggregateFunctionDataHelper<AggregateFunctionUniqCombinedData<UInt64, K>,
AggregateFunctionUniqCombinedVariadic<is_exact, argument_is_tuple, K>>
{
private:
size_t num_args = 0;
public:
explicit AggregateFunctionUniqCombinedVariadic(const DataTypes & arguments)
{
if (argument_is_tuple)
num_args = typeid_cast<const DataTypeTuple &>(*arguments[0]).getElements().size();
else
num_args = arguments.size();
}
String getName() const override
{
return "uniqCombined";
}
DataTypePtr getReturnType() const override
{
return std::make_shared<DataTypeUInt64>();
}
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
{
this->data(place).set.insert(typename AggregateFunctionUniqCombinedData<UInt64, K>::Set::value_type(
UniqVariadicHash<is_exact, argument_is_tuple>::apply(num_args, columns, row_num)));
}
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
{
this->data(place).set.merge(this->data(rhs).set);
}
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
{
this->data(place).set.write(buf);
}
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
{
this->data(place).set.read(buf);
}
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
{
static_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
}
const char * getHeaderFilePath() const override
{
return __FILE__;
}
};
} // namespace DB

4
dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h
View File

@ -16,10 +16,12 @@
namespace DB
{
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
struct ComparePairFirst final
@ -191,7 +193,7 @@ public:
const auto time_arg = arguments.front().get();
if (!WhichDataType(time_arg).isDateTime() && !WhichDataType(time_arg).isUInt32())
throw Exception{"Illegal type " + time_arg->getName() + " of first argument of aggregate function " + getName()
+ ", must be DateTime or UInt32"};
+ ", must be DateTime or UInt32", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
for (const auto i : ext::range(1, arguments.size()))
{

2
dbms/src/AggregateFunctions/CMakeLists.txt
View File

@ -20,5 +20,5 @@ list(REMOVE_ITEM clickhouse_aggregate_functions_headers
)
add_library(clickhouse_aggregate_functions ${LINK_MODE} ${clickhouse_aggregate_functions_sources})
target_link_libraries(clickhouse_aggregate_functions dbms)
target_link_libraries(clickhouse_aggregate_functions PRIVATE dbms)
target_include_directories (clickhouse_aggregate_functions BEFORE PRIVATE ${COMMON_INCLUDE_DIR})

31
dbms/src/AggregateFunctions/FactoryHelpers.cpp
View File

@ -1,31 +0,0 @@
#include <AggregateFunctions/FactoryHelpers.h>
namespace DB
{
namespace ErrorCodes
{
extern const int AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
void assertNoParameters(const std::string & name, const Array & parameters)
{
if (!parameters.empty())
throw Exception("Aggregate function " + name + " cannot have parameters", ErrorCodes::AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS);
}
void assertUnary(const std::string & name, const DataTypes & argument_types)
{
if (argument_types.size() != 1)
throw Exception("Aggregate function " + name + " require single argument", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}
void assertBinary(const std::string & name, const DataTypes & argument_types)
{
if (argument_types.size() != 2)
throw Exception("Aggregate function " + name + " require two arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}
}

26
dbms/src/AggregateFunctions/FactoryHelpers.h
View File

@ -7,8 +7,28 @@
namespace DB
{
void assertNoParameters(const std::string & name, const Array & parameters);
void assertUnary(const std::string & name, const DataTypes & argument_types);
void assertBinary(const std::string & name, const DataTypes & argument_types);
namespace ErrorCodes
{
extern const int AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
inline void assertNoParameters(const std::string & name, const Array & parameters)
{
if (!parameters.empty())
throw Exception("Aggregate function " + name + " cannot have parameters", ErrorCodes::AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS);
}
inline void assertUnary(const std::string & name, const DataTypes & argument_types)
{
if (argument_types.size() != 1)
throw Exception("Aggregate function " + name + " require single argument", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}
inline void assertBinary(const std::string & name, const DataTypes & argument_types)
{
if (argument_types.size() != 2)
throw Exception("Aggregate function " + name + " require two arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}
}

2
dbms/src/AggregateFunctions/registerAggregateFunctions.cpp
View File

@ -21,6 +21,7 @@ void registerAggregateFunctionsStatisticsSimple(AggregateFunctionFactory &);
void registerAggregateFunctionSum(AggregateFunctionFactory &);
void registerAggregateFunctionSumMap(AggregateFunctionFactory &);
void registerAggregateFunctionsUniq(AggregateFunctionFactory &);
void registerAggregateFunctionUniqCombined(AggregateFunctionFactory &);
void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &);
void registerAggregateFunctionTopK(AggregateFunctionFactory &);
void registerAggregateFunctionsBitwise(AggregateFunctionFactory &);
@ -55,6 +56,7 @@ void registerAggregateFunctions()
registerAggregateFunctionSum(factory);
registerAggregateFunctionSumMap(factory);
registerAggregateFunctionsUniq(factory);
registerAggregateFunctionUniqCombined(factory);
registerAggregateFunctionUniqUpTo(factory);
registerAggregateFunctionTopK(factory);
registerAggregateFunctionsBitwise(factory);

4
dbms/src/Client/Connection.cpp
View File

@ -108,14 +108,14 @@ void Connection::connect()
disconnect();
/// Add server address to exception. Also Exception will remember stack trace. It's a pity that more precise exception type is lost.
throw NetException(e.displayText(), "(" + getDescription() + ")", ErrorCodes::NETWORK_ERROR);
throw NetException(e.displayText() + " (" + getDescription() + ")", ErrorCodes::NETWORK_ERROR);
}
catch (Poco::TimeoutException & e)
{
disconnect();
/// Add server address to exception. Also Exception will remember stack trace. It's a pity that more precise exception type is lost.
throw NetException(e.displayText(), "(" + getDescription() + ")", ErrorCodes::SOCKET_TIMEOUT);
throw NetException(e.displayText() + " (" + getDescription() + ")", ErrorCodes::SOCKET_TIMEOUT);
}
}

2
dbms/src/Columns/ColumnConst.h
View File

@ -209,7 +209,7 @@ public:
Field getField() const { return getDataColumn()[0]; }
template <typename T>
T getValue() const { return getField().safeGet<typename NearestFieldType<T>::Type>(); }
T getValue() const { return getField().safeGet<NearestFieldType<T>>(); }
};
}

8
dbms/src/Columns/ColumnDecimal.cpp
View File

@ -213,8 +213,8 @@ void ColumnDecimal<T>::getExtremes(Field & min, Field & max) const
{
if (data.size() == 0)
{
min = typename NearestFieldType<T>::Type(0, scale);
max = typename NearestFieldType<T>::Type(0, scale);
min = NearestFieldType<T>(0, scale);
max = NearestFieldType<T>(0, scale);
return;
}
@ -229,8 +229,8 @@ void ColumnDecimal<T>::getExtremes(Field & min, Field & max) const
cur_max = x;
}
min = typename NearestFieldType<T>::Type(cur_min, scale);
max = typename NearestFieldType<T>::Type(cur_max, scale);
min = NearestFieldType<T>(cur_min, scale);
max = NearestFieldType<T>(cur_max, scale);
}
template class ColumnDecimal<Decimal32>;

2
dbms/src/Columns/ColumnDecimal.h
View File

@ -91,7 +91,7 @@ public:
void insertFrom(const IColumn & src, size_t n) override { data.push_back(static_cast<const Self &>(src).getData()[n]); }
void insertData(const char * pos, size_t /*length*/) override;
void insertDefault() override { data.push_back(T()); }
void insert(const Field & x) override { data.push_back(DB::get<typename NearestFieldType<T>::Type>(x)); }
void insert(const Field & x) override { data.push_back(DB::get<NearestFieldType<T>>(x)); }
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
void popBack(size_t n) override { data.resize_assume_reserved(data.size() - n); }

2
dbms/src/Columns/ColumnFunction.cpp
View File

@ -183,7 +183,7 @@ void ColumnFunction::appendArgument(const ColumnWithTypeAndName & column)
auto index = captured_columns.size();
if (!column.type->equals(*argumnet_types[index]))
throw Exception("Cannot capture column " + std::to_string(argumnet_types.size()) +
"because it has incompatible type: got " + column.type->getName() +
" because it has incompatible type: got " + column.type->getName() +
", but " + argumnet_types[index]->getName() + " is expected.", ErrorCodes::LOGICAL_ERROR);
captured_columns.push_back(column);

84
dbms/src/Columns/ColumnUnique.h
View File

@ -81,12 +81,14 @@ public:
{
return column_holder->allocatedBytes()
+ index.allocatedBytes()
+ (cached_null_mask ? cached_null_mask->allocatedBytes() : 0);
+ (nested_null_mask ? nested_null_mask->allocatedBytes() : 0);
}
void forEachSubcolumn(IColumn::ColumnCallback callback) override
{
callback(column_holder);
index.setColumn(getRawColumnPtr());
if (is_nullable)
nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask);
}
const UInt64 * tryGetSavedHash() const override { return index.tryGetSavedHash(); }
@ -100,8 +102,8 @@ private:
ReverseIndex<UInt64, ColumnType> index;
/// For DataTypeNullable, stores null map.
mutable ColumnPtr cached_null_mask;
mutable ColumnPtr cached_column_nullable;
ColumnPtr nested_null_mask;
ColumnPtr nested_column_nullable;
class IncrementalHash
{
@ -118,6 +120,9 @@ private:
mutable IncrementalHash hash;
void createNullMask();
void updateNullMask();
static size_t numSpecialValues(bool is_nullable) { return is_nullable ? 2 : 1; }
size_t numSpecialValues() const { return numSpecialValues(is_nullable); }
@ -148,6 +153,7 @@ ColumnUnique<ColumnType>::ColumnUnique(const ColumnUnique & other)
, index(numSpecialValues(is_nullable), 0)
{
index.setColumn(getRawColumnPtr());
createNullMask();
}
template <typename ColumnType>
@ -158,6 +164,7 @@ ColumnUnique<ColumnType>::ColumnUnique(const IDataType & type)
const auto & holder_type = is_nullable ? *static_cast<const DataTypeNullable &>(type).getNestedType() : type;
column_holder = holder_type.createColumn()->cloneResized(numSpecialValues());
index.setColumn(getRawColumnPtr());
createNullMask();
}
template <typename ColumnType>
@ -172,32 +179,51 @@ ColumnUnique<ColumnType>::ColumnUnique(MutableColumnPtr && holder, bool is_nulla
throw Exception("Holder column for ColumnUnique can't be nullable.", ErrorCodes::ILLEGAL_COLUMN);
index.setColumn(getRawColumnPtr());
createNullMask();
}
template <typename ColumnType>
void ColumnUnique<ColumnType>::createNullMask()
{
if (is_nullable)
{
size_t size = getRawColumnPtr()->size();
if (!nested_null_mask)
{
ColumnUInt8::MutablePtr null_mask = ColumnUInt8::create(size, UInt8(0));
null_mask->getData()[getNullValueIndex()] = 1;
nested_null_mask = std::move(null_mask);
nested_column_nullable = ColumnNullable::create(column_holder, nested_null_mask);
}
else
throw Exception("Null mask for ColumnUnique is already created.", ErrorCodes::LOGICAL_ERROR);
}
}
template <typename ColumnType>
void ColumnUnique<ColumnType>::updateNullMask()
{
if (is_nullable)
{
if (!nested_null_mask)
throw Exception("Null mask for ColumnUnique is was not created.", ErrorCodes::LOGICAL_ERROR);
size_t size = getRawColumnPtr()->size();
if (nested_null_mask->size() != size)
{
IColumn & null_mask = nested_null_mask->assumeMutableRef();
static_cast<ColumnUInt8 &>(null_mask).getData().resize_fill(size);
}
}
}
template <typename ColumnType>
const ColumnPtr & ColumnUnique<ColumnType>::getNestedColumn() const
{
if (is_nullable)
{
size_t size = getRawColumnPtr()->size();
if (!cached_null_mask)
{
ColumnUInt8::MutablePtr null_mask = ColumnUInt8::create(size, UInt8(0));
null_mask->getData()[getNullValueIndex()] = 1;
cached_null_mask = std::move(null_mask);
cached_column_nullable = ColumnNullable::create(column_holder, cached_null_mask);
}
return nested_column_nullable;
if (cached_null_mask->size() != size)
{
MutableColumnPtr null_mask = (*std::move(cached_null_mask)).mutate();
static_cast<ColumnUInt8 &>(*null_mask).getData().resize_fill(size);
cached_null_mask = std::move(null_mask);
cached_column_nullable = ColumnNullable::create(column_holder, cached_null_mask);
}
return cached_column_nullable;
}
return column_holder;
}
@ -227,6 +253,8 @@ size_t ColumnUnique<ColumnType>::uniqueInsert(const Field & x)
if (pos != prev_size)
column->popBack(1);
updateNullMask();
return pos;
}
@ -260,6 +288,8 @@ size_t ColumnUnique<ColumnType>::uniqueInsertData(const char * pos, size_t lengt
index.insertFromLastRow();
}
updateNullMask();
return insertion_point;
}
@ -288,6 +318,8 @@ size_t ColumnUnique<ColumnType>::uniqueInsertDataWithTerminatingZero(const char
if (position != prev_size)
column->popBack(1);
updateNullMask();
return static_cast<size_t>(position);
}
@ -305,7 +337,7 @@ StringRef ColumnUnique<ColumnType>::serializeValueIntoArena(size_t n, Arena & ar
size_t nested_size = 0;
if (n == getNullValueIndex())
if (n != getNullValueIndex())
nested_size = column_holder->serializeValueIntoArena(n, arena, begin).size;
return StringRef(pos, sizeof(null_flag) + nested_size);
@ -343,6 +375,8 @@ size_t ColumnUnique<ColumnType>::uniqueDeserializeAndInsertFromArena(const char
if (index_pos != prev_size)
column->popBack(1);
updateNullMask();
return static_cast<size_t>(index_pos);
}
@ -533,6 +567,8 @@ MutableColumnPtr ColumnUnique<ColumnType>::uniqueInsertRangeFrom(const IColumn &
if (!positions_column)
throw Exception("Can't find index type for ColumnUnique", ErrorCodes::LOGICAL_ERROR);
updateNullMask();
return positions_column;
}
@ -577,6 +613,8 @@ IColumnUnique::IndexesWithOverflow ColumnUnique<ColumnType>::uniqueInsertRangeWi
if (!positions_column)
throw Exception("Can't find index type for ColumnUnique", ErrorCodes::LOGICAL_ERROR);
updateNullMask();
IColumnUnique::IndexesWithOverflow indexes_with_overflow;
indexes_with_overflow.indexes = std::move(positions_column);
indexes_with_overflow.overflowed_keys = std::move(overflowed_keys);

13
dbms/src/Columns/ColumnVector.cpp
View File

@ -1,19 +1,16 @@
#include "ColumnVector.h"
#include <cstring>
#include <cmath>
#include <common/unaligned.h>
#include <Common/Exception.h>
#include <Common/Arena.h>
#include <Common/SipHash.h>
#include <Common/NaNUtils.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteHelpers.h>
#include <Columns/ColumnVector.h>
#include <Columns/ColumnsCommon.h>
#include <DataStreams/ColumnGathererStream.h>
#include <ext/bit_cast.h>
#if __SSE2__
@ -314,8 +311,8 @@ void ColumnVector<T>::getExtremes(Field & min, Field & max) const
cur_max = x;
}
min = typename NearestFieldType<T>::Type(cur_min);
max = typename NearestFieldType<T>::Type(cur_max);
min = NearestFieldType<T>(cur_min);
max = NearestFieldType<T>(cur_max);
}
/// Explicit template instantiations - to avoid code bloat in headers.

2
dbms/src/Columns/ColumnVector.h
View File

@ -244,7 +244,7 @@ public:
void insert(const Field & x) override
{
data.push_back(DB::get<typename NearestFieldType<T>::Type>(x));
data.push_back(DB::get<NearestFieldType<T>>(x));
}
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;

1
dbms/src/Common/ClickHouseRevision.cpp
View File

@ -4,4 +4,5 @@
namespace ClickHouseRevision
{
unsigned get() { return VERSION_REVISION; }
unsigned getVersionInteger() { return VERSION_INTEGER; }
}

1
dbms/src/Common/ClickHouseRevision.h
View File

@ -3,4 +3,5 @@
namespace ClickHouseRevision
{
unsigned get();
unsigned getVersionInteger();
}

5
dbms/src/Common/Config/CMakeLists.txt
View File

@ -1,9 +1,8 @@
include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
add_headers_and_sources(clickhouse_common_config .)
add_library(clickhouse_common_config ${LINK_MODE} ${clickhouse_common_config_headers} ${clickhouse_common_config_sources})
target_link_libraries (clickhouse_common_config clickhouse_common_zookeeper string_utils ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY})
target_include_directories (clickhouse_common_config PRIVATE ${DBMS_INCLUDE_DIR})
target_link_libraries(clickhouse_common_config PUBLIC common PRIVATE clickhouse_common_zookeeper string_utils PUBLIC ${Poco_XML_LIBRARY} ${Poco_Util_LIBRARY})
target_include_directories(clickhouse_common_config PUBLIC ${DBMS_INCLUDE_DIR})

22
dbms/src/Common/CounterInFile.h
View File

@ -18,7 +18,15 @@
#include <common/Types.h>
#define SMALL_READ_WRITE_BUFFER_SIZE 16
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_OPEN_FILE;
extern const int CANNOT_READ_ALL_DATA;
extern const int ATTEMPT_TO_READ_AFTER_EOF;
}
}
/** Stores a number in the file.
@ -26,6 +34,9 @@
*/
class CounterInFile
{
private:
static inline constexpr size_t SMALL_READ_WRITE_BUFFER_SIZE = 16;
public:
/// path - the name of the file, including the path
CounterInFile(const std::string & path_) : path(path_) {}
@ -56,13 +67,13 @@ public:
int fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0666);
if (-1 == fd)
DB::throwFromErrno("Cannot open file " + path);
DB::throwFromErrno("Cannot open file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
try
{
int flock_ret = flock(fd, LOCK_EX);
if (-1 == flock_ret)
DB::throwFromErrno("Cannot lock file " + path);
DB::throwFromErrno("Cannot lock file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
if (!file_doesnt_exists)
{
@ -130,7 +141,7 @@ public:
int fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0666);
if (-1 == fd)
DB::throwFromErrno("Cannot open file " + path);
DB::throwFromErrno("Cannot open file " + path, DB::ErrorCodes::CANNOT_OPEN_FILE);
try
{
@ -178,6 +189,3 @@ private:
std::string path;
std::mutex mutex;
};
#undef SMALL_READ_WRITE_BUFFER_SIZE

3
dbms/src/Common/CurrentMetrics.cpp
View File

@ -39,7 +39,8 @@
M(StorageBufferRows, "Number of rows in buffers of Buffer tables") \
M(StorageBufferBytes, "Number of bytes in buffers of Buffer tables") \
M(DictCacheRequests, "Number of requests in fly to data sources of dictionaries of cache type.") \
M(Revision, "Revision of the server. It is a number incremented for every release or release candidate.") \
M(Revision, "Revision of the server. It is a number incremented for every release or release candidate except patch releases.") \
M(VersionInteger, "Version of the server in a single integer number in base-1000. For example, version 11.22.33 is translated to 11022033.") \
M(RWLockWaitingReaders, "Number of threads waiting for read on a table RWLock.") \
M(RWLockWaitingWriters, "Number of threads waiting for write on a table RWLock.") \
M(RWLockActiveReaders, "Number of threads holding read lock in a table RWLock.") \

5
dbms/src/Common/ErrorCodes.cpp
View File

@ -396,6 +396,11 @@ namespace ErrorCodes
extern const int MULTIPLE_ASSIGNMENTS_TO_COLUMN = 419;
extern const int CANNOT_UPDATE_COLUMN = 420;
extern const int CANNOT_ADD_DIFFERENT_AGGREGATE_STATES = 421;
extern const int UNSUPPORTED_URI_SCHEME = 422;
extern const int CANNOT_GETTIMEOFDAY = 423;
extern const int CANNOT_LINK = 424;
extern const int SYSTEM_ERROR = 425;
extern const int NULL_POINTER_DEREFERENCE = 426;
extern const int KEEPER_EXCEPTION = 999;
extern const int POCO_EXCEPTION = 1000;

2
dbms/src/Common/Exception.cpp
View File

@ -52,7 +52,7 @@ std::string errnoToString(int code, int e)
void throwFromErrno(const std::string & s, int code, int e)
{
throw ErrnoException(s + ", " + errnoToString(code, e));
throw ErrnoException(s + ", " + errnoToString(code, e), code, e);
}
void tryLogCurrentException(const char * log_name, const std::string & start_of_message)

38
dbms/src/Common/Exception.h
View File

@ -14,19 +14,25 @@ namespace Poco { class Logger; }
namespace DB
{
namespace ErrorCodes
{
extern const int POCO_EXCEPTION;
}
class Exception : public Poco::Exception
{
public:
Exception() {} /// For deferred initialization.
Exception(const std::string & msg, int code = 0) : Poco::Exception(msg, code) {}
Exception(const std::string & msg, const std::string & arg, int code = 0) : Poco::Exception(msg, arg, code) {}
Exception(const std::string & msg, const Exception & exc, int code = 0) : Poco::Exception(msg, exc, code), trace(exc.trace) {}
explicit Exception(const Poco::Exception & exc) : Poco::Exception(exc.displayText()) {}
Exception(const std::string & msg, int code) : Poco::Exception(msg, code) {}
Exception(const std::string & msg, const Exception & nested_exception, int code)
: Poco::Exception(msg, nested_exception, code), trace(nested_exception.trace) {}
const char * name() const throw() override { return "DB::Exception"; }
const char * className() const throw() override { return "DB::Exception"; }
DB::Exception * clone() const override { return new DB::Exception(*this); }
enum CreateFromPocoTag { CreateFromPoco };
Exception(CreateFromPocoTag, const Poco::Exception & exc) : Poco::Exception(exc.displayText(), ErrorCodes::POCO_EXCEPTION) {}
Exception * clone() const override { return new Exception(*this); }
void rethrow() const override { throw *this; }
const char * name() const throw() override { return "DB::Exception"; }
/// Add something to the existing message.
void addMessage(const std::string & arg) { extendedMessage(arg); }
@ -35,6 +41,8 @@ public:
private:
StackTrace trace;
const char * className() const throw() override { return "DB::Exception"; }
};
@ -42,25 +50,27 @@ private:
class ErrnoException : public Exception
{
public:
ErrnoException(const std::string & msg, int code = 0, int saved_errno_ = 0)
ErrnoException(const std::string & msg, int code, int saved_errno_)
: Exception(msg, code), saved_errno(saved_errno_) {}
ErrnoException(const std::string & msg, const std::string & arg, int code = 0, int saved_errno_ = 0)
: Exception(msg, arg, code), saved_errno(saved_errno_) {}
ErrnoException(const std::string & msg, const Exception & exc, int code = 0, int saved_errno_ = 0)
: Exception(msg, exc, code), saved_errno(saved_errno_) {}
ErrnoException * clone() const override { return new ErrnoException(*this); }
void rethrow() const override { throw *this; }
int getErrno() const { return saved_errno; }
private:
int saved_errno;
const char * name() const throw() override { return "DB::ErrnoException"; }
const char * className() const throw() override { return "DB::ErrnoException"; }
};
using Exceptions = std::vector<std::exception_ptr>;
std::string errnoToString(int code = 0, int the_errno = errno);
[[noreturn]] void throwFromErrno(const std::string & s, int code = 0, int the_errno = errno);
std::string errnoToString(int code, int the_errno = errno);
[[noreturn]] void throwFromErrno(const std::string & s, int code, int the_errno = errno);
/** Try to write an exception to the log (and forget about it).

16
dbms/src/Common/HashTable/SmallTable.h
View File

@ -3,6 +3,15 @@
#include <Common/HashTable/HashMap.h>
namespace DB
{
namespace ErrorCodes
{
extern const int INCORRECT_DATA;
}
}
/** Replacement of the hash table for a small number (<10) of keys.
* Implemented as an array with linear search.
* The array is located inside the object.
@ -13,7 +22,6 @@
* you should check if the table is not full,
* and do a `fallback` in this case (for example, use a real hash table).
*/
template
<
typename Key,
@ -86,7 +94,7 @@ public:
DB::readVarUInt(size, in);
if (size > capacity)
throw DB::Exception("Illegal size");
throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA);
is_initialized = true;
}
@ -306,7 +314,7 @@ public:
DB::readVarUInt(new_size, rb);
if (new_size > capacity)
throw DB::Exception("Illegal size");
throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA);
for (size_t i = 0; i < new_size; ++i)
buf[i].read(rb);
@ -324,7 +332,7 @@ public:
DB::readText(new_size, rb);
if (new_size > capacity)
throw DB::Exception("Illegal size");
throw DB::Exception("Illegal size", DB::ErrorCodes::INCORRECT_DATA);
for (size_t i = 0; i < new_size; ++i)
{

13
dbms/src/Common/NetException.h
View File

@ -6,20 +6,17 @@
namespace DB
{
class NetException : public DB::Exception
class NetException : public Exception
{
public:
NetException(const std::string & msg, int code = 0) : DB::Exception(msg, code) {}
NetException(const std::string & msg, const std::string & arg, int code = 0) : DB::Exception(msg, arg, code) {}
NetException(const std::string & msg, const DB::Exception & exc, int code = 0) : DB::Exception(msg, exc, code) {}
NetException(const std::string & msg, int code) : Exception(msg, code) {}
explicit NetException(const DB::Exception & exc) : DB::Exception(exc) {}
explicit NetException(const Poco::Exception & exc) : DB::Exception(exc.displayText()) {}
NetException * clone() const override { return new NetException(*this); }
void rethrow() const override { throw *this; }
private:
const char * name() const throw() override { return "DB::NetException"; }
const char * className() const throw() override { return "DB::NetException"; }
DB::NetException * clone() const override { return new DB::NetException(*this); }
void rethrow() const override { throw *this; }
};
}

2
dbms/src/Common/PoolWithFailoverBase.h
View File

@ -305,7 +305,7 @@ void PoolWithFailoverBase<TNestedPool>::reportError(const Entry & entry)
return;
}
}
throw DB::Exception("Can't find pool to report error.");
throw DB::Exception("Can't find pool to report error", DB::ErrorCodes::LOGICAL_ERROR);
}
template <typename TNestedPool>

1
dbms/src/Common/ProfileEvents.cpp
View File

@ -170,6 +170,7 @@
M(OSWriteBytes, "Number of bytes written to disks or block devices. Doesn't include bytes that are in page cache dirty pages. May not include data that was written by OS asynchronously.") \
M(OSReadChars, "Number of bytes read from filesystem, including page cache.") \
M(OSWriteChars, "Number of bytes written to filesystem, including page cache.") \
M(CreatedHTTPConnections, "Total amount of created HTTP connections (closed or opened).") \
namespace ProfileEvents
{

30
dbms/src/Common/ShellCommand.cpp
View File

@ -4,9 +4,11 @@
#include <dlfcn.h>
#include <Common/Exception.h>
#include <Common/ShellCommand.h>
#include <common/logger_useful.h>
#include <IO/WriteBufferFromVector.h>
#include <IO/WriteHelpers.h>
#include <port/unistd.h>
#include <csignal>
namespace DB
@ -75,13 +77,27 @@ namespace
namespace DB
{
ShellCommand::ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd, bool terminate_in_destructor_)
: pid(pid)
, terminate_in_destructor(terminate_in_destructor_)
, log(&Poco::Logger::get("ShellCommand"))
, in(in_fd)
, out(out_fd)
, err(err_fd) {}
ShellCommand::~ShellCommand()
{
if (!wait_called)
if (terminate_in_destructor)
{
int retcode = kill(pid, SIGTERM);
if (retcode != 0)
LOG_WARNING(log, "Cannot kill pid " << pid << " errno '" << errnoToString(retcode) << "'");
}
else if (!wait_called)
tryWait();
}
std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only)
std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor)
{
/** Here it is written that with a normal call `vfork`, there is a chance of deadlock in multithreaded programs,
* because of the resolving of characters in the shared library
@ -128,7 +144,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, c
_exit(int(ReturnCodes::CANNOT_EXEC));
}
std::unique_ptr<ShellCommand> res(new ShellCommand(pid, pipe_stdin.write_fd, pipe_stdout.read_fd, pipe_stderr.read_fd));
std::unique_ptr<ShellCommand> res(new ShellCommand(pid, pipe_stdin.write_fd, pipe_stdout.read_fd, pipe_stderr.read_fd, terminate_in_destructor));
/// Now the ownership of the file descriptors is passed to the result.
pipe_stdin.write_fd = -1;
@ -139,7 +155,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, c
}
std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command, bool pipe_stdin_only)
std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command, bool pipe_stdin_only, bool terminate_in_destructor)
{
/// Arguments in non-constant chunks of memory (as required for `execv`).
/// Moreover, their copying must be done before calling `vfork`, so after `vfork` do a minimum of things.
@ -149,11 +165,11 @@ std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command,
char * const argv[] = { argv0.data(), argv1.data(), argv2.data(), nullptr };
return executeImpl("/bin/sh", argv, pipe_stdin_only);
return executeImpl("/bin/sh", argv, pipe_stdin_only, terminate_in_destructor);
}
std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & path, const std::vector<std::string> & arguments)
std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & path, const std::vector<std::string> & arguments, bool terminate_in_destructor)
{
size_t argv_sum_size = path.size() + 1;
for (const auto & arg : arguments)
@ -174,7 +190,7 @@ std::unique_ptr<ShellCommand> ShellCommand::executeDirect(const std::string & pa
argv[arguments.size() + 1] = nullptr;
return executeImpl(path.data(), argv.data(), false);
return executeImpl(path.data(), argv.data(), false, terminate_in_destructor);
}

16
dbms/src/Common/ShellCommand.h
View File

@ -28,11 +28,13 @@ class ShellCommand
private:
pid_t pid;
bool wait_called = false;
bool terminate_in_destructor;
ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd)
: pid(pid), in(in_fd), out(out_fd), err(err_fd) {}
Poco::Logger * log;
static std::unique_ptr<ShellCommand> executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only);
ShellCommand(pid_t pid, int in_fd, int out_fd, int err_fd, bool terminate_in_destructor_);
static std::unique_ptr<ShellCommand> executeImpl(const char * filename, char * const argv[], bool pipe_stdin_only, bool terminate_in_destructor);
public:
WriteBufferFromFile in; /// If the command reads from stdin, do not forget to call in.close() after writing all the data there.
@ -41,11 +43,13 @@ public:
~ShellCommand();
/// Run the command using /bin/sh -c
static std::unique_ptr<ShellCommand> execute(const std::string & command, bool pipe_stdin_only = false);
/// Run the command using /bin/sh -c.
/// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process.
static std::unique_ptr<ShellCommand> execute(const std::string & command, bool pipe_stdin_only = false, bool terminate_in_destructor = false);
/// Run the executable with the specified arguments. `arguments` - without argv[0].
static std::unique_ptr<ShellCommand> executeDirect(const std::string & path, const std::vector<std::string> & arguments);
/// If terminate_in_destructor is true, send terminate signal in destructor and don't wait process.
static std::unique_ptr<ShellCommand> executeDirect(const std::string & path, const std::vector<std::string> & arguments, bool terminate_in_destructor = false);
/// Wait for the process to end, throw an exception if the code is not 0 or if the process was not completed by itself.
void wait();

22
dbms/src/Common/StatusFile.cpp
View File

@ -20,6 +20,14 @@
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_OPEN_FILE;
extern const int CANNOT_CLOSE_FILE;
extern const int CANNOT_TRUNCATE_FILE;
extern const int CANNOT_SEEK_THROUGH_FILE;
}
StatusFile::StatusFile(const std::string & path_)
: path(path_)
@ -43,7 +51,7 @@ StatusFile::StatusFile(const std::string & path_)
fd = ::open(path.c_str(), O_WRONLY | O_CREAT, 0666);
if (-1 == fd)
throwFromErrno("Cannot open file " + path);
throwFromErrno("Cannot open file " + path, ErrorCodes::CANNOT_OPEN_FILE);
try
{
@ -51,16 +59,16 @@ StatusFile::StatusFile(const std::string & path_)
if (-1 == flock_ret)
{
if (errno == EWOULDBLOCK)
throw Exception("Cannot lock file " + path + ". Another server instance in same directory is already running.");
throw Exception("Cannot lock file " + path + ". Another server instance in same directory is already running.", ErrorCodes::CANNOT_OPEN_FILE);
else
throwFromErrno("Cannot lock file " + path);
throwFromErrno("Cannot lock file " + path, ErrorCodes::CANNOT_OPEN_FILE);
}
if (0 != ftruncate(fd, 0))
throwFromErrno("Cannot ftruncate " + path);
throwFromErrno("Cannot ftruncate " + path, ErrorCodes::CANNOT_TRUNCATE_FILE);
if (0 != lseek(fd, 0, SEEK_SET))
throwFromErrno("Cannot lseek " + path);
throwFromErrno("Cannot lseek " + path, ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
/// Write information about current server instance to the file.
{
@ -82,10 +90,10 @@ StatusFile::StatusFile(const std::string & path_)
StatusFile::~StatusFile()
{
if (0 != close(fd))
LOG_ERROR(&Logger::get("StatusFile"), "Cannot close file " << path << ", " << errnoToString());
LOG_ERROR(&Logger::get("StatusFile"), "Cannot close file " << path << ", " << errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
if (0 != unlink(path.c_str()))
LOG_ERROR(&Logger::get("StatusFile"), "Cannot unlink file " << path << ", " << errnoToString());
LOG_ERROR(&Logger::get("StatusFile"), "Cannot unlink file " << path << ", " << errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
}
}

57
dbms/src/Common/XDBCBridgeHelper.h
View File

@ -68,6 +68,7 @@ protected:
public:
using Configuration = Poco::Util::AbstractConfiguration;
Context & context;
const Configuration & config;
static constexpr inline auto DEFAULT_HOST = "localhost";
@ -78,8 +79,8 @@ public:
static constexpr inline auto IDENTIFIER_QUOTE_HANDLER = "/identifier_quote";
static constexpr inline auto PING_OK_ANSWER = "Ok.";
XDBCBridgeHelper(const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_)
: http_timeout(http_timeout_), connection_string(connection_string_), config(config_)
XDBCBridgeHelper(Context & global_context_, const Poco::Timespan & http_timeout_, const std::string & connection_string_)
: http_timeout(http_timeout_), connection_string(connection_string_), context(global_context_), config(context.getConfigRef())
{
size_t bridge_port = config.getUInt(BridgeHelperMixin::configPrefix() + ".port", DEFAULT_PORT);
std::string bridge_host = config.getString(BridgeHelperMixin::configPrefix() + ".host", DEFAULT_HOST);
@ -210,7 +211,8 @@ private:
/* Contains logic for instantiation of the bridge instance */
void startBridge() const
{
BridgeHelperMixin::startBridge(config, log, http_timeout);
auto cmd = BridgeHelperMixin::startBridge(config, log, http_timeout);
context.addXDBCBridgeCommand(std::move(cmd));
}
};
@ -230,7 +232,7 @@ struct JDBCBridgeMixin
return "JDBC";
}
static void startBridge(const Poco::Util::AbstractConfiguration &, const Poco::Logger *, const Poco::Timespan &)
static std::unique_ptr<ShellCommand> startBridge(const Poco::Util::AbstractConfiguration &, const Poco::Logger *, const Poco::Timespan &)
{
throw Exception("jdbc-bridge is not running. Please, start it manually", ErrorCodes::EXTERNAL_SERVER_IS_NOT_RESPONDING);
}
@ -253,11 +255,13 @@ struct ODBCBridgeMixin
return "ODBC";
}
static void startBridge(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log, const Poco::Timespan & http_timeout)
static std::unique_ptr<ShellCommand> startBridge(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log, const Poco::Timespan & http_timeout)
{
/// Path to executable folder
Poco::Path path{config.getString("application.dir", "/usr/bin")};
std::vector<std::string> cmd_args;
path.setFileName(
#if CLICKHOUSE_SPLIT_BINARY
"clickhouse-odbc-bridge"
@ -268,34 +272,35 @@ struct ODBCBridgeMixin
std::stringstream command;
command << path.toString() <<
#if CLICKHOUSE_SPLIT_BINARY
" "
#else
" odbc-bridge "
#if !CLICKHOUSE_SPLIT_BINARY
cmd_args.push_back("odbc-bridge");
#endif
;
command << "--http-port " << config.getUInt(configPrefix() + ".port", DEFAULT_PORT) << ' ';
command << "--listen-host " << config.getString(configPrefix() + ".listen_host", XDBCBridgeHelper<ODBCBridgeMixin>::DEFAULT_HOST)
<< ' ';
command << "--http-timeout " << http_timeout.totalMicroseconds() << ' ';
cmd_args.push_back("--http-port");
cmd_args.push_back(std::to_string(config.getUInt(configPrefix() + ".port", DEFAULT_PORT)));
cmd_args.push_back("--listen-host");
cmd_args.push_back(config.getString(configPrefix() + ".listen_host", XDBCBridgeHelper<ODBCBridgeMixin>::DEFAULT_HOST));
cmd_args.push_back("--http-timeout");
cmd_args.push_back(std::to_string(http_timeout.totalMicroseconds()));
if (config.has("logger." + configPrefix() + "_log"))
command << "--log-path " << config.getString("logger." + configPrefix() + "_log") << ' ';
{
cmd_args.push_back("--log-path");
cmd_args.push_back(config.getString("logger." + configPrefix() + "_log"));
}
if (config.has("logger." + configPrefix() + "_errlog"))
command << "--err-log-path " << config.getString("logger." + configPrefix() + "_errlog") << ' ';
{
cmd_args.push_back("--err-log-path");
cmd_args.push_back(config.getString("logger." + configPrefix() + "_errlog"));
}
if (config.has("logger." + configPrefix() + "_level"))
command << "--log-level " << config.getString("logger." + configPrefix() + "_level") << ' ';
command << "&"; /// we don't want to wait this process
{
cmd_args.push_back("--log-level");
cmd_args.push_back(config.getString("logger." + configPrefix() + "_level"));
}
auto command_str = command.str();
LOG_TRACE(log, "Starting " + serviceAlias());
std::cerr << command_str << std::endl;
LOG_TRACE(log, "Starting " + serviceAlias() + " with command: " << command_str);
auto cmd = ShellCommand::execute(command_str);
cmd->wait();
return ShellCommand::executeDirect(path.toString(), cmd_args, true);
}
};
}

3
dbms/src/Common/ZooKeeper/CMakeLists.txt
View File

@ -4,7 +4,8 @@ add_headers_and_sources(clickhouse_common_zookeeper .)
add_library(clickhouse_common_zookeeper ${LINK_MODE} ${clickhouse_common_zookeeper_headers} ${clickhouse_common_zookeeper_sources})
target_link_libraries (clickhouse_common_zookeeper clickhouse_common_io)
target_link_libraries (clickhouse_common_zookeeper PUBLIC clickhouse_common_io common PRIVATE string_utils PUBLIC ${Poco_Util_LIBRARY})
target_include_directories(clickhouse_common_zookeeper PUBLIC ${DBMS_INCLUDE_DIR})
if (ENABLE_TESTS)
add_subdirectory (tests)

18
dbms/src/Common/ZooKeeper/ZooKeeperHolder.cpp
View File

@ -1,5 +1,15 @@
#include "ZooKeeperHolder.h"
namespace DB
{
namespace ErrorCodes
{
extern const int NULL_POINTER_DEREFERENCE;
}
}
using namespace zkutil;
ZooKeeperHolder::UnstorableZookeeperHandler ZooKeeperHolder::getZooKeeper()
@ -47,7 +57,7 @@ ZooKeeperHolder::UnstorableZookeeperHandler::UnstorableZookeeperHandler(ZooKeepe
ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->()
{
if (zk_ptr == nullptr)
throw DB::Exception(nullptr_exception_message);
throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
return zk_ptr.get();
}
@ -55,20 +65,20 @@ ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->()
const ZooKeeper * ZooKeeperHolder::UnstorableZookeeperHandler::operator->() const
{
if (zk_ptr == nullptr)
throw DB::Exception(nullptr_exception_message);
throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
return zk_ptr.get();
}
ZooKeeper & ZooKeeperHolder::UnstorableZookeeperHandler::operator*()
{
if (zk_ptr == nullptr)
throw DB::Exception(nullptr_exception_message);
throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
return *zk_ptr;
}
const ZooKeeper & ZooKeeperHolder::UnstorableZookeeperHandler::operator*() const
{
if (zk_ptr == nullptr)
throw DB::Exception(nullptr_exception_message);
throw DB::Exception(nullptr_exception_message, DB::ErrorCodes::NULL_POINTER_DEREFERENCE);
return *zk_ptr;
}

16
dbms/src/Common/ZooKeeper/tests/CMakeLists.txt
View File

@ -1,23 +1,23 @@
add_executable(zkutil_test_commands zkutil_test_commands.cpp)
target_link_libraries(zkutil_test_commands clickhouse_common_zookeeper)
target_link_libraries(zkutil_test_commands PRIVATE clickhouse_common_zookeeper)
add_executable(zkutil_test_commands_new_lib zkutil_test_commands_new_lib.cpp)
target_link_libraries(zkutil_test_commands_new_lib clickhouse_common_zookeeper)
target_link_libraries(zkutil_test_commands_new_lib PRIVATE clickhouse_common_zookeeper)
add_executable(zkutil_test_lock zkutil_test_lock.cpp)
target_link_libraries(zkutil_test_lock clickhouse_common_zookeeper)
target_link_libraries(zkutil_test_lock PRIVATE clickhouse_common_zookeeper)
add_executable(zkutil_expiration_test zkutil_expiration_test.cpp)
target_link_libraries(zkutil_expiration_test clickhouse_common_zookeeper)
target_link_libraries(zkutil_expiration_test PRIVATE clickhouse_common_zookeeper)
add_executable(zkutil_test_async zkutil_test_async.cpp)
target_link_libraries(zkutil_test_async clickhouse_common_zookeeper)
target_link_libraries(zkutil_test_async PRIVATE clickhouse_common_zookeeper)
add_executable(zkutil_zookeeper_holder zkutil_zookeeper_holder.cpp)
target_link_libraries(zkutil_zookeeper_holder clickhouse_common_zookeeper)
target_link_libraries(zkutil_zookeeper_holder PRIVATE clickhouse_common_zookeeper)
add_executable (zk_many_watches_reconnect zk_many_watches_reconnect.cpp)
target_link_libraries (zk_many_watches_reconnect clickhouse_common_zookeeper clickhouse_common_config)
target_link_libraries (zk_many_watches_reconnect PRIVATE clickhouse_common_zookeeper clickhouse_common_config)
add_executable (zookeeper_impl zookeeper_impl.cpp)
target_link_libraries (zookeeper_impl clickhouse_common_zookeeper)
target_link_libraries (zookeeper_impl PRIVATE clickhouse_common_zookeeper)

1
dbms/src/Common/config_version.h.in
View File

@ -23,6 +23,7 @@
#cmakedefine VERSION_FULL "@VERSION_FULL@"
#cmakedefine VERSION_DESCRIBE "@VERSION_DESCRIBE@"
#cmakedefine VERSION_GITHASH "@VERSION_GITHASH@"
#cmakedefine VERSION_INTEGER @VERSION_INTEGER@
#if defined(VERSION_MAJOR)
#define DBMS_VERSION_MAJOR VERSION_MAJOR

Some files were not shown because too many files have changed in this diff Show More