thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into alter-order-by

Browse Source 
Conflicts:
	dbms/src/Parsers/ASTAlterQuery.h
	dbms/src/Parsers/ParserAlterQuery.cpp
	dbms/src/Storages/AlterCommands.cpp
	dbms/src/Storages/AlterCommands.h
This commit is contained in:
Alexey Zatelepin 2018-11-26 16:01:48 +03:00
commit d3d9242462
630 changed files with 11459 additions and 9489 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.clang-format
View File

@ -1,6 +1,6 @@
BasedOnStyle: WebKit
Language: Cpp
AlignAfterOpenBracket: false
AlignAfterOpenBracket: AlwaysBreak
BreakBeforeBraces: Custom
BraceWrapping:
AfterClass: true
@ -25,7 +25,7 @@ Standard: Cpp11
PointerAlignment: Middle
MaxEmptyLinesToKeep: 2
KeepEmptyLinesAtTheStartOfBlocks: false
AllowShortFunctionsOnASingleLine: Empty
AllowShortFunctionsOnASingleLine: InlineOnly
AlwaysBreakTemplateDeclarations: true
IndentCaseLabels: true
SpaceAfterTemplateKeyword: true

1
.gitignore vendored
View File

@ -11,6 +11,7 @@
/build
/build_*
/build-*
/docs/build
/docs/edit
/docs/tools/venv/

3
.gitmodules vendored
View File

@ -46,3 +46,6 @@
[submodule "contrib/unixodbc"]
path = contrib/unixodbc
url = https://github.com/ClickHouse-Extras/UnixODBC.git
[submodule "contrib/base64"]
path = contrib/base64
url = https://github.com/aklomp/base64.git

0
.travis.yml → .travis.yml.bak
View File

2237
CHANGELOG.md
View File

File diff suppressed because it is too large Load Diff

13
CHANGELOG_RU.md
View File

@ -1,3 +1,16 @@
## ClickHouse release 18.14.15, 2018-11-21
### Исправления ошибок:
* При чтении столбцов типа `Array(String)`, размер требуемого куска памяти оценивался слишком большим, что приводило к исключению "Memory limit exceeded" при выполнении запроса. Ошибка появилась в версии 18.12.13. [#3589](https://github.com/yandex/ClickHouse/issues/3589)
## ClickHouse release 18.14.14, 2018-11-20
### Исправления ошибок:
* Исправлена работа запросов `ON CLUSTER` в случае, когда в конфигурации кластера включено шифрование (флаг `<secure>`). [#3599](https://github.com/yandex/ClickHouse/pull/3599)
### Улучшения процесса сборки ClickHouse:
* Испрпавлены проблемы сборки (llvm-7 из системы, macos) [#3582](https://github.com/yandex/ClickHouse/pull/3582)
## ClickHouse release 18.14.13, 2018-11-08
### Исправления ошибок:

3
CMakeLists.txt
View File

@ -115,7 +115,7 @@ endif ()
include (cmake/test_cpu.cmake)
option (ARCH_NATIVE "Enable -march=native compiler flag" OFF)
option (ARCH_NATIVE "Enable -march=native compiler flag" ${ARCH_ARM})
if (ARCH_NATIVE)
set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
endif ()
@ -256,6 +256,7 @@ include (cmake/find_capnp.cmake)
include (cmake/find_llvm.cmake)
include (cmake/find_cpuid.cmake)
include (cmake/find_consistent-hashing.cmake)
include (cmake/find_base64.cmake)
if (ENABLE_TESTS)
include (cmake/find_gtest.cmake)
endif ()

3
README.md
View File

@ -2,11 +2,10 @@
ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time.
🎤🥂 **ClickHouse Meetup in [Amsterdam on November 15](https://events.yandex.com/events/meetings/15-11-2018/)** 🍰🔥🐻
## Useful Links
* [Official website](https://clickhouse.yandex/) has quick high-level overview of ClickHouse on main page.
* [Tutorial](https://clickhouse.yandex/tutorial.html) shows how to set up and query small ClickHouse cluster.
* [Documentation](https://clickhouse.yandex/docs/en/) provides more in-depth information.
* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
* [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any.

12
cmake/find_base64.cmake Normal file
View File

@ -0,0 +1,12 @@
option (ENABLE_BASE64 "Enable base64" ON)
if (ENABLE_BASE64)
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64")
message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
else()
set (BASE64_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64/include)
set (BASE64_LIBRARY base64)
set (USE_BASE64 1)
endif()
endif ()

6
cmake/find_llvm.cmake
View File

@ -18,12 +18,12 @@ if (ENABLE_EMBEDDED_COMPILER)
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
find_package(LLVM ${CMAKE_CXX_COMPILER_VERSION} CONFIG PATHS ${LLVM_PATHS})
else ()
find_package (LLVM 6 CONFIG PATHS ${LLVM_PATHS})
find_package (LLVM 7 CONFIG PATHS ${LLVM_PATHS})
if (NOT LLVM_FOUND)
find_package (LLVM 5 CONFIG PATHS ${LLVM_PATHS})
find_package (LLVM 6 CONFIG PATHS ${LLVM_PATHS})
endif ()
if (NOT LLVM_FOUND)
find_package (LLVM 7 CONFIG PATHS ${LLVM_PATHS})
find_package (LLVM 5 CONFIG PATHS ${LLVM_PATHS})
endif ()
endif ()

8
cmake/find_odbc.cmake
View File

@ -71,10 +71,10 @@ if (ENABLE_ODBC)
)
# MinGW find usually fails
if(MINGW)
if (MINGW)
set(ODBC_INCLUDE_DIRECTORIES ".")
set(ODBC_LIBRARIES odbc32)
endif()
endif ()
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(ODBC
@ -82,6 +82,10 @@ if (ENABLE_ODBC)
ODBC_INCLUDE_DIRECTORIES
ODBC_LIBRARIES)
if (USE_STATIC_LIBRARIES)
list(APPEND ODBC_LIBRARIES ${LTDL_LIBRARY})
endif ()
mark_as_advanced(ODBC_FOUND ODBC_LIBRARIES ODBC_INCLUDE_DIRECTORIES)
endif ()
endif ()

4
cmake/find_poco.cmake
View File

@ -93,8 +93,8 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
endif ()
if (OPENSSL_FOUND AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL))
set (Poco_NetSSL_LIBRARY PocoNetSSL)
set (Poco_Crypto_LIBRARY PocoCrypto)
set (Poco_NetSSL_LIBRARY PocoNetSSL ${OPENSSL_LIBRARIES})
set (Poco_Crypto_LIBRARY PocoCrypto ${OPENSSL_LIBRARIES})
endif ()
if (USE_STATIC_LIBRARIES AND USE_INTERNAL_ZLIB_LIBRARY)

1
cmake/lib_name.cmake
View File

@ -1,5 +1,4 @@
set(DIVIDE_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libdivide)
set(CITYHASH_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/cityhash102/include)
set(COMMON_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/libs/libcommon/include ${ClickHouse_BINARY_DIR}/libs/libcommon/include)
set(DBMS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/dbms/src ${ClickHouse_BINARY_DIR}/dbms/src)
set(DOUBLE_CONVERSION_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/double-conversion)

3
cmake/print_include_directories.cmake
View File

@ -10,6 +10,9 @@ list(APPEND dirs ${dirs1})
get_property (dirs1 TARGET common PROPERTY INCLUDE_DIRECTORIES)
list(APPEND dirs ${dirs1})
get_property (dirs1 TARGET cityhash PROPERTY INCLUDE_DIRECTORIES)
list(APPEND dirs ${dirs1})
if (USE_INTERNAL_BOOST_LIBRARY)
get_property (dirs1 TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY} PROPERTY INCLUDE_DIRECTORIES)
list(APPEND dirs ${dirs1})

34
cmake/test_cpu.cmake
View File

@ -45,6 +45,38 @@ if (HAVE_SSE42)
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
endif ()
set (TEST_FLAG "-mssse3")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <tmmintrin.h>
int main() {
__m64 a = _mm_abs_pi8(__m64());
(void)a;
return 0;
}
" HAVE_SSSE3)
set (TEST_FLAG "-mavx")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <immintrin.h>
int main() {
auto a = _mm256_insert_epi8(__m256i(), 0, 0);
(void)a;
return 0;
}
" HAVE_AVX)
set (TEST_FLAG "-mavx2")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <immintrin.h>
int main() {
auto a = _mm256_add_epi16(__m256i(), __m256i());
(void)a;
return 0;
}
" HAVE_AVX2)
# gcc -dM -E -mpopcnt - < /dev/null | sort > gcc-dump-popcnt
#define __POPCNT__ 1
@ -65,5 +97,3 @@ if (HAVE_POPCNT AND NOT ARCH_AARCH64)
endif ()
cmake_pop_check_state ()
# TODO: add here sse3 test if you want use it

38
contrib/CMakeLists.txt vendored
View File

@ -159,16 +159,6 @@ if (USE_INTERNAL_POCO_LIBRARY)
endif ()
endif ()
if (USE_INTERNAL_LLVM_LIBRARY)
# ld: unknown option: --color-diagnostics
if (APPLE)
set (LINKER_SUPPORTS_COLOR_DIAGNOSTICS 0 CACHE INTERNAL "")
endif ()
set (LLVM_ENABLE_EH 1 CACHE INTERNAL "")
set (LLVM_ENABLE_RTTI 1 CACHE INTERNAL "")
add_subdirectory (llvm/llvm)
endif ()
if (USE_INTERNAL_GTEST_LIBRARY)
# Google Test from sources
add_subdirectory(${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest ${CMAKE_CURRENT_BINARY_DIR}/googletest)
@ -176,3 +166,31 @@ if (USE_INTERNAL_GTEST_LIBRARY)
target_compile_definitions (gtest INTERFACE GTEST_HAS_POSIX_RE=0)
target_include_directories (gtest SYSTEM INTERFACE ${ClickHouse_SOURCE_DIR}/contrib/googletest/include)
endif ()
if (USE_INTERNAL_LLVM_LIBRARY)
file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/empty.cpp CONTENT " ")
add_library(LLVM0 ${CMAKE_CURRENT_BINARY_DIR}/empty.cpp) # silly cmake bug fix
# ld: unknown option: --color-diagnostics
if (APPLE)
set (LINKER_SUPPORTS_COLOR_DIAGNOSTICS 0 CACHE INTERNAL "")
endif ()
set (LLVM_ENABLE_EH 1 CACHE INTERNAL "")
set (LLVM_ENABLE_RTTI 1 CACHE INTERNAL "")
set (LLVM_INCLUDE_TESTS 0 CACHE INTERNAL "")
set (LLVM_INCLUDE_EXAMPLES 0 CACHE INTERNAL "")
set (LLVM_INCLUDE_TOOLS 0 CACHE INTERNAL "")
set (LLVM_INSTALL_TOOLCHAIN_ONLY 0 CACHE INTERNAL "")
set (CLANG_BUILT_STANDALONE 0 CACHE INTERNAL "")
set (LLDB_BUILT_STANDALONE 0 CACHE INTERNAL "")
set (CLANG_ENABLE_STATIC_ANALYZER 0 CACHE INTERNAL "")
set (CLANG_ENABLE_ARCMT 0 CACHE INTERNAL "")
set (CLANG_BUILD_TOOLS 0 CACHE INTERNAL "")
set (BENCHMARK_ENABLE_GTEST_TESTS 0 CACHE INTERNAL "")
set (BENCHMARK_ENABLE_ASSEMBLY_TESTS 0 CACHE INTERNAL "")
set (LLVM_TARGETS_TO_BUILD "X86;AArch64" CACHE INTERNAL "")
add_subdirectory (llvm/llvm)
endif ()
if (USE_BASE64)
add_subdirectory (base64-cmake)
endif()

1
contrib/base64 vendored Submodule

@ -0,0 +1 @@
Subproject commit a27c565d1b6c676beaf297fe503c4518185666f7

1
contrib/base64-cmake/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
config.h

52
contrib/base64-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,52 @@
SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64)
set(base64_compile_instructions "")
LIST(LENGTH base64_compile_instructions 0)
macro(cast_to_bool var instruction)
if (HAVE_${var})
set(base64_${var} 1)
set(base64_${var}_opt ${instruction})
else()
set(base64_${var} 0)
endif()
endmacro()
cast_to_bool(SSSE3 "-mssse3")
cast_to_bool(SSE41 "-msse4.1")
cast_to_bool(SSE42 "-msse4.2")
cast_to_bool(AVX "-mavx")
cast_to_bool(AVX2 "-mavx2")
# write config.h file, to include it in application
file(READ config-header.tpl header)
file(WRITE config.h ${header})
file(APPEND config.h "#define HAVE_SSSE3 ${base64_SSSE3}\n")
file(APPEND config.h "#define HAVE_SSE41 ${base64_SSE41}\n")
file(APPEND config.h "#define HAVE_SSE42 ${base64_SSE42}\n")
file(APPEND config.h "#define HAVE_AVX ${base64_AVX}\n")
file(APPEND config.h "#define HAVE_AVX2 ${base64_AVX2}\n")
set(HAVE_FAST_UNALIGNED_ACCESS 0)
if (${base64_SSSE3} OR ${base64_SSE41} OR ${base64_SSE42} OR ${base64_AVX} OR ${base64_AVX2})
set(HAVE_FAST_UNALIGNED_ACCESS 1)
endif ()
file(APPEND config.h "#define HAVE_FAST_UNALIGNED_ACCESS " ${HAVE_FAST_UNALIGNED_ACCESS} "\n")
add_library(base64 ${LINK_MODE}
${LIBRARY_DIR}/lib/lib.c
${LIBRARY_DIR}/lib/codec_choose.c
${LIBRARY_DIR}/lib/arch/avx/codec.c
${LIBRARY_DIR}/lib/arch/avx2/codec.c
${LIBRARY_DIR}/lib/arch/generic/codec.c
${LIBRARY_DIR}/lib/arch/neon32/codec.c
${LIBRARY_DIR}/lib/arch/neon64/codec.c
${LIBRARY_DIR}/lib/arch/sse41/codec.c
${LIBRARY_DIR}/lib/arch/sse42/codec.c
${LIBRARY_DIR}/lib/arch/ssse3/codec.c
${LIBRARY_DIR}/lib/codecs.h
config.h)
target_compile_options(base64 PRIVATE ${base64_SSSE3_opt} ${base64_SSE41_opt} ${base64_SSE42_opt} ${base64_AVX_opt} ${base64_AVX2_opt})
target_include_directories(base64 PRIVATE ${LIBRARY_DIR}/include .)

2
contrib/base64-cmake/config-header.tpl Normal file
View File

@ -0,0 +1,2 @@
#define HAVE_NEON32 0
#define HAVE_NEON64 0

5
contrib/cityhash102/CMakeLists.txt
View File

@ -1,9 +1,8 @@
add_library(cityhash
src/city.cc
include/citycrc.h
include/city.h
src/config.h)
target_include_directories (cityhash BEFORE PUBLIC include)
target_include_directories (cityhash PRIVATE src)
target_include_directories(cityhash BEFORE PUBLIC include)
target_include_directories(cityhash PRIVATE src)

2
contrib/poco vendored

@ -1 +1 @@
Subproject commit d7a4383c4d85b51938b62ed5812bc0935245edb3
Subproject commit 20c1d877773b6a672f1bbfe3290dfea42a117ed5

2
contrib/ssl vendored

@ -1 +1 @@
Subproject commit 919f6f1331d500bfdd26f8bbbf88e92c0119879b
Subproject commit dbbbcdbbd17785566f8f9c107b714f9e213d7293

53
dbms/CMakeLists.txt
View File

@ -59,7 +59,6 @@ add_headers_and_sources(clickhouse_common_io src/Common)
add_headers_and_sources(clickhouse_common_io src/Common/HashTable)
add_headers_and_sources(clickhouse_common_io src/IO)
add_headers_and_sources(dbms src/Analyzers)
add_headers_and_sources(dbms src/Core)
add_headers_and_sources(dbms src/DataStreams)
add_headers_and_sources(dbms src/DataTypes)
@ -87,7 +86,6 @@ list (APPEND dbms_sources
src/AggregateFunctions/AggregateFunctionFactory.cpp
src/AggregateFunctions/AggregateFunctionCombinatorFactory.cpp
src/AggregateFunctions/AggregateFunctionState.cpp
src/AggregateFunctions/FactoryHelpers.cpp
src/AggregateFunctions/parseAggregateFunctionParameters.cpp)
list (APPEND dbms_headers
@ -121,7 +119,7 @@ endif ()
if (USE_EMBEDDED_COMPILER)
llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
target_link_libraries (dbms ${REQUIRED_LLVM_LIBRARIES})
target_link_libraries (dbms PRIVATE ${REQUIRED_LLVM_LIBRARIES})
target_include_directories (dbms SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
endif ()
@ -152,33 +150,48 @@ if (NOT ARCH_ARM AND CPUID_LIBRARY)
endif()
target_link_libraries (clickhouse_common_io
PUBLIC
common
PRIVATE
string_utils
widechar_width
${LINK_LIBRARIES_ONLY_ON_X86_64}
${LZ4_LIBRARY}
${ZSTD_LIBRARY}
${DOUBLE_CONVERSION_LIBRARIES}
pocoext
PUBLIC
${Poco_Net_LIBRARY}
${Poco_Util_LIBRARY}
${Poco_Foundation_LIBRARY}
${RE2_LIBRARY}
${RE2_ST_LIBRARY}
${CITYHASH_LIBRARIES}
PRIVATE
${ZLIB_LIBRARIES}
${EXECINFO_LIBRARY}
${ELF_LIBRARY}
PUBLIC
${Boost_SYSTEM_LIBRARY}
PRIVATE
apple_rt
${CMAKE_DL_LIBS}
)
target_link_libraries (dbms
PRIVATE
clickhouse_parsers
clickhouse_common_config
PUBLIC
clickhouse_common_io
pocoext
PUBLIC
${MYSQLXX_LIBRARY}
${RE2_LIBRARY}
${RE2_ST_LIBRARY}
PRIVATE
${BTRIE_LIBRARIES}
${Boost_PROGRAM_OPTIONS_LIBRARY}
PUBLIC
${Boost_SYSTEM_LIBRARY}
)
if (NOT USE_INTERNAL_RE2_LIBRARY)
@ -195,8 +208,8 @@ if (Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
endif()
if (USE_POCO_SQLODBC)
target_link_libraries (clickhouse_common_io ${Poco_SQL_LIBRARY})
target_link_libraries (dbms ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
target_link_libraries (clickhouse_common_io PRIVATE ${Poco_SQL_LIBRARY})
target_link_libraries (dbms PRIVATE ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY})
if (NOT USE_INTERNAL_POCO_LIBRARY)
target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIR})
target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR} PUBLIC ${Poco_SQL_INCLUDE_DIR})
@ -210,48 +223,44 @@ if (Poco_Data_FOUND)
endif()
if (USE_POCO_DATAODBC)
target_link_libraries (clickhouse_common_io ${Poco_Data_LIBRARY})
target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
target_link_libraries (clickhouse_common_io PRIVATE ${Poco_Data_LIBRARY})
target_link_libraries (dbms PRIVATE ${Poco_DataODBC_LIBRARY})
if (NOT USE_INTERNAL_POCO_LIBRARY)
target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
endif()
endif()
if (USE_POCO_MONGODB)
target_link_libraries (dbms ${Poco_MongoDB_LIBRARY})
target_link_libraries (dbms PRIVATE ${Poco_MongoDB_LIBRARY})
endif()
if (USE_POCO_NETSSL)
target_link_libraries (clickhouse_common_io ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
target_link_libraries (dbms ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
target_link_libraries (clickhouse_common_io PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
target_link_libraries (dbms PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
endif()
target_link_libraries (dbms ${Poco_Foundation_LIBRARY})
target_link_libraries (dbms PRIVATE ${Poco_Foundation_LIBRARY})
if (USE_ICU)
target_link_libraries (dbms ${ICU_LIBS})
target_link_libraries (dbms PRIVATE ${ICU_LIBS})
target_include_directories (dbms SYSTEM PRIVATE ${ICU_INCLUDE_DIR})
endif ()
if (USE_CAPNP)
target_link_libraries (dbms ${CAPNP_LIBRARY})
target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARY})
if (NOT USE_INTERNAL_CAPNP_LIBRARY)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR})
endif ()
endif ()
if (USE_RDKAFKA)
target_link_libraries (dbms ${RDKAFKA_LIBRARY})
target_link_libraries (dbms PRIVATE ${RDKAFKA_LIBRARY})
if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
endif ()
endif ()
target_link_libraries(dbms ${OPENSSL_CRYPTO_LIBRARY})
target_link_libraries (dbms
Threads::Threads
)
target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY} Threads::Threads)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
@ -287,6 +296,6 @@ if (ENABLE_TESTS)
# attach all dbms gtest sources
grep_gtest_sources(${ClickHouse_SOURCE_DIR}/dbms dbms_gtest_sources)
add_executable(unit_tests_dbms ${dbms_gtest_sources})
target_link_libraries(unit_tests_dbms gtest_main dbms)
target_link_libraries(unit_tests_dbms PRIVATE gtest_main dbms clickhouse_common_zookeeper)
add_check(unit_tests_dbms)
endif ()

2
dbms/cmake/version.cmake
View File

@ -22,3 +22,5 @@ endif ()
set (VERSION_NAME "${PROJECT_NAME}" CACHE STRING "")
set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}" CACHE STRING "")
set (VERSION_SO "${VERSION_STRING}" CACHE STRING "")
math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000")

26
dbms/programs/CMakeLists.txt
View File

@ -48,45 +48,45 @@ else ()
link_directories (${LLVM_LIBRARY_DIRS})
endif ()
add_executable (clickhouse main.cpp)
target_link_libraries (clickhouse clickhouse_common_io)
target_link_libraries (clickhouse PRIVATE clickhouse_common_io)
target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
if (ENABLE_CLICKHOUSE_SERVER)
target_link_libraries (clickhouse clickhouse-server-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-server-lib)
endif ()
if (ENABLE_CLICKHOUSE_CLIENT)
target_link_libraries (clickhouse clickhouse-client-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-client-lib)
endif ()
if (ENABLE_CLICKHOUSE_LOCAL)
target_link_libraries (clickhouse clickhouse-local-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-local-lib)
endif ()
if (ENABLE_CLICKHOUSE_BENCHMARK)
target_link_libraries (clickhouse clickhouse-benchmark-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-benchmark-lib)
endif ()
if (ENABLE_CLICKHOUSE_PERFORMANCE)
target_link_libraries (clickhouse clickhouse-performance-test-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-performance-test-lib)
endif ()
if (ENABLE_CLICKHOUSE_COPIER)
target_link_libraries (clickhouse clickhouse-copier-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-copier-lib)
endif ()
if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
target_link_libraries (clickhouse clickhouse-extract-from-config-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-extract-from-config-lib)
endif ()
if (ENABLE_CLICKHOUSE_COMPRESSOR)
target_link_libraries (clickhouse clickhouse-compressor-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-compressor-lib)
endif ()
if (ENABLE_CLICKHOUSE_FORMAT)
target_link_libraries (clickhouse clickhouse-format-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-format-lib)
endif ()
if (ENABLE_CLICKHOUSE_OBFUSCATOR)
target_link_libraries (clickhouse clickhouse-obfuscator-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-obfuscator-lib)
endif ()
if (USE_EMBEDDED_COMPILER)
target_link_libraries (clickhouse clickhouse-compiler-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-compiler-lib)
endif ()
if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
target_link_libraries (clickhouse clickhouse-odbc-bridge-lib)
target_link_libraries (clickhouse PRIVATE clickhouse-odbc-bridge-lib)
endif()
set (CLICKHOUSE_BUNDLE)

6
dbms/programs/benchmark/Benchmark.cpp
View File

@ -42,10 +42,8 @@ namespace DB
namespace ErrorCodes
{
extern const int POCO_EXCEPTION;
extern const int STD_EXCEPTION;
extern const int UNKNOWN_EXCEPTION;
extern const int BAD_ARGUMENTS;
extern const int EMPTY_DATA_PASSED;
}
class Benchmark
@ -170,7 +168,7 @@ private:
}
if (queries.empty())
throw Exception("Empty list of queries.");
throw Exception("Empty list of queries.", ErrorCodes::EMPTY_DATA_PASSED);
std::cerr << "Loaded " << queries.size() << " queries.\n";
}

4
dbms/programs/benchmark/CMakeLists.txt
View File

@ -1,8 +1,8 @@
add_library (clickhouse-benchmark-lib ${LINK_MODE} Benchmark.cpp)
target_link_libraries (clickhouse-benchmark-lib clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-benchmark-lib PRIVATE clickhouse-client-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_include_directories (clickhouse-benchmark-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-benchmark clickhouse-benchmark.cpp)
target_link_libraries (clickhouse-benchmark clickhouse-benchmark-lib clickhouse_aggregate_functions)
target_link_libraries (clickhouse-benchmark PRIVATE clickhouse-benchmark-lib clickhouse_aggregate_functions)
endif ()

4
dbms/programs/clang/CMakeLists.txt
View File

@ -6,9 +6,9 @@ if (CLICKHOUSE_SPLIT_BINARY)
if (USE_EMBEDDED_COMPILER)
link_directories (${LLVM_LIBRARY_DIRS})
add_executable (clickhouse-clang clickhouse-clang.cpp)
target_link_libraries (clickhouse-clang clickhouse-compiler-lib)
target_link_libraries (clickhouse-clang PRIVATE clickhouse-compiler-lib)
add_executable (clickhouse-lld clickhouse-lld.cpp)
target_link_libraries (clickhouse-lld clickhouse-compiler-lib)
target_link_libraries (clickhouse-lld PRIVATE clickhouse-compiler-lib)
install (TARGETS clickhouse-clang clickhouse-lld RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
endif ()
endif ()

2
dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt
View File

@ -1,5 +1,7 @@
add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG)
link_directories(${LLVM_LIBRARY_DIRS})
add_library(clickhouse-compiler-lib ${LINK_MODE}
driver.cpp
cc1_main.cpp

2
dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt
View File

@ -1,5 +1,7 @@
add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG)
link_directories(${LLVM_LIBRARY_DIRS})
add_library(clickhouse-compiler-lib ${LINK_MODE}
driver.cpp
cc1_main.cpp

2
dbms/programs/clang/Compiler-6.0.0/cc1as_main.cpp
View File

@ -441,7 +441,7 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
auto Pair = StringRef(S).split('=');
auto Sym = Pair.first;
auto Val = Pair.second;
int64_t Value;
int64_t Value = 0;
// We have already error checked this in the driver.
Val.getAsInteger(0, Value);
Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value);

4
dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt
View File

@ -1,8 +1,11 @@
add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG)
link_directories(${LLVM_LIBRARY_DIRS})
add_library(clickhouse-compiler-lib ${LINK_MODE}
driver.cpp
cc1_main.cpp
cc1gen_reproducer_main.cpp
cc1as_main.cpp
lld.cpp)
@ -19,7 +22,6 @@ target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE
# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.
target_link_libraries(clickhouse-compiler-lib PRIVATE
clangBasic clangCodeGen clangDriver
clangFrontend
clangFrontendTool

24
dbms/programs/clang/Compiler-7.0.0/cc1_main.cpp
View File

@ -16,6 +16,7 @@
#include "llvm/Option/Arg.h"
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
#include "clang/Config/config.h"
#include "clang/Basic/Stack.h"
#include "clang/Driver/DriverDiagnostic.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/CompilerInstance.h"
@ -76,13 +77,6 @@ void initializePollyPasses(llvm::PassRegistry &Registry);
#endif
#ifdef CLANG_HAVE_RLIMITS
// The amount of stack we think is "sufficient". If less than this much is
// available, we may be unable to reach our template instantiation depth
// limit and other similar limits.
// FIXME: Unify this with the stack we request when spawning a thread to build
// a module.
static const int kSufficientStack = 8 << 20;
#if defined(__linux__) && defined(__PIE__)
static size_t getCurrentStackAllocation() {
// If we can't compute the current stack usage, allow for 512K of command
@ -120,7 +114,7 @@ static size_t getCurrentStackAllocation() {
#include <alloca.h>
LLVM_ATTRIBUTE_NOINLINE
static void ensureStackAddressSpace(int ExtraChunks = 0) {
static void ensureStackAddressSpace() {
// Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary
// relatively close to the stack (they are only guaranteed to be 128MiB
// apart). This results in crashes if we happen to heap-allocate more than
@ -129,7 +123,7 @@ static void ensureStackAddressSpace(int ExtraChunks = 0) {
// To avoid these crashes, ensure that we have sufficient virtual memory
// pages allocated before we start running.
size_t Curr = getCurrentStackAllocation();
const int kTargetStack = kSufficientStack - 256 * 1024;
const int kTargetStack = DesiredStackSize - 256 * 1024;
if (Curr < kTargetStack) {
volatile char *volatile Alloc =
static_cast<volatile char *>(alloca(kTargetStack - Curr));
@ -149,21 +143,23 @@ static void ensureSufficientStack() {
// Increase the soft stack limit to our desired level, if necessary and
// possible.
if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) {
if (rlim.rlim_cur != RLIM_INFINITY &&
rlim.rlim_cur < rlim_t(DesiredStackSize)) {
// Try to allocate sufficient stack.
if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack)
rlim.rlim_cur = kSufficientStack;
if (rlim.rlim_max == RLIM_INFINITY ||
rlim.rlim_max >= rlim_t(DesiredStackSize))
rlim.rlim_cur = DesiredStackSize;
else if (rlim.rlim_cur == rlim.rlim_max)
return;
else
rlim.rlim_cur = rlim.rlim_max;
if (setrlimit(RLIMIT_STACK, &rlim) != 0 ||
rlim.rlim_cur != kSufficientStack)
rlim.rlim_cur != DesiredStackSize)
return;
}
// We should now have a stack of size at least kSufficientStack. Ensure
// We should now have a stack of size at least DesiredStackSize. Ensure
// that we can actually use that much, if necessary.
ensureStackAddressSpace();
}

57
dbms/programs/clang/Compiler-7.0.0/cc1as_main.cpp
View File

@ -29,6 +29,7 @@
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCInstrInfo.h"
#include "llvm/MC/MCObjectFileInfo.h"
#include "llvm/MC/MCObjectWriter.h"
#include "llvm/MC/MCParser/MCAsmParser.h"
#include "llvm/MC/MCParser/MCTargetAsmParser.h"
#include "llvm/MC/MCRegisterInfo.h"
@ -59,10 +60,9 @@ using namespace clang::driver::options;
using namespace llvm;
using namespace llvm::opt;
namespace {
/// \brief Helper class for representing a single invocation of the assembler.
/// Helper class for representing a single invocation of the assembler.
struct AssemblerInvocation {
/// @name Target Options
/// @{
@ -94,9 +94,11 @@ struct AssemblerInvocation {
std::string DwarfDebugFlags;
std::string DwarfDebugProducer;
std::string DebugCompilationDir;
std::map<const std::string, const std::string> DebugPrefixMap;
llvm::DebugCompressionType CompressDebugSections =
llvm::DebugCompressionType::None;
std::string MainFileName;
std::string SplitDwarfFile;
/// @}
/// @name Frontend Options
@ -232,6 +234,9 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir);
Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name);
for (const auto &Arg : Args.getAllArgValues(OPT_fdebug_prefix_map_EQ))
Opts.DebugPrefixMap.insert(StringRef(Arg).split('='));
// Frontend Options
if (Args.hasArg(OPT_INPUT)) {
bool First = true;
@ -247,6 +252,7 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
}
Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm);
Opts.OutputPath = Args.getLastArgValue(OPT_o);
Opts.SplitDwarfFile = Args.getLastArgValue(OPT_split_dwarf_file);
if (Arg *A = Args.getLastArg(OPT_filetype)) {
StringRef Name = A->getValue();
unsigned OutputType = StringSwitch<unsigned>(Name)
@ -282,22 +288,17 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
}
static std::unique_ptr<raw_fd_ostream>
getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags,
bool Binary) {
if (Opts.OutputPath.empty())
Opts.OutputPath = "-";
getOutputStream(StringRef Path, DiagnosticsEngine &Diags, bool Binary) {
// Make sure that the Out file gets unlinked from the disk if we get a
// SIGINT.
if (Opts.OutputPath != "-")
sys::RemoveFileOnSignal(Opts.OutputPath);
if (Path != "-")
sys::RemoveFileOnSignal(Path);
std::error_code EC;
auto Out = llvm::make_unique<raw_fd_ostream>(
Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text));
Path, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text));
if (EC) {
Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath
<< EC.message();
Diags.Report(diag::err_fe_unable_to_open_output) << Path << EC.message();
return nullptr;
}
@ -342,9 +343,15 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations);
bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj;
std::unique_ptr<raw_fd_ostream> FDOS = getOutputStream(Opts, Diags, IsBinary);
if (Opts.OutputPath.empty())
Opts.OutputPath = "-";
std::unique_ptr<raw_fd_ostream> FDOS =
getOutputStream(Opts.OutputPath, Diags, IsBinary);
if (!FDOS)
return true;
std::unique_ptr<raw_fd_ostream> DwoOS;
if (!Opts.SplitDwarfFile.empty())
DwoOS = getOutputStream(Opts.SplitDwarfFile, Diags, IsBinary);
// FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and
// MCObjectFileInfo needs a MCContext reference in order to initialize itself.
@ -374,6 +381,9 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer));
if (!Opts.DebugCompilationDir.empty())
Ctx.setCompilationDir(Opts.DebugCompilationDir);
if (!Opts.DebugPrefixMap.empty())
for (const auto &KV : Opts.DebugPrefixMap)
Ctx.addDebugPrefixMapEntry(KV.first, KV.second);
if (!Opts.MainFileName.empty())
Ctx.setMainFileName(StringRef(Opts.MainFileName));
Ctx.setDwarfVersion(Opts.DwarfVersion);
@ -427,11 +437,14 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
MCTargetOptions MCOptions;
std::unique_ptr<MCAsmBackend> MAB(
TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
std::unique_ptr<MCObjectWriter> OW =
DwoOS ? MAB->createDwoObjectWriter(*Out, *DwoOS)
: MAB->createObjectWriter(*Out);
Triple T(Opts.Triple);
Str.reset(TheTarget->createMCObjectStreamer(
T, Ctx, std::move(MAB), *Out, std::move(CE), *STI, Opts.RelaxAll,
Opts.IncrementalLinkerCompatible,
T, Ctx, std::move(MAB), std::move(OW), std::move(CE), *STI,
Opts.RelaxAll, Opts.IncrementalLinkerCompatible,
/*DWARFMustBeAtTheEnd*/ true));
Str.get()->InitSections(Opts.NoExecStack);
}
@ -456,7 +469,7 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
auto Pair = StringRef(S).split('=');
auto Sym = Pair.first;
auto Val = Pair.second;
int64_t Value;
int64_t Value = 1;
// We have already error checked this in the driver.
Val.getAsInteger(0, Value);
Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value);
@ -475,14 +488,18 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
FDOS.reset();
// Delete output file if there were errors.
if (Failed && Opts.OutputPath != "-")
sys::fs::remove(Opts.OutputPath);
if (Failed) {
if (Opts.OutputPath != "-")
sys::fs::remove(Opts.OutputPath);
if (!Opts.SplitDwarfFile.empty() && Opts.SplitDwarfFile != "-")
sys::fs::remove(Opts.SplitDwarfFile);
}
return Failed;
}
static void LLVMErrorHandler(void *UserData, const std::string &Message,
bool /*GenCrashDiag*/) {
bool GenCrashDiag) {
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
Diags.Report(diag::err_fe_error_backend) << Message;
@ -491,7 +508,7 @@ static void LLVMErrorHandler(void *UserData, const std::string &Message,
exit(1);
}
int cc1as_main(ArrayRef<const char *> Argv, const char */*Argv0*/, void */*MainAddr*/) {
int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
// Initialize targets and assembly printers/parsers.
InitializeAllTargetInfos();
InitializeAllTargetMCs();

196
dbms/programs/clang/Compiler-7.0.0/cc1gen_reproducer_main.cpp Normal file
View File

@ -0,0 +1,196 @@
//===-- cc1gen_reproducer_main.cpp - Clang reproducer generator ----------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the clang -cc1gen-reproducer functionality, which
// generates reproducers for invocations for clang-based tools.
//
//===----------------------------------------------------------------------===//
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/LLVM.h"
#include "clang/Basic/VirtualFileSystem.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/Driver.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/YAMLTraits.h"
#include "llvm/Support/raw_ostream.h"
using namespace clang;
namespace {
struct UnsavedFileHash {
std::string Name;
std::string MD5;
};
struct ClangInvocationInfo {
std::string Toolchain;
std::string LibclangOperation;
std::string LibclangOptions;
std::vector<std::string> Arguments;
std::vector<std::string> InvocationArguments;
std::vector<UnsavedFileHash> UnsavedFileHashes;
bool Dump = false;
};
} // end anonymous namespace
LLVM_YAML_IS_SEQUENCE_VECTOR(UnsavedFileHash)
namespace llvm {
namespace yaml {
template <> struct MappingTraits<UnsavedFileHash> {
static void mapping(IO &IO, UnsavedFileHash &Info) {
IO.mapRequired("name", Info.Name);
IO.mapRequired("md5", Info.MD5);
}
};
template <> struct MappingTraits<ClangInvocationInfo> {
static void mapping(IO &IO, ClangInvocationInfo &Info) {
IO.mapRequired("toolchain", Info.Toolchain);
IO.mapOptional("libclang.operation", Info.LibclangOperation);
IO.mapOptional("libclang.opts", Info.LibclangOptions);
IO.mapRequired("args", Info.Arguments);
IO.mapOptional("invocation-args", Info.InvocationArguments);
IO.mapOptional("unsaved_file_hashes", Info.UnsavedFileHashes);
}
};
} // end namespace yaml
} // end namespace llvm
static std::string generateReproducerMetaInfo(const ClangInvocationInfo &Info) {
std::string Result;
llvm::raw_string_ostream OS(Result);
OS << '{';
bool NeedComma = false;
auto EmitKey = [&](StringRef Key) {
if (NeedComma)
OS << ", ";
NeedComma = true;
OS << '"' << Key << "\": ";
};
auto EmitStringKey = [&](StringRef Key, StringRef Value) {
if (Value.empty())
return;
EmitKey(Key);
OS << '"' << Value << '"';
};
EmitStringKey("libclang.operation", Info.LibclangOperation);
EmitStringKey("libclang.opts", Info.LibclangOptions);
if (!Info.InvocationArguments.empty()) {
EmitKey("invocation-args");
OS << '[';
for (const auto &Arg : llvm::enumerate(Info.InvocationArguments)) {
if (Arg.index())
OS << ',';
OS << '"' << Arg.value() << '"';
}
OS << ']';
}
OS << '}';
// FIXME: Compare unsaved file hashes and report mismatch in the reproducer.
if (Info.Dump)
llvm::outs() << "REPRODUCER METAINFO: " << OS.str() << "\n";
return std::move(OS.str());
}
/// Generates a reproducer for a set of arguments from a specific invocation.
static llvm::Optional<driver::Driver::CompilationDiagnosticReport>
generateReproducerForInvocationArguments(ArrayRef<const char *> Argv,
const ClangInvocationInfo &Info) {
using namespace driver;
auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(Argv[0]);
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions;
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
DiagnosticsEngine Diags(DiagID, &*DiagOpts, new IgnoringDiagConsumer());
ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false);
Driver TheDriver(Argv[0], llvm::sys::getDefaultTargetTriple(), Diags);
TheDriver.setTargetAndMode(TargetAndMode);
std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(Argv));
if (C && !C->containsError()) {
for (const auto &J : C->getJobs()) {
if (const Command *Cmd = dyn_cast<Command>(&J)) {
Driver::CompilationDiagnosticReport Report;
TheDriver.generateCompilationDiagnostics(
*C, *Cmd, generateReproducerMetaInfo(Info), &Report);
return Report;
}
}
}
return None;
}
std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes);
static void printReproducerInformation(
llvm::raw_ostream &OS, const ClangInvocationInfo &Info,
const driver::Driver::CompilationDiagnosticReport &Report) {
OS << "REPRODUCER:\n";
OS << "{\n";
OS << R"("files":[)";
for (const auto &File : llvm::enumerate(Report.TemporaryFiles)) {
if (File.index())
OS << ',';
OS << '"' << File.value() << '"';
}
OS << "]\n}\n";
}
int cc1gen_reproducer_main(ArrayRef<const char *> Argv, const char *Argv0,
void *MainAddr) {
if (Argv.size() < 1) {
llvm::errs() << "error: missing invocation file\n";
return 1;
}
// Parse the invocation descriptor.
StringRef Input = Argv[0];
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
llvm::MemoryBuffer::getFile(Input);
if (!Buffer) {
llvm::errs() << "error: failed to read " << Input << ": "
<< Buffer.getError().message() << "\n";
return 1;
}
llvm::yaml::Input YAML(Buffer.get()->getBuffer());
ClangInvocationInfo InvocationInfo;
YAML >> InvocationInfo;
if (Argv.size() > 1 && Argv[1] == StringRef("-v"))
InvocationInfo.Dump = true;
// Create an invocation that will produce the reproducer.
std::vector<const char *> DriverArgs;
for (const auto &Arg : InvocationInfo.Arguments)
DriverArgs.push_back(Arg.c_str());
std::string Path = GetExecutablePath(Argv0, /*CanonicalPrefixes=*/true);
DriverArgs[0] = Path.c_str();
llvm::Optional<driver::Driver::CompilationDiagnosticReport> Report =
generateReproducerForInvocationArguments(DriverArgs, InvocationInfo);
// Emit the information about the reproduce files to stdout.
int Result = 1;
if (Report) {
printReproducerInformation(llvm::outs(), InvocationInfo, *Report);
Result = 0;
}
// Remove the input file.
llvm::sys::fs::remove(Input);
return Result;
}

4
dbms/programs/clang/Compiler-7.0.0/driver.cpp
View File

@ -310,6 +310,8 @@ static int ExecuteCC1Tool(ArrayRef<const char *> argv, StringRef Tool) {
return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP);
if (Tool == "as")
return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP);
if (Tool == "gen-reproducer")
return cc1gen_reproducer_main(argv.slice(2), argv[0], GetExecutablePathVP);
// Reject unknown tools.
llvm::errs() << "error: unknown integrated tool '" << Tool << "'. "
@ -317,7 +319,7 @@ static int ExecuteCC1Tool(ArrayRef<const char *> argv, StringRef Tool) {
return 1;
}
int mainEntryClickHouseClang(int argc_, char **argv_) {
int mainEntryClickHouseClang(int argc_, /* const */ char **argv_) {
llvm::InitLLVM X(argc_, argv_);
SmallVector<const char *, 256> argv(argv_, argv_ + argc_);

156
dbms/programs/clang/Compiler-7.0.0/lld.cpp
View File

@ -1,10 +1,150 @@
#include "lld/Common/Driver.h"
#include "llvm/Support/InitLLVM.h"
#include <vector>
//===- tools/lld/lld.cpp - Linker Driver Dispatcher -----------------------===//
//
// The LLVM Linker
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file contains the main function of the lld executable. The main
// function is a thin wrapper which dispatches to the platform specific
// driver.
//
// lld is a single executable that contains four different linkers for ELF,
// COFF, WebAssembly and Mach-O. The main function dispatches according to
// argv[0] (i.e. command name). The most common name for each target is shown
// below:
//
// - ld.lld: ELF (Unix)
// - ld64: Mach-O (macOS)
// - lld-link: COFF (Windows)
// - ld-wasm: WebAssembly
//
// lld can be invoked as "lld" along with "-flavor" option. This is for
// backward compatibility and not recommended.
//
//===----------------------------------------------------------------------===//
int mainEntryClickHouseLLD(int argc, char ** argv)
{
llvm::InitLLVM X(argc, argv);
std::vector<const char *> args(argv, argv + argc);
return !lld::elf::link(args, false);
#include "lld/Common/Driver.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Support/InitLLVM.h"
#include "llvm/Support/Path.h"
#include <cstdlib>
using namespace lld;
using namespace llvm;
using namespace llvm::sys;
/*
enum Flavor {
Invalid,
Gnu, // -flavor gnu
WinLink, // -flavor link
Darwin, // -flavor darwin
Wasm, // -flavor wasm
};
LLVM_ATTRIBUTE_NORETURN static void die(const Twine &S) {
errs() << S << "\n";
exit(1);
}
static Flavor getFlavor(StringRef S) {
return StringSwitch<Flavor>(S)
.CasesLower("ld", "ld.lld", "gnu", Gnu)
.CasesLower("wasm", "ld-wasm", Wasm)
.CaseLower("link", WinLink)
.CasesLower("ld64", "ld64.lld", "darwin", Darwin)
.Default(Invalid);
}
static bool isPETarget(const std::vector<const char *> &V) {
for (auto It = V.begin(); It + 1 != V.end(); ++It) {
if (StringRef(*It) != "-m")
continue;
StringRef S = *(It + 1);
return S == "i386pe" || S == "i386pep" || S == "thumb2pe" || S == "arm64pe";
}
return false;
}
static Flavor parseProgname(StringRef Progname) {
#if __APPLE__
// Use Darwin driver for "ld" on Darwin.
if (Progname == "ld")
return Darwin;
#endif
#if LLVM_ON_UNIX
// Use GNU driver for "ld" on other Unix-like system.
if (Progname == "ld")
return Gnu;
#endif
// Progname may be something like "lld-gnu". Parse it.
SmallVector<StringRef, 3> V;
Progname.split(V, "-");
for (StringRef S : V)
if (Flavor F = getFlavor(S))
return F;
return Invalid;
}
static Flavor parseFlavor(std::vector<const char *> &V) {
// Parse -flavor option.
if (V.size() > 1 && V[1] == StringRef("-flavor")) {
if (V.size() <= 2)
die("missing arg value for '-flavor'");
Flavor F = getFlavor(V[2]);
if (F == Invalid)
die("Unknown flavor: " + StringRef(V[2]));
V.erase(V.begin() + 1, V.begin() + 3);
return F;
}
// Deduct the flavor from argv[0].
StringRef Arg0 = path::filename(V[0]);
if (Arg0.endswith_lower(".exe"))
Arg0 = Arg0.drop_back(4);
return parseProgname(Arg0);
}
*/
// If this function returns true, lld calls _exit() so that it quickly
// exits without invoking destructors of globally allocated objects.
//
// We don't want to do that if we are running tests though, because
// doing that breaks leak sanitizer. So, lit sets this environment variable,
// and we use it to detect whether we are running tests or not.
static bool canExitEarly() { return StringRef(getenv("LLD_IN_TEST")) != "1"; }
/// Universal linker main(). This linker emulates the gnu, darwin, or
/// windows linker based on the argv[0] or -flavor option.
int mainEntryClickHouseLLD(int Argc, /* const */ char **Argv) {
InitLLVM X(Argc, Argv);
std::vector<const char *> Args(Argv, Argv + Argc);
/*
switch (parseFlavor(Args)) {
case Gnu:
if (isPETarget(Args))
return !mingw::link(Args);
*/
return !elf::link(Args, canExitEarly());
/*
case WinLink:
return !coff::link(Args, canExitEarly());
case Darwin:
return !mach_o::link(Args, canExitEarly());
case Wasm:
return !wasm::link(Args, canExitEarly());
default:
die("lld is a generic driver.\n"
"Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-lld"
" (WebAssembly) instead");
}
*/
}

1
dbms/programs/clang/Compiler-7.0.0bundled
View File

@ -1 +0,0 @@
Compiler-7.0.0

49
dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt Normal file
View File

@ -0,0 +1,49 @@
add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG)
link_directories(${LLVM_LIBRARY_DIRS})
add_library(clickhouse-compiler-lib ${LINK_MODE}
driver.cpp
cc1_main.cpp
cc1as_main.cpp
lld.cpp)
target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0)
string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti
llvm_libs_all(REQUIRED_LLVM_LIBRARIES)
message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS})
# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.
target_link_libraries(clickhouse-compiler-lib PRIVATE
clangBasic clangCodeGen clangDriver
clangFrontend
clangFrontendTool
clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend
clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers
clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic
clangCrossTU clangIndex
lldCOFF
lldDriver
lldELF
lldMinGW
lldMachO
lldReaderWriter
lldYAML
lldCommon
lldCore
${REQUIRED_LLVM_LIBRARIES}
PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads
${MALLOC_LIBRARIES}
${GLIBC_COMPATIBILITY_LIBRARIES}
${MEMCPY_LIBRARIES}
)

243
dbms/programs/clang/Compiler-7.0.0bundled/cc1_main.cpp Normal file
View File

@ -0,0 +1,243 @@
//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the clang -cc1 functionality, which implements the
// core compiler functionality along with a number of additional tools for
// demonstration and testing purposes.
//
//===----------------------------------------------------------------------===//
#include "llvm/Option/Arg.h"
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
#include "clang/Config/config.h"
#include "clang/Driver/DriverDiagnostic.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "clang/Frontend/TextDiagnosticBuffer.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "clang/FrontendTool/Utils.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/LinkAllPasses.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/raw_ostream.h"
#include <cstdio>
#ifdef CLANG_HAVE_RLIMITS
#include <sys/resource.h>
#endif
// have no .a version in packages
#undef LINK_POLLY_INTO_TOOLS
using namespace clang;
using namespace llvm::opt;
//===----------------------------------------------------------------------===//
// Main driver
//===----------------------------------------------------------------------===//
static void LLVMErrorHandler(void *UserData, const std::string &Message,
bool GenCrashDiag) {
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
Diags.Report(diag::err_fe_error_backend) << Message;
// Run the interrupt handlers to make sure any special cleanups get done, in
// particular that we remove files registered with RemoveFileOnSignal.
llvm::sys::RunInterruptHandlers();
// We cannot recover from llvm errors. When reporting a fatal error, exit
// with status 70 to generate crash diagnostics. For BSD systems this is
// defined as an internal software error. Otherwise, exit with status 1.
exit(GenCrashDiag ? 70 : 1);
}
#ifdef LINK_POLLY_INTO_TOOLS
namespace polly {
void initializePollyPasses(llvm::PassRegistry &Registry);
}
#endif
#ifdef CLANG_HAVE_RLIMITS
// The amount of stack we think is "sufficient". If less than this much is
// available, we may be unable to reach our template instantiation depth
// limit and other similar limits.
// FIXME: Unify this with the stack we request when spawning a thread to build
// a module.
static const int kSufficientStack = 8 << 20;
#if defined(__linux__) && defined(__PIE__)
static size_t getCurrentStackAllocation() {
// If we can't compute the current stack usage, allow for 512K of command
// line arguments and environment.
size_t Usage = 512 * 1024;
if (FILE *StatFile = fopen("/proc/self/stat", "r")) {
// We assume that the stack extends from its current address to the end of
// the environment space. In reality, there is another string literal (the
// program name) after the environment, but this is close enough (we only
// need to be within 100K or so).
unsigned long StackPtr, EnvEnd;
// Disable silly GCC -Wformat warning that complains about length
// modifiers on ignored format specifiers. We want to retain these
// for documentation purposes even though they have no effect.
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wformat"
#endif
if (fscanf(StatFile,
"%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu "
"%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu "
"%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d "
"%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d",
&StackPtr, &EnvEnd) == 2) {
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic pop
#endif
Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd;
}
fclose(StatFile);
}
return Usage;
}
#include <alloca.h>
LLVM_ATTRIBUTE_NOINLINE
static void ensureStackAddressSpace(int ExtraChunks = 0) {
// Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary
// relatively close to the stack (they are only guaranteed to be 128MiB
// apart). This results in crashes if we happen to heap-allocate more than
// 128MiB before we reach our stack high-water mark.
//
// To avoid these crashes, ensure that we have sufficient virtual memory
// pages allocated before we start running.
size_t Curr = getCurrentStackAllocation();
const int kTargetStack = kSufficientStack - 256 * 1024;
if (Curr < kTargetStack) {
volatile char *volatile Alloc =
static_cast<volatile char *>(alloca(kTargetStack - Curr));
Alloc[0] = 0;
Alloc[kTargetStack - Curr - 1] = 0;
}
}
#else
static void ensureStackAddressSpace() {}
#endif
/// Attempt to ensure that we have at least 8MiB of usable stack space.
static void ensureSufficientStack() {
struct rlimit rlim;
if (getrlimit(RLIMIT_STACK, &rlim) != 0)
return;
// Increase the soft stack limit to our desired level, if necessary and
// possible.
if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) {
// Try to allocate sufficient stack.
if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack)
rlim.rlim_cur = kSufficientStack;
else if (rlim.rlim_cur == rlim.rlim_max)
return;
else
rlim.rlim_cur = rlim.rlim_max;
if (setrlimit(RLIMIT_STACK, &rlim) != 0 ||
rlim.rlim_cur != kSufficientStack)
return;
}
// We should now have a stack of size at least kSufficientStack. Ensure
// that we can actually use that much, if necessary.
ensureStackAddressSpace();
}
#else
static void ensureSufficientStack() {}
#endif
int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
ensureSufficientStack();
std::unique_ptr<CompilerInstance> Clang(new CompilerInstance());
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
// Register the support for object-file-wrapped Clang modules.
auto PCHOps = Clang->getPCHContainerOperations();
PCHOps->registerWriter(llvm::make_unique<ObjectFilePCHContainerWriter>());
PCHOps->registerReader(llvm::make_unique<ObjectFilePCHContainerReader>());
// Initialize targets first, so that --version shows registered targets.
llvm::InitializeAllTargets();
llvm::InitializeAllTargetMCs();
llvm::InitializeAllAsmPrinters();
llvm::InitializeAllAsmParsers();
#ifdef LINK_POLLY_INTO_TOOLS
llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry();
polly::initializePollyPasses(Registry);
#endif
// Buffer diagnostics from argument parsing so that we can output them using a
// well formed diagnostic object.
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer;
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer);
bool Success = CompilerInvocation::CreateFromArgs(
Clang->getInvocation(), Argv.begin(), Argv.end(), Diags);
// Infer the builtin include path if unspecified.
if (Clang->getHeaderSearchOpts().UseBuiltinIncludes &&
Clang->getHeaderSearchOpts().ResourceDir.empty())
Clang->getHeaderSearchOpts().ResourceDir =
CompilerInvocation::GetResourcesPath(Argv0, MainAddr);
// Create the actual diagnostics engine.
Clang->createDiagnostics();
if (!Clang->hasDiagnostics())
return 1;
// Set an error handler, so that any LLVM backend diagnostics go through our
// error handler.
llvm::install_fatal_error_handler(LLVMErrorHandler,
static_cast<void*>(&Clang->getDiagnostics()));
DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics());
if (!Success)
return 1;
// Execute the frontend actions.
Success = ExecuteCompilerInvocation(Clang.get());
// If any timers were active but haven't been destroyed yet, print their
// results now. This happens in -disable-free mode.
llvm::TimerGroup::printAll(llvm::errs());
// Our error handler depends on the Diagnostics object, which we're
// potentially about to delete. Uninstall the handler now so that any
// later errors use the default handling behavior instead.
llvm::remove_fatal_error_handler();
// When running with -disable-free, don't do any destruction or shutdown.
if (Clang->getFrontendOpts().DisableFree) {
BuryPointer(std::move(Clang));
return !Success;
}
return !Success;
}

555
dbms/programs/clang/Compiler-7.0.0bundled/cc1as_main.cpp Normal file
View File

@ -0,0 +1,555 @@
//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the clang -cc1as functionality, which implements
// the direct interface to the LLVM MC based assembler.
//
//===----------------------------------------------------------------------===//
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Driver/DriverDiagnostic.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Triple.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/MC/MCAsmBackend.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCCodeEmitter.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCInstrInfo.h"
#include "llvm/MC/MCObjectFileInfo.h"
#include "llvm/MC/MCParser/MCAsmParser.h"
#include "llvm/MC/MCParser/MCTargetAsmParser.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/MCSubtargetInfo.h"
#include "llvm/MC/MCTargetOptions.h"
#include "llvm/Option/Arg.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/FormattedStream.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/TargetRegistry.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/raw_ostream.h"
#include <memory>
#include <system_error>
using namespace clang;
using namespace clang::driver;
using namespace clang::driver::options;
using namespace llvm;
using namespace llvm::opt;
namespace {
/// \brief Helper class for representing a single invocation of the assembler.
struct AssemblerInvocation {
/// @name Target Options
/// @{
/// The name of the target triple to assemble for.
std::string Triple;
/// If given, the name of the target CPU to determine which instructions
/// are legal.
std::string CPU;
/// The list of target specific features to enable or disable -- this should
/// be a list of strings starting with '+' or '-'.
std::vector<std::string> Features;
/// The list of symbol definitions.
std::vector<std::string> SymbolDefs;
/// @}
/// @name Language Options
/// @{
std::vector<std::string> IncludePaths;
unsigned NoInitialTextSection : 1;
unsigned SaveTemporaryLabels : 1;
unsigned GenDwarfForAssembly : 1;
unsigned RelaxELFRelocations : 1;
unsigned DwarfVersion;
std::string DwarfDebugFlags;
std::string DwarfDebugProducer;
std::string DebugCompilationDir;
llvm::DebugCompressionType CompressDebugSections =
llvm::DebugCompressionType::None;
std::string MainFileName;
/// @}
/// @name Frontend Options
/// @{
std::string InputFile;
std::vector<std::string> LLVMArgs;
std::string OutputPath;
enum FileType {
FT_Asm, ///< Assembly (.s) output, transliterate mode.
FT_Null, ///< No output, for timing purposes.
FT_Obj ///< Object file output.
};
FileType OutputType;
unsigned ShowHelp : 1;
unsigned ShowVersion : 1;
/// @}
/// @name Transliterate Options
/// @{
unsigned OutputAsmVariant;
unsigned ShowEncoding : 1;
unsigned ShowInst : 1;
/// @}
/// @name Assembler Options
/// @{
unsigned RelaxAll : 1;
unsigned NoExecStack : 1;
unsigned FatalWarnings : 1;
unsigned IncrementalLinkerCompatible : 1;
/// The name of the relocation model to use.
std::string RelocationModel;
/// @}
public:
AssemblerInvocation() {
Triple = "";
NoInitialTextSection = 0;
InputFile = "-";
OutputPath = "-";
OutputType = FT_Asm;
OutputAsmVariant = 0;
ShowInst = 0;
ShowEncoding = 0;
RelaxAll = 0;
NoExecStack = 0;
FatalWarnings = 0;
IncrementalLinkerCompatible = 0;
DwarfVersion = 0;
}
static bool CreateFromArgs(AssemblerInvocation &Res,
ArrayRef<const char *> Argv,
DiagnosticsEngine &Diags);
};
}
bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
ArrayRef<const char *> Argv,
DiagnosticsEngine &Diags) {
bool Success = true;
// Parse the arguments.
std::unique_ptr<OptTable> OptTbl(createDriverOptTable());
const unsigned IncludedFlagsBitmask = options::CC1AsOption;
unsigned MissingArgIndex, MissingArgCount;
InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount,
IncludedFlagsBitmask);
// Check for missing argument error.
if (MissingArgCount) {
Diags.Report(diag::err_drv_missing_argument)
<< Args.getArgString(MissingArgIndex) << MissingArgCount;
Success = false;
}
// Issue errors on unknown arguments.
for (const Arg *A : Args.filtered(OPT_UNKNOWN)) {
auto ArgString = A->getAsString(Args);
std::string Nearest;
if (OptTbl->findNearest(ArgString, Nearest, IncludedFlagsBitmask) > 1)
Diags.Report(diag::err_drv_unknown_argument) << ArgString;
else
Diags.Report(diag::err_drv_unknown_argument_with_suggestion)
<< ArgString << Nearest;
Success = false;
}
// Construct the invocation.
// Target Options
Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple));
Opts.CPU = Args.getLastArgValue(OPT_target_cpu);
Opts.Features = Args.getAllArgValues(OPT_target_feature);
// Use the default target triple if unspecified.
if (Opts.Triple.empty())
Opts.Triple = llvm::sys::getDefaultTargetTriple();
// Language Options
Opts.IncludePaths = Args.getAllArgValues(OPT_I);
Opts.NoInitialTextSection = Args.hasArg(OPT_n);
Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels);
// Any DebugInfoKind implies GenDwarfForAssembly.
Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ);
if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections,
OPT_compress_debug_sections_EQ)) {
if (A->getOption().getID() == OPT_compress_debug_sections) {
// TODO: be more clever about the compression type auto-detection
Opts.CompressDebugSections = llvm::DebugCompressionType::GNU;
} else {
Opts.CompressDebugSections =
llvm::StringSwitch<llvm::DebugCompressionType>(A->getValue())
.Case("none", llvm::DebugCompressionType::None)
.Case("zlib", llvm::DebugCompressionType::Z)
.Case("zlib-gnu", llvm::DebugCompressionType::GNU)
.Default(llvm::DebugCompressionType::None);
}
}
Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations);
Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags);
Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags);
Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer);
Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir);
Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name);
// Frontend Options
if (Args.hasArg(OPT_INPUT)) {
bool First = true;
for (const Arg *A : Args.filtered(OPT_INPUT)) {
if (First) {
Opts.InputFile = A->getValue();
First = false;
} else {
Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args);
Success = false;
}
}
}
Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm);
Opts.OutputPath = Args.getLastArgValue(OPT_o);
if (Arg *A = Args.getLastArg(OPT_filetype)) {
StringRef Name = A->getValue();
unsigned OutputType = StringSwitch<unsigned>(Name)
.Case("asm", FT_Asm)
.Case("null", FT_Null)
.Case("obj", FT_Obj)
.Default(~0U);
if (OutputType == ~0U) {
Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name;
Success = false;
} else
Opts.OutputType = FileType(OutputType);
}
Opts.ShowHelp = Args.hasArg(OPT_help);
Opts.ShowVersion = Args.hasArg(OPT_version);
// Transliterate Options
Opts.OutputAsmVariant =
getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags);
Opts.ShowEncoding = Args.hasArg(OPT_show_encoding);
Opts.ShowInst = Args.hasArg(OPT_show_inst);
// Assemble Options
Opts.RelaxAll = Args.hasArg(OPT_mrelax_all);
Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack);
Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings);
Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic");
Opts.IncrementalLinkerCompatible =
Args.hasArg(OPT_mincremental_linker_compatible);
Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym);
return Success;
}
static std::unique_ptr<raw_fd_ostream>
getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags,
bool Binary) {
if (Opts.OutputPath.empty())
Opts.OutputPath = "-";
// Make sure that the Out file gets unlinked from the disk if we get a
// SIGINT.
if (Opts.OutputPath != "-")
sys::RemoveFileOnSignal(Opts.OutputPath);
std::error_code EC;
auto Out = llvm::make_unique<raw_fd_ostream>(
Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text));
if (EC) {
Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath
<< EC.message();
return nullptr;
}
return Out;
}
static bool ExecuteAssembler(AssemblerInvocation &Opts,
DiagnosticsEngine &Diags) {
// Get the target specific parser.
std::string Error;
const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error);
if (!TheTarget)
return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
ErrorOr<std::unique_ptr<MemoryBuffer>> Buffer =
MemoryBuffer::getFileOrSTDIN(Opts.InputFile);
if (std::error_code EC = Buffer.getError()) {
Error = EC.message();
return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile;
}
SourceMgr SrcMgr;
// Tell SrcMgr about this buffer, which is what the parser will pick up.
SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc());
// Record the location of the include directories so that the lexer can find
// it later.
SrcMgr.setIncludeDirs(Opts.IncludePaths);
std::unique_ptr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(Opts.Triple));
assert(MRI && "Unable to create target register info!");
std::unique_ptr<MCAsmInfo> MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple));
assert(MAI && "Unable to create target asm info!");
// Ensure MCAsmInfo initialization occurs before any use, otherwise sections
// may be created with a combination of default and explicit settings.
MAI->setCompressDebugSections(Opts.CompressDebugSections);
MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations);
bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj;
std::unique_ptr<raw_fd_ostream> FDOS = getOutputStream(Opts, Diags, IsBinary);
if (!FDOS)
return true;
// FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and
// MCObjectFileInfo needs a MCContext reference in order to initialize itself.
std::unique_ptr<MCObjectFileInfo> MOFI(new MCObjectFileInfo());
MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr);
bool PIC = false;
if (Opts.RelocationModel == "static") {
PIC = false;
} else if (Opts.RelocationModel == "pic") {
PIC = true;
} else {
assert(Opts.RelocationModel == "dynamic-no-pic" &&
"Invalid PIC model!");
PIC = false;
}
MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx);
if (Opts.SaveTemporaryLabels)
Ctx.setAllowTemporaryLabels(false);
if (Opts.GenDwarfForAssembly)
Ctx.setGenDwarfForAssembly(true);
if (!Opts.DwarfDebugFlags.empty())
Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags));
if (!Opts.DwarfDebugProducer.empty())
Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer));
if (!Opts.DebugCompilationDir.empty())
Ctx.setCompilationDir(Opts.DebugCompilationDir);
if (!Opts.MainFileName.empty())
Ctx.setMainFileName(StringRef(Opts.MainFileName));
Ctx.setDwarfVersion(Opts.DwarfVersion);
// Build up the feature string from the target feature list.
std::string FS;
if (!Opts.Features.empty()) {
FS = Opts.Features[0];
for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i)
FS += "," + Opts.Features[i];
}
std::unique_ptr<MCStreamer> Str;
std::unique_ptr<MCInstrInfo> MCII(TheTarget->createMCInstrInfo());
std::unique_ptr<MCSubtargetInfo> STI(
TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS));
raw_pwrite_stream *Out = FDOS.get();
std::unique_ptr<buffer_ostream> BOS;
// FIXME: There is a bit of code duplication with addPassesToEmitFile.
if (Opts.OutputType == AssemblerInvocation::FT_Asm) {
MCInstPrinter *IP = TheTarget->createMCInstPrinter(
llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI);
std::unique_ptr<MCCodeEmitter> CE;
if (Opts.ShowEncoding)
CE.reset(TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx));
MCTargetOptions MCOptions;
std::unique_ptr<MCAsmBackend> MAB(
TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
auto FOut = llvm::make_unique<formatted_raw_ostream>(*Out);
Str.reset(TheTarget->createAsmStreamer(
Ctx, std::move(FOut), /*asmverbose*/ true,
/*useDwarfDirectory*/ true, IP, std::move(CE), std::move(MAB),
Opts.ShowInst));
} else if (Opts.OutputType == AssemblerInvocation::FT_Null) {
Str.reset(createNullStreamer(Ctx));
} else {
assert(Opts.OutputType == AssemblerInvocation::FT_Obj &&
"Invalid file type!");
if (!FDOS->supportsSeeking()) {
BOS = make_unique<buffer_ostream>(*FDOS);
Out = BOS.get();
}
std::unique_ptr<MCCodeEmitter> CE(
TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx));
MCTargetOptions MCOptions;
std::unique_ptr<MCAsmBackend> MAB(
TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
Triple T(Opts.Triple);
Str.reset(TheTarget->createMCObjectStreamer(
T, Ctx, std::move(MAB), *Out, std::move(CE), *STI, Opts.RelaxAll,
Opts.IncrementalLinkerCompatible,
/*DWARFMustBeAtTheEnd*/ true));
Str.get()->InitSections(Opts.NoExecStack);
}
// Assembly to object compilation should leverage assembly info.
Str->setUseAssemblerInfoForParsing(true);
bool Failed = false;
std::unique_ptr<MCAsmParser> Parser(
createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI));
// FIXME: init MCTargetOptions from sanitizer flags here.
MCTargetOptions Options;
std::unique_ptr<MCTargetAsmParser> TAP(
TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options));
if (!TAP)
Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
// Set values for symbols, if any.
for (auto &S : Opts.SymbolDefs) {
auto Pair = StringRef(S).split('=');
auto Sym = Pair.first;
auto Val = Pair.second;
int64_t Value;
// We have already error checked this in the driver.
Val.getAsInteger(0, Value);
Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value);
}
if (!Failed) {
Parser->setTargetParser(*TAP.get());
Failed = Parser->Run(Opts.NoInitialTextSection);
}
// Close Streamer first.
// It might have a reference to the output stream.
Str.reset();
// Close the output stream early.
BOS.reset();
FDOS.reset();
// Delete output file if there were errors.
if (Failed && Opts.OutputPath != "-")
sys::fs::remove(Opts.OutputPath);
return Failed;
}
static void LLVMErrorHandler(void *UserData, const std::string &Message,
bool /*GenCrashDiag*/) {
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
Diags.Report(diag::err_fe_error_backend) << Message;
// We cannot recover from llvm errors.
exit(1);
}
int cc1as_main(ArrayRef<const char *> Argv, const char */*Argv0*/, void */*MainAddr*/) {
// Initialize targets and assembly printers/parsers.
InitializeAllTargetInfos();
InitializeAllTargetMCs();
InitializeAllAsmParsers();
// Construct our diagnostic client.
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
TextDiagnosticPrinter *DiagClient
= new TextDiagnosticPrinter(errs(), &*DiagOpts);
DiagClient->setPrefix("clang -cc1as");
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
// Set an error handler, so that any LLVM backend diagnostics go through our
// error handler.
ScopedFatalErrorHandler FatalErrorHandler
(LLVMErrorHandler, static_cast<void*>(&Diags));
// Parse the arguments.
AssemblerInvocation Asm;
if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags))
return 1;
if (Asm.ShowHelp) {
std::unique_ptr<OptTable> Opts(driver::createDriverOptTable());
Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler",
/*Include=*/driver::options::CC1AsOption, /*Exclude=*/0,
/*ShowAllAliases=*/false);
return 0;
}
// Honor -version.
//
// FIXME: Use a better -version message?
if (Asm.ShowVersion) {
llvm::cl::PrintVersionMessage();
return 0;
}
// Honor -mllvm.
//
// FIXME: Remove this, one day.
if (!Asm.LLVMArgs.empty()) {
unsigned NumArgs = Asm.LLVMArgs.size();
auto Args = llvm::make_unique<const char*[]>(NumArgs + 2);
Args[0] = "clang (LLVM option parsing)";
for (unsigned i = 0; i != NumArgs; ++i)
Args[i + 1] = Asm.LLVMArgs[i].c_str();
Args[NumArgs + 1] = nullptr;
llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get());
}
// Execute the invocation, unless there were parsing errors.
bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags);
// If any timers were active but haven't been destroyed yet, print their
// results now.
TimerGroup::printAll(errs());
return !!Failed;
}

512
dbms/programs/clang/Compiler-7.0.0bundled/driver.cpp Normal file
View File

@ -0,0 +1,512 @@
//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the clang driver; it is a thin wrapper
// for functionality in the Driver clang library.
//
//===----------------------------------------------------------------------===//
#include "clang/Driver/Driver.h"
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/DriverDiagnostic.h"
#include "clang/Driver/Options.h"
#include "clang/Driver/ToolChain.h"
#include "clang/Frontend/ChainedDiagnosticConsumer.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/SerializedDiagnosticPrinter.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Option/Option.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/InitLLVM.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/Program.h"
#include "llvm/Support/Regex.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/StringSaver.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/raw_ostream.h"
#include <memory>
#include <set>
#include <system_error>
using namespace clang;
using namespace clang::driver;
using namespace llvm::opt;
std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) {
if (!CanonicalPrefixes) {
SmallString<128> ExecutablePath(Argv0);
// Do a PATH lookup if Argv0 isn't a valid path.
if (!llvm::sys::fs::exists(ExecutablePath))
if (llvm::ErrorOr<std::string> P =
llvm::sys::findProgramByName(ExecutablePath))
ExecutablePath = *P;
return ExecutablePath.str();
}
// This just needs to be some symbol in the binary; C++ doesn't
// allow taking the address of ::main however.
void *P = (void*) (intptr_t) GetExecutablePath;
return llvm::sys::fs::getMainExecutable(Argv0, P);
}
static const char *GetStableCStr(std::set<std::string> &SavedStrings,
StringRef S) {
return SavedStrings.insert(S).first->c_str();
}
/// ApplyQAOverride - Apply a list of edits to the input argument lists.
///
/// The input string is a space separate list of edits to perform,
/// they are applied in order to the input argument lists. Edits
/// should be one of the following forms:
///
/// '#': Silence information about the changes to the command line arguments.
///
/// '^': Add FOO as a new argument at the beginning of the command line.
///
/// '+': Add FOO as a new argument at the end of the command line.
///
/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command
/// line.
///
/// 'xOPTION': Removes all instances of the literal argument OPTION.
///
/// 'XOPTION': Removes all instances of the literal argument OPTION,
/// and the following argument.
///
/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox'
/// at the end of the command line.
///
/// \param OS - The stream to write edit information to.
/// \param Args - The vector of command line arguments.
/// \param Edit - The override command to perform.
/// \param SavedStrings - Set to use for storing string representations.
static void ApplyOneQAOverride(raw_ostream &OS,
SmallVectorImpl<const char*> &Args,
StringRef Edit,
std::set<std::string> &SavedStrings) {
// This does not need to be efficient.
if (Edit[0] == '^') {
const char *Str =
GetStableCStr(SavedStrings, Edit.substr(1));
OS << "### Adding argument " << Str << " at beginning\n";
Args.insert(Args.begin() + 1, Str);
} else if (Edit[0] == '+') {
const char *Str =
GetStableCStr(SavedStrings, Edit.substr(1));
OS << "### Adding argument " << Str << " at end\n";
Args.push_back(Str);
} else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") &&
Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) {
StringRef MatchPattern = Edit.substr(2).split('/').first;
StringRef ReplPattern = Edit.substr(2).split('/').second;
ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1);
for (unsigned i = 1, e = Args.size(); i != e; ++i) {
// Ignore end-of-line response file markers
if (Args[i] == nullptr)
continue;
std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]);
if (Repl != Args[i]) {
OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n";
Args[i] = GetStableCStr(SavedStrings, Repl);
}
}
} else if (Edit[0] == 'x' || Edit[0] == 'X') {
auto Option = Edit.substr(1);
for (unsigned i = 1; i < Args.size();) {
if (Option == Args[i]) {
OS << "### Deleting argument " << Args[i] << '\n';
Args.erase(Args.begin() + i);
if (Edit[0] == 'X') {
if (i < Args.size()) {
OS << "### Deleting argument " << Args[i] << '\n';
Args.erase(Args.begin() + i);
} else
OS << "### Invalid X edit, end of command line!\n";
}
} else
++i;
}
} else if (Edit[0] == 'O') {
for (unsigned i = 1; i < Args.size();) {
const char *A = Args[i];
// Ignore end-of-line response file markers
if (A == nullptr)
continue;
if (A[0] == '-' && A[1] == 'O' &&
(A[2] == '\0' ||
(A[3] == '\0' && (A[2] == 's' || A[2] == 'z' ||
('0' <= A[2] && A[2] <= '9'))))) {
OS << "### Deleting argument " << Args[i] << '\n';
Args.erase(Args.begin() + i);
} else
++i;
}
OS << "### Adding argument " << Edit << " at end\n";
Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str()));
} else {
OS << "### Unrecognized edit: " << Edit << "\n";
}
}
/// ApplyQAOverride - Apply a comma separate list of edits to the
/// input argument lists. See ApplyOneQAOverride.
static void ApplyQAOverride(SmallVectorImpl<const char*> &Args,
const char *OverrideStr,
std::set<std::string> &SavedStrings) {
raw_ostream *OS = &llvm::errs();
if (OverrideStr[0] == '#') {
++OverrideStr;
OS = &llvm::nulls();
}
*OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n";
// This does not need to be efficient.
const char *S = OverrideStr;
while (*S) {
const char *End = ::strchr(S, ' ');
if (!End)
End = S + strlen(S);
if (End != S)
ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings);
S = End;
if (*S != '\0')
++S;
}
}
extern int cc1_main(ArrayRef<const char *> Argv, const char *Argv0,
void *MainAddr);
extern int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0,
void *MainAddr);
extern int cc1gen_reproducer_main(ArrayRef<const char *> Argv,
const char *Argv0, void *MainAddr);
static void insertTargetAndModeArgs(const ParsedClangName &NameParts,
SmallVectorImpl<const char *> &ArgVector,
std::set<std::string> &SavedStrings) {
// Put target and mode arguments at the start of argument list so that
// arguments specified in command line could override them. Avoid putting
// them at index 0, as an option like '-cc1' must remain the first.
int InsertionPoint = 0;
if (ArgVector.size() > 0)
++InsertionPoint;
if (NameParts.DriverMode) {
// Add the mode flag to the arguments.
ArgVector.insert(ArgVector.begin() + InsertionPoint,
GetStableCStr(SavedStrings, NameParts.DriverMode));
}
if (NameParts.TargetIsValid) {
const char *arr[] = {"-target", GetStableCStr(SavedStrings,
NameParts.TargetPrefix)};
ArgVector.insert(ArgVector.begin() + InsertionPoint,
std::begin(arr), std::end(arr));
}
}
static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver,
SmallVectorImpl<const char *> &Opts) {
llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts);
// The first instance of '#' should be replaced with '=' in each option.
for (const char *Opt : Opts)
if (char *NumberSignPtr = const_cast<char *>(::strchr(Opt, '#')))
*NumberSignPtr = '=';
}
static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) {
// Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE.
TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS");
if (TheDriver.CCPrintOptions)
TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE");
// Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE.
TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS");
if (TheDriver.CCPrintHeaders)
TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE");
// Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE.
TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS");
if (TheDriver.CCLogDiagnostics)
TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE");
}
static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient,
const std::string &Path) {
// If the clang binary happens to be named cl.exe for compatibility reasons,
// use clang-cl.exe as the prefix to avoid confusion between clang and MSVC.
StringRef ExeBasename(llvm::sys::path::filename(Path));
if (ExeBasename.equals_lower("cl.exe"))
ExeBasename = "clang-cl.exe";
DiagClient->setPrefix(ExeBasename);
}
// This lets us create the DiagnosticsEngine with a properly-filled-out
// DiagnosticOptions instance.
static DiagnosticOptions *
CreateAndPopulateDiagOpts(ArrayRef<const char *> argv) {
auto *DiagOpts = new DiagnosticOptions;
std::unique_ptr<OptTable> Opts(createDriverOptTable());
unsigned MissingArgIndex, MissingArgCount;
InputArgList Args =
Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount);
// We ignore MissingArgCount and the return value of ParseDiagnosticArgs.
// Any errors that would be diagnosed here will also be diagnosed later,
// when the DiagnosticsEngine actually exists.
(void)ParseDiagnosticArgs(*DiagOpts, Args);
return DiagOpts;
}
static void SetInstallDir(SmallVectorImpl<const char *> &argv,
Driver &TheDriver, bool CanonicalPrefixes) {
// Attempt to find the original path used to invoke the driver, to determine
// the installed path. We do this manually, because we want to support that
// path being a symlink.
SmallString<128> InstalledPath(argv[0]);
// Do a PATH lookup, if there are no directory components.
if (llvm::sys::path::filename(InstalledPath) == InstalledPath)
if (llvm::ErrorOr<std::string> Tmp = llvm::sys::findProgramByName(
llvm::sys::path::filename(InstalledPath.str())))
InstalledPath = *Tmp;
// FIXME: We don't actually canonicalize this, we just make it absolute.
if (CanonicalPrefixes)
llvm::sys::fs::make_absolute(InstalledPath);
StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath));
if (llvm::sys::fs::exists(InstalledPathParent))
TheDriver.setInstalledDir(InstalledPathParent);
}
static int ExecuteCC1Tool(ArrayRef<const char *> argv, StringRef Tool) {
void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath;
if (Tool == "")
return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP);
if (Tool == "as")
return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP);
// Reject unknown tools.
llvm::errs() << "error: unknown integrated tool '" << Tool << "'. "
<< "Valid tools include '-cc1' and '-cc1as'.\n";
return 1;
}
int mainEntryClickHouseClang(int argc_, char **argv_) {
llvm::InitLLVM X(argc_, argv_);
SmallVector<const char *, 256> argv(argv_, argv_ + argc_);
if (llvm::sys::Process::FixupStandardFileDescriptors())
return 1;
llvm::InitializeAllTargets();
auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]);
llvm::BumpPtrAllocator A;
llvm::StringSaver Saver(A);
// Parse response files using the GNU syntax, unless we're in CL mode. There
// are two ways to put clang in CL compatibility mode: argv[0] is either
// clang-cl or cl, or --driver-mode=cl is on the command line. The normal
// command line parsing can't happen until after response file parsing, so we
// have to manually search for a --driver-mode=cl argument the hard way.
// Finally, our -cc1 tools don't care which tokenization mode we use because
// response files written by clang will tokenize the same way in either mode.
bool ClangCLMode = false;
if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") ||
std::find_if(argv.begin(), argv.end(), [](const char *F) {
return F && strcmp(F, "--driver-mode=cl") == 0;
}) != argv.end()) {
ClangCLMode = true;
}
enum { Default, POSIX, Windows } RSPQuoting = Default;
for (const char *F : argv) {
if (strcmp(F, "--rsp-quoting=posix") == 0)
RSPQuoting = POSIX;
else if (strcmp(F, "--rsp-quoting=windows") == 0)
RSPQuoting = Windows;
}
// Determines whether we want nullptr markers in argv to indicate response
// files end-of-lines. We only use this for the /LINK driver argument with
// clang-cl.exe on Windows.
bool MarkEOLs = ClangCLMode;
llvm::cl::TokenizerCallback Tokenizer;
if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode))
Tokenizer = &llvm::cl::TokenizeWindowsCommandLine;
else
Tokenizer = &llvm::cl::TokenizeGNUCommandLine;
if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1"))
MarkEOLs = false;
llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs);
// Handle -cc1 integrated tools, even if -cc1 was expanded from a response
// file.
auto FirstArg = std::find_if(argv.begin() + 1, argv.end(),
[](const char *A) { return A != nullptr; });
if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) {
// If -cc1 came from a response file, remove the EOL sentinels.
if (MarkEOLs) {
auto newEnd = std::remove(argv.begin(), argv.end(), nullptr);
argv.resize(newEnd - argv.begin());
}
return ExecuteCC1Tool(argv, argv[1] + 4);
}
bool CanonicalPrefixes = true;
for (int i = 1, size = argv.size(); i < size; ++i) {
// Skip end-of-line response file markers
if (argv[i] == nullptr)
continue;
if (StringRef(argv[i]) == "-no-canonical-prefixes") {
CanonicalPrefixes = false;
break;
}
}
// Handle CL and _CL_ which permits additional command line options to be
// prepended or appended.
if (ClangCLMode) {
// Arguments in "CL" are prepended.
llvm::Optional<std::string> OptCL = llvm::sys::Process::GetEnv("CL");
if (OptCL.hasValue()) {
SmallVector<const char *, 8> PrependedOpts;
getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts);
// Insert right after the program name to prepend to the argument list.
argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end());
}
// Arguments in "_CL_" are appended.
llvm::Optional<std::string> Opt_CL_ = llvm::sys::Process::GetEnv("_CL_");
if (Opt_CL_.hasValue()) {
SmallVector<const char *, 8> AppendedOpts;
getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts);
// Insert at the end of the argument list to append.
argv.append(AppendedOpts.begin(), AppendedOpts.end());
}
}
std::set<std::string> SavedStrings;
// Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the
// scenes.
if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) {
// FIXME: Driver shouldn't take extra initial argument.
ApplyQAOverride(argv, OverrideStr, SavedStrings);
}
std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes);
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts =
CreateAndPopulateDiagOpts(argv);
TextDiagnosticPrinter *DiagClient
= new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
FixupDiagPrefixExeName(DiagClient, Path);
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
if (!DiagOpts->DiagnosticSerializationFile.empty()) {
auto SerializedConsumer =
clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile,
&*DiagOpts, /*MergeChildRecords=*/true);
Diags.setClient(new ChainedDiagnosticConsumer(
Diags.takeClient(), std::move(SerializedConsumer)));
}
ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false);
Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags);
SetInstallDir(argv, TheDriver, CanonicalPrefixes);
TheDriver.setTargetAndMode(TargetAndMode);
insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings);
SetBackdoorDriverOutputsFromEnvVars(TheDriver);
std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(argv));
int Res = 1;
if (C && !C->containsError()) {
SmallVector<std::pair<int, const Command *>, 4> FailingCommands;
Res = TheDriver.ExecuteCompilation(*C, FailingCommands);
// Force a crash to test the diagnostics.
if (TheDriver.GenReproducer) {
Diags.Report(diag::err_drv_force_crash)
<< !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH");
// Pretend that every command failed.
FailingCommands.clear();
for (const auto &J : C->getJobs())
if (const Command *C = dyn_cast<Command>(&J))
FailingCommands.push_back(std::make_pair(-1, C));
}
for (const auto &P : FailingCommands) {
int CommandRes = P.first;
const Command *FailingCommand = P.second;
if (!Res)
Res = CommandRes;
// If result status is < 0, then the driver command signalled an error.
// If result status is 70, then the driver command reported a fatal error.
// On Windows, abort will return an exit code of 3. In these cases,
// generate additional diagnostic information if possible.
bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70;
#ifdef _WIN32
DiagnoseCrash |= CommandRes == 3;
#endif
if (DiagnoseCrash) {
TheDriver.generateCompilationDiagnostics(*C, *FailingCommand);
break;
}
}
}
Diags.getClient()->finish();
// If any timers were active but haven't been destroyed yet, print their
// results now. This happens in -disable-free mode.
llvm::TimerGroup::printAll(llvm::errs());
#ifdef _WIN32
// Exit status should not be negative on Win32, unless abnormal termination.
// Once abnormal termiation was caught, negative status should not be
// propagated.
if (Res < 0)
Res = 1;
#endif
// If we have multiple failing commands, we return the result of the first
// failing command.
return Res;
}

10
dbms/programs/clang/Compiler-7.0.0bundled/lld.cpp Normal file
View File

@ -0,0 +1,10 @@
#include "lld/Common/Driver.h"
#include "llvm/Support/InitLLVM.h"
#include <vector>
int mainEntryClickHouseLLD(int argc, char ** argv)
{
llvm::InitLLVM X(argc, argv);
std::vector<const char *> args(argv, argv + argc);
return !lld::elf::link(args, false);
}

5
dbms/programs/client/CMakeLists.txt
View File

@ -1,13 +1,12 @@
add_library (clickhouse-client-lib ${LINK_MODE} Client.cpp)
target_link_libraries (clickhouse-client-lib clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-client-lib PRIVATE clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (READLINE_INCLUDE_DIR)
target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
endif ()
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-client clickhouse-client.cpp)
target_link_libraries (clickhouse-client clickhouse-client-lib)
target_link_libraries (clickhouse-client PRIVATE clickhouse-client-lib)
endif ()
install (FILES clickhouse-client.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-client COMPONENT clickhouse-client RENAME config.xml)

9
dbms/programs/client/Client.cpp
View File

@ -18,7 +18,7 @@
#include <Poco/File.h>
#include <Poco/Util/Application.h>
#include <common/readline_use.h>
#include <common/find_first_symbols.h>
#include <common/find_symbols.h>
#include <Common/ClickHouseRevision.h>
#include <Common/Stopwatch.h>
#include <Common/Exception.h>
@ -86,9 +86,6 @@ namespace DB
namespace ErrorCodes
{
extern const int POCO_EXCEPTION;
extern const int STD_EXCEPTION;
extern const int UNKNOWN_EXCEPTION;
extern const int NETWORK_ERROR;
extern const int NO_DATA_TO_INSERT;
extern const int BAD_ARGUMENTS;
@ -529,7 +526,7 @@ private:
if (max_client_network_bandwidth)
{
ThrottlerPtr throttler = std::make_shared<Throttler>(max_client_network_bandwidth, 0, "");
ThrottlerPtr throttler = std::make_shared<Throttler>(max_client_network_bandwidth, 0, "");
connection->setThrottler(throttler);
}
@ -1527,7 +1524,7 @@ public:
min_description_length = std::min(min_description_length, line_length - 2);
}
#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, po::value<std::string> (), DESCRIPTION)
#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, po::value<std::string>(), DESCRIPTION)
/// Main commandline options related to client functionality and all parameters from Settings.
po::options_description main_description("Main options", line_length, min_description_length);

2
dbms/programs/client/Suggest.h
View File

@ -184,7 +184,7 @@ public:
}
catch (...)
{
std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false) << "\n";
std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n";
}
/// Note that keyword suggestions are available even if we cannot load data from server.

4
dbms/programs/compressor/CMakeLists.txt
View File

@ -1,8 +1,8 @@
add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp)
target_link_libraries (clickhouse-compressor-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
# Also in utils
add_executable (clickhouse-compressor clickhouse-compressor.cpp)
target_link_libraries (clickhouse-compressor clickhouse-compressor-lib)
target_link_libraries (clickhouse-compressor PRIVATE clickhouse-compressor-lib)
endif ()

2
dbms/programs/copier/CMakeLists.txt
View File

@ -1,5 +1,5 @@
add_library (clickhouse-copier-lib ${LINK_MODE} ClusterCopier.cpp)
target_link_libraries (clickhouse-copier-lib clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions)
target_link_libraries (clickhouse-copier-lib PRIVATE clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions daemon)
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-copier clickhouse-copier.cpp)

5
dbms/programs/copier/ClusterCopier.cpp
View File

@ -323,7 +323,7 @@ struct TaskTable
struct TaskCluster
{
TaskCluster(const String & task_zookeeper_path_, const String & default_local_database_)
: task_zookeeper_path(task_zookeeper_path_), default_local_database(default_local_database_) {}
: task_zookeeper_path(task_zookeeper_path_), default_local_database(default_local_database_) {}
void loadTasks(const Poco::Util::AbstractConfiguration & config, const String & base_key = "");
@ -410,8 +410,7 @@ BlockInputStreamPtr squashStreamIntoOneBlock(const BlockInputStreamPtr & stream)
return std::make_shared<SquashingBlockInputStream>(
stream,
std::numeric_limits<size_t>::max(),
std::numeric_limits<size_t>::max()
);
std::numeric_limits<size_t>::max());
}
Block getBlockWithAllStreamData(const BlockInputStreamPtr & stream)

4
dbms/programs/extract-from-config/CMakeLists.txt
View File

@ -1,7 +1,7 @@
add_library (clickhouse-extract-from-config-lib ${LINK_MODE} ExtractFromConfig.cpp)
target_link_libraries (clickhouse-extract-from-config-lib clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-extract-from-config-lib PRIVATE clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-extract-from-config clickhouse-extract-from-config.cpp)
target_link_libraries (clickhouse-extract-from-config clickhouse-extract-from-config-lib)
target_link_libraries (clickhouse-extract-from-config PRIVATE clickhouse-extract-from-config-lib)
endif ()

4
dbms/programs/format/CMakeLists.txt
View File

@ -1,6 +1,6 @@
add_library (clickhouse-format-lib ${LINK_MODE} Format.cpp)
target_link_libraries (clickhouse-format-lib dbms clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-format-lib PRIVATE dbms clickhouse_common_io clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-format clickhouse-format.cpp)
target_link_libraries (clickhouse-format clickhouse-format-lib)
target_link_libraries (clickhouse-format PRIVATE clickhouse-format-lib)
endif ()

4
dbms/programs/local/CMakeLists.txt
View File

@ -1,7 +1,7 @@
add_library (clickhouse-local-lib ${LINK_MODE} LocalServer.cpp)
target_link_libraries (clickhouse-local-lib clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-local-lib PRIVATE clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-local clickhouse-local.cpp)
target_link_libraries (clickhouse-local clickhouse-local-lib)
target_link_libraries (clickhouse-local PRIVATE clickhouse-local-lib)
endif ()

2
dbms/programs/main.cpp
View File

@ -1,7 +1,7 @@
#include <iostream>
#include <vector>
#include <string>
#include <utility> /// pair
#include <utility> /// pair
#if __has_include("config_tools.h")
#include "config_tools.h"

4
dbms/programs/obfuscator/CMakeLists.txt
View File

@ -1,8 +1,8 @@
add_library (clickhouse-obfuscator-lib ${LINK_MODE} Obfuscator.cpp)
target_link_libraries (clickhouse-obfuscator-lib dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-obfuscator-lib PRIVATE dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-obfuscator clickhouse-obfuscator.cpp)
set_target_properties(clickhouse-obfuscator PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
target_link_libraries (clickhouse-obfuscator clickhouse-obfuscator-lib)
target_link_libraries (clickhouse-obfuscator PRIVATE clickhouse-obfuscator-lib)
endif ()

12
dbms/programs/odbc-bridge/CMakeLists.txt
View File

@ -9,23 +9,23 @@ add_library (clickhouse-odbc-bridge-lib ${LINK_MODE}
validateODBCConnectionString.cpp
)
target_link_libraries (clickhouse-odbc-bridge-lib clickhouse_common_io daemon dbms)
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE clickhouse_common_io daemon dbms)
target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
if (USE_POCO_SQLODBC)
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQLODBC_LIBRARY})
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQLODBC_LIBRARY})
target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR})
endif ()
if (Poco_SQL_FOUND)
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQL_LIBRARY})
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQL_LIBRARY})
endif ()
if (USE_POCO_DATAODBC)
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_DataODBC_LIBRARY})
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_DataODBC_LIBRARY})
target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
endif()
if (Poco_Data_FOUND)
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_Data_LIBRARY})
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_Data_LIBRARY})
endif ()
@ -35,5 +35,5 @@ endif ()
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-odbc-bridge odbc-bridge.cpp)
target_link_libraries (clickhouse-odbc-bridge clickhouse-odbc-bridge-lib)
target_link_libraries (clickhouse-odbc-bridge PRIVATE clickhouse-odbc-bridge-lib)
endif ()

4
dbms/programs/odbc-bridge/ColumnInfoHandler.cpp
View File

@ -124,9 +124,9 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques
auto identifier_quote = getIdentifierQuote(hdbc);
if (identifier_quote.length() == 0)
settings.identifier_quoting_style = IdentifierQuotingStyle::None;
else if(identifier_quote[0] == '`')
else if (identifier_quote[0] == '`')
settings.identifier_quoting_style = IdentifierQuotingStyle::Backticks;
else if(identifier_quote[0] == '"')
else if (identifier_quote[0] == '"')
settings.identifier_quoting_style = IdentifierQuotingStyle::DoubleQuotes;
else
throw Exception("Can not map quote identifier '" + identifier_quote + "' to IdentifierQuotingStyle value", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);

2
dbms/programs/odbc-bridge/HandlerFactory.cpp
View File

@ -25,7 +25,7 @@ Poco::Net::HTTPRequestHandler * HandlerFactory::createRequestHandler(const Poco:
#else
return nullptr;
#endif
else if(uri.getPath() == "/identifier_quote")
else if (uri.getPath() == "/identifier_quote")
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
return new IdentifierQuoteHandler(keep_alive_timeout, context);
#else

2
dbms/programs/odbc-bridge/tests/CMakeLists.txt
View File

@ -1,2 +1,2 @@
add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp)
target_link_libraries (validate-odbc-connection-string clickhouse-odbc-bridge-lib)
target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse-odbc-bridge-lib clickhouse_common_io)

2
dbms/programs/odbc-bridge/validateODBCConnectionString.cpp
View File

@ -2,7 +2,7 @@
#include <cstring>
#include <algorithm>
#include <Poco/String.h>
#include <common/find_first_symbols.h>
#include <common/find_symbols.h>
#include <Common/Exception.h>
#include <Common/StringUtils/StringUtils.h>
#include "validateODBCConnectionString.h"

4
dbms/programs/performance-test/CMakeLists.txt
View File

@ -1,8 +1,8 @@
add_library (clickhouse-performance-test-lib ${LINK_MODE} PerformanceTest.cpp)
target_link_libraries (clickhouse-performance-test-lib clickhouse_common_io dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-performance-test-lib PRIVATE dbms clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_include_directories (clickhouse-performance-test-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-performance-test clickhouse-performance-test.cpp)
target_link_libraries (clickhouse-performance-test clickhouse-performance-test-lib dbms)
target_link_libraries (clickhouse-performance-test PRIVATE clickhouse-performance-test-lib)
endif ()

46
dbms/programs/performance-test/PerformanceTest.cpp
View File

@ -49,10 +49,10 @@ namespace DB
{
namespace ErrorCodes
{
extern const int POCO_EXCEPTION;
extern const int STD_EXCEPTION;
extern const int UNKNOWN_EXCEPTION;
extern const int NOT_IMPLEMENTED;
extern const int LOGICAL_ERROR;
extern const int BAD_ARGUMENTS;
extern const int FILE_DOESNT_EXIST;
}
static String pad(size_t padding)
@ -156,7 +156,7 @@ struct StopConditionsSet
else if (key == "average_speed_not_changing_for_ms")
average_speed_not_changing_for_ms.value = stop_conditions_view->getUInt64(key);
else
throw DB::Exception("Met unkown stop condition: " + key);
throw DB::Exception("Met unkown stop condition: " + key, DB::ErrorCodes::LOGICAL_ERROR);
++initialized_count;
}
@ -521,7 +521,7 @@ public:
{
if (input_files.size() < 1)
{
throw DB::Exception("No tests were specified", 0);
throw DB::Exception("No tests were specified", DB::ErrorCodes::BAD_ARGUMENTS);
}
std::string name;
@ -694,7 +694,7 @@ private:
size_t ram_size_needed = config->getUInt64("preconditions.ram_size");
size_t actual_ram = getMemoryAmount();
if (!actual_ram)
throw DB::Exception("ram_size precondition not available on this platform", ErrorCodes::NOT_IMPLEMENTED);
throw DB::Exception("ram_size precondition not available on this platform", DB::ErrorCodes::NOT_IMPLEMENTED);
if (ram_size_needed > actual_ram)
{
@ -868,12 +868,12 @@ private:
if (!test_config->has("query") && !test_config->has("query_file"))
{
throw DB::Exception("Missing query fields in test's config: " + test_name);
throw DB::Exception("Missing query fields in test's config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
}
if (test_config->has("query") && test_config->has("query_file"))
{
throw DB::Exception("Found both query and query_file fields. Choose only one");
throw DB::Exception("Found both query and query_file fields. Choose only one", DB::ErrorCodes::BAD_ARGUMENTS);
}
if (test_config->has("query"))
@ -885,7 +885,7 @@ private:
{
const String filename = test_config->getString("query_file");
if (filename.empty())
throw DB::Exception("Empty file name");
throw DB::Exception("Empty file name", DB::ErrorCodes::BAD_ARGUMENTS);
bool tsv = fs::path(filename).extension().string() == ".tsv";
@ -909,7 +909,7 @@ private:
if (queries.empty())
{
throw DB::Exception("Did not find any query to execute: " + test_name);
throw DB::Exception("Did not find any query to execute: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
}
if (test_config->has("substitutions"))
@ -929,7 +929,7 @@ private:
if (!test_config->has("type"))
{
throw DB::Exception("Missing type property in config: " + test_name);
throw DB::Exception("Missing type property in config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
}
String config_exec_type = test_config->getString("type");
@ -938,7 +938,7 @@ private:
else if (config_exec_type == "once")
exec_type = ExecutionType::Once;
else
throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name);
throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
times_to_run = test_config->getUInt("times_to_run", 1);
@ -951,7 +951,7 @@ private:
}
if (stop_conditions_template.empty())
throw DB::Exception("No termination conditions were found in config");
throw DB::Exception("No termination conditions were found in config", DB::ErrorCodes::BAD_ARGUMENTS);
for (size_t i = 0; i < times_to_run * queries.size(); ++i)
stop_conditions_by_run.push_back(stop_conditions_template);
@ -978,7 +978,7 @@ private:
else
{
if (lite_output)
throw DB::Exception("Specify main_metric for lite output");
throw DB::Exception("Specify main_metric for lite output", DB::ErrorCodes::BAD_ARGUMENTS);
}
if (metrics.size() > 0)
@ -1023,22 +1023,14 @@ private:
if (exec_type == ExecutionType::Loop)
{
for (const String & metric : metrics)
{
if (std::find(non_loop_metrics.begin(), non_loop_metrics.end(), metric) != non_loop_metrics.end())
{
throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")");
}
}
throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS);
}
else
{
for (const String & metric : metrics)
{
if (std::find(loop_metrics.begin(), loop_metrics.end(), metric) != loop_metrics.end())
{
throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")");
}
}
throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS);
}
}
@ -1439,7 +1431,7 @@ try
if (input_files.empty())
{
std::cerr << std::endl;
throw DB::Exception("Did not find any xml files");
throw DB::Exception("Did not find any xml files", DB::ErrorCodes::BAD_ARGUMENTS);
}
else
std::cerr << " found " << input_files.size() << " files." << std::endl;
@ -1454,7 +1446,7 @@ try
fs::path file(filename);
if (!fs::exists(file))
throw DB::Exception("File '" + filename + "' does not exist");
throw DB::Exception("File '" + filename + "' does not exist", DB::ErrorCodes::FILE_DOESNT_EXIST);
if (fs::is_directory(file))
{
@ -1463,7 +1455,7 @@ try
else
{
if (file.extension().string() != ".xml")
throw DB::Exception("File '" + filename + "' does not have .xml extension");
throw DB::Exception("File '" + filename + "' does not have .xml extension", DB::ErrorCodes::BAD_ARGUMENTS);
collected_files.push_back(filename);
}
}

12
dbms/programs/server/CMakeLists.txt
View File

@ -10,17 +10,21 @@ add_library (clickhouse-server-lib ${LINK_MODE}
TCPHandler.cpp
)
target_link_libraries (clickhouse-server-lib clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions)
target_link_libraries (clickhouse-server-lib PRIVATE clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY})
if (USE_POCO_NETSSL)
target_link_libraries (clickhouse-server-lib PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
endif ()
target_include_directories (clickhouse-server-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-server clickhouse-server.cpp)
target_link_libraries (clickhouse-server clickhouse-server-lib)
target_link_libraries (clickhouse-server PRIVATE clickhouse-server-lib)
install (TARGETS clickhouse-server ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
endif ()
if (OS_LINUX)
set (GLIBC_MAX_REQUIRED 2.4)
if (OS_LINUX AND MAKE_STATIC_LIBRARIES)
set (GLIBC_MAX_REQUIRED 2.4 CACHE INTERNAL "")
add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-server | grep '@GLIBC' | grep -oP 'GLIBC_[\\d\\.]+' | sort | uniq | sort -r | perl -lnE 'exit 1 if $_ gt q{GLIBC_${GLIBC_MAX_REQUIRED}}'")
endif ()

1
dbms/programs/server/HTTPHandler.cpp
View File

@ -270,7 +270,6 @@ void HTTPHandler::processQuery(
std::string query_id = params.get("query_id", "");
context.setUser(user, password, request.clientAddress(), quota_key);
context.setCurrentQueryId(query_id);
CurrentThread::attachQueryContext(context);
/// The user could specify session identifier and session timeout.
/// It allows to modify settings, create temporary tables and reuse them in subsequent requests.

3
dbms/programs/server/InterserverIOHTTPHandler.cpp
View File

@ -19,9 +19,6 @@ namespace DB
namespace ErrorCodes
{
extern const int ABORTED;
extern const int POCO_EXCEPTION;
extern const int STD_EXCEPTION;
extern const int UNKNOWN_EXCEPTION;
extern const int TOO_MANY_SIMULTANEOUS_QUERIES;
}

49
dbms/programs/server/Server.cpp
View File

@ -42,6 +42,11 @@
#include <Common/StatusFile.h>
#include "TCPHandlerFactory.h"
#if defined(__linux__)
#include <Common/hasLinuxCapability.h>
#include <sys/mman.h>
#endif
#if USE_POCO_NETSSL
#include <Poco/Net/Context.h>
#include <Poco/Net/SecureServerSocket.h>
@ -50,6 +55,7 @@
namespace CurrentMetrics
{
extern const Metric Revision;
extern const Metric VersionInteger;
}
namespace DB
@ -61,6 +67,8 @@ namespace ErrorCodes
extern const int SUPPORT_IS_DISABLED;
extern const int ARGUMENT_OUT_OF_BOUND;
extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
extern const int INVALID_CONFIG_PARAMETER;
extern const int SYSTEM_ERROR;
}
@ -68,7 +76,7 @@ static std::string getCanonicalPath(std::string && path)
{
Poco::trimInPlace(path);
if (path.empty())
throw Exception("path configuration parameter is empty");
throw Exception("path configuration parameter is empty", ErrorCodes::INVALID_CONFIG_PARAMETER);
if (path.back() != '/')
path += '/';
return std::move(path);
@ -103,6 +111,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
registerStorages();
CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get());
CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
/** Context contains all that query execution is dependent:
* settings, available functions, data types, aggregate functions, databases...
@ -126,6 +135,32 @@ int Server::main(const std::vector<std::string> & /*args*/)
config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
}
const auto memory_amount = getMemoryAmount();
#if defined(__linux__)
/// After full config loaded
{
if (config().getBool("mlock_executable", false))
{
if (hasLinuxCapability(CAP_IPC_LOCK))
{
LOG_TRACE(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds.");
if (0 != mlockall(MCL_CURRENT))
LOG_WARNING(log, "Failed mlockall: " + errnoToString(ErrorCodes::SYSTEM_ERROR));
else
LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed");
}
else
{
LOG_INFO(log, "It looks like the process has no CAP_IPC_LOCK capability, binary mlock will be disabled."
" It could happen due to incorrect ClickHouse package installation."
" You could resolve the problem manually with 'sudo setcap cap_ipc_lock=+ep /usr/bin/clickhouse'."
" Note that it will not work on 'nosuid' mounted filesystems.");
}
}
}
#endif
std::string path = getCanonicalPath(config().getString("path"));
std::string default_database = config().getString("default_database", "default");
@ -527,10 +562,10 @@ int Server::main(const std::vector<std::string> & /*args*/)
socket.setReceiveTimeout(settings.receive_timeout);
socket.setSendTimeout(settings.send_timeout);
servers.emplace_back(new Poco::Net::TCPServer(
new TCPHandlerFactory(*this, /* secure= */ true ),
server_pool,
socket,
new Poco::Net::TCPServerParams));
new TCPHandlerFactory(*this, /* secure= */ true),
server_pool,
socket,
new Poco::Net::TCPServerParams));
LOG_INFO(log, "Listening tcp_secure: " + address.toString());
#else
throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
@ -602,10 +637,10 @@ int Server::main(const std::vector<std::string> & /*args*/)
{
std::stringstream message;
message << "Available RAM = " << formatReadableSizeWithBinarySuffix(getMemoryAmount()) << ";"
message << "Available RAM = " << formatReadableSizeWithBinarySuffix(memory_amount) << ";"
<< " physical cores = " << getNumberOfPhysicalCPUCores() << ";"
// on ARM processors it can show only enabled at current moment cores
<< " threads = " << std::thread::hardware_concurrency() << ".";
<< " threads = " << std::thread::hardware_concurrency() << ".";
LOG_INFO(log, message.str());
}

4
dbms/programs/server/TCPHandler.cpp
View File

@ -181,7 +181,7 @@ void TCPHandler::runImpl()
/// Reset the input stream, as we received an empty block while receiving external table data.
/// So, the stream has been marked as cancelled and we can't read from it anymore.
state.block_in.reset();
state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker.
state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker.
});
/// Processing Query
@ -718,7 +718,7 @@ bool TCPHandler::receiveData()
{
NamesAndTypesList columns = block.getNamesAndTypesList();
storage = StorageMemory::create(external_table_name,
ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}});
ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, ColumnComments{}});
storage->startup();
query_context.addExternalTable(external_table_name, storage);
}

2
dbms/programs/server/TCPHandler.h
View File

@ -89,7 +89,7 @@ public:
, connection_context(server.context())
, query_context(server.context())
{
server_display_name = server.config().getString("display_name", getFQDNOrHostName());
server_display_name = server.config().getString("display_name", getFQDNOrHostName());
}
void run();

20
dbms/programs/server/config.xml
View File

@ -145,6 +145,12 @@
-->
<!-- <umask>022</umask> -->
<!-- Perform mlockall after startup to lower first queries latency
and to prevent clickhouse executable from being paged out under high IO load.
Enabling this option is recommended but will lead to increased startup time for up to a few seconds.
-->
<mlock_executable>false</mlock_executable>
<!-- Configuration of clusters that could be used in Distributed tables.
https://clickhouse.yandex/docs/en/table_engines/distributed/
-->
@ -158,6 +164,20 @@
</replica>
</shard>
</test_shard_localhost>
<test_cluster_two_shards_localhost>
<shard>
<replica>
<host>localhost</host>
<port>9000</port>
</replica>
</shard>
<shard>
<replica>
<host>localhost</host>
<port>9000</port>
</replica>
</shard>
</test_cluster_two_shards_localhost>
<test_shard_localhost_secure>
<shard>
<replica>

2
dbms/programs/server/users.xml
View File

@ -56,7 +56,7 @@
Each element of list has one of the following forms:
<ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
<host> Hostname. Example: server01.yandex.ru.
To check access, DNS query is performed, and all received addresses compared to peer address.
<host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$

2
dbms/scripts/gen-bias-data.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/python3.4
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import sys

2
dbms/src/AggregateFunctions/AggregateFunctionAvg.cpp
View File

@ -12,7 +12,7 @@ namespace
template <typename T>
struct Avg
{
using FieldType = std::conditional_t<IsDecimalNumber<T>, Decimal128, typename NearestFieldType<T>::Type>;
using FieldType = std::conditional_t<IsDecimalNumber<T>, Decimal128, NearestFieldType<T>>;
using Function = AggregateFunctionAvg<T, AggregateFunctionAvgData<FieldType>>;
};

4
dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp
View File

@ -55,10 +55,10 @@ static AggregateFunctionPtr createAggregateFunctionGroupArray(const std::string
else if (parameters.size() == 1)
{
auto type = parameters[0].getType();
if (type != Field::Types::Int64 && type != Field::Types::UInt64)
if (type != Field::Types::Int64 && type != Field::Types::UInt64)
throw Exception("Parameter for aggregate function " + name + " should be positive number", ErrorCodes::BAD_ARGUMENTS);
if ((type == Field::Types::Int64 && parameters[0].get<Int64>() < 0) ||
if ((type == Field::Types::Int64 && parameters[0].get<Int64>() < 0) ||
(type == Field::Types::UInt64 && parameters[0].get<UInt64>() == 0))
throw Exception("Parameter for aggregate function " + name + " should be positive number", ErrorCodes::BAD_ARGUMENTS);

48
dbms/src/AggregateFunctions/AggregateFunctionQuantile.cpp
View File

@ -1,11 +1,7 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/Helpers.h>
#include <AggregateFunctions/FactoryHelpers.h>
#include <AggregateFunctions/AggregateFunctionQuantile.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/Helpers.h>
namespace DB
@ -52,13 +48,11 @@ static constexpr bool SupportDecimal()
}
template <template <typename> class Function, bool have_second_arg>
template <template <typename> class Function>
AggregateFunctionPtr createAggregateFunctionQuantile(const std::string & name, const DataTypes & argument_types, const Array & params)
{
if constexpr (have_second_arg)
assertBinary(name, argument_types);
else
assertUnary(name, argument_types);
/// Second argument type check doesn't depend on the type of the first one.
Function<void>::assertSecondArg(argument_types);
const DataTypePtr & argument_type = argument_types[0];
WhichDataType which(argument_type);
@ -86,29 +80,29 @@ AggregateFunctionPtr createAggregateFunctionQuantile(const std::string & name, c
void registerAggregateFunctionsQuantile(AggregateFunctionFactory & factory)
{
factory.registerFunction(NameQuantile::name, createAggregateFunctionQuantile<FuncQuantile, false>);
factory.registerFunction(NameQuantiles::name, createAggregateFunctionQuantile<FuncQuantiles, false>);
factory.registerFunction(NameQuantile::name, createAggregateFunctionQuantile<FuncQuantile>);
factory.registerFunction(NameQuantiles::name, createAggregateFunctionQuantile<FuncQuantiles>);
factory.registerFunction(NameQuantileDeterministic::name, createAggregateFunctionQuantile<FuncQuantileDeterministic, true>);
factory.registerFunction(NameQuantilesDeterministic::name, createAggregateFunctionQuantile<FuncQuantilesDeterministic, true>);
factory.registerFunction(NameQuantileDeterministic::name, createAggregateFunctionQuantile<FuncQuantileDeterministic>);
factory.registerFunction(NameQuantilesDeterministic::name, createAggregateFunctionQuantile<FuncQuantilesDeterministic>);
factory.registerFunction(NameQuantileExact::name, createAggregateFunctionQuantile<FuncQuantileExact, false>);
factory.registerFunction(NameQuantilesExact::name, createAggregateFunctionQuantile<FuncQuantilesExact, false>);
factory.registerFunction(NameQuantileExact::name, createAggregateFunctionQuantile<FuncQuantileExact>);
factory.registerFunction(NameQuantilesExact::name, createAggregateFunctionQuantile<FuncQuantilesExact>);
factory.registerFunction(NameQuantileExactWeighted::name, createAggregateFunctionQuantile<FuncQuantileExactWeighted, true>);
factory.registerFunction(NameQuantilesExactWeighted::name, createAggregateFunctionQuantile<FuncQuantilesExactWeighted, true>);
factory.registerFunction(NameQuantileExactWeighted::name, createAggregateFunctionQuantile<FuncQuantileExactWeighted>);
factory.registerFunction(NameQuantilesExactWeighted::name, createAggregateFunctionQuantile<FuncQuantilesExactWeighted>);
factory.registerFunction(NameQuantileTiming::name, createAggregateFunctionQuantile<FuncQuantileTiming, false>);
factory.registerFunction(NameQuantilesTiming::name, createAggregateFunctionQuantile<FuncQuantilesTiming, false>);
factory.registerFunction(NameQuantileTiming::name, createAggregateFunctionQuantile<FuncQuantileTiming>);
factory.registerFunction(NameQuantilesTiming::name, createAggregateFunctionQuantile<FuncQuantilesTiming>);
factory.registerFunction(NameQuantileTimingWeighted::name, createAggregateFunctionQuantile<FuncQuantileTimingWeighted, true>);
factory.registerFunction(NameQuantilesTimingWeighted::name, createAggregateFunctionQuantile<FuncQuantilesTimingWeighted, true>);
factory.registerFunction(NameQuantileTimingWeighted::name, createAggregateFunctionQuantile<FuncQuantileTimingWeighted>);
factory.registerFunction(NameQuantilesTimingWeighted::name, createAggregateFunctionQuantile<FuncQuantilesTimingWeighted>);
factory.registerFunction(NameQuantileTDigest::name, createAggregateFunctionQuantile<FuncQuantileTDigest, false>);
factory.registerFunction(NameQuantilesTDigest::name, createAggregateFunctionQuantile<FuncQuantilesTDigest, false>);
factory.registerFunction(NameQuantileTDigest::name, createAggregateFunctionQuantile<FuncQuantileTDigest>);
factory.registerFunction(NameQuantilesTDigest::name, createAggregateFunctionQuantile<FuncQuantilesTDigest>);
factory.registerFunction(NameQuantileTDigestWeighted::name, createAggregateFunctionQuantile<FuncQuantileTDigestWeighted, true>);
factory.registerFunction(NameQuantilesTDigestWeighted::name, createAggregateFunctionQuantile<FuncQuantilesTDigestWeighted, true>);
factory.registerFunction(NameQuantileTDigestWeighted::name, createAggregateFunctionQuantile<FuncQuantileTDigestWeighted>);
factory.registerFunction(NameQuantilesTDigestWeighted::name, createAggregateFunctionQuantile<FuncQuantilesTDigestWeighted>);
/// 'median' is an alias for 'quantile'
factory.registerAlias("median", NameQuantile::name);

60
dbms/src/AggregateFunctions/AggregateFunctionQuantile.h
View File

@ -1,19 +1,28 @@
#pragma once
#include <type_traits>
#include <AggregateFunctions/FactoryHelpers.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeArray.h>
/// These must be exposed in header for the purpose of dynamic compilation.
#include <AggregateFunctions/QuantileReservoirSampler.h>
#include <AggregateFunctions/QuantileReservoirSamplerDeterministic.h>
#include <AggregateFunctions/QuantileExact.h>
#include <AggregateFunctions/QuantileExactWeighted.h>
#include <AggregateFunctions/QuantileTiming.h>
#include <AggregateFunctions/QuantileTDigest.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <AggregateFunctions/QuantilesCommon.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnDecimal.h>
#include <Columns/ColumnsNumber.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypesNumber.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <type_traits>
namespace DB
@ -36,10 +45,10 @@ template <
typename Data,
/// Structure with static member "name", containing the name of the aggregate function.
typename Name,
/// If true, the function accept second argument
/// If true, the function accepts the second argument
/// (in can be "weight" to calculate quantiles or "determinator" that is used instead of PRNG).
/// Second argument is always obtained through 'getUInt' method.
bool have_second_arg,
bool has_second_arg,
/// If non-void, the function will return float of specified type with possibly interpolated results and NaN if there was no values.
/// Otherwise it will return Value type and default value if there was no values.
/// As an example, the function cannot return floats, if the SQL type of argument is Date or DateTime.
@ -49,12 +58,14 @@ template <
bool returns_many
>
class AggregateFunctionQuantile final : public IAggregateFunctionDataHelper<Data,
AggregateFunctionQuantile<Value, Data, Name, have_second_arg, FloatReturnType, returns_many>>
AggregateFunctionQuantile<Value, Data, Name, has_second_arg, FloatReturnType, returns_many>>
{
private:
using ColVecType = std::conditional_t<IsDecimalNumber<Value>, ColumnDecimal<Value>, ColumnVector<Value>>;
static constexpr bool returns_float = !std::is_same_v<FloatReturnType, void>;
static constexpr bool returns_float = !(std::is_same_v<FloatReturnType, void>)
&& (!(std::is_same_v<Value, DataTypeDate::FieldType> || std::is_same_v<Value, DataTypeDateTime::FieldType>)
|| std::is_same_v<Data, QuantileTiming<Value>>);
static_assert(!IsDecimalNumber<Value> || !returns_float);
QuantileLevels<Float64> levels;
@ -92,7 +103,7 @@ public:
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
{
const auto & column = static_cast<const ColVecType &>(*columns[0]);
if constexpr (have_second_arg)
if constexpr (has_second_arg)
this->data(place).add(
column.getData()[row_num],
columns[1]->getUInt(row_num));
@ -159,22 +170,17 @@ public:
}
const char * getHeaderFilePath() const override { return __FILE__; }
static void assertSecondArg(const DataTypes & argument_types)
{
if constexpr (has_second_arg)
/// TODO: check that second argument is of numerical type.
assertBinary(Name::name, argument_types);
else
assertUnary(Name::name, argument_types);
}
};
}
/// These must be exposed in header for the purpose of dynamic compilation.
#include <AggregateFunctions/QuantileReservoirSampler.h>
#include <AggregateFunctions/QuantileReservoirSamplerDeterministic.h>
#include <AggregateFunctions/QuantileExact.h>
#include <AggregateFunctions/QuantileExactWeighted.h>
#include <AggregateFunctions/QuantileTiming.h>
#include <AggregateFunctions/QuantileTDigest.h>
namespace DB
{
struct NameQuantile { static constexpr auto name = "quantile"; };
struct NameQuantiles { static constexpr auto name = "quantiles"; };
struct NameQuantileDeterministic { static constexpr auto name = "quantileDeterministic"; };

2
dbms/src/AggregateFunctions/AggregateFunctionStatistics.h
View File

@ -274,7 +274,7 @@ public:
++count;
left_mean += left_delta / count;
left_mean += left_delta / count;
right_mean += right_delta / count;
co_moment += (left_val - left_mean) * (right_val - old_right_mean);

2
dbms/src/AggregateFunctions/AggregateFunctionSum.cpp
View File

@ -14,7 +14,7 @@ template <typename T>
struct SumSimple
{
/// @note It uses slow Decimal128 (cause we need such a variant). sumWithOverflow is faster for Decimal32/64
using ResultType = std::conditional_t<IsDecimalNumber<T>, Decimal128, typename NearestFieldType<T>::Type>;
using ResultType = std::conditional_t<IsDecimalNumber<T>, Decimal128, NearestFieldType<T>>;
using AggregateDataType = AggregateFunctionSumData<ResultType>;
using Function = AggregateFunctionSum<T, ResultType, AggregateDataType>;
};

2
dbms/src/AggregateFunctions/AggregateFunctionSumMap.h
View File

@ -52,7 +52,7 @@ struct AggregateFunctionSumMapData
template <typename T>
class AggregateFunctionSumMap final : public IAggregateFunctionDataHelper<
AggregateFunctionSumMapData<typename NearestFieldType<T>::Type>, AggregateFunctionSumMap<T>>
AggregateFunctionSumMapData<NearestFieldType<T>>, AggregateFunctionSumMap<T>>
{
private:
using ColVecType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;

3
dbms/src/AggregateFunctions/AggregateFunctionUniq.cpp
View File

@ -130,9 +130,6 @@ void registerAggregateFunctionsUniq(AggregateFunctionFactory & factory)
factory.registerFunction("uniqExact",
createAggregateFunctionUniq<true, AggregateFunctionUniqExactData, AggregateFunctionUniqExactData<String>>);
factory.registerFunction("uniqCombined",
createAggregateFunctionUniq<false, AggregateFunctionUniqCombinedData, AggregateFunctionUniqCombinedData<UInt64>>);
}
}

88
dbms/src/AggregateFunctions/AggregateFunctionUniq.h
View File

@ -22,7 +22,6 @@
#include <Common/typeid_cast.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <AggregateFunctions/UniqCombinedBiasData.h>
#include <AggregateFunctions/UniqVariadicHash.h>
@ -124,46 +123,6 @@ struct AggregateFunctionUniqExactData<String>
static String getName() { return "uniqExact"; }
};
template <typename T>
struct AggregateFunctionUniqCombinedData
{
using Key = UInt32;
using Set = CombinedCardinalityEstimator<
Key,
HashSet<Key, TrivialHash, HashTableGrower<>>,
16,
14,
17,
TrivialHash,
UInt32,
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
HyperLogLogMode::FullFeatured>;
Set set;
static String getName() { return "uniqCombined"; }
};
template <>
struct AggregateFunctionUniqCombinedData<String>
{
using Key = UInt64;
using Set = CombinedCardinalityEstimator<
Key,
HashSet<Key, TrivialHash, HashTableGrower<>>,
16,
14,
17,
TrivialHash,
UInt64,
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
HyperLogLogMode::FullFeatured>;
Set set;
static String getName() { return "uniqCombined"; }
};
namespace detail
{
@ -199,39 +158,6 @@ template <> struct AggregateFunctionUniqTraits<Float64>
}
};
/** Hash function for uniqCombined.
*/
template <typename T> struct AggregateFunctionUniqCombinedTraits
{
static UInt32 hash(T x) { return static_cast<UInt32>(intHash64(x)); }
};
template <> struct AggregateFunctionUniqCombinedTraits<UInt128>
{
static UInt32 hash(UInt128 x)
{
return sipHash64(x);
}
};
template <> struct AggregateFunctionUniqCombinedTraits<Float32>
{
static UInt32 hash(Float32 x)
{
UInt64 res = ext::bit_cast<UInt64>(x);
return static_cast<UInt32>(intHash64(res));
}
};
template <> struct AggregateFunctionUniqCombinedTraits<Float64>
{
static UInt32 hash(Float64 x)
{
UInt64 res = ext::bit_cast<UInt64>(x);
return static_cast<UInt32>(intHash64(res));
}
};
/** The structure for the delegation work to add one element to the `uniq` aggregate functions.
* Used for partial specialization to add strings.
@ -255,19 +181,6 @@ struct OneAdder
data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
}
}
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqCombinedData<T>>)
{
if constexpr (!std::is_same_v<T, String>)
{
const auto & value = static_cast<const ColumnVector<T> &>(column).getData()[row_num];
data.set.insert(AggregateFunctionUniqCombinedTraits<T>::hash(value));
}
else
{
StringRef value = column.getDataAt(row_num);
data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
}
}
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqExactData<T>>)
{
if constexpr (!std::is_same_v<T, String>)
@ -387,5 +300,4 @@ public:
const char * getHeaderFilePath() const override { return __FILE__; }
};
}

127
dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.cpp Normal file
View File

@ -0,0 +1,127 @@
#include <AggregateFunctions/AggregateFunctionUniqCombined.h>
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/Helpers.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int ARGUMENT_OUT_OF_BOUND;
}
namespace
{
template <UInt8 K>
struct WithK
{
template <typename T>
using AggregateFunction = AggregateFunctionUniqCombined<T, K>;
template <bool is_exact, bool argument_is_tuple>
using AggregateFunctionVariadic = AggregateFunctionUniqCombinedVariadic<is_exact, argument_is_tuple, K>;
};
template <UInt8 K>
AggregateFunctionPtr createAggregateFunctionWithK(const DataTypes & argument_types)
{
/// We use exact hash function if the arguments are not contiguous in memory, because only exact hash function has support for this case.
bool use_exact_hash_function = !isAllArgumentsContiguousInMemory(argument_types);
if (argument_types.size() == 1)
{
const IDataType & argument_type = *argument_types[0];
AggregateFunctionPtr res(createWithNumericType<WithK<K>::template AggregateFunction>(*argument_types[0]));
WhichDataType which(argument_type);
if (res)
return res;
else if (which.isDate())
return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeDate::FieldType>>();
else if (which.isDateTime())
return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeDateTime::FieldType>>();
else if (which.isStringOrFixedString())
return std::make_shared<typename WithK<K>::template AggregateFunction<String>>();
else if (which.isUUID())
return std::make_shared<typename WithK<K>::template AggregateFunction<DataTypeUUID::FieldType>>();
else if (which.isTuple())
{
if (use_exact_hash_function)
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<true, true>>(argument_types);
else
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<false, true>>(argument_types);
}
}
/// "Variadic" method also works as a fallback generic case for a single argument.
if (use_exact_hash_function)
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<true, false>>(argument_types);
else
return std::make_shared<typename WithK<K>::template AggregateFunctionVariadic<false, false>>(argument_types);
}
AggregateFunctionPtr createAggregateFunctionUniqCombined(
const std::string & name, const DataTypes & argument_types, const Array & params)
{
/// log2 of the number of cells in HyperLogLog.
/// Reasonable default value, selected to be comparable in quality with "uniq" aggregate function.
UInt8 precision = 17;
if (!params.empty())
{
if (params.size() != 1)
throw Exception(
"Aggregate function " + name + " requires one parameter or less.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
UInt64 precision_param = applyVisitor(FieldVisitorConvertToNumber<UInt64>(), params[0]);
// This range is hardcoded below
if (precision_param > 20 || precision_param < 12)
throw Exception(
"Parameter for aggregate function " + name + "is out or range: [12, 20].", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
precision = precision_param;
}
if (argument_types.empty())
throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
switch (precision)
{
case 12:
return createAggregateFunctionWithK<12>(argument_types);
case 13:
return createAggregateFunctionWithK<13>(argument_types);
case 14:
return createAggregateFunctionWithK<14>(argument_types);
case 15:
return createAggregateFunctionWithK<15>(argument_types);
case 16:
return createAggregateFunctionWithK<16>(argument_types);
case 17:
return createAggregateFunctionWithK<17>(argument_types);
case 18:
return createAggregateFunctionWithK<18>(argument_types);
case 19:
return createAggregateFunctionWithK<19>(argument_types);
case 20:
return createAggregateFunctionWithK<20>(argument_types);
}
__builtin_unreachable();
}
} // namespace
void registerAggregateFunctionUniqCombined(AggregateFunctionFactory & factory)
{
factory.registerFunction("uniqCombined", createAggregateFunctionUniqCombined);
}
} // namespace DB

229
dbms/src/AggregateFunctions/AggregateFunctionUniqCombined.h Normal file
View File

@ -0,0 +1,229 @@
#pragma once
#include <Common/CombinedCardinalityEstimator.h>
#include <DataTypes/DataTypeTuple.h>
#include <DataTypes/DataTypeUUID.h>
#include <DataTypes/DataTypesNumber.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <AggregateFunctions/UniqCombinedBiasData.h>
#include <AggregateFunctions/UniqVariadicHash.h>
#include <ext/bit_cast.h>
#include <Columns/ColumnVector.h>
#include <Columns/ColumnsNumber.h>
#include <Common/FieldVisitors.h>
#include <Common/SipHash.h>
#include <Common/typeid_cast.h>
namespace DB
{
namespace detail
{
/** Hash function for uniqCombined.
*/
template <typename T>
struct AggregateFunctionUniqCombinedTraits
{
static UInt32 hash(T x)
{
return static_cast<UInt32>(intHash64(x));
}
};
template <>
struct AggregateFunctionUniqCombinedTraits<UInt128>
{
static UInt32 hash(UInt128 x)
{
return sipHash64(x);
}
};
template <>
struct AggregateFunctionUniqCombinedTraits<Float32>
{
static UInt32 hash(Float32 x)
{
UInt64 res = ext::bit_cast<UInt64>(x);
return static_cast<UInt32>(intHash64(res));
}
};
template <>
struct AggregateFunctionUniqCombinedTraits<Float64>
{
static UInt32 hash(Float64 x)
{
UInt64 res = ext::bit_cast<UInt64>(x);
return static_cast<UInt32>(intHash64(res));
}
};
} // namespace detail
template <typename Key, UInt8 K>
struct AggregateFunctionUniqCombinedDataWithKey
{
// TODO(ilezhankin): pre-generate values for |UniqCombinedBiasData|,
// at the moment gen-bias-data.py script doesn't work.
// We want to migrate from |HashSet| to |HyperLogLogCounter| when the sizes in memory become almost equal.
// The size per element in |HashSet| is sizeof(Key)*2 bytes, and the overall size of |HyperLogLogCounter| is 2^K * 6 bits.
// For Key=UInt32 we can calculate: 2^X * 4 * 2 ≤ 2^(K-3) * 6 ⇒ X ≤ K-4.
using Set = CombinedCardinalityEstimator<Key, HashSet<Key, TrivialHash, HashTableGrower<>>, 16, K - 4, K, TrivialHash, Key>;
Set set;
};
template <typename Key>
struct AggregateFunctionUniqCombinedDataWithKey<Key, 17>
{
using Set = CombinedCardinalityEstimator<Key,
HashSet<Key, TrivialHash, HashTableGrower<>>,
16,
13,
17,
TrivialHash,
Key,
HyperLogLogBiasEstimator<UniqCombinedBiasData>,
HyperLogLogMode::FullFeatured>;
Set set;
};
template <typename T, UInt8 K>
struct AggregateFunctionUniqCombinedData : public AggregateFunctionUniqCombinedDataWithKey<UInt32, K>
{
};
template <UInt8 K>
struct AggregateFunctionUniqCombinedData<String, K> : public AggregateFunctionUniqCombinedDataWithKey<UInt64, K>
{
};
template <typename T, UInt8 K>
class AggregateFunctionUniqCombined final
: public IAggregateFunctionDataHelper<AggregateFunctionUniqCombinedData<T, K>, AggregateFunctionUniqCombined<T, K>>
{
public:
String getName() const override
{
return "uniqCombined";
}
DataTypePtr getReturnType() const override
{
return std::make_shared<DataTypeUInt64>();
}
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
{
if constexpr (!std::is_same_v<T, String>)
{
const auto & value = static_cast<const ColumnVector<T> &>(*columns[0]).getData()[row_num];
this->data(place).set.insert(detail::AggregateFunctionUniqCombinedTraits<T>::hash(value));
}
else
{
StringRef value = columns[0]->getDataAt(row_num);
this->data(place).set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
}
}
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
{
this->data(place).set.merge(this->data(rhs).set);
}
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
{
this->data(place).set.write(buf);
}
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
{
this->data(place).set.read(buf);
}
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
{
static_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
}
const char * getHeaderFilePath() const override
{
return __FILE__;
}
};
/** For multiple arguments. To compute, hashes them.
* You can pass multiple arguments as is; You can also pass one argument - a tuple.
* But (for the possibility of efficient implementation), you can not pass several arguments, among which there are tuples.
*/
template <bool is_exact, bool argument_is_tuple, UInt8 K>
class AggregateFunctionUniqCombinedVariadic final : public IAggregateFunctionDataHelper<AggregateFunctionUniqCombinedData<UInt64, K>,
AggregateFunctionUniqCombinedVariadic<is_exact, argument_is_tuple, K>>
{
private:
size_t num_args = 0;
public:
explicit AggregateFunctionUniqCombinedVariadic(const DataTypes & arguments)
{
if (argument_is_tuple)
num_args = typeid_cast<const DataTypeTuple &>(*arguments[0]).getElements().size();
else
num_args = arguments.size();
}
String getName() const override
{
return "uniqCombined";
}
DataTypePtr getReturnType() const override
{
return std::make_shared<DataTypeUInt64>();
}
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
{
this->data(place).set.insert(typename AggregateFunctionUniqCombinedData<UInt64, K>::Set::value_type(
UniqVariadicHash<is_exact, argument_is_tuple>::apply(num_args, columns, row_num)));
}
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
{
this->data(place).set.merge(this->data(rhs).set);
}
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
{
this->data(place).set.write(buf);
}
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
{
this->data(place).set.read(buf);
}
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
{
static_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
}
const char * getHeaderFilePath() const override
{
return __FILE__;
}
};
} // namespace DB

4
dbms/src/AggregateFunctions/AggregateFunctionWindowFunnel.h
View File

@ -16,10 +16,12 @@
namespace DB
{
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
struct ComparePairFirst final
@ -191,7 +193,7 @@ public:
const auto time_arg = arguments.front().get();
if (!WhichDataType(time_arg).isDateTime() && !WhichDataType(time_arg).isUInt32())
throw Exception{"Illegal type " + time_arg->getName() + " of first argument of aggregate function " + getName()
+ ", must be DateTime or UInt32"};
+ ", must be DateTime or UInt32", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
for (const auto i : ext::range(1, arguments.size()))
{

2
dbms/src/AggregateFunctions/CMakeLists.txt
View File

@ -20,5 +20,5 @@ list(REMOVE_ITEM clickhouse_aggregate_functions_headers
)
add_library(clickhouse_aggregate_functions ${LINK_MODE} ${clickhouse_aggregate_functions_sources})
target_link_libraries(clickhouse_aggregate_functions dbms)
target_link_libraries(clickhouse_aggregate_functions PRIVATE dbms)
target_include_directories (clickhouse_aggregate_functions BEFORE PRIVATE ${COMMON_INCLUDE_DIR})

31
dbms/src/AggregateFunctions/FactoryHelpers.cpp
View File

@ -1,31 +0,0 @@
#include <AggregateFunctions/FactoryHelpers.h>
namespace DB
{
namespace ErrorCodes
{
extern const int AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
void assertNoParameters(const std::string & name, const Array & parameters)
{
if (!parameters.empty())
throw Exception("Aggregate function " + name + " cannot have parameters", ErrorCodes::AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS);
}
void assertUnary(const std::string & name, const DataTypes & argument_types)
{
if (argument_types.size() != 1)
throw Exception("Aggregate function " + name + " require single argument", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}
void assertBinary(const std::string & name, const DataTypes & argument_types)
{
if (argument_types.size() != 2)
throw Exception("Aggregate function " + name + " require two arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}
}

26
dbms/src/AggregateFunctions/FactoryHelpers.h
View File

@ -7,8 +7,28 @@
namespace DB
{
void assertNoParameters(const std::string & name, const Array & parameters);
void assertUnary(const std::string & name, const DataTypes & argument_types);
void assertBinary(const std::string & name, const DataTypes & argument_types);
namespace ErrorCodes
{
extern const int AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
inline void assertNoParameters(const std::string & name, const Array & parameters)
{
if (!parameters.empty())
throw Exception("Aggregate function " + name + " cannot have parameters", ErrorCodes::AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS);
}
inline void assertUnary(const std::string & name, const DataTypes & argument_types)
{
if (argument_types.size() != 1)
throw Exception("Aggregate function " + name + " require single argument", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}
inline void assertBinary(const std::string & name, const DataTypes & argument_types)
{
if (argument_types.size() != 2)
throw Exception("Aggregate function " + name + " require two arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}
}

2
dbms/src/AggregateFunctions/registerAggregateFunctions.cpp
View File

@ -21,6 +21,7 @@ void registerAggregateFunctionsStatisticsSimple(AggregateFunctionFactory &);
void registerAggregateFunctionSum(AggregateFunctionFactory &);
void registerAggregateFunctionSumMap(AggregateFunctionFactory &);
void registerAggregateFunctionsUniq(AggregateFunctionFactory &);
void registerAggregateFunctionUniqCombined(AggregateFunctionFactory &);
void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &);
void registerAggregateFunctionTopK(AggregateFunctionFactory &);
void registerAggregateFunctionsBitwise(AggregateFunctionFactory &);
@ -55,6 +56,7 @@ void registerAggregateFunctions()
registerAggregateFunctionSum(factory);
registerAggregateFunctionSumMap(factory);
registerAggregateFunctionsUniq(factory);
registerAggregateFunctionUniqCombined(factory);
registerAggregateFunctionUniqUpTo(factory);
registerAggregateFunctionTopK(factory);
registerAggregateFunctionsBitwise(factory);

46
dbms/src/Analyzers/AnalyzeArrayJoins.h
View File

@ -1,46 +0,0 @@
#pragma once
#include <Parsers/IAST.h>
#include <unordered_map>
#include <vector>
namespace DB
{
class WriteBuffer;
struct CollectAliases;
struct CollectTables;
/** For every ARRAY JOIN, collect a map:
* result alias -> source
*
* There could be several variants:
*
* SELECT elem FROM t ARRAY JOIN array AS elem elem -> array
* SELECT n.elem FROM t ARRAY JOIN nested AS n n -> nested
* SELECT array FROM t ARRAY JOIN array array -> array
* SELECT nested.elem FROM t ARRAY JOIN nested nested -> nested
* SELECT elem FROM t ARRAY JOIN [1, 2, 3] AS elem elem -> [1, 2, 3]
*
* Does not analyze arrayJoin functions.
*/
struct AnalyzeArrayJoins
{
void process(const ASTPtr & ast);
struct SourceInfo
{
String column_name;
ASTPtr node;
};
using ResultToSource = std::unordered_map<String, SourceInfo>;
using ArrayJoins = std::vector<ResultToSource>;
/// Debug output
void dump(WriteBuffer & out) const;
};
}

455
dbms/src/Analyzers/AnalyzeColumns.cpp
View File

@ -1,455 +0,0 @@
#include <vector>
#include <Analyzers/AnalyzeColumns.h>
#include <Analyzers/CollectAliases.h>
#include <Parsers/formatAST.h>
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Parsers/ASTAsterisk.h>
#include <Parsers/ASTQualifiedAsterisk.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTFunction.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteHelpers.h>
#include <Poco/String.h>
#include <Common/typeid_cast.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int NOT_IMPLEMENTED;
extern const int AMBIGUOUS_TABLE_NAME;
extern const int AMBIGUOUS_COLUMN_NAME;
extern const int UNKNOWN_TABLE;
extern const int THERE_IS_NO_COLUMN;
}
namespace
{
/// Find by fully qualified name, like db.table.column
const CollectTables::TableInfo * findTableByDatabaseAndTableName(
const CollectTables & tables, const String & database_name, const String & table_name)
{
for (const auto & table : tables.tables)
if (table.database_name == database_name && table.table_name == table_name)
return &table;
return nullptr;
}
/** Find by single-qualified name, like table.column or alias.column.
*
* There are primary matches:
* when name is alias like
* SELECT name.column FROM (SELECT 1) AS name
* or name is table in current database like
* SELECT name.column FROM name
*
* And secondary matches:
* when name is name of table in explicitly specified database like
* SELECT name.column FROM db.name
*
* If there is only one primary match - return it.
* If there is many primary matches - ambiguity.
* If there is no primary matches and only one secondary match - return it.
* If there is no primary matches and many secondary matches - ambiguity.
* If there is no any matches - not found.
*/
const CollectTables::TableInfo * findTableByNameOrAlias(
const CollectTables & tables, const String & name)
{
const CollectTables::TableInfo * primary_match = nullptr;
const CollectTables::TableInfo * secondary_match = nullptr;
for (const auto & table : tables.tables)
{
if (table.alias == name
|| (table.database_name.empty() && table.table_name == name))
{
if (primary_match)
throw Exception("Table name " + backQuoteIfNeed(name) + " is ambiguous", ErrorCodes::AMBIGUOUS_TABLE_NAME);
primary_match = &table;
}
else if (!primary_match && table.table_name == name)
{
if (secondary_match)
throw Exception("Table name " + backQuoteIfNeed(name) + " is ambiguous", ErrorCodes::AMBIGUOUS_TABLE_NAME);
secondary_match = &table;
}
}
if (primary_match)
return primary_match;
if (secondary_match)
return secondary_match;
return nullptr;
}
/** Find table in case when its name is not specified. Like just
* SELECT column FROM t1, t2
* Select a table, where specified column exists.
* If more than one such table - ambiguity.
*/
const CollectTables::TableInfo * findTableWithUnqualifiedName(const CollectTables & tables, const String & column_name)
{
const CollectTables::TableInfo * res = nullptr;
for (const auto & table : tables.tables)
{
if (table.structure_of_subquery)
{
if (table.structure_of_subquery.has(column_name))
{
if (res)
throw Exception("Ambiguous column name " + backQuoteIfNeed(column_name), ErrorCodes::AMBIGUOUS_COLUMN_NAME);
res = &table;
break;
}
}
else if (table.storage)
{
if (table.storage->hasColumn(column_name))
{
if (res)
throw Exception("Ambiguous column name " + backQuoteIfNeed(column_name), ErrorCodes::AMBIGUOUS_COLUMN_NAME);
res = &table;
}
}
else
throw Exception("Logical error: no storage and no structure of subquery is specified for table", ErrorCodes::LOGICAL_ERROR);
}
return res;
}
/// Create maximum-qualified identifier for column in table.
ASTPtr createASTIdentifierForColumnInTable(const String & column, const CollectTables::TableInfo & table)
{
ASTPtr database_name_identifier_node;
if (!table.database_name.empty())
database_name_identifier_node = std::make_shared<ASTIdentifier>(table.database_name);
ASTPtr table_name_identifier_node;
String table_name_or_alias;
if (!table.table_name.empty())
table_name_or_alias = table.table_name;
else if (table.database_name.empty() && !table.alias.empty())
table_name_or_alias = table.alias;
if (!table_name_or_alias.empty())
table_name_identifier_node = std::make_shared<ASTIdentifier>(table_name_or_alias);
ASTPtr column_identifier_node = std::make_shared<ASTIdentifier>(column);
String compound_name;
if (database_name_identifier_node)
compound_name += table.database_name + ".";
if (table_name_identifier_node)
compound_name += table_name_or_alias + ".";
compound_name += column;
auto elem = std::make_shared<ASTIdentifier>(compound_name);
if (database_name_identifier_node)
elem->children.emplace_back(std::move(database_name_identifier_node));
if (table_name_identifier_node)
elem->children.emplace_back(std::move(table_name_identifier_node));
if (!elem->children.empty())
elem->children.emplace_back(std::move(column_identifier_node));
return elem;
}
void createASTsForAllColumnsInTable(const CollectTables::TableInfo & table, ASTs & res)
{
if (table.storage)
for (const auto & name : table.storage->getColumns().getNamesOfPhysical())
res.emplace_back(createASTIdentifierForColumnInTable(name, table));
else
for (size_t i = 0, size = table.structure_of_subquery.columns(); i < size; ++i)
res.emplace_back(createASTIdentifierForColumnInTable(table.structure_of_subquery.getByPosition(i).name, table));
}
ASTs expandUnqualifiedAsterisk(const CollectTables & tables)
{
ASTs res;
for (const auto & table : tables.tables)
createASTsForAllColumnsInTable(table, res);
return res;
}
ASTs expandQualifiedAsterisk(
const IAST & ast, const CollectTables & tables)
{
if (ast.children.size() != 1)
throw Exception("Logical error: AST node for qualified asterisk has number of children not equal to one", ErrorCodes::LOGICAL_ERROR);
const ASTIdentifier & qualifier = static_cast<const ASTIdentifier &>(*ast.children[0]);
const CollectTables::TableInfo * table = nullptr;
if (qualifier.children.empty())
table = findTableByNameOrAlias(tables, qualifier.name);
else if (qualifier.children.size() == 2)
table = findTableByDatabaseAndTableName(tables,
static_cast<const ASTIdentifier &>(*qualifier.children[0]).name,
static_cast<const ASTIdentifier &>(*qualifier.children[1]).name);
else
throw Exception("Unsupported number of components in asterisk qualifier", ErrorCodes::NOT_IMPLEMENTED);
/// TODO Implement for case table.nested.* and database.table.nested.*
if (!table)
throw Exception("There is no table " + qualifier.name + " in query", ErrorCodes::UNKNOWN_TABLE);
ASTs res;
createASTsForAllColumnsInTable(*table, res);
return res;
}
void processIdentifier(
const ASTPtr & ast, AnalyzeColumns::Columns & columns, const CollectAliases & aliases, const CollectTables & tables)
{
const ASTIdentifier & identifier = static_cast<const ASTIdentifier &>(*ast);
if (aliases.aliases.count(identifier.name))
return;
if (columns.count(identifier.name))
return;
const CollectTables::TableInfo * table = nullptr;
String column_name;
if (identifier.children.empty())
{
/** Lambda parameters are not columns from table. Just skip them.
* This step requires AnalyzeLambdas to be done on AST.
*/
if (startsWith(identifier.name, "_lambda"))
return;
table = findTableWithUnqualifiedName(tables, identifier.name);
if (table)
column_name = identifier.name;
}
else if (identifier.children.size() == 2)
{
const String & first = static_cast<const ASTIdentifier &>(*identifier.children[0]).name;
const String & second = static_cast<const ASTIdentifier &>(*identifier.children[1]).name;
/// table.column
table = findTableByNameOrAlias(tables, first);
if (table)
{
column_name = second;
}
else
{
/// column.nested
table = findTableWithUnqualifiedName(tables, identifier.name);
if (table)
column_name = identifier.name;
}
}
else if (identifier.children.size() == 3)
{
const String & first = static_cast<const ASTIdentifier &>(*identifier.children[0]).name;
const String & second = static_cast<const ASTIdentifier &>(*identifier.children[1]).name;
const String & third = static_cast<const ASTIdentifier &>(*identifier.children[2]).name;
/// database.table.column
table = findTableByDatabaseAndTableName(tables, first, second);
if (table)
{
column_name = third;
}
else
{
/// table.column.nested
table = findTableByNameOrAlias(tables, first);
if (table)
{
column_name = second + "." + third;
}
else
{
/// column.nested.nested
table = findTableWithUnqualifiedName(tables, identifier.name);
if (table)
column_name = identifier.name;
}
}
}
if (!table)
throw Exception("Cannot find column " + identifier.name, ErrorCodes::THERE_IS_NO_COLUMN);
AnalyzeColumns::ColumnInfo info;
info.node = ast;
info.table = *table;
info.name_in_table = column_name;
if (table->structure_of_subquery)
{
if (!table->structure_of_subquery.has(column_name))
throw Exception("Cannot find column " + backQuoteIfNeed(column_name) + " in subquery", ErrorCodes::LOGICAL_ERROR);
info.data_type = table->structure_of_subquery.getByName(column_name).type;
}
else if (table->storage)
{
info.data_type = table->storage->getColumn(column_name).type;
}
else
throw Exception("Logical error: no storage and no structure of subquery is specified for table", ErrorCodes::LOGICAL_ERROR);
columns[identifier.name] = info;
}
void processImpl(ASTPtr & ast, AnalyzeColumns::Columns & columns, const CollectAliases & aliases, const CollectTables & tables)
{
/// Don't go into subqueries and table-like expressions.
if (typeid_cast<const ASTSelectQuery *>(ast.get())
|| typeid_cast<const ASTTableExpression *>(ast.get()))
{
return;
}
else if (const ASTFunction * func = typeid_cast<const ASTFunction *>(ast.get()))
{
String func_name_lowercase = Poco::toLower(func->name);
/// As special case, treat count(*) as count(), not as count(list of all columns).
if (func_name_lowercase == "count" && func->arguments->children.size() == 1
&& typeid_cast<const ASTAsterisk *>(func->arguments->children[0].get()))
{
func->arguments->children.clear();
}
}
else if (typeid_cast<ASTExpressionList *>(ast.get()))
{
/// Replace asterisks to list of columns.
ASTs & asts = ast->children;
for (int i = static_cast<int>(asts.size()) - 1; i >= 0; --i)
{
if (typeid_cast<ASTAsterisk *>(asts[i].get()))
{
ASTs expanded = expandUnqualifiedAsterisk(tables);
asts.erase(asts.begin() + i);
asts.insert(asts.begin() + i, expanded.begin(), expanded.end());
}
else if (ASTQualifiedAsterisk * asterisk = typeid_cast<ASTQualifiedAsterisk *>(asts[i].get()))
{
ASTs expanded = expandQualifiedAsterisk(*asterisk, tables);
asts.erase(asts.begin() + i);
asts.insert(asts.begin() + i, expanded.begin(), expanded.end());
}
}
}
else if (typeid_cast<const ASTIdentifier *>(ast.get()))
{
processIdentifier(ast, columns, aliases, tables);
return;
}
for (auto & child : ast->children)
processImpl(child, columns, aliases, tables);
}
}
void AnalyzeColumns::process(ASTPtr & ast, const CollectAliases & aliases, const CollectTables & tables)
{
/// If this is SELECT query, don't go into FORMAT and SETTINGS clauses
/// - they contain identifiers that are not columns.
const ASTSelectQuery * select = typeid_cast<const ASTSelectQuery *>(ast.get());
for (auto & child : ast->children)
{
if (select && child.get() == select->settings.get())
continue;
processImpl(child, columns, aliases, tables);
}
}
void AnalyzeColumns::dump(WriteBuffer & out) const
{
/// For need of tests, we need to dump result in some fixed order.
std::vector<Columns::const_iterator> vec;
vec.reserve(columns.size());
for (auto it = columns.begin(); it != columns.end(); ++it)
vec.emplace_back(it);
std::sort(vec.begin(), vec.end(), [](const auto & a, const auto & b) { return a->first < b->first; });
for (const auto & it : vec)
{
writeString(it->first, out);
writeCString(" -> ", out);
writeProbablyBackQuotedString(it->second.name_in_table, out);
writeCString(" ", out);
writeProbablyBackQuotedString(it->second.data_type->getName(), out);
const auto & table = it->second.table;
writeCString(". Database name: ", out);
if (table.database_name.empty())
writeCString("(none)", out);
else
writeProbablyBackQuotedString(table.database_name, out);
writeCString(". Table name: ", out);
if (table.table_name.empty())
writeCString("(none)", out);
else
writeProbablyBackQuotedString(table.table_name, out);
writeCString(". Alias: ", out);
if (table.alias.empty())
writeCString("(none)", out);
else
writeProbablyBackQuotedString(table.alias, out);
writeCString(". Storage: ", out);
if (!table.storage)
writeCString("(none)", out);
else
writeProbablyBackQuotedString(table.storage->getName(), out);
writeCString(". AST: ", out);
if (it->second.node)
{
std::stringstream formatted_ast;
formatAST(*it->second.node, formatted_ast, false, true);
writeString(formatted_ast.str(), out);
}
else
writeCString("(none)", out);
writeChar('\n', out);
}
}
}

48
dbms/src/Analyzers/AnalyzeColumns.h
View File

@ -1,48 +0,0 @@
#pragma once
#include <Parsers/IAST.h>
#include <Analyzers/CollectTables.h>
#include <unordered_map>
namespace DB
{
class WriteBuffer;
struct CollectAliases;
struct CollectTables;
/** For every identifier, that is not an alias,
* determine from what table it comes,
* its original name in table,
* and its data type.
*
* Also:
* - expand asterisks (such as *, t.*, db.table.* and (TODO) even db.table.nested.*) to corresponding list of columns;
* - translate count(*) to count();
* - TODO expand alias columns that come from table definition;
* - TODO replace column names to fully qualified names: identical columns will have same names.
*
* If column is not found or in case of ambiguity, throw an exception.
*/
struct AnalyzeColumns
{
void process(ASTPtr & ast, const CollectAliases & aliases, const CollectTables & tables);
struct ColumnInfo
{
ASTPtr node;
CollectTables::TableInfo table;
String name_in_table;
DataTypePtr data_type;
};
using Columns = std::unordered_map<String, ColumnInfo>;
Columns columns;
/// Debug output
void dump(WriteBuffer & out) const;
};
}

201
dbms/src/Analyzers/AnalyzeLambdas.cpp
View File

@ -1,201 +0,0 @@
#include <vector>
#include <Analyzers/AnalyzeLambdas.h>
#include <Parsers/formatAST.h>
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTFunction.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteHelpers.h>
#include <Common/typeid_cast.h>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_LAMBDA;
extern const int RESERVED_IDENTIFIER_NAME;
}
AnalyzeLambdas::LambdaParameters AnalyzeLambdas::extractLambdaParameters(ASTPtr & ast)
{
/// Lambda parameters could be specified in AST in two forms:
/// - just as single parameter: x -> x + 1
/// - parameters in tuple: (x, y) -> x + 1
#define LAMBDA_ERROR_MESSAGE " There are two valid forms of lambda expressions: x -> ... and (x, y...) -> ..."
if (!ast->tryGetAlias().empty())
throw Exception("Lambda parameters cannot have aliases."
LAMBDA_ERROR_MESSAGE, ErrorCodes::BAD_LAMBDA);
if (const ASTIdentifier * identifier = typeid_cast<const ASTIdentifier *>(ast.get()))
{
return { identifier->name };
}
else if (const ASTFunction * function = typeid_cast<const ASTFunction *>(ast.get()))
{
if (function->name != "tuple")
throw Exception("Left hand side of '->' or first argument of 'lambda' is a function, but this function is not tuple."
LAMBDA_ERROR_MESSAGE " Found function '" + function->name + "' instead.", ErrorCodes::BAD_LAMBDA);
if (!function->arguments || function->arguments->children.empty())
throw Exception("Left hand side of '->' or first argument of 'lambda' is empty tuple."
LAMBDA_ERROR_MESSAGE, ErrorCodes::BAD_LAMBDA);
LambdaParameters res;
res.reserve(function->arguments->children.size());
for (const ASTPtr & arg : function->arguments->children)
{
const ASTIdentifier * arg_identifier = typeid_cast<const ASTIdentifier *>(arg.get());
if (!arg_identifier)
throw Exception("Left hand side of '->' or first argument of 'lambda' contains something that is not just identifier."
LAMBDA_ERROR_MESSAGE, ErrorCodes::BAD_LAMBDA);
if (!arg_identifier->children.empty())
throw Exception("Left hand side of '->' or first argument of 'lambda' contains compound identifier."
LAMBDA_ERROR_MESSAGE, ErrorCodes::BAD_LAMBDA);
if (!arg_identifier->alias.empty())
throw Exception("Lambda parameters cannot have aliases."
LAMBDA_ERROR_MESSAGE, ErrorCodes::BAD_LAMBDA);
res.emplace_back(arg_identifier->name);
}
return res;
}
else
throw Exception("Unexpected left hand side of '->' or first argument of 'lambda'."
LAMBDA_ERROR_MESSAGE, ErrorCodes::BAD_LAMBDA);
#undef LAMBDA_ERROR_MESSAGE
}
namespace
{
/// Currently visible parameters in all scopes of lambda expressions.
/// Lambda expressions could be nested: arrayMap(x -> arrayMap(y -> x[y], x), [[1], [2, 3]])
using LambdaScopes = std::vector<AnalyzeLambdas::LambdaParameters>;
void processIdentifier(ASTPtr & ast, LambdaScopes & lambda_scopes)
{
ASTIdentifier & identifier = static_cast<ASTIdentifier &>(*ast);
if (identifier.children.empty())
{
bool found = false;
/// From most inner scope towards outer scopes.
for (ssize_t num_scopes = lambda_scopes.size(), scope_idx = num_scopes - 1; scope_idx >= 0; --scope_idx)
{
for (size_t arg_idx = 0, num_args = lambda_scopes[scope_idx].size(); arg_idx < num_args; ++arg_idx)
{
if (lambda_scopes[scope_idx][arg_idx] == identifier.name)
{
identifier.name = "_lambda" + toString(scope_idx) + "_arg" + toString(arg_idx);
found = true;
break;
}
}
if (found)
break;
}
if (!found && startsWith(identifier.name, "_lambda"))
throw Exception("Identifier names starting with '_lambda' are reserved for parameters of lambda expressions.",
ErrorCodes::RESERVED_IDENTIFIER_NAME);
}
}
void processImpl(
ASTPtr & ast,
LambdaScopes & lambda_scopes,
const ASTPtr & parent_function_for_this_argument,
AnalyzeLambdas::HigherOrderFunctions & higher_order_functions)
{
/// Don't go into subqueries and table-like expressions.
if (typeid_cast<const ASTSelectQuery *>(ast.get())
|| typeid_cast<const ASTTableExpression *>(ast.get()))
{
return;
}
else if (ASTFunction * func = typeid_cast<ASTFunction *>(ast.get()))
{
/** We must memoize parameters from left hand side (x, y) and then analyze right hand side.
*/
if (func->name == "lambda")
{
auto num_arguments = func->arguments->children.size();
if (num_arguments != 2)
throw Exception("Lambda expression ('->' or 'lambda' function) must have exactly two arguments."
" Found " + toString(num_arguments) + " instead.", ErrorCodes::BAD_LAMBDA);
lambda_scopes.emplace_back(AnalyzeLambdas::extractLambdaParameters(func->arguments->children[0]));
for (size_t i = 0; i < num_arguments; ++i)
processImpl(func->arguments->children[i], lambda_scopes, nullptr, higher_order_functions);
lambda_scopes.pop_back();
if (!parent_function_for_this_argument)
throw Exception("Lambda expression ('->' or 'lambda' function) must be presented as an argument of higher-order function."
" Found standalone lambda expression instead.", ErrorCodes::BAD_LAMBDA);
higher_order_functions.emplace_back(parent_function_for_this_argument);
}
else
{
/// When diving into function arguments, pass current ast node.
if (func->arguments)
for (auto & child : func->arguments->children)
processImpl(child, lambda_scopes, ast, higher_order_functions);
if (func->parameters)
for (auto & child : func->parameters->children)
processImpl(child, lambda_scopes, nullptr, higher_order_functions);
}
return;
}
else if (typeid_cast<ASTIdentifier *>(ast.get()))
{
processIdentifier(ast, lambda_scopes);
return;
}
for (auto & child : ast->children)
processImpl(child, lambda_scopes, nullptr, higher_order_functions);
}
}
void AnalyzeLambdas::process(ASTPtr & ast)
{
LambdaScopes lambda_scopes;
for (auto & child : ast->children)
processImpl(child, lambda_scopes, nullptr, higher_order_functions);
}
void AnalyzeLambdas::dump(WriteBuffer & out) const
{
for (const auto & ast : higher_order_functions)
{
writeString(ast->getColumnName(), out);
writeChar('\n', out);
}
}
}

35
dbms/src/Analyzers/AnalyzeLambdas.h
View File

@ -1,35 +0,0 @@
#pragma once
#include <Parsers/IAST.h>
#include <vector>
namespace DB
{
class WriteBuffer;
/** For every lambda expression, rename its parameters to '_lambda0_arg0' form.
* Check correctness of lambda expressions.
* Find functions, that have lambda expressions as arguments (they are called "higher order" functions).
*
* This should be done before CollectAliases.
*/
struct AnalyzeLambdas
{
void process(ASTPtr & ast);
/// Parameters of lambda expressions.
using LambdaParameters = std::vector<String>;
static LambdaParameters extractLambdaParameters(ASTPtr & ast);
using HigherOrderFunctions = std::vector<ASTPtr>;
HigherOrderFunctions higher_order_functions;
/// Debug output
void dump(WriteBuffer & out) const;
};
}

70
dbms/src/Analyzers/AnalyzeResultOfQuery.cpp
View File

@ -1,70 +0,0 @@
#include <Analyzers/AnalyzeResultOfQuery.h>
#include <Analyzers/CollectAliases.h>
#include <Analyzers/CollectTables.h>
#include <Analyzers/AnalyzeLambdas.h>
#include <Analyzers/AnalyzeColumns.h>
#include <Analyzers/TypeAndConstantInference.h>
#include <Interpreters/Context.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteHelpers.h>
#include <Parsers/ASTSelectQuery.h>
#include <Common/typeid_cast.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int UNEXPECTED_AST_STRUCTURE;
}
void AnalyzeResultOfQuery::process(ASTPtr & ast, const Context & context, ExecuteTableFunctions & table_functions)
{
const ASTSelectQuery * select = typeid_cast<const ASTSelectQuery *>(ast.get());
if (!select)
throw Exception("AnalyzeResultOfQuery::process was called for not a SELECT query", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
if (!select->select_expression_list)
throw Exception("SELECT query doesn't have select_expression_list", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
AnalyzeLambdas analyze_lambdas;
analyze_lambdas.process(ast);
CollectAliases collect_aliases;
collect_aliases.process(ast);
CollectTables collect_tables;
collect_tables.process(ast, context, collect_aliases, table_functions);
AnalyzeColumns analyze_columns;
analyze_columns.process(ast, collect_aliases, collect_tables);
TypeAndConstantInference inference;
inference.process(ast, context, collect_aliases, analyze_columns, analyze_lambdas, table_functions);
for (const ASTPtr & child : select->select_expression_list->children)
{
auto it = inference.info.find(child->getColumnName());
if (it == inference.info.end())
throw Exception("Logical error: type information for result column of SELECT query was not inferred", ErrorCodes::LOGICAL_ERROR);
String name = child->getAliasOrColumnName();
const TypeAndConstantInference::ExpressionInfo & info = it->second;
result.insert(ColumnWithTypeAndName(
info.is_constant_expression ? info.data_type->createColumnConst(1, info.value) : nullptr,
info.data_type,
std::move(name)));
}
}
void AnalyzeResultOfQuery::dump(WriteBuffer & out) const
{
writeString(result.dumpStructure(), out);
}
}

32
dbms/src/Analyzers/AnalyzeResultOfQuery.h
View File

@ -1,32 +0,0 @@
#pragma once
#include <Parsers/IAST.h>
#include <Core/Block.h>
namespace DB
{
class WriteBuffer;
class Context;
struct ExecuteTableFunctions;
/** For SELECT query, determine names and types of columns of result,
* and if some columns are constant expressions, calculate their values.
*
* NOTE It's possible to memoize calculations, that happens under the hood
* and could be duplicated in subsequent analysis of subqueries.
*/
struct AnalyzeResultOfQuery
{
void process(ASTPtr & ast, const Context & context, ExecuteTableFunctions & table_functions);
/// Block will have non-nullptr columns for constant expressions.
Block result;
/// Debug output
void dump(WriteBuffer & out) const;
};
}

3
dbms/src/Analyzers/CMakeLists.txt
View File

@ -1,3 +0,0 @@
if(ENABLE_TESTS)
add_subdirectory(tests)
endif()

109
dbms/src/Analyzers/CollectAliases.cpp
View File

@ -1,109 +0,0 @@
#include <Analyzers/CollectAliases.h>
#include <Parsers/formatAST.h>
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteHelpers.h>
#include <Common/typeid_cast.h>
namespace DB
{
namespace ErrorCodes
{
extern const int MULTIPLE_EXPRESSIONS_FOR_ALIAS;
}
static void processImpl(const ASTPtr & ast, CollectAliases::Aliases & aliases, CollectAliases::Kind kind, size_t keep_kind_for_depth)
{
String alias = ast->tryGetAlias();
if (!alias.empty())
{
auto it_inserted = aliases.emplace(alias, CollectAliases::AliasInfo(ast, kind));
if (!it_inserted.second && ast->getTreeHash() != it_inserted.first->second.node->getTreeHash())
{
std::stringstream message;
message << "Different expressions with the same alias " << backQuoteIfNeed(alias) << ":\n";
formatAST(*it_inserted.first->second.node, message, false, true);
message << "\nand\n";
formatAST(*ast, message, false, true);
message << "\n";
throw Exception(message.str(), ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS);
}
}
for (auto & child : ast->children)
{
if (typeid_cast<const ASTSelectQuery *>(child.get()))
{
/// Don't go into subqueries.
}
else if (typeid_cast<const ASTTableExpression *>(child.get()))
{
processImpl(child, aliases, CollectAliases::Kind::Table, 1);
}
else if (typeid_cast<const ASTArrayJoin *>(child.get()))
{
/// ASTArrayJoin -> ASTExpressionList -> element of expression AS alias
processImpl(child, aliases, CollectAliases::Kind::ArrayJoin, 3);
}
else if (keep_kind_for_depth > 0)
{
processImpl(child, aliases, kind, keep_kind_for_depth - 1);
}
else
{
processImpl(child, aliases, CollectAliases::Kind::Expression, 0);
}
}
}
void CollectAliases::process(const ASTPtr & ast)
{
processImpl(ast, aliases, Kind::Expression, 0);
}
void CollectAliases::dump(WriteBuffer & out) const
{
/// For need of tests, we need to dump result in some fixed order.
std::vector<Aliases::const_iterator> vec;
vec.reserve(aliases.size());
for (auto it = aliases.begin(); it != aliases.end(); ++it)
vec.emplace_back(it);
std::sort(vec.begin(), vec.end(), [](const auto & a, const auto & b) { return a->first < b->first; });
for (const auto & it : vec)
{
writeProbablyBackQuotedString(it->first, out);
writeCString(" -> ", out);
switch (it->second.kind)
{
case Kind::Expression:
writeCString("(expression) ", out);
break;
case Kind::Table:
writeCString("(table) ", out);
break;
case Kind::ArrayJoin:
writeCString("(array join) ", out);
break;
}
std::stringstream formatted_ast;
formatAST(*it->second.node, formatted_ast, false, true);
writeString(formatted_ast.str(), out);
writeChar('\n', out);
}
}
}

54
dbms/src/Analyzers/CollectAliases.h
View File

@ -1,54 +0,0 @@
#pragma once
#include <Parsers/IAST.h>
#include <unordered_map>
namespace DB
{
class WriteBuffer;
/** Build a map: alias -> AST node.
*
* Also fill information about what kind each alias has:
* - expression alias;
* - table alias;
* - ARRAY JOIN alias.
*
* As extension to standard SQL, aliases could be specified and used in any part of query.
* Example: SELECT a, (1 AS a) + 1 AS b FROM t GROUP BY a, b
* Alias could be used in query before it is defined.
*
* Aliases could not be redefined. Example: 1 AS a, a + 1 AS a - is prohibited.
*
* Don't descend into subqueries (as aliases are local inside them).
*/
struct CollectAliases
{
void process(const ASTPtr & ast);
enum class Kind
{
Expression, /// Example: SELECT a AS b, f(x) AS y
Table, /// Example: SELECT t.* FROM (SELECT 1) AS t
ArrayJoin /// Example: SELECT t.x.a FROM t ARRAY JOIN arr AS x
};
struct AliasInfo
{
ASTPtr node;
Kind kind;
AliasInfo(const ASTPtr & node, Kind kind) : node(node), kind(kind) {}
};
using Aliases = std::unordered_map<String, AliasInfo>;
Aliases aliases;
/// Debug output
void dump(WriteBuffer & out) const;
};
}

186
dbms/src/Analyzers/CollectTables.cpp
View File

@ -1,186 +0,0 @@
#include <Analyzers/CollectTables.h>
#include <Analyzers/CollectAliases.h>
#include <Analyzers/ExecuteTableFunctions.h>
#include <Analyzers/AnalyzeResultOfQuery.h>
#include <Interpreters/Context.h>
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/formatAST.h>
#include <Parsers/ASTSubquery.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteHelpers.h>
#include <Common/typeid_cast.h>
namespace DB
{
namespace ErrorCodes
{
extern const int UNEXPECTED_AST_STRUCTURE;
extern const int LOGICAL_ERROR;
}
static CollectTables::TableInfo processOrdinaryTable(const ASTPtr & ast_database_and_table_name, const Context & context)
{
const ASTIdentifier & identifier = static_cast<const ASTIdentifier &>(*ast_database_and_table_name);
CollectTables::TableInfo res;
res.node = ast_database_and_table_name;
res.alias = identifier.tryGetAlias();
if (ast_database_and_table_name->children.empty())
{
res.table_name = identifier.name;
}
else
{
if (ast_database_and_table_name->children.size() != 2)
throw Exception("Logical error: number of components in table expression not equal to two", ErrorCodes::LOGICAL_ERROR);
res.database_name = static_cast<const ASTIdentifier &>(*identifier.children[0]).name;
res.table_name = static_cast<const ASTIdentifier &>(*identifier.children[1]).name;
}
res.storage = context.getTable(res.database_name, res.table_name);
return res;
}
static CollectTables::TableInfo processTableFunction(const ASTPtr & ast_table_function, const ExecuteTableFunctions & table_functions)
{
const ASTFunction & function = typeid_cast<const ASTFunction &>(*ast_table_function);
IAST::Hash ast_hash = ast_table_function->getTreeHash();
auto it = table_functions.tables.find(ast_hash);
if (table_functions.tables.end() == it)
throw Exception("Table function " + function.name + " was not executed in advance.", ErrorCodes::LOGICAL_ERROR);
CollectTables::TableInfo res;
res.node = ast_table_function;
res.alias = function.tryGetAlias();
res.storage = it->second;
return res;
}
static CollectTables::TableInfo processNoTables(const Context & context)
{
/// No FROM section. Interpret it as FROM system.one.
CollectTables::TableInfo res;
res.database_name = "system";
res.table_name = "one";
res.storage = context.getTable(res.database_name, res.table_name);
return res;
}
static CollectTables::TableInfo processSubquery(ASTPtr & ast_subquery, const Context & context, ExecuteTableFunctions & table_functions)
{
AnalyzeResultOfQuery analyzer;
analyzer.process(typeid_cast<ASTSubquery &>(*ast_subquery).children.at(0), context, table_functions);
CollectTables::TableInfo res;
res.node = ast_subquery;
res.alias = ast_subquery->tryGetAlias();
res.structure_of_subquery = analyzer.result;
return res;
}
void CollectTables::process(ASTPtr & ast, const Context & context, const CollectAliases & /*aliases*/, ExecuteTableFunctions & table_functions)
{
const ASTSelectQuery * select = typeid_cast<const ASTSelectQuery *>(ast.get());
if (!select)
throw Exception("CollectTables::process was called for not a SELECT query", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
if (!select->tables)
{
tables.emplace_back(processNoTables(context));
return;
}
for (auto & child : select->tables->children)
{
ASTTablesInSelectQueryElement & element = static_cast<ASTTablesInSelectQueryElement &>(*child);
if (!element.table_expression) /// This is ARRAY JOIN
continue;
ASTTableExpression & table_expression = static_cast<ASTTableExpression &>(*element.table_expression);
if (table_expression.database_and_table_name)
{
tables.emplace_back(processOrdinaryTable(table_expression.database_and_table_name, context));
/// TODO It could be alias to another table expression.
}
else if (table_expression.table_function)
{
tables.emplace_back(processTableFunction(table_expression.table_function, table_functions));
}
else if (table_expression.subquery)
{
tables.emplace_back(processSubquery(table_expression.subquery, context, table_functions));
}
else
throw Exception("Logical error: no known elements in ASTTableExpression", ErrorCodes::LOGICAL_ERROR);
}
/// TODO Control that tables don't have conflicting names.
}
void CollectTables::dump(WriteBuffer & out) const
{
for (const auto & table : tables)
{
writeCString("Database name: ", out);
if (table.database_name.empty())
writeCString("(none)", out);
else
writeProbablyBackQuotedString(table.database_name, out);
writeCString(". Table name: ", out);
if (table.table_name.empty())
writeCString("(none)", out);
else
writeProbablyBackQuotedString(table.table_name, out);
writeCString(". Alias: ", out);
if (table.alias.empty())
writeCString("(none)", out);
else
writeProbablyBackQuotedString(table.alias, out);
writeCString(". Storage: ", out);
if (!table.storage)
writeCString("(none)", out);
else
writeProbablyBackQuotedString(table.storage->getName(), out);
writeCString(". Structure of subquery: ", out);
if (!table.structure_of_subquery)
writeCString("(none)", out);
else
writeString(table.structure_of_subquery.dumpStructure(), out);
writeCString(". AST: ", out);
if (!table.node)
writeCString("(none)", out);
else
{
std::stringstream formatted_ast;
formatAST(*table.node, formatted_ast, false, true);
writeString(formatted_ast.str(), out);
}
writeChar('\n', out);
}
}
}

52
dbms/src/Analyzers/CollectTables.h
View File

@ -1,52 +0,0 @@
#pragma once
#include <Parsers/IAST.h>
#include <Storages/IStorage.h>
#include <vector>
namespace DB
{
class Context;
struct CollectAliases;
struct ExecuteTableFunctions;
class WriteBuffer;
/** Collect and analyze table-like expressions in section FROM in a query.
* For every expression, keep its alias.
*
* For ordinary tables, determine database and table name, obtain and keep StoragePtr.
* For subqueries, determine result structure. This requires analysis of subquery, such as type inference.
* For table functions, grab them from prepared ExecuteTableFunctions object.
*/
struct CollectTables
{
void process(ASTPtr & ast, const Context & context, const CollectAliases & aliases, ExecuteTableFunctions & table_functions);
enum class Kind
{
OrdinaryTable,
TableFunction,
Subquery
};
struct TableInfo
{
ASTPtr node;
String database_name;
String table_name;
String alias;
StoragePtr storage;
Block structure_of_subquery;
};
using Tables = std::vector<TableInfo>;
Tables tables;
/// Debug output
void dump(WriteBuffer & out) const;
};
}

80
dbms/src/Analyzers/ExecuteTableFunctions.cpp
View File

@ -1,80 +0,0 @@
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Parsers/ASTFunction.h>
#include <TableFunctions/ITableFunction.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Interpreters/Context.h>
#include <IO/WriteHelpers.h>
#include <Common/typeid_cast.h>
#include <Analyzers/ExecuteTableFunctions.h>
namespace DB
{
namespace ErrorCodes
{
extern const int UNEXPECTED_AST_STRUCTURE;
extern const int LOGICAL_ERROR;
}
/// Allows to execute exactly same table functions only once.
using ASTTreeToTable = std::map<IAST::Hash, StoragePtr>;
static void processTableFunction(const ASTPtr & ast_table_function, const Context & context, ExecuteTableFunctions::Tables & result_map)
{
const ASTFunction & function = typeid_cast<const ASTFunction &>(*ast_table_function);
/// If already executed.
IAST::Hash ast_hash = ast_table_function->getTreeHash();
if (result_map.count(ast_hash))
return;
/// Obtain table function
TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(function.name, context);
/// Execute it and store result
StoragePtr table = table_function_ptr->execute(ast_table_function, context);
result_map[ast_hash] = table;
}
void ExecuteTableFunctions::process(ASTPtr & ast, const Context & context)
{
const ASTSelectQuery * select = typeid_cast<const ASTSelectQuery *>(ast.get());
if (!select)
throw Exception("ExecuteTableFunctions::process was called for not a SELECT query", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
if (!select->tables)
return;
for (auto & child : select->tables->children)
{
ASTTablesInSelectQueryElement & element = static_cast<ASTTablesInSelectQueryElement &>(*child);
if (!element.table_expression) /// This is ARRAY JOIN
continue;
ASTTableExpression & table_expression = static_cast<ASTTableExpression &>(*element.table_expression);
if (!table_expression.table_function)
continue;
processTableFunction(table_expression.table_function, context, tables);
}
}
void ExecuteTableFunctions::dump(WriteBuffer & out) const
{
for (const auto & table : tables)
{
writeString(table.second->getName(), out);
writeCString("\n\n", out);
writeString(table.second->getColumns().getAllPhysical().toString(), out);
writeCString("\n", out);
}
}
}

Some files were not shown because too many files have changed in this diff Show More