thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into add-grpc-contrib

Browse Source
This commit is contained in:
levysh 2020-02-02 14:47:16 +03:00 committed by GitHub
commit 9928b0bbb2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1426 changed files with 27715 additions and 11225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.github/PULL_REQUEST_TEMPLATE.md vendored
View File

@ -7,16 +7,20 @@ Changelog category (leave one):
- Performance Improvement
- Backward Incompatible Change
- Build/Testing/Packaging Improvement
- Documentation
- Documentation (changelog entry is not required)
- Other
- Non-significant (changelog entry is not needed)
- Non-significant (changelog entry is not required)
Changelog entry (up to few sentences, required except for Non-significant/Documentation categories):
Changelog entry (a user-readable short description of the changes that goes to CHANGELOG.md):
...
Detailed description (optional):
Detailed description / Documentation draft:
...
By adding documentation, you'll allow users to try your new feature immediately, not when someone else will have time to document it later. Documentation is necessary for all features that affect user experience in any way. You can add brief documentation draft above, or add documentation right into your patch as Markdown files in [docs](https://github.com/ClickHouse/ClickHouse/tree/master/docs) folder.
If you are doing this for the first time, it's recommended to read the lightweight [Contributing to ClickHouse Documentation](https://github.com/ClickHouse/ClickHouse/tree/master/docs/README.md) guide first.

9
.gitignore vendored
View File

@ -13,12 +13,14 @@
/build_*
/build-*
/docs/build
/docs/publish
/docs/edit
/docs/tools/venv/
/docs/en/development/build/
/docs/ru/development/build/
/docs/en/single.md
/docs/ru/single.md
/docs/zh/single.md
/docs/ja/single.md
/docs/fa/single.md
# callgrind files
callgrind.out.*
@ -244,6 +246,9 @@ website/package-lock.json
# ccls cache
/.ccls-cache
# clangd cache
/.clangd
/compile_commands.json
# Toolchains

9
.gitmodules vendored
View File

@ -49,7 +49,7 @@
url = https://github.com/ClickHouse-Extras/boost.git
[submodule "contrib/base64"]
path = contrib/base64
url = https://github.com/aklomp/base64.git
url = https://github.com/powturbo/Turbo-Base64.git
[submodule "contrib/arrow"]
path = contrib/arrow
url = https://github.com/apache/arrow
@ -137,6 +137,13 @@
[submodule "contrib/libc-headers"]
path = contrib/libc-headers
url = https://github.com/ClickHouse-Extras/libc-headers.git
[submodule "contrib/replxx"]
path = contrib/replxx
url = https://github.com/AmokHuginnsson/replxx.git
[submodule "contrib/ryu"]
path = contrib/ryu
url = https://github.com/ClickHouse-Extras/ryu.git
[submodule "contrib/avro"]
path = contrib/avro
url = https://github.com/ClickHouse-Extras/avro.git
ignore = untracked

45
AUTHORS
View File

@ -1,43 +1,2 @@
The following authors have created the source code of "ClickHouse"
published and distributed by YANDEX LLC as the owner:
Alexander Makarov <asealback@yandex-team.ru>
Alexander Prudaev <aprudaev@yandex-team.ru>
Alexey Arno <af-arno@yandex-team.ru>
Alexey Milovidov <milovidov@yandex-team.ru>
Alexey Tronov <vkusny@yandex-team.ru>
Alexey Vasiliev <loudhorr@yandex-team.ru>
Alexey Zatelepin <ztlpn@yandex-team.ru>
Amy Krishnevsky <krishnevsky@yandex-team.ru>
Andrey M <hertz@yandex-team.ru>
Andrey Mironov <hertz@yandex-team.ru>
Andrey Urusov <drobus@yandex-team.ru>
Anton Tikhonov <rokerjoker@yandex-team.ru>
Dmitry Bilunov <kmeaw@yandex-team.ru>
Dmitry Galuza <galuza@yandex-team.ru>
Eugene Konkov <konkov@yandex-team.ru>
Evgeniy Gatov <egatov@yandex-team.ru>
Ilya Khomutov <robert@yandex-team.ru>
Ilya Korolev <breeze@yandex-team.ru>
Ivan Blinkov <blinkov@yandex-team.ru>
Maxim Nikulin <mnikulin@yandex-team.ru>
Michael Kolupaev <mkolupaev@yandex-team.ru>
Michael Razuvaev <razuvaev@yandex-team.ru>
Nikolai Kochetov <nik-kochetov@yandex-team.ru>
Nikolay Vasiliev <lonlylocly@yandex-team.ru>
Nikolay Volosatov <bamx23@yandex-team.ru>
Pavel Artemkin <stanly@yandex-team.ru>
Pavel Kartaviy <kartavyy@yandex-team.ru>
Roman Nozdrin <drrtuy@yandex-team.ru>
Roman Peshkurov <peshkurov@yandex-team.ru>
Sergey Fedorov <fets@yandex-team.ru>
Sergey Lazarev <hamilkar@yandex-team.ru>
Sergey Magidovich <mgsergio@yandex-team.ru>
Sergey Serebryanik <serebrserg@yandex-team.ru>
Sergey Veletskiy <velom@yandex-team.ru>
Vasily Okunev <okunev@yandex-team.ru>
Vitaliy Lyudvichenko <vludv@yandex-team.ru>
Vladimir Chebotarev <chebotarev@yandex-team.ru>
Vsevolod Orlov <vorloff@yandex-team.ru>
Vyacheslav Alipov <alipov@yandex-team.ru>
Yuriy Galitskiy <orantius@yandex-team.ru>
To see the list of authors who created the source code of ClickHouse, published and distributed by YANDEX LLC as the owner,
run "SELECT * FROM system.contributors;" query on any ClickHouse server.

1135
CHANGELOG.md
View File

File diff suppressed because it is too large Load Diff

8
CMakeLists.txt
View File

@ -95,6 +95,8 @@ if (CMAKE_GENERATOR STREQUAL "Ninja")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always")
endif ()
include (cmake/add_warning.cmake)
if (NOT MSVC)
set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror is also added inside directories with our own code.
endif ()
@ -224,8 +226,8 @@ else ()
set(NOT_UNBUNDLED 1)
endif ()
# Using system libs can cause lot of warnings in includes (on macro expansion).
if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32)
# Using system libs can cause a lot of warnings in includes (on macro expansion).
if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN) OR ARCH_32)
option (NO_WERROR "Disable -Werror compiler option" ON)
endif ()
@ -326,7 +328,6 @@ include (cmake/find/xxhash.cmake)
include (cmake/find/sparsehash.cmake)
include (cmake/find/rt.cmake)
include (cmake/find/execinfo.cmake)
include (cmake/find/readline_edit.cmake)
include (cmake/find/re2.cmake)
include (cmake/find/libgsasl.cmake)
include (cmake/find/rdkafka.cmake)
@ -352,6 +353,7 @@ include (cmake/find/simdjson.cmake)
include (cmake/find/rapidjson.cmake)
include (cmake/find/fastops.cmake)
include (cmake/find/orc.cmake)
include (cmake/find/avro.cmake)
find_contrib_lib(cityhash)
find_contrib_lib(farmhash)

27
CONTRIBUTING.md
View File

@ -1,24 +1,30 @@
# Contributing to ClickHouse
## Technical info
Developer guide for writing code for ClickHouse is published on official website alongside the usage and operations documentation:
https://clickhouse.yandex/docs/en/development/architecture/
ClickHouse is an open project, and you can contribute to it in many ways. You can help with ideas, code, or documentation. We appreciate any efforts that help us to make the project better.
## Legal info
Thank you.
In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here:
## Technical Info
We have a [developer's guide](https://clickhouse.yandex/docs/en/development/developer_instruction/) for writing code for ClickHouse. Besides this guide, you can find [Overview of ClickHouse Architecture](https://clickhouse.yandex/docs/en/development/architecture/) and instructions on how to build ClickHouse in different environments.
If you want to contribute to documentation, read the [Contributing to ClickHouse Documentation](docs/README.md) guide.
## Legal Info
In order for us (YANDEX LLC) to accept patches and other contributions from you, you may adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here:
1) https://yandex.ru/legal/cla/?lang=en (in English) and
2) https://yandex.ru/legal/cla/?lang=ru (in Russian).
By adopting the CLA, you state the following:
* You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA,
* You has read the terms and conditions of the CLA and agree with them in full,
* You have read the terms and conditions of the CLA and agree with them in full,
* You are legally able to provide and license your contributions as stated,
* We may use your contributions for our open source projects and for any other our project too,
* We rely on your assurances concerning the rights of third parties in relation to your contributes.
* We rely on your assurances concerning the rights of third parties in relation to your contributions.
If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you has already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA.
If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you have already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA.
If you have already adopted terms and conditions of the CLA, you are able to provide your contributes. When you submit your pull request, please add the following information into it:
@ -31,4 +37,7 @@ Replace the bracketed text as follows:
It is enough to provide us such notification once.
If you don't agree with the CLA, you still can open a pull request to provide your contributions.
As an alternative, you can provide DCO instead of CLA. You can find the text of DCO here: https://developercertificate.org/
It is enough to read and copy it verbatim to your pull request.
If you don't agree with the CLA and don't want to provide DCO, you still can open a pull request to provide your contributions.

4
LICENSE
View File

@ -1,4 +1,4 @@
Copyright 2016-2019 Yandex LLC
Copyright 2016-2020 Yandex LLC
Apache License
Version 2.0, January 2004
@ -188,7 +188,7 @@ Copyright 2016-2019 Yandex LLC
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2016-2019 Yandex LLC
Copyright 2016-2020 Yandex LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

5
README.md
View File

@ -11,3 +11,8 @@ ClickHouse is an open-source column-oriented database management system that all
* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
* [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any.
* You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person.
## Upcoming Events
* [ClickHouse Meetup in San Francisco](https://www.eventbrite.com/e/clickhouse-february-meetup-registration-88496227599) on February 5.
* [ClickHouse Meetup in New York](https://www.meetup.com/Uber-Engineering-Events-New-York/events/268328663/) on February 11.

18
cmake/add_warning.cmake Normal file
View File

@ -0,0 +1,18 @@
include (CheckCXXCompilerFlag)
# Try to add -Wflag if compiler supports it
macro (add_warning flag)
string (REPLACE "-" "_" underscored_flag ${flag})
string (REPLACE "+" "x" underscored_flag ${underscored_flag})
check_cxx_compiler_flag("-W${flag}" SUPPORTS_FLAG_${underscored_flag})
if (SUPPORTS_FLAG_${underscored_flag})
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W${flag}")
else ()
message (WARNING "Flag -W${flag} is unsupported")
endif ()
endmacro ()
# Try to add -Wno flag if compiler supports it
macro (no_warning flag)
add_warning(no-${flag})
endmacro ()

10
cmake/darwin/default_libs.cmake
View File

@ -13,12 +13,12 @@ set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
# Minimal supported SDK version
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mmacosx-version-min=10.14")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mmacosx-version-min=10.14")
set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -mmacosx-version-min=10.14")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mmacosx-version-min=10.15")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mmacosx-version-min=10.15")
set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -mmacosx-version-min=10.15")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -mmacosx-version-min=10.14")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -mmacosx-version-min=10.14")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -mmacosx-version-min=10.15")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -mmacosx-version-min=10.15")
# Global libraries

28
cmake/find/avro.cmake Normal file
View File

@ -0,0 +1,28 @@
option (ENABLE_AVRO "Enable Avro" ${ENABLE_LIBRARIES})
if (ENABLE_AVRO)
option (USE_INTERNAL_AVRO_LIBRARY "Set to FALSE to use system avro library instead of bundled" ${NOT_UNBUNDLED})
if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/CMakeLists.txt")
if(USE_INTERNAL_AVRO_LIBRARY)
message(WARNING "submodule contrib/avro is missing. to fix try run: \n git submodule update --init --recursive")
endif()
set(MISSING_INTERNAL_AVRO_LIBRARY 1)
set(USE_INTERNAL_AVRO_LIBRARY 0)
endif()
if (NOT USE_INTERNAL_AVRO_LIBRARY)
elseif(NOT MISSING_INTERNAL_AVRO_LIBRARY)
include(cmake/find/snappy.cmake)
set(AVROCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/include")
set(AVROCPP_LIBRARY avrocpp)
endif ()
if (AVROCPP_LIBRARY AND AVROCPP_INCLUDE_DIR)
set(USE_AVRO 1)
endif()
endif()
message (STATUS "Using avro=${USE_AVRO}: ${AVROCPP_INCLUDE_DIR} : ${AVROCPP_LIBRARY}")

3
cmake/find/base64.cmake
View File

@ -1,4 +1,4 @@
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64/lib/lib.c")
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64/LICENSE")
set (MISSING_INTERNAL_BASE64_LIBRARY 1)
message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
endif ()
@ -11,7 +11,6 @@ if (ENABLE_BASE64)
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64")
message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
else()
set (BASE64_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64/include)
set (BASE64_LIBRARY base64)
set (USE_BASE64 1)
endif()

3
cmake/find/boost.cmake
View File

@ -31,6 +31,7 @@ if (NOT Boost_SYSTEM_LIBRARY AND NOT MISSING_INTERNAL_BOOST_LIBRARY)
set (Boost_SYSTEM_LIBRARY boost_system_internal)
set (Boost_PROGRAM_OPTIONS_LIBRARY boost_program_options_internal)
set (Boost_FILESYSTEM_LIBRARY boost_filesystem_internal ${Boost_SYSTEM_LIBRARY})
set (Boost_IOSTREAMS_LIBRARY boost_iostreams_internal)
set (Boost_REGEX_LIBRARY boost_regex_internal)
set (Boost_INCLUDE_DIRS)
@ -48,4 +49,4 @@ if (NOT Boost_SYSTEM_LIBRARY AND NOT MISSING_INTERNAL_BOOST_LIBRARY)
list (APPEND Boost_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/boost")
endif ()
message (STATUS "Using Boost: ${Boost_INCLUDE_DIRS} : ${Boost_PROGRAM_OPTIONS_LIBRARY},${Boost_SYSTEM_LIBRARY},${Boost_FILESYSTEM_LIBRARY},${Boost_REGEX_LIBRARY}")
message (STATUS "Using Boost: ${Boost_INCLUDE_DIRS} : ${Boost_PROGRAM_OPTIONS_LIBRARY},${Boost_SYSTEM_LIBRARY},${Boost_FILESYSTEM_LIBRARY},${Boost_IOSTREAMS_LIBRARY},${Boost_REGEX_LIBRARY}")

9
cmake/find/cxx.cmake
View File

@ -1,5 +1,12 @@
set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT ${NOT_UNBUNDLED})
if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/CMakeLists.txt")
message(WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive")
set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT 0)
endif()
option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${NOT_UNBUNDLED})
option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled" ${NOT_UNBUNDLED})
option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled" ${USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT})
if (USE_LIBCXX)
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build.

14
cmake/find/poco.cmake
View File

@ -14,6 +14,7 @@ if (NOT ENABLE_LIBRARIES)
set (ENABLE_POCO_REDIS ${ENABLE_LIBRARIES} CACHE BOOL "")
set (ENABLE_POCO_ODBC ${ENABLE_LIBRARIES} CACHE BOOL "")
set (ENABLE_POCO_SQL ${ENABLE_LIBRARIES} CACHE BOOL "")
set (ENABLE_POCO_JSON ${ENABLE_LIBRARIES} CACHE BOOL "")
endif ()
set (POCO_COMPONENTS Net XML SQL Data)
@ -34,6 +35,9 @@ if (NOT DEFINED ENABLE_POCO_ODBC OR ENABLE_POCO_ODBC)
list (APPEND POCO_COMPONENTS DataODBC)
list (APPEND POCO_COMPONENTS SQLODBC)
endif ()
if (NOT DEFINED ENABLE_POCO_JSON OR ENABLE_POCO_JSON)
list (APPEND POCO_COMPONENTS JSON)
endif ()
if (NOT USE_INTERNAL_POCO_LIBRARY)
find_package (Poco COMPONENTS ${POCO_COMPONENTS})
@ -112,6 +116,11 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY)
endif ()
endif ()
if (NOT DEFINED ENABLE_POCO_JSON OR ENABLE_POCO_JSON)
set (Poco_JSON_LIBRARY PocoJSON)
set (Poco_JSON_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/poco/JSON/include/")
endif ()
if (OPENSSL_FOUND AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL))
set (Poco_NetSSL_LIBRARY PocoNetSSL ${OPENSSL_LIBRARIES})
set (Poco_Crypto_LIBRARY PocoCrypto ${OPENSSL_LIBRARIES})
@ -145,8 +154,11 @@ endif ()
if (Poco_SQLODBC_LIBRARY AND ODBC_FOUND)
set (USE_POCO_SQLODBC 1)
endif ()
if (Poco_JSON_LIBRARY)
set (USE_POCO_JSON 1)
endif ()
message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_Crypto_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_SQL_LIBRARY},${Poco_SQLODBC_LIBRARY},${Poco_MongoDB_LIBRARY},${Poco_Redis_LIBRARY}; MongoDB=${USE_POCO_MONGODB}, Redis=${USE_POCO_REDIS}, DataODBC=${USE_POCO_DATAODBC}, NetSSL=${USE_POCO_NETSSL}")
message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_Crypto_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_SQL_LIBRARY},${Poco_SQLODBC_LIBRARY},${Poco_MongoDB_LIBRARY},${Poco_Redis_LIBRARY},${Poco_JSON_LIBRARY}; MongoDB=${USE_POCO_MONGODB}, Redis=${USE_POCO_REDIS}, DataODBC=${USE_POCO_DATAODBC}, NetSSL=${USE_POCO_NETSSL}, JSON=${USE_POCO_JSON}")
# How to make sutable poco:
# use branch:

60
cmake/find/readline_edit.cmake
View File

@ -1,60 +0,0 @@
include (CMakePushCheckState)
cmake_push_check_state ()
option (ENABLE_READLINE "Enable readline" ${ENABLE_LIBRARIES})
if (ENABLE_READLINE)
set (READLINE_PATHS "/usr/local/opt/readline/lib")
# First try find custom lib for macos users (default lib without history support)
find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS} NO_DEFAULT_PATH)
if (NOT READLINE_LIB)
find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS})
endif ()
list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.2)
find_library (EDIT_LIB NAMES edit)
set(READLINE_INCLUDE_PATHS "/usr/local/opt/readline/include")
if (READLINE_LIB AND TERMCAP_LIBRARY)
find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS} NO_DEFAULT_PATH)
if (NOT READLINE_INCLUDE_DIR)
find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS})
endif ()
if (READLINE_INCLUDE_DIR AND READLINE_LIB)
set (USE_READLINE 1)
set (LINE_EDITING_LIBS ${READLINE_LIB} ${TERMCAP_LIBRARY})
message (STATUS "Using line editing libraries (readline): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}")
endif ()
elseif (EDIT_LIB AND TERMCAP_LIBRARY)
find_library (CURSES_LIB NAMES curses)
find_path (READLINE_INCLUDE_DIR NAMES editline/readline.h PATHS ${READLINE_INCLUDE_PATHS})
if (CURSES_LIB AND READLINE_INCLUDE_DIR)
set (USE_LIBEDIT 1)
set (LINE_EDITING_LIBS ${EDIT_LIB} ${CURSES_LIB} ${TERMCAP_LIBRARY})
message (STATUS "Using line editing libraries (edit): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}")
endif ()
endif ()
endif ()
if (LINE_EDITING_LIBS AND READLINE_INCLUDE_DIR)
include (CheckCXXSourceRuns)
set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${LINE_EDITING_LIBS})
set (CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES} ${READLINE_INCLUDE_DIR})
check_cxx_source_runs ("
#include <stdio.h>
#include <readline/readline.h>
#include <readline/history.h>
int main() {
add_history(NULL);
append_history(1,NULL);
return 0;
}
" HAVE_READLINE_HISTORY)
else ()
message (STATUS "Not using any library for line editing.")
endif ()
cmake_pop_check_state ()

2
cmake/sanitize.cmake
View File

@ -48,12 +48,12 @@ if (SANITIZE)
set (ENABLE_EMBEDDED_COMPILER 0 CACHE BOOL "")
set (USE_INTERNAL_CAPNP_LIBRARY 0 CACHE BOOL "")
set (USE_SIMDJSON 0 CACHE BOOL "")
set (ENABLE_READLINE 0 CACHE BOOL "")
set (ENABLE_ORC 0 CACHE BOOL "")
set (ENABLE_PARQUET 0 CACHE BOOL "")
set (USE_CAPNP 0 CACHE BOOL "")
set (USE_INTERNAL_ORC_LIBRARY 0 CACHE BOOL "")
set (USE_ORC 0 CACHE BOOL "")
set (USE_AVRO 0 CACHE BOOL "")
set (ENABLE_SSL 0 CACHE BOOL "")
elseif (SANITIZE STREQUAL "thread")

1
cmake/target.cmake
View File

@ -15,7 +15,6 @@ if (CMAKE_CROSSCOMPILING)
set (USE_SNAPPY OFF CACHE INTERNAL "")
set (ENABLE_PROTOBUF OFF CACHE INTERNAL "")
set (ENABLE_PARQUET OFF CACHE INTERNAL "")
set (ENABLE_READLINE OFF CACHE INTERNAL "")
set (ENABLE_ICU OFF CACHE INTERNAL "")
set (ENABLE_FASTOPS OFF CACHE INTERNAL "")
elseif (OS_LINUX)

36
contrib/CMakeLists.txt vendored
View File

@ -146,6 +146,20 @@ if (ENABLE_ICU AND USE_INTERNAL_ICU_LIBRARY)
add_subdirectory (icu-cmake)
endif ()
if(USE_INTERNAL_SNAPPY_LIBRARY)
set(SNAPPY_BUILD_TESTS 0 CACHE INTERNAL "")
if (NOT MAKE_STATIC_LIBRARIES)
set(BUILD_SHARED_LIBS 1) # TODO: set at root dir
endif()
add_subdirectory(snappy)
set (SNAPPY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/snappy")
if(SANITIZE STREQUAL "undefined")
target_compile_options(${SNAPPY_LIBRARY} PRIVATE -fno-sanitize=undefined)
endif()
endif()
if (USE_INTERNAL_PARQUET_LIBRARY)
if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE)
# We dont use arrow's cmakefiles because they uses too many depends and download some libs in compile time
@ -189,20 +203,6 @@ if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE)
endif()
else()
if(USE_INTERNAL_SNAPPY_LIBRARY)
set(SNAPPY_BUILD_TESTS 0 CACHE INTERNAL "")
if (NOT MAKE_STATIC_LIBRARIES)
set(BUILD_SHARED_LIBS 1) # TODO: set at root dir
endif()
add_subdirectory(snappy)
set (SNAPPY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/snappy")
if(SANITIZE STREQUAL "undefined")
target_compile_options(${SNAPPY_LIBRARY} PRIVATE -fno-sanitize=undefined)
endif()
endif()
add_subdirectory(arrow-cmake)
# The library is large - avoid bloat.
@ -212,6 +212,10 @@ else()
endif()
endif()
if (USE_INTERNAL_AVRO_LIBRARY)
add_subdirectory(avro-cmake)
endif()
if (USE_INTERNAL_POCO_LIBRARY)
set (POCO_VERBOSE_MESSAGES 0 CACHE INTERNAL "")
set (save_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
@ -302,7 +306,7 @@ if (USE_INTERNAL_AWS_S3_LIBRARY)
# The library is large - avoid bloat.
target_compile_options (aws_s3 PRIVATE -g0)
target_compile_options (aws_s3_checksums PRIVATE -g0)
target_compile_options (libcurl PRIVATE -g0)
target_compile_options (curl PRIVATE -g0)
endif ()
if (USE_BASE64)
@ -333,3 +337,5 @@ if (USE_FASTOPS)
endif()
add_subdirectory(grpc-cmake)
add_subdirectory(replxx-cmake)

1
contrib/avro vendored Submodule

@ -0,0 +1 @@
Subproject commit 5b2752041c8d2f75eb5c1dbec8b4c25fc0e24d12

70
contrib/avro-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,70 @@
set(AVROCPP_ROOT_DIR ${CMAKE_SOURCE_DIR}/contrib/avro/lang/c++)
set(AVROCPP_INCLUDE_DIR ${AVROCPP_ROOT_DIR}/api)
set(AVROCPP_SOURCE_DIR ${AVROCPP_ROOT_DIR}/impl)
set (CMAKE_CXX_STANDARD 17)
if (EXISTS ${AVROCPP_ROOT_DIR}/../../share/VERSION.txt)
file(READ "${AVROCPP_ROOT_DIR}/../../share/VERSION.txt"
AVRO_VERSION)
endif()
string(REPLACE "\n" "" AVRO_VERSION ${AVRO_VERSION})
set (AVRO_VERSION_MAJOR ${AVRO_VERSION})
set (AVRO_VERSION_MINOR "0")
set (AVROCPP_SOURCE_FILES
${AVROCPP_SOURCE_DIR}/Compiler.cc
${AVROCPP_SOURCE_DIR}/Node.cc
${AVROCPP_SOURCE_DIR}/LogicalType.cc
${AVROCPP_SOURCE_DIR}/NodeImpl.cc
${AVROCPP_SOURCE_DIR}/ResolverSchema.cc
${AVROCPP_SOURCE_DIR}/Schema.cc
${AVROCPP_SOURCE_DIR}/Types.cc
${AVROCPP_SOURCE_DIR}/ValidSchema.cc
${AVROCPP_SOURCE_DIR}/Zigzag.cc
${AVROCPP_SOURCE_DIR}/BinaryEncoder.cc
${AVROCPP_SOURCE_DIR}/BinaryDecoder.cc
${AVROCPP_SOURCE_DIR}/Stream.cc
${AVROCPP_SOURCE_DIR}/FileStream.cc
${AVROCPP_SOURCE_DIR}/Generic.cc
${AVROCPP_SOURCE_DIR}/GenericDatum.cc
${AVROCPP_SOURCE_DIR}/DataFile.cc
${AVROCPP_SOURCE_DIR}/parsing/Symbol.cc
${AVROCPP_SOURCE_DIR}/parsing/ValidatingCodec.cc
${AVROCPP_SOURCE_DIR}/parsing/JsonCodec.cc
${AVROCPP_SOURCE_DIR}/parsing/ResolvingDecoder.cc
${AVROCPP_SOURCE_DIR}/json/JsonIO.cc
${AVROCPP_SOURCE_DIR}/json/JsonDom.cc
${AVROCPP_SOURCE_DIR}/Resolver.cc
${AVROCPP_SOURCE_DIR}/Validator.cc
)
add_library (avrocpp ${AVROCPP_SOURCE_FILES})
set_target_properties (avrocpp PROPERTIES VERSION ${AVRO_VERSION_MAJOR}.${AVRO_VERSION_MINOR})
target_include_directories(avrocpp SYSTEM PUBLIC ${AVROCPP_INCLUDE_DIR})
target_include_directories(avrocpp SYSTEM PUBLIC ${Boost_INCLUDE_DIRS})
target_link_libraries (avrocpp ${Boost_IOSTREAMS_LIBRARY})
if (SNAPPY_INCLUDE_DIR AND SNAPPY_LIBRARY)
target_compile_definitions (avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE)
target_include_directories (avrocpp PRIVATE ${SNAPPY_INCLUDE_DIR})
target_link_libraries (avrocpp ${SNAPPY_LIBRARY})
endif ()
if (COMPILER_GCC)
set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor)
elseif (COMPILER_CLANG)
set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor)
endif ()
target_compile_options(avrocpp PRIVATE ${SUPPRESS_WARNINGS})
# create a symlink to include headers with <avro/...>
ADD_CUSTOM_TARGET(avro_symlink_headers ALL
COMMAND ${CMAKE_COMMAND} -E make_directory ${AVROCPP_ROOT_DIR}/include
COMMAND ${CMAKE_COMMAND} -E create_symlink ${AVROCPP_ROOT_DIR}/api ${AVROCPP_ROOT_DIR}/include/avro
)
add_dependencies(avrocpp avro_symlink_headers)

5
contrib/aws-s3-cmake/CMakeLists.txt
View File

@ -74,7 +74,6 @@ file(GLOB S3_UNIFIED_SRC
)
set(S3_INCLUDES
"${CMAKE_CURRENT_SOURCE_DIR}/include/"
"${AWS_COMMON_LIBRARY_DIR}/include/"
"${AWS_EVENT_STREAM_LIBRARY_DIR}/include/"
"${AWS_S3_LIBRARY_DIR}/include/"
@ -96,11 +95,11 @@ target_compile_definitions(aws_s3 PUBLIC -DENABLE_CURL_CLIENT)
target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MAJOR=1")
target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MINOR=7")
target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_PATCH=231")
target_include_directories(aws_s3 PUBLIC ${S3_INCLUDES} "${CMAKE_BINARY_DIR}/install")
target_include_directories(aws_s3 PUBLIC ${S3_INCLUDES})
if (OPENSSL_FOUND)
target_compile_definitions(aws_s3 PUBLIC -DENABLE_OPENSSL_ENCRYPTION)
target_link_libraries(aws_s3 PRIVATE ${OPENSSL_LIBRARIES})
endif()
target_link_libraries(aws_s3 PRIVATE aws_s3_checksums libcurl)
target_link_libraries(aws_s3 PRIVATE aws_s3_checksums curl)

2
contrib/base64 vendored

@ -1 +1 @@
Subproject commit 32a87227aa254e2f02332c0ed8bc375522414fa3
Subproject commit 5257626d2be17a3eb23f79be17fe55ebba394ad2

1
contrib/base64-cmake/.gitignore vendored
View File

@ -1 +0,0 @@
config.h

76
contrib/base64-cmake/CMakeLists.txt
View File

@ -1,59 +1,33 @@
SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64)
macro(cast_to_bool var instruction)
if (HAVE_${var})
set(base64_${var} 1)
set(base64_${var}_opt ${instruction})
else()
set(base64_${var} 0)
endif()
endmacro()
add_library(base64_scalar OBJECT ${LIBRARY_DIR}/turbob64c.c ${LIBRARY_DIR}/turbob64d.c)
add_library(base64_ssse3 OBJECT ${LIBRARY_DIR}/turbob64sse.c) # This file also contains code for ARM NEON
cast_to_bool(NEON32 "") # TODO flags
cast_to_bool(NEON64 "") # TODO flags
cast_to_bool(SSSE3 "-mssse3")
cast_to_bool(SSE41 "-msse4.1")
cast_to_bool(SSE42 "-msse4.2")
cast_to_bool(AVX "-mavx")
cast_to_bool(AVX2 "-mavx2")
set(HAVE_FAST_UNALIGNED_ACCESS 0)
if(HAVE_SSSE3 OR HAVE_SSE41 OR HAVE_SSE42 OR HAVE_AVX OR HAVE_AVX2)
set(HAVE_FAST_UNALIGNED_ACCESS 1)
if (ARCH_AMD64)
add_library(base64_avx OBJECT ${LIBRARY_DIR}/turbob64sse.c) # This is not a mistake. One file is compiled twice.
add_library(base64_avx2 OBJECT ${LIBRARY_DIR}/turbob64avx2.c)
endif ()
configure_file(config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h)
target_compile_options(base64_scalar PRIVATE -falign-loops)
add_library(base64
${LIBRARY_DIR}/lib/lib.c
${LIBRARY_DIR}/lib/codec_choose.c
${LIBRARY_DIR}/lib/arch/avx/codec.c
${LIBRARY_DIR}/lib/arch/avx2/codec.c
${LIBRARY_DIR}/lib/arch/generic/codec.c
${LIBRARY_DIR}/lib/arch/neon32/codec.c
${LIBRARY_DIR}/lib/arch/neon64/codec.c
${LIBRARY_DIR}/lib/arch/sse41/codec.c
${LIBRARY_DIR}/lib/arch/sse42/codec.c
${LIBRARY_DIR}/lib/arch/ssse3/codec.c
if (ARCH_AMD64)
target_compile_options(base64_ssse3 PRIVATE -mssse3 -falign-loops)
target_compile_options(base64_avx PRIVATE -falign-loops -mavx)
target_compile_options(base64_avx2 PRIVATE -falign-loops -mavx2)
else ()
target_compile_options(base64_ssse3 PRIVATE -falign-loops)
endif ()
${LIBRARY_DIR}/lib/codecs.h
${CMAKE_CURRENT_BINARY_DIR}/config.h
${CMAKE_CURRENT_SOURCE_DIR}/tables.h)
if (ARCH_AMD64)
add_library(base64
$<TARGET_OBJECTS:base64_scalar>
$<TARGET_OBJECTS:base64_ssse3>
$<TARGET_OBJECTS:base64_avx>
$<TARGET_OBJECTS:base64_avx2>)
else ()
add_library(base64
$<TARGET_OBJECTS:base64_scalar>
$<TARGET_OBJECTS:base64_ssse3>)
endif ()
if(HAVE_AVX)
set_source_files_properties(${LIBRARY_DIR}/lib/arch/avx/codec.c PROPERTIES COMPILE_FLAGS -mavx)
endif()
if(HAVE_AVX2)
set_source_files_properties(${LIBRARY_DIR}/lib/arch/avx2/codec.c PROPERTIES COMPILE_FLAGS -mavx2)
endif()
if(HAVE_SSE41)
set_source_files_properties(${LIBRARY_DIR}/lib/arch/sse41/codec.c PROPERTIES COMPILE_FLAGS -msse4.1)
endif()
if(HAVE_SSE42)
set_source_files_properties(${LIBRARY_DIR}/lib/arch/sse42/codec.c PROPERTIES COMPILE_FLAGS -msse4.2)
endif()
if(HAVE_SSSE3)
set_source_files_properties(${LIBRARY_DIR}/lib/arch/ssse3/codec.c PROPERTIES COMPILE_FLAGS -mssse3)
endif()
target_include_directories(base64 PRIVATE ${LIBRARY_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
target_include_directories(base64 SYSTEM PUBLIC ${LIBRARY_DIR})

5
contrib/base64-cmake/README.md
View File

@ -1,5 +0,0 @@
`tables.h` file was generated by the base64 library with it's makefile:
```
cc -std=c99 -O3 -Wall -Wextra -pedantic -o lib/table_generator lib/table_generator.c
./lib/table_generator > lib/tables.h
```

8
contrib/base64-cmake/config.h.in
View File

@ -1,8 +0,0 @@
#define HAVE_NEON32 @base64_NEON32@
#define HAVE_NEON64 @base64_NEON64@
#cmakedefine HAVE_SSSE3 @base64_SSSE3@
#cmakedefine HAVE_SSE41 @base64_SSE41@
#cmakedefine HAVE_SSE42 @base64_SSE42@
#cmakedefine HAVE_AVX @base64_AVX@
#cmakedefine HAVE_AVX2 @base64_AVX2@
#cmakedefine HAVE_FAST_UNALIGNED_ACCESS @HAVE_FAST_UNALIGNED_ACCESS@

393
contrib/base64-cmake/tables.h
View File

@ -1,393 +0,0 @@
#include <stdint.h>
#define CHAR62 '+'
#define CHAR63 '/'
#define CHARPAD '='
#if BASE64_LITTLE_ENDIAN
/* SPECIAL DECODE TABLES FOR LITTLE ENDIAN (INTEL) CPUS */
const uint32_t base64_table_dec_d0[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x000000f8, 0xffffffff, 0xffffffff, 0xffffffff, 0x000000fc,
0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4,
0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018,
0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030,
0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048,
0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060,
0x00000064, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078,
0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090,
0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8,
0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0,
0x000000c4, 0x000000c8, 0x000000cc, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d1[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x0000e003, 0xffffffff, 0xffffffff, 0xffffffff, 0x0000f003,
0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003,
0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000,
0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000,
0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001,
0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001,
0x00009001, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001,
0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002,
0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002,
0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003,
0x00001003, 0x00002003, 0x00003003, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d2[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00800f00, 0xffffffff, 0xffffffff, 0xffffffff, 0x00c00f00,
0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00,
0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100,
0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300,
0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400,
0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600,
0x00400600, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700,
0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900,
0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00,
0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00,
0x00400c00, 0x00800c00, 0x00c00c00, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d3[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x003e0000, 0xffffffff, 0xffffffff, 0xffffffff, 0x003f0000,
0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000,
0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000,
0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000,
0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000,
0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000,
0x00190000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000,
0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000,
0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000,
0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000,
0x00310000, 0x00320000, 0x00330000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
#else
/* SPECIAL DECODE TABLES FOR BIG ENDIAN (IBM/MOTOROLA/SUN) CPUS */
const uint32_t base64_table_dec_d0[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xf8000000, 0xffffffff, 0xffffffff, 0xffffffff, 0xfc000000,
0xd0000000, 0xd4000000, 0xd8000000, 0xdc000000, 0xe0000000, 0xe4000000,
0xe8000000, 0xec000000, 0xf0000000, 0xf4000000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x04000000, 0x08000000, 0x0c000000, 0x10000000, 0x14000000, 0x18000000,
0x1c000000, 0x20000000, 0x24000000, 0x28000000, 0x2c000000, 0x30000000,
0x34000000, 0x38000000, 0x3c000000, 0x40000000, 0x44000000, 0x48000000,
0x4c000000, 0x50000000, 0x54000000, 0x58000000, 0x5c000000, 0x60000000,
0x64000000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x68000000, 0x6c000000, 0x70000000, 0x74000000, 0x78000000,
0x7c000000, 0x80000000, 0x84000000, 0x88000000, 0x8c000000, 0x90000000,
0x94000000, 0x98000000, 0x9c000000, 0xa0000000, 0xa4000000, 0xa8000000,
0xac000000, 0xb0000000, 0xb4000000, 0xb8000000, 0xbc000000, 0xc0000000,
0xc4000000, 0xc8000000, 0xcc000000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d1[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x03e00000, 0xffffffff, 0xffffffff, 0xffffffff, 0x03f00000,
0x03400000, 0x03500000, 0x03600000, 0x03700000, 0x03800000, 0x03900000,
0x03a00000, 0x03b00000, 0x03c00000, 0x03d00000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00100000, 0x00200000, 0x00300000, 0x00400000, 0x00500000, 0x00600000,
0x00700000, 0x00800000, 0x00900000, 0x00a00000, 0x00b00000, 0x00c00000,
0x00d00000, 0x00e00000, 0x00f00000, 0x01000000, 0x01100000, 0x01200000,
0x01300000, 0x01400000, 0x01500000, 0x01600000, 0x01700000, 0x01800000,
0x01900000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x01a00000, 0x01b00000, 0x01c00000, 0x01d00000, 0x01e00000,
0x01f00000, 0x02000000, 0x02100000, 0x02200000, 0x02300000, 0x02400000,
0x02500000, 0x02600000, 0x02700000, 0x02800000, 0x02900000, 0x02a00000,
0x02b00000, 0x02c00000, 0x02d00000, 0x02e00000, 0x02f00000, 0x03000000,
0x03100000, 0x03200000, 0x03300000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d2[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x000f8000, 0xffffffff, 0xffffffff, 0xffffffff, 0x000fc000,
0x000d0000, 0x000d4000, 0x000d8000, 0x000dc000, 0x000e0000, 0x000e4000,
0x000e8000, 0x000ec000, 0x000f0000, 0x000f4000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00004000, 0x00008000, 0x0000c000, 0x00010000, 0x00014000, 0x00018000,
0x0001c000, 0x00020000, 0x00024000, 0x00028000, 0x0002c000, 0x00030000,
0x00034000, 0x00038000, 0x0003c000, 0x00040000, 0x00044000, 0x00048000,
0x0004c000, 0x00050000, 0x00054000, 0x00058000, 0x0005c000, 0x00060000,
0x00064000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00068000, 0x0006c000, 0x00070000, 0x00074000, 0x00078000,
0x0007c000, 0x00080000, 0x00084000, 0x00088000, 0x0008c000, 0x00090000,
0x00094000, 0x00098000, 0x0009c000, 0x000a0000, 0x000a4000, 0x000a8000,
0x000ac000, 0x000b0000, 0x000b4000, 0x000b8000, 0x000bc000, 0x000c0000,
0x000c4000, 0x000c8000, 0x000cc000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d3[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00003e00, 0xffffffff, 0xffffffff, 0xffffffff, 0x00003f00,
0x00003400, 0x00003500, 0x00003600, 0x00003700, 0x00003800, 0x00003900,
0x00003a00, 0x00003b00, 0x00003c00, 0x00003d00, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00000100, 0x00000200, 0x00000300, 0x00000400, 0x00000500, 0x00000600,
0x00000700, 0x00000800, 0x00000900, 0x00000a00, 0x00000b00, 0x00000c00,
0x00000d00, 0x00000e00, 0x00000f00, 0x00001000, 0x00001100, 0x00001200,
0x00001300, 0x00001400, 0x00001500, 0x00001600, 0x00001700, 0x00001800,
0x00001900, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00001a00, 0x00001b00, 0x00001c00, 0x00001d00, 0x00001e00,
0x00001f00, 0x00002000, 0x00002100, 0x00002200, 0x00002300, 0x00002400,
0x00002500, 0x00002600, 0x00002700, 0x00002800, 0x00002900, 0x00002a00,
0x00002b00, 0x00002c00, 0x00002d00, 0x00002e00, 0x00002f00, 0x00003000,
0x00003100, 0x00003200, 0x00003300, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
#endif

2
contrib/boost vendored

@ -1 +1 @@
Subproject commit 830e51edb59c4f37a8638138581e1e56c29ac44f
Subproject commit 86be2aef20bee2356b744e5569eed6eaded85dbe

5
contrib/boost-cmake/CMakeLists.txt
View File

@ -37,3 +37,8 @@ target_link_libraries(boost_filesystem_internal PRIVATE boost_system_internal)
if (USE_INTERNAL_PARQUET_LIBRARY)
add_boost_lib(regex)
endif()
if (USE_INTERNAL_AVRO_LIBRARY)
add_boost_lib(iostreams)
target_link_libraries(boost_iostreams_internal PUBLIC ${ZLIB_LIBRARIES})
endif()

61
contrib/curl-cmake/CMake/CurlSymbolHiding.cmake
View File

@ -1,61 +0,0 @@
include(CheckCSourceCompiles)
option(CURL_HIDDEN_SYMBOLS "Set to ON to hide libcurl internal symbols (=hide all symbols that aren't officially external)." ON)
mark_as_advanced(CURL_HIDDEN_SYMBOLS)
if(CURL_HIDDEN_SYMBOLS)
set(SUPPORTS_SYMBOL_HIDING FALSE)
if(CMAKE_C_COMPILER_ID MATCHES "Clang")
set(SUPPORTS_SYMBOL_HIDING TRUE)
set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))")
set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden")
elseif(CMAKE_COMPILER_IS_GNUCC)
if(NOT CMAKE_VERSION VERSION_LESS 2.8.10)
set(GCC_VERSION ${CMAKE_C_COMPILER_VERSION})
else()
execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion
OUTPUT_VARIABLE GCC_VERSION)
endif()
if(NOT GCC_VERSION VERSION_LESS 3.4)
# note: this is considered buggy prior to 4.0 but the autotools don't care, so let's ignore that fact
set(SUPPORTS_SYMBOL_HIDING TRUE)
set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))")
set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden")
endif()
elseif(CMAKE_C_COMPILER_ID MATCHES "SunPro" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 8.0)
set(SUPPORTS_SYMBOL_HIDING TRUE)
set(_SYMBOL_EXTERN "__global")
set(_CFLAG_SYMBOLS_HIDE "-xldscope=hidden")
elseif(CMAKE_C_COMPILER_ID MATCHES "Intel" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.0)
# note: this should probably just check for version 9.1.045 but I'm not 100% sure
# so let's do it the same way autotools do.
set(SUPPORTS_SYMBOL_HIDING TRUE)
set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))")
set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden")
check_c_source_compiles("#include <stdio.h>
int main (void) { printf(\"icc fvisibility bug test\"); return 0; }" _no_bug)
if(NOT _no_bug)
set(SUPPORTS_SYMBOL_HIDING FALSE)
set(_SYMBOL_EXTERN "")
set(_CFLAG_SYMBOLS_HIDE "")
endif()
elseif(MSVC)
set(SUPPORTS_SYMBOL_HIDING TRUE)
endif()
set(HIDES_CURL_PRIVATE_SYMBOLS ${SUPPORTS_SYMBOL_HIDING})
elseif(MSVC)
if(NOT CMAKE_VERSION VERSION_LESS 3.7)
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) #present since 3.4.3 but broken
set(HIDES_CURL_PRIVATE_SYMBOLS FALSE)
else()
message(WARNING "Hiding private symbols regardless CURL_HIDDEN_SYMBOLS being disabled.")
set(HIDES_CURL_PRIVATE_SYMBOLS TRUE)
endif()
else()
set(HIDES_CURL_PRIVATE_SYMBOLS FALSE)
endif()
set(CURL_CFLAG_SYMBOLS_HIDE ${_CFLAG_SYMBOLS_HIDE})
set(CURL_EXTERN_SYMBOL ${_SYMBOL_EXTERN})

617
contrib/curl-cmake/CMake/CurlTests.c
View File

@ -1,617 +0,0 @@
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at https://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
#ifdef TIME_WITH_SYS_TIME
/* Time with sys/time test */
#include <sys/types.h>
#include <sys/time.h>
#include <time.h>
int
main ()
{
if ((struct tm *) 0)
return 0;
;
return 0;
}
#endif
#ifdef HAVE_FCNTL_O_NONBLOCK
/* headers for FCNTL_O_NONBLOCK test */
#include <sys/types.h>
#include <unistd.h>
#include <fcntl.h>
/* */
#if defined(sun) || defined(__sun__) || \
defined(__SUNPRO_C) || defined(__SUNPRO_CC)
# if defined(__SVR4) || defined(__srv4__)
# define PLATFORM_SOLARIS
# else
# define PLATFORM_SUNOS4
# endif
#endif
#if (defined(_AIX) || defined(__xlC__)) && !defined(_AIX41)
# define PLATFORM_AIX_V3
#endif
/* */
#if defined(PLATFORM_SUNOS4) || defined(PLATFORM_AIX_V3) || defined(__BEOS__)
#error "O_NONBLOCK does not work on this platform"
#endif
int
main ()
{
/* O_NONBLOCK source test */
int flags = 0;
if(0 != fcntl(0, F_SETFL, flags | O_NONBLOCK))
return 1;
return 0;
}
#endif
/* tests for gethostbyaddr_r or gethostbyname_r */
#if defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT) || \
defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT) || \
defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT) || \
defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT) || \
defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT) || \
defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT)
# define _REENTRANT
/* no idea whether _REENTRANT is always set, just invent a new flag */
# define TEST_GETHOSTBYFOO_REENTRANT
#endif
#if defined(HAVE_GETHOSTBYADDR_R_5) || \
defined(HAVE_GETHOSTBYADDR_R_7) || \
defined(HAVE_GETHOSTBYADDR_R_8) || \
defined(HAVE_GETHOSTBYNAME_R_3) || \
defined(HAVE_GETHOSTBYNAME_R_5) || \
defined(HAVE_GETHOSTBYNAME_R_6) || \
defined(TEST_GETHOSTBYFOO_REENTRANT)
#include <sys/types.h>
#include <netdb.h>
int main(void)
{
char *address = "example.com";
int length = 0;
int type = 0;
struct hostent h;
int rc = 0;
#if defined(HAVE_GETHOSTBYADDR_R_5) || \
defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT) || \
\
defined(HAVE_GETHOSTBYNAME_R_3) || \
defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT)
struct hostent_data hdata;
#elif defined(HAVE_GETHOSTBYADDR_R_7) || \
defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT) || \
defined(HAVE_GETHOSTBYADDR_R_8) || \
defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT) || \
\
defined(HAVE_GETHOSTBYNAME_R_5) || \
defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT) || \
defined(HAVE_GETHOSTBYNAME_R_6) || \
defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT)
char buffer[8192];
int h_errnop;
struct hostent *hp;
#endif
#ifndef gethostbyaddr_r
(void)gethostbyaddr_r;
#endif
#if defined(HAVE_GETHOSTBYADDR_R_5) || \
defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT)
rc = gethostbyaddr_r(address, length, type, &h, &hdata);
(void)rc;
#elif defined(HAVE_GETHOSTBYADDR_R_7) || \
defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT)
hp = gethostbyaddr_r(address, length, type, &h, buffer, 8192, &h_errnop);
(void)hp;
#elif defined(HAVE_GETHOSTBYADDR_R_8) || \
defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT)
rc = gethostbyaddr_r(address, length, type, &h, buffer, 8192, &hp, &h_errnop);
(void)rc;
#endif
#if defined(HAVE_GETHOSTBYNAME_R_3) || \
defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT)
rc = gethostbyname_r(address, &h, &hdata);
#elif defined(HAVE_GETHOSTBYNAME_R_5) || \
defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT)
rc = gethostbyname_r(address, &h, buffer, 8192, &h_errnop);
(void)hp; /* not used for test */
#elif defined(HAVE_GETHOSTBYNAME_R_6) || \
defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT)
rc = gethostbyname_r(address, &h, buffer, 8192, &hp, &h_errnop);
#endif
(void)length;
(void)type;
(void)rc;
return 0;
}
#endif
#ifdef HAVE_SOCKLEN_T
#ifdef _WIN32
#include <ws2tcpip.h>
#else
#include <sys/types.h>
#include <sys/socket.h>
#endif
int
main ()
{
if ((socklen_t *) 0)
return 0;
if (sizeof (socklen_t))
return 0;
;
return 0;
}
#endif
#ifdef HAVE_IN_ADDR_T
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
int
main ()
{
if ((in_addr_t *) 0)
return 0;
if (sizeof (in_addr_t))
return 0;
;
return 0;
}
#endif
#ifdef HAVE_BOOL_T
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_STDBOOL_H
#include <stdbool.h>
#endif
int
main ()
{
if (sizeof (bool *) )
return 0;
;
return 0;
}
#endif
#ifdef STDC_HEADERS
#include <stdlib.h>
#include <stdarg.h>
#include <string.h>
#include <float.h>
int main() { return 0; }
#endif
#ifdef RETSIGTYPE_TEST
#include <sys/types.h>
#include <signal.h>
#ifdef signal
# undef signal
#endif
#ifdef __cplusplus
extern "C" void (*signal (int, void (*)(int)))(int);
#else
void (*signal ()) ();
#endif
int
main ()
{
return 0;
}
#endif
#ifdef HAVE_INET_NTOA_R_DECL
#include <arpa/inet.h>
typedef void (*func_type)();
int main()
{
#ifndef inet_ntoa_r
func_type func;
func = (func_type)inet_ntoa_r;
(void)func;
#endif
return 0;
}
#endif
#ifdef HAVE_INET_NTOA_R_DECL_REENTRANT
#define _REENTRANT
#include <arpa/inet.h>
typedef void (*func_type)();
int main()
{
#ifndef inet_ntoa_r
func_type func;
func = (func_type)&inet_ntoa_r;
(void)func;
#endif
return 0;
}
#endif
#ifdef HAVE_GETADDRINFO
#include <netdb.h>
#include <sys/types.h>
#include <sys/socket.h>
int main(void) {
struct addrinfo hints, *ai;
int error;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
#ifndef getaddrinfo
(void)getaddrinfo;
#endif
error = getaddrinfo("127.0.0.1", "8080", &hints, &ai);
if (error) {
return 1;
}
return 0;
}
#endif
#ifdef HAVE_FILE_OFFSET_BITS
#ifdef _FILE_OFFSET_BITS
#undef _FILE_OFFSET_BITS
#endif
#define _FILE_OFFSET_BITS 64
#include <sys/types.h>
/* Check that off_t can represent 2**63 - 1 correctly.
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
int main () { ; return 0; }
#endif
#ifdef HAVE_IOCTLSOCKET
/* includes start */
#ifdef HAVE_WINDOWS_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
# else
# ifdef HAVE_WINSOCK_H
# include <winsock.h>
# endif
# endif
#endif
int
main ()
{
/* ioctlsocket source code */
int socket;
unsigned long flags = ioctlsocket(socket, FIONBIO, &flags);
;
return 0;
}
#endif
#ifdef HAVE_IOCTLSOCKET_CAMEL
/* includes start */
#ifdef HAVE_WINDOWS_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
# else
# ifdef HAVE_WINSOCK_H
# include <winsock.h>
# endif
# endif
#endif
int
main ()
{
/* IoctlSocket source code */
if(0 != IoctlSocket(0, 0, 0))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_IOCTLSOCKET_CAMEL_FIONBIO
/* includes start */
#ifdef HAVE_WINDOWS_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
# else
# ifdef HAVE_WINSOCK_H
# include <winsock.h>
# endif
# endif
#endif
int
main ()
{
/* IoctlSocket source code */
long flags = 0;
if(0 != ioctlsocket(0, FIONBIO, &flags))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_IOCTLSOCKET_FIONBIO
/* includes start */
#ifdef HAVE_WINDOWS_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
# else
# ifdef HAVE_WINSOCK_H
# include <winsock.h>
# endif
# endif
#endif
int
main ()
{
int flags = 0;
if(0 != ioctlsocket(0, FIONBIO, &flags))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_IOCTL_FIONBIO
/* headers for FIONBIO test */
/* includes start */
#ifdef HAVE_SYS_TYPES_H
# include <sys/types.h>
#endif
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
# include <sys/socket.h>
#endif
#ifdef HAVE_SYS_IOCTL_H
# include <sys/ioctl.h>
#endif
#ifdef HAVE_STROPTS_H
# include <stropts.h>
#endif
int
main ()
{
int flags = 0;
if(0 != ioctl(0, FIONBIO, &flags))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_IOCTL_SIOCGIFADDR
/* headers for FIONBIO test */
/* includes start */
#ifdef HAVE_SYS_TYPES_H
# include <sys/types.h>
#endif
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
# include <sys/socket.h>
#endif
#ifdef HAVE_SYS_IOCTL_H
# include <sys/ioctl.h>
#endif
#ifdef HAVE_STROPTS_H
# include <stropts.h>
#endif
#include <net/if.h>
int
main ()
{
struct ifreq ifr;
if(0 != ioctl(0, SIOCGIFADDR, &ifr))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_SETSOCKOPT_SO_NONBLOCK
/* includes start */
#ifdef HAVE_WINDOWS_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
# else
# ifdef HAVE_WINSOCK_H
# include <winsock.h>
# endif
# endif
#endif
/* includes start */
#ifdef HAVE_SYS_TYPES_H
# include <sys/types.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
# include <sys/socket.h>
#endif
/* includes end */
int
main ()
{
if(0 != setsockopt(0, SOL_SOCKET, SO_NONBLOCK, 0, 0))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_GLIBC_STRERROR_R
#include <string.h>
#include <errno.h>
void check(char c) {}
int
main () {
char buffer[1024];
/* This will not compile if strerror_r does not return a char* */
check(strerror_r(EACCES, buffer, sizeof(buffer))[0]);
return 0;
}
#endif
#ifdef HAVE_POSIX_STRERROR_R
#include <string.h>
#include <errno.h>
/* float, because a pointer can't be implicitly cast to float */
void check(float f) {}
int
main () {
char buffer[1024];
/* This will not compile if strerror_r does not return an int */
check(strerror_r(EACCES, buffer, sizeof(buffer)));
return 0;
}
#endif
#ifdef HAVE_FSETXATTR_6
#include <sys/xattr.h> /* header from libc, not from libattr */
int
main() {
fsetxattr(0, 0, 0, 0, 0, 0);
return 0;
}
#endif
#ifdef HAVE_FSETXATTR_5
#include <sys/xattr.h> /* header from libc, not from libattr */
int
main() {
fsetxattr(0, 0, 0, 0, 0);
return 0;
}
#endif
#ifdef HAVE_CLOCK_GETTIME_MONOTONIC
#include <time.h>
int
main() {
struct timespec ts = {0, 0};
clock_gettime(CLOCK_MONOTONIC, &ts);
return 0;
}
#endif
#ifdef HAVE_BUILTIN_AVAILABLE
int
main() {
if(__builtin_available(macOS 10.12, *)) {}
return 0;
}
#endif
#ifdef HAVE_VARIADIC_MACROS_C99
#define c99_vmacro3(first, ...) fun3(first, __VA_ARGS__)
#define c99_vmacro2(first, ...) fun2(first, __VA_ARGS__)
int fun3(int arg1, int arg2, int arg3);
int fun2(int arg1, int arg2);
int fun3(int arg1, int arg2, int arg3) {
return arg1 + arg2 + arg3;
}
int fun2(int arg1, int arg2) {
return arg1 + arg2;
}
int
main() {
int res3 = c99_vmacro3(1, 2, 3);
int res2 = c99_vmacro2(1, 2);
(void)res3;
(void)res2;
return 0;
}
#endif
#ifdef HAVE_VARIADIC_MACROS_GCC
#define gcc_vmacro3(first, args...) fun3(first, args)
#define gcc_vmacro2(first, args...) fun2(first, args)
int fun3(int arg1, int arg2, int arg3);
int fun2(int arg1, int arg2);
int fun3(int arg1, int arg2, int arg3) {
return arg1 + arg2 + arg3;
}
int fun2(int arg1, int arg2) {
return arg1 + arg2;
}
int
main() {
int res3 = gcc_vmacro3(1, 2, 3);
int res2 = gcc_vmacro2(1, 2);
(void)res3;
(void)res2;
return 0;
}
#endif

84
contrib/curl-cmake/CMake/Macros.cmake
View File

@ -1,84 +0,0 @@
#File defines convenience macros for available feature testing
# This macro checks if the symbol exists in the library and if it
# does, it prepends library to the list. It is intended to be called
# multiple times with a sequence of possibly dependent libraries in
# order of least-to-most-dependent. Some libraries depend on others
# to link correctly.
macro(check_library_exists_concat LIBRARY SYMBOL VARIABLE)
check_library_exists("${LIBRARY};${CURL_LIBS}" ${SYMBOL} "${CMAKE_LIBRARY_PATH}"
${VARIABLE})
if(${VARIABLE})
set(CURL_LIBS ${LIBRARY} ${CURL_LIBS})
endif()
endmacro()
# Check if header file exists and add it to the list.
# This macro is intended to be called multiple times with a sequence of
# possibly dependent header files. Some headers depend on others to be
# compiled correctly.
macro(check_include_file_concat FILE VARIABLE)
check_include_files("${CURL_INCLUDES};${FILE}" ${VARIABLE})
if(${VARIABLE})
set(CURL_INCLUDES ${CURL_INCLUDES} ${FILE})
set(CURL_TEST_DEFINES "${CURL_TEST_DEFINES} -D${VARIABLE}")
endif()
endmacro()
# For other curl specific tests, use this macro.
macro(curl_internal_test CURL_TEST)
if(NOT DEFINED "${CURL_TEST}")
set(MACRO_CHECK_FUNCTION_DEFINITIONS
"-D${CURL_TEST} ${CURL_TEST_DEFINES} ${CMAKE_REQUIRED_FLAGS}")
if(CMAKE_REQUIRED_LIBRARIES)
set(CURL_TEST_ADD_LIBRARIES
"-DLINK_LIBRARIES:STRING=${CMAKE_REQUIRED_LIBRARIES}")
endif()
try_compile(${CURL_TEST}
${CMAKE_BINARY_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/CMake/CurlTests.c
CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${MACRO_CHECK_FUNCTION_DEFINITIONS}
"${CURL_TEST_ADD_LIBRARIES}"
OUTPUT_VARIABLE OUTPUT)
if(${CURL_TEST})
set(${CURL_TEST} 1 CACHE INTERNAL "Curl test ${FUNCTION}")
file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log
"Performing Curl Test ${CURL_TEST} passed with the following output:\n"
"${OUTPUT}\n")
else()
set(${CURL_TEST} "" CACHE INTERNAL "Curl test ${FUNCTION}")
file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log
"Performing Curl Test ${CURL_TEST} failed with the following output:\n"
"${OUTPUT}\n")
endif()
endif()
endmacro()
macro(curl_nroff_check)
find_program(NROFF NAMES gnroff nroff)
if(NROFF)
# Need a way to write to stdin, this will do
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt" "test")
# Tests for a valid nroff option to generate a manpage
foreach(_MANOPT "-man" "-mandoc")
execute_process(COMMAND "${NROFF}" ${_MANOPT}
OUTPUT_VARIABLE NROFF_MANOPT_OUTPUT
INPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt"
ERROR_QUIET)
# Save the option if it was valid
if(NROFF_MANOPT_OUTPUT)
set(NROFF_MANOPT ${_MANOPT})
set(NROFF_USEFUL ON)
break()
endif()
endforeach()
# No need for the temporary file
file(REMOVE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt")
if(NOT NROFF_USEFUL)
message(WARNING "Found no *nroff option to get plaintext from man pages")
endif()
else()
message(WARNING "Found no *nroff program")
endif()
endmacro()

260
contrib/curl-cmake/CMake/OtherTests.cmake
View File

@ -1,260 +0,0 @@
include(CheckCSourceCompiles)
# The begin of the sources (macros and includes)
set(_source_epilogue "#undef inline")
macro(add_header_include check header)
if(${check})
set(_source_epilogue "${_source_epilogue}\n#include <${header}>")
endif()
endmacro()
set(signature_call_conv)
if(HAVE_WINDOWS_H)
add_header_include(HAVE_WINSOCK2_H "winsock2.h")
add_header_include(HAVE_WINDOWS_H "windows.h")
add_header_include(HAVE_WINSOCK_H "winsock.h")
set(_source_epilogue
"${_source_epilogue}\n#ifndef WIN32_LEAN_AND_MEAN\n#define WIN32_LEAN_AND_MEAN\n#endif")
set(signature_call_conv "PASCAL")
if(HAVE_LIBWS2_32)
set(CMAKE_REQUIRED_LIBRARIES ws2_32)
endif()
else()
add_header_include(HAVE_SYS_TYPES_H "sys/types.h")
add_header_include(HAVE_SYS_SOCKET_H "sys/socket.h")
endif()
set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
check_c_source_compiles("${_source_epilogue}
int main(void) {
recv(0, 0, 0, 0);
return 0;
}" curl_cv_recv)
if(curl_cv_recv)
if(NOT DEFINED curl_cv_func_recv_args OR "${curl_cv_func_recv_args}" STREQUAL "unknown")
foreach(recv_retv "int" "ssize_t" )
foreach(recv_arg1 "SOCKET" "int" )
foreach(recv_arg2 "char *" "void *" )
foreach(recv_arg3 "int" "size_t" "socklen_t" "unsigned int")
foreach(recv_arg4 "int" "unsigned int")
if(NOT curl_cv_func_recv_done)
unset(curl_cv_func_recv_test CACHE)
check_c_source_compiles("
${_source_epilogue}
extern ${recv_retv} ${signature_call_conv}
recv(${recv_arg1}, ${recv_arg2}, ${recv_arg3}, ${recv_arg4});
int main(void) {
${recv_arg1} s=0;
${recv_arg2} buf=0;
${recv_arg3} len=0;
${recv_arg4} flags=0;
${recv_retv} res = recv(s, buf, len, flags);
(void) res;
return 0;
}"
curl_cv_func_recv_test)
if(curl_cv_func_recv_test)
set(curl_cv_func_recv_args
"${recv_arg1},${recv_arg2},${recv_arg3},${recv_arg4},${recv_retv}")
set(RECV_TYPE_ARG1 "${recv_arg1}")
set(RECV_TYPE_ARG2 "${recv_arg2}")
set(RECV_TYPE_ARG3 "${recv_arg3}")
set(RECV_TYPE_ARG4 "${recv_arg4}")
set(RECV_TYPE_RETV "${recv_retv}")
set(HAVE_RECV 1)
set(curl_cv_func_recv_done 1)
endif()
endif()
endforeach()
endforeach()
endforeach()
endforeach()
endforeach()
else()
string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG1 "${curl_cv_func_recv_args}")
string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG2 "${curl_cv_func_recv_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,([^,]*),[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG3 "${curl_cv_func_recv_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,([^,]*),[^,]*$" "\\1" RECV_TYPE_ARG4 "${curl_cv_func_recv_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,([^,]*)$" "\\1" RECV_TYPE_RETV "${curl_cv_func_recv_args}")
endif()
if("${curl_cv_func_recv_args}" STREQUAL "unknown")
message(FATAL_ERROR "Cannot find proper types to use for recv args")
endif()
else()
message(FATAL_ERROR "Unable to link function recv")
endif()
set(curl_cv_func_recv_args "${curl_cv_func_recv_args}" CACHE INTERNAL "Arguments for recv")
set(HAVE_RECV 1)
check_c_source_compiles("${_source_epilogue}
int main(void) {
send(0, 0, 0, 0);
return 0;
}" curl_cv_send)
if(curl_cv_send)
if(NOT DEFINED curl_cv_func_send_args OR "${curl_cv_func_send_args}" STREQUAL "unknown")
foreach(send_retv "int" "ssize_t" )
foreach(send_arg1 "SOCKET" "int" "ssize_t" )
foreach(send_arg2 "const char *" "const void *" "void *" "char *")
foreach(send_arg3 "int" "size_t" "socklen_t" "unsigned int")
foreach(send_arg4 "int" "unsigned int")
if(NOT curl_cv_func_send_done)
unset(curl_cv_func_send_test CACHE)
check_c_source_compiles("
${_source_epilogue}
extern ${send_retv} ${signature_call_conv}
send(${send_arg1}, ${send_arg2}, ${send_arg3}, ${send_arg4});
int main(void) {
${send_arg1} s=0;
${send_arg2} buf=0;
${send_arg3} len=0;
${send_arg4} flags=0;
${send_retv} res = send(s, buf, len, flags);
(void) res;
return 0;
}"
curl_cv_func_send_test)
if(curl_cv_func_send_test)
string(REGEX REPLACE "(const) .*" "\\1" send_qual_arg2 "${send_arg2}")
string(REGEX REPLACE "const (.*)" "\\1" send_arg2 "${send_arg2}")
set(curl_cv_func_send_args
"${send_arg1},${send_arg2},${send_arg3},${send_arg4},${send_retv},${send_qual_arg2}")
set(SEND_TYPE_ARG1 "${send_arg1}")
set(SEND_TYPE_ARG2 "${send_arg2}")
set(SEND_TYPE_ARG3 "${send_arg3}")
set(SEND_TYPE_ARG4 "${send_arg4}")
set(SEND_TYPE_RETV "${send_retv}")
set(HAVE_SEND 1)
set(curl_cv_func_send_done 1)
endif()
endif()
endforeach()
endforeach()
endforeach()
endforeach()
endforeach()
else()
string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG1 "${curl_cv_func_send_args}")
string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG2 "${curl_cv_func_send_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,([^,]*),[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG3 "${curl_cv_func_send_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,([^,]*),[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG4 "${curl_cv_func_send_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,([^,]*),[^,]*$" "\\1" SEND_TYPE_RETV "${curl_cv_func_send_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,[^,]*,([^,]*)$" "\\1" SEND_QUAL_ARG2 "${curl_cv_func_send_args}")
endif()
if("${curl_cv_func_send_args}" STREQUAL "unknown")
message(FATAL_ERROR "Cannot find proper types to use for send args")
endif()
set(SEND_QUAL_ARG2 "const")
else()
message(FATAL_ERROR "Unable to link function send")
endif()
set(curl_cv_func_send_args "${curl_cv_func_send_args}" CACHE INTERNAL "Arguments for send")
set(HAVE_SEND 1)
check_c_source_compiles("${_source_epilogue}
int main(void) {
int flag = MSG_NOSIGNAL;
(void)flag;
return 0;
}" HAVE_MSG_NOSIGNAL)
if(NOT HAVE_WINDOWS_H)
add_header_include(HAVE_SYS_TIME_H "sys/time.h")
add_header_include(TIME_WITH_SYS_TIME "time.h")
add_header_include(HAVE_TIME_H "time.h")
endif()
check_c_source_compiles("${_source_epilogue}
int main(void) {
struct timeval ts;
ts.tv_sec = 0;
ts.tv_usec = 0;
(void)ts;
return 0;
}" HAVE_STRUCT_TIMEVAL)
set(HAVE_SIG_ATOMIC_T 1)
set(CMAKE_REQUIRED_FLAGS)
if(HAVE_SIGNAL_H)
set(CMAKE_REQUIRED_FLAGS "-DHAVE_SIGNAL_H")
set(CMAKE_EXTRA_INCLUDE_FILES "signal.h")
endif()
check_type_size("sig_atomic_t" SIZEOF_SIG_ATOMIC_T)
if(HAVE_SIZEOF_SIG_ATOMIC_T)
check_c_source_compiles("
#ifdef HAVE_SIGNAL_H
# include <signal.h>
#endif
int main(void) {
static volatile sig_atomic_t dummy = 0;
(void)dummy;
return 0;
}" HAVE_SIG_ATOMIC_T_NOT_VOLATILE)
if(NOT HAVE_SIG_ATOMIC_T_NOT_VOLATILE)
set(HAVE_SIG_ATOMIC_T_VOLATILE 1)
endif()
endif()
if(HAVE_WINDOWS_H)
set(CMAKE_EXTRA_INCLUDE_FILES winsock2.h)
else()
set(CMAKE_EXTRA_INCLUDE_FILES)
if(HAVE_SYS_SOCKET_H)
set(CMAKE_EXTRA_INCLUDE_FILES sys/socket.h)
endif()
endif()
check_type_size("struct sockaddr_storage" SIZEOF_STRUCT_SOCKADDR_STORAGE)
if(HAVE_SIZEOF_STRUCT_SOCKADDR_STORAGE)
set(HAVE_STRUCT_SOCKADDR_STORAGE 1)
endif()
unset(CMAKE_TRY_COMPILE_TARGET_TYPE)
if(NOT DEFINED CMAKE_TOOLCHAIN_FILE)
# if not cross-compilation...
include(CheckCSourceRuns)
set(CMAKE_REQUIRED_FLAGS "")
if(HAVE_SYS_POLL_H)
set(CMAKE_REQUIRED_FLAGS "-DHAVE_SYS_POLL_H")
elseif(HAVE_POLL_H)
set(CMAKE_REQUIRED_FLAGS "-DHAVE_POLL_H")
endif()
check_c_source_runs("
#include <stdlib.h>
#include <sys/time.h>
#ifdef HAVE_SYS_POLL_H
# include <sys/poll.h>
#elif HAVE_POLL_H
# include <poll.h>
#endif
int main(void)
{
if(0 != poll(0, 0, 10)) {
return 1; /* fail */
}
else {
/* detect the 10.12 poll() breakage */
struct timeval before, after;
int rc;
size_t us;
gettimeofday(&before, NULL);
rc = poll(NULL, 0, 500);
gettimeofday(&after, NULL);
us = (after.tv_sec - before.tv_sec) * 1000000 +
(after.tv_usec - before.tv_usec);
if(us < 400000) {
return 1;
}
}
return 0;
}" HAVE_POLL_FINE)
endif()

28
contrib/curl-cmake/CMake/README.md
View File

@ -1,28 +0,0 @@
#***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
# | (__| |_| | _ <| |___
# \___|\___/|_| \_\_____|
#
# Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at https://curl.haxx.se/docs/copyright.html.
#
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
# copies of the Software, and permit persons to whom the Software is
# furnished to do so, under the terms of the COPYING file.
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
# KIND, either express or implied.
#
###########################################################################
CMake files under this directory were reused from project curl.
Here are links to original source files:
https://github.com/curl/curl/blob/master/CMake/CurlSymbolHiding.cmake
https://github.com/curl/curl/blob/master/CMake/CurlTests,c
https://github.com/curl/curl/blob/master/CMake/Macros.cmake
https://github.com/curl/curl/blob/master/CMake/OtherTests.cmake

1509
contrib/curl-cmake/CMakeLists.patch
View File

File diff suppressed because it is too large Load Diff

753
contrib/curl-cmake/CMakeLists.txt
View File

@ -1,605 +1,148 @@
#***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
# | (__| |_| | _ <| |___
# \___|\___/|_| \_\_____|
#
# Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at https://curl.haxx.se/docs/copyright.html.
#
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
# copies of the Software, and permit persons to whom the Software is
# furnished to do so, under the terms of the COPYING file.
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
# KIND, either express or implied.
#
###########################################################################
# NOTE:
# This file is shrinked and reworked version of original curl CMakeLists.txt
# Original file link https://github.com/curl/curl/blob/3b8bbbbd1609c638a3d3d0acb148a33dedb67be3/CMakeLists.txt
# If you need to update curl building you can find patch file in this directory
# and apply it to fresh original CMakeLists.txt file.
cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
SET(CURL_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl)
SET(CURL_LIBRARY_DIR ${CURL_SOURCE_DIR}/lib)
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake;${CMAKE_MODULE_PATH}")
# Disable status messages when perform checks
set(CMAKE_REQUIRED_QUIET TRUE)
include(Macros)
include(CMakeDependentOption)
include(CheckCCompilerFlag)
file(READ ${CURL_SOURCE_DIR}/include/curl/curlver.h CURL_VERSION_H_CONTENTS)
string(REGEX MATCH "#define LIBCURL_VERSION \"[^\"]*"
CURL_VERSION ${CURL_VERSION_H_CONTENTS})
string(REGEX REPLACE "[^\"]+\"" "" CURL_VERSION ${CURL_VERSION})
string(REGEX MATCH "#define LIBCURL_VERSION_NUM 0x[0-9a-fA-F]+"
CURL_VERSION_NUM ${CURL_VERSION_H_CONTENTS})
string(REGEX REPLACE "[^0]+0x" "" CURL_VERSION_NUM ${CURL_VERSION_NUM})
message(STATUS "Use curl version=[${CURL_VERSION}]")
set(OPERATING_SYSTEM "${CMAKE_SYSTEM_NAME}")
set(OS "\"${CMAKE_SYSTEM_NAME}\"")
option(PICKY_COMPILER "Enable picky compiler options" ON)
option(ENABLE_THREADED_RESOLVER "Set to ON to enable threaded DNS lookup" ON)
if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)
if(PICKY_COMPILER)
foreach(_CCOPT -pedantic -Wall -W -Wpointer-arith -Wwrite-strings -Wunused -Wshadow -Winline -Wnested-externs -Wmissing-declarations -Wmissing-prototypes -Wno-long-long -Wfloat-equal -Wno-multichar -Wsign-compare -Wundef -Wno-format-nonliteral -Wendif-labels -Wstrict-prototypes -Wdeclaration-after-statement -Wstrict-aliasing=3 -Wcast-align -Wtype-limits -Wold-style-declaration -Wmissing-parameter-type -Wempty-body -Wclobbered -Wignored-qualifiers -Wconversion -Wno-sign-conversion -Wvla -Wdouble-promotion -Wno-system-headers -Wno-pedantic-ms-format)
# surprisingly, CHECK_C_COMPILER_FLAG needs a new variable to store each new
# test result in.
check_c_compiler_flag(${_CCOPT} OPT${_CCOPT})
if(OPT${_CCOPT})
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_CCOPT}")
endif()
endforeach()
endif()
endif()
# initialize CURL_LIBS
set(CURL_LIBS "")
include(CurlSymbolHiding)
# Http only
set(CURL_DISABLE_FTP ON)
set(CURL_DISABLE_LDAP ON)
set(CURL_DISABLE_LDAPS ON)
set(CURL_DISABLE_TELNET ON)
set(CURL_DISABLE_DICT ON)
set(CURL_DISABLE_FILE ON)
set(CURL_DISABLE_TFTP ON)
set(CURL_DISABLE_RTSP ON)
set(CURL_DISABLE_POP3 ON)
set(CURL_DISABLE_IMAP ON)
set(CURL_DISABLE_SMTP ON)
set(CURL_DISABLE_GOPHER ON)
option(CURL_DISABLE_COOKIES "to disable cookies support" OFF)
mark_as_advanced(CURL_DISABLE_COOKIES)
option(CURL_DISABLE_CRYPTO_AUTH "to disable cryptographic authentication" OFF)
mark_as_advanced(CURL_DISABLE_CRYPTO_AUTH)
option(CURL_DISABLE_VERBOSE_STRINGS "to disable verbose strings" OFF)
mark_as_advanced(CURL_DISABLE_VERBOSE_STRINGS)
option(ENABLE_IPV6 "Define if you want to enable IPv6 support" ON)
mark_as_advanced(ENABLE_IPV6)
if(ENABLE_IPV6 AND NOT WIN32)
include(CheckStructHasMember)
check_struct_has_member("struct sockaddr_in6" sin6_addr "netinet/in.h"
HAVE_SOCKADDR_IN6_SIN6_ADDR)
check_struct_has_member("struct sockaddr_in6" sin6_scope_id "netinet/in.h"
HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID)
if(NOT HAVE_SOCKADDR_IN6_SIN6_ADDR)
message(WARNING "struct sockaddr_in6 not available, disabling IPv6 support")
# Force the feature off as this name is used as guard macro...
set(ENABLE_IPV6 OFF
CACHE BOOL "Define if you want to enable IPv6 support" FORCE)
endif()
endif()
# We need ansi c-flags, especially on HP
set(CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS}")
set(CMAKE_REQUIRED_FLAGS ${CMAKE_ANSI_CFLAGS})
# Include all the necessary files for macros
include(CheckFunctionExists)
include(CheckIncludeFile)
include(CheckIncludeFiles)
include(CheckLibraryExists)
include(CheckSymbolExists)
include(CheckTypeSize)
include(CheckCSourceCompiles)
if(ENABLE_THREADED_RESOLVER)
find_package(Threads REQUIRED)
set(USE_THREADS_POSIX ${CMAKE_USE_PTHREADS_INIT})
set(HAVE_PTHREAD_H ${CMAKE_USE_PTHREADS_INIT})
set(CURL_LIBS ${CURL_LIBS} ${CMAKE_THREAD_LIBS_INIT})
endif()
# Check for all needed libraries
# We don't want any plugin loading at runtime. It is harmful.
#check_library_exists_concat("${CMAKE_DL_LIBS}" dlopen HAVE_LIBDL)
# This is unneeded.
#check_library_exists_concat("socket" connect HAVE_LIBSOCKET)
set (NOT_NEED_LIBNSL 1)
set (gethostname HAVE_GETHOSTNAME 1)
# From cmake/find/ssl.cmake
if (OPENSSL_FOUND)
set(SSL_ENABLED ON)
set(USE_OPENSSL ON)
list(APPEND CURL_LIBS ${OPENSSL_LIBRARIES})
check_include_file("openssl/crypto.h" HAVE_OPENSSL_CRYPTO_H)
check_include_file("openssl/err.h" HAVE_OPENSSL_ERR_H)
check_include_file("openssl/pem.h" HAVE_OPENSSL_PEM_H)
check_include_file("openssl/rsa.h" HAVE_OPENSSL_RSA_H)
check_include_file("openssl/ssl.h" HAVE_OPENSSL_SSL_H)
check_include_file("openssl/x509.h" HAVE_OPENSSL_X509_H)
check_include_file("openssl/rand.h" HAVE_OPENSSL_RAND_H)
check_symbol_exists(RAND_status "${CURL_INCLUDES}" HAVE_RAND_STATUS)
check_symbol_exists(RAND_screen "${CURL_INCLUDES}" HAVE_RAND_SCREEN)
check_symbol_exists(RAND_egd "${CURL_INCLUDES}" HAVE_RAND_EGD)
endif()
# Check for idn
# No, we don't need that.
# check_library_exists_concat("idn2" idn2_lookup_ul HAVE_LIBIDN2)
# Check for symbol dlopen (same as HAVE_LIBDL)
# We don't want any plugin loading at runtime. It is harmful.
# check_library_exists("${CURL_LIBS}" dlopen "" HAVE_DLOPEN)
# From /cmake/find/zlib.cmake
if (ZLIB_FOUND)
set(HAVE_ZLIB_H ON)
set(HAVE_LIBZ ON)
set(USE_ZLIB ON)
list(APPEND CURL_LIBS ${ZLIB_LIBRARIES})
endif()
option(ENABLE_UNIX_SOCKETS "Define if you want Unix domain sockets support" OFF)
if(ENABLE_UNIX_SOCKETS)
include(CheckStructHasMember)
check_struct_has_member("struct sockaddr_un" sun_path "sys/un.h" USE_UNIX_SOCKETS)
else()
unset(USE_UNIX_SOCKETS CACHE)
endif()
# CA handling
# Explicitly set to most common case
if (OPENSSL_FOUND)
set(CURL_CA_BUNDLE "/etc/ssl/certs/ca-certificates.crt")
set(CURL_CA_BUNDLE_SET TRUE CACHE BOOL "Path to the CA bundle has been set")
set(CURL_CA_PATH "/etc/ssl/certs")
set(CURL_CA_PATH_SET TRUE CACHE BOOL "Path to the CA bundle has been set")
endif()
check_include_file_concat("stdio.h" HAVE_STDIO_H)
check_include_file_concat("inttypes.h" HAVE_INTTYPES_H)
check_include_file_concat("sys/filio.h" HAVE_SYS_FILIO_H)
check_include_file_concat("sys/ioctl.h" HAVE_SYS_IOCTL_H)
check_include_file_concat("sys/param.h" HAVE_SYS_PARAM_H)
check_include_file_concat("sys/poll.h" HAVE_SYS_POLL_H)
check_include_file_concat("sys/resource.h" HAVE_SYS_RESOURCE_H)
check_include_file_concat("sys/select.h" HAVE_SYS_SELECT_H)
check_include_file_concat("sys/socket.h" HAVE_SYS_SOCKET_H)
check_include_file_concat("sys/sockio.h" HAVE_SYS_SOCKIO_H)
check_include_file_concat("sys/stat.h" HAVE_SYS_STAT_H)
check_include_file_concat("sys/time.h" HAVE_SYS_TIME_H)
check_include_file_concat("sys/types.h" HAVE_SYS_TYPES_H)
check_include_file_concat("sys/uio.h" HAVE_SYS_UIO_H)
check_include_file_concat("sys/un.h" HAVE_SYS_UN_H)
check_include_file_concat("sys/utime.h" HAVE_SYS_UTIME_H)
check_include_file_concat("sys/xattr.h" HAVE_SYS_XATTR_H)
check_include_file_concat("alloca.h" HAVE_ALLOCA_H)
check_include_file_concat("arpa/inet.h" HAVE_ARPA_INET_H)
#check_include_file_concat("arpa/tftp.h" HAVE_ARPA_TFTP_H)
check_include_file_concat("assert.h" HAVE_ASSERT_H)
check_include_file_concat("crypto.h" HAVE_CRYPTO_H)
check_include_file_concat("des.h" HAVE_DES_H)
check_include_file_concat("err.h" HAVE_ERR_H)
check_include_file_concat("errno.h" HAVE_ERRNO_H)
check_include_file_concat("fcntl.h" HAVE_FCNTL_H)
#check_include_file_concat("idn2.h" HAVE_IDN2_H)
check_include_file_concat("ifaddrs.h" HAVE_IFADDRS_H)
check_include_file_concat("io.h" HAVE_IO_H)
check_include_file_concat("krb.h" HAVE_KRB_H)
check_include_file_concat("libgen.h" HAVE_LIBGEN_H)
check_include_file_concat("locale.h" HAVE_LOCALE_H)
check_include_file_concat("net/if.h" HAVE_NET_IF_H)
check_include_file_concat("netdb.h" HAVE_NETDB_H)
check_include_file_concat("netinet/in.h" HAVE_NETINET_IN_H)
check_include_file_concat("netinet/tcp.h" HAVE_NETINET_TCP_H)
check_include_file_concat("pem.h" HAVE_PEM_H)
check_include_file_concat("poll.h" HAVE_POLL_H)
check_include_file_concat("pwd.h" HAVE_PWD_H)
check_include_file_concat("rsa.h" HAVE_RSA_H)
check_include_file_concat("setjmp.h" HAVE_SETJMP_H)
check_include_file_concat("sgtty.h" HAVE_SGTTY_H)
check_include_file_concat("signal.h" HAVE_SIGNAL_H)
check_include_file_concat("ssl.h" HAVE_SSL_H)
check_include_file_concat("stdbool.h" HAVE_STDBOOL_H)
check_include_file_concat("stdint.h" HAVE_STDINT_H)
check_include_file_concat("stdio.h" HAVE_STDIO_H)
check_include_file_concat("stdlib.h" HAVE_STDLIB_H)
check_include_file_concat("string.h" HAVE_STRING_H)
check_include_file_concat("strings.h" HAVE_STRINGS_H)
check_include_file_concat("stropts.h" HAVE_STROPTS_H)
check_include_file_concat("termio.h" HAVE_TERMIO_H)
check_include_file_concat("termios.h" HAVE_TERMIOS_H)
check_include_file_concat("time.h" HAVE_TIME_H)
check_include_file_concat("unistd.h" HAVE_UNISTD_H)
check_include_file_concat("utime.h" HAVE_UTIME_H)
check_include_file_concat("x509.h" HAVE_X509_H)
check_include_file_concat("process.h" HAVE_PROCESS_H)
check_include_file_concat("stddef.h" HAVE_STDDEF_H)
#check_include_file_concat("dlfcn.h" HAVE_DLFCN_H)
check_include_file_concat("malloc.h" HAVE_MALLOC_H)
check_include_file_concat("memory.h" HAVE_MEMORY_H)
check_include_file_concat("netinet/if_ether.h" HAVE_NETINET_IF_ETHER_H)
check_include_file_concat("stdint.h" HAVE_STDINT_H)
check_include_file_concat("sockio.h" HAVE_SOCKIO_H)
check_include_file_concat("sys/utsname.h" HAVE_SYS_UTSNAME_H)
check_type_size(size_t SIZEOF_SIZE_T)
check_type_size(ssize_t SIZEOF_SSIZE_T)
check_type_size("long long" SIZEOF_LONG_LONG)
check_type_size("long" SIZEOF_LONG)
check_type_size("short" SIZEOF_SHORT)
check_type_size("int" SIZEOF_INT)
check_type_size("__int64" SIZEOF___INT64)
check_type_size("long double" SIZEOF_LONG_DOUBLE)
check_type_size("time_t" SIZEOF_TIME_T)
set(HAVE_LONGLONG 1)
set(HAVE_LL 1)
set(RANDOM_FILE /dev/urandom)
check_symbol_exists(basename "${CURL_INCLUDES}" HAVE_BASENAME)
check_symbol_exists(socket "${CURL_INCLUDES}" HAVE_SOCKET)
check_symbol_exists(select "${CURL_INCLUDES}" HAVE_SELECT)
check_symbol_exists(poll "${CURL_INCLUDES}" HAVE_POLL)
check_symbol_exists(strdup "${CURL_INCLUDES}" HAVE_STRDUP)
check_symbol_exists(strstr "${CURL_INCLUDES}" HAVE_STRSTR)
check_symbol_exists(strtok_r "${CURL_INCLUDES}" HAVE_STRTOK_R)
check_symbol_exists(strftime "${CURL_INCLUDES}" HAVE_STRFTIME)
check_symbol_exists(uname "${CURL_INCLUDES}" HAVE_UNAME)
check_symbol_exists(strcasecmp "${CURL_INCLUDES}" HAVE_STRCASECMP)
#check_symbol_exists(stricmp "${CURL_INCLUDES}" HAVE_STRICMP)
#check_symbol_exists(strcmpi "${CURL_INCLUDES}" HAVE_STRCMPI)
#check_symbol_exists(strncmpi "${CURL_INCLUDES}" HAVE_STRNCMPI)
check_symbol_exists(alarm "${CURL_INCLUDES}" HAVE_ALARM)
#check_symbol_exists(gethostbyaddr "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR)
check_symbol_exists(gethostbyaddr_r "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR_R)
check_symbol_exists(gettimeofday "${CURL_INCLUDES}" HAVE_GETTIMEOFDAY)
check_symbol_exists(inet_addr "${CURL_INCLUDES}" HAVE_INET_ADDR)
#check_symbol_exists(inet_ntoa "${CURL_INCLUDES}" HAVE_INET_NTOA)
check_symbol_exists(inet_ntoa_r "${CURL_INCLUDES}" HAVE_INET_NTOA_R)
check_symbol_exists(tcsetattr "${CURL_INCLUDES}" HAVE_TCSETATTR)
check_symbol_exists(tcgetattr "${CURL_INCLUDES}" HAVE_TCGETATTR)
check_symbol_exists(perror "${CURL_INCLUDES}" HAVE_PERROR)
check_symbol_exists(closesocket "${CURL_INCLUDES}" HAVE_CLOSESOCKET)
check_symbol_exists(setvbuf "${CURL_INCLUDES}" HAVE_SETVBUF)
check_symbol_exists(sigsetjmp "${CURL_INCLUDES}" HAVE_SIGSETJMP)
check_symbol_exists(getpass_r "${CURL_INCLUDES}" HAVE_GETPASS_R)
#check_symbol_exists(strlcat "${CURL_INCLUDES}" HAVE_STRLCAT)
#check_symbol_exists(getpwuid "${CURL_INCLUDES}" HAVE_GETPWUID)
check_symbol_exists(getpwuid_r "${CURL_INCLUDES}" HAVE_GETPWUID_R)
check_symbol_exists(geteuid "${CURL_INCLUDES}" HAVE_GETEUID)
check_symbol_exists(usleep "${CURL_INCLUDES}" HAVE_USLEEP)
check_symbol_exists(utime "${CURL_INCLUDES}" HAVE_UTIME)
check_symbol_exists(gmtime_r "${CURL_INCLUDES}" HAVE_GMTIME_R)
check_symbol_exists(localtime_r "${CURL_INCLUDES}" HAVE_LOCALTIME_R)
#check_symbol_exists(gethostbyname "${CURL_INCLUDES}" HAVE_GETHOSTBYNAME)
check_symbol_exists(gethostbyname_r "${CURL_INCLUDES}" HAVE_GETHOSTBYNAME_R)
check_symbol_exists(signal "${CURL_INCLUDES}" HAVE_SIGNAL_FUNC)
check_symbol_exists(SIGALRM "${CURL_INCLUDES}" HAVE_SIGNAL_MACRO)
set(HAVE_SIGNAL 1)
check_symbol_exists(uname "${CURL_INCLUDES}" HAVE_UNAME)
check_symbol_exists(strtoll "${CURL_INCLUDES}" HAVE_STRTOLL)
#check_symbol_exists(_strtoi64 "${CURL_INCLUDES}" HAVE__STRTOI64)
check_symbol_exists(strerror_r "${CURL_INCLUDES}" HAVE_STRERROR_R)
check_symbol_exists(siginterrupt "${CURL_INCLUDES}" HAVE_SIGINTERRUPT)
check_symbol_exists(perror "${CURL_INCLUDES}" HAVE_PERROR)
check_symbol_exists(fork "${CURL_INCLUDES}" HAVE_FORK)
check_symbol_exists(getaddrinfo "${CURL_INCLUDES}" HAVE_GETADDRINFO)
check_symbol_exists(freeaddrinfo "${CURL_INCLUDES}" HAVE_FREEADDRINFO)
check_symbol_exists(freeifaddrs "${CURL_INCLUDES}" HAVE_FREEIFADDRS)
check_symbol_exists(pipe "${CURL_INCLUDES}" HAVE_PIPE)
check_symbol_exists(ftruncate "${CURL_INCLUDES}" HAVE_FTRUNCATE)
check_symbol_exists(getprotobyname "${CURL_INCLUDES}" HAVE_GETPROTOBYNAME)
check_symbol_exists(getpeername "${CURL_INCLUDES}" HAVE_GETPEERNAME)
check_symbol_exists(getsockname "${CURL_INCLUDES}" HAVE_GETSOCKNAME)
check_symbol_exists(if_nametoindex "${CURL_INCLUDES}" HAVE_IF_NAMETOINDEX)
check_symbol_exists(getrlimit "${CURL_INCLUDES}" HAVE_GETRLIMIT)
check_symbol_exists(setlocale "${CURL_INCLUDES}" HAVE_SETLOCALE)
check_symbol_exists(setmode "${CURL_INCLUDES}" HAVE_SETMODE)
check_symbol_exists(setrlimit "${CURL_INCLUDES}" HAVE_SETRLIMIT)
check_symbol_exists(fcntl "${CURL_INCLUDES}" HAVE_FCNTL)
check_symbol_exists(ioctl "${CURL_INCLUDES}" HAVE_IOCTL)
check_symbol_exists(setsockopt "${CURL_INCLUDES}" HAVE_SETSOCKOPT)
check_function_exists(mach_absolute_time HAVE_MACH_ABSOLUTE_TIME)
check_symbol_exists(fsetxattr "${CURL_INCLUDES}" HAVE_FSETXATTR)
if(HAVE_FSETXATTR)
foreach(CURL_TEST HAVE_FSETXATTR_5 HAVE_FSETXATTR_6)
curl_internal_test(${CURL_TEST})
endforeach()
endif()
# sigaction and sigsetjmp are special. Use special mechanism for
# detecting those, but only if previous attempt failed.
if(HAVE_SIGNAL_H)
check_symbol_exists(sigaction "signal.h" HAVE_SIGACTION)
endif()
if(NOT HAVE_SIGSETJMP)
if(HAVE_SETJMP_H)
check_symbol_exists(sigsetjmp "setjmp.h" HAVE_MACRO_SIGSETJMP)
if(HAVE_MACRO_SIGSETJMP)
set(HAVE_SIGSETJMP 1)
endif()
endif()
endif()
# If there is no stricmp(), do not allow LDAP to parse URLs
if(NOT HAVE_STRICMP)
set(HAVE_LDAP_URL_PARSE 1)
endif()
# Do curl specific tests
foreach(CURL_TEST
HAVE_FCNTL_O_NONBLOCK
HAVE_IOCTLSOCKET
HAVE_IOCTLSOCKET_CAMEL
HAVE_IOCTLSOCKET_CAMEL_FIONBIO
HAVE_IOCTLSOCKET_FIONBIO
HAVE_IOCTL_FIONBIO
HAVE_IOCTL_SIOCGIFADDR
HAVE_SETSOCKOPT_SO_NONBLOCK
HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID
TIME_WITH_SYS_TIME
HAVE_O_NONBLOCK
HAVE_GETHOSTBYADDR_R_5
HAVE_GETHOSTBYADDR_R_7
HAVE_GETHOSTBYADDR_R_8
HAVE_GETHOSTBYADDR_R_5_REENTRANT
HAVE_GETHOSTBYADDR_R_7_REENTRANT
HAVE_GETHOSTBYADDR_R_8_REENTRANT
HAVE_GETHOSTBYNAME_R_3
HAVE_GETHOSTBYNAME_R_5
HAVE_GETHOSTBYNAME_R_6
HAVE_GETHOSTBYNAME_R_3_REENTRANT
HAVE_GETHOSTBYNAME_R_5_REENTRANT
HAVE_GETHOSTBYNAME_R_6_REENTRANT
HAVE_IN_ADDR_T
HAVE_BOOL_T
STDC_HEADERS
RETSIGTYPE_TEST
HAVE_INET_NTOA_R_DECL
HAVE_INET_NTOA_R_DECL_REENTRANT
HAVE_GETADDRINFO
HAVE_FILE_OFFSET_BITS
HAVE_VARIADIC_MACROS_C99
HAVE_VARIADIC_MACROS_GCC
)
curl_internal_test(${CURL_TEST})
endforeach()
if(HAVE_FILE_OFFSET_BITS)
set(_FILE_OFFSET_BITS 64)
set(CMAKE_REQUIRED_FLAGS "-D_FILE_OFFSET_BITS=64")
endif()
check_type_size("off_t" SIZEOF_OFF_T)
# include this header to get the type
set(CMAKE_REQUIRED_INCLUDES "${CURL_SOURCE_DIR}/include")
set(CMAKE_EXTRA_INCLUDE_FILES "curl/system.h")
check_type_size("curl_off_t" SIZEOF_CURL_OFF_T)
set(CMAKE_EXTRA_INCLUDE_FILES "")
foreach(CURL_TEST
HAVE_GLIBC_STRERROR_R
HAVE_POSIX_STRERROR_R
)
curl_internal_test(${CURL_TEST})
endforeach()
# Check for reentrant
foreach(CURL_TEST
HAVE_GETHOSTBYADDR_R_5
HAVE_GETHOSTBYADDR_R_7
HAVE_GETHOSTBYADDR_R_8
HAVE_GETHOSTBYNAME_R_3
HAVE_GETHOSTBYNAME_R_5
HAVE_GETHOSTBYNAME_R_6
HAVE_INET_NTOA_R_DECL_REENTRANT)
if(NOT ${CURL_TEST})
if(${CURL_TEST}_REENTRANT)
set(NEED_REENTRANT 1)
endif()
endif()
endforeach()
if(NEED_REENTRANT)
foreach(CURL_TEST
HAVE_GETHOSTBYADDR_R_5
HAVE_GETHOSTBYADDR_R_7
HAVE_GETHOSTBYADDR_R_8
HAVE_GETHOSTBYNAME_R_3
HAVE_GETHOSTBYNAME_R_5
HAVE_GETHOSTBYNAME_R_6)
set(${CURL_TEST} 0)
if(${CURL_TEST}_REENTRANT)
set(${CURL_TEST} 1)
endif()
endforeach()
endif()
if(HAVE_INET_NTOA_R_DECL_REENTRANT)
set(HAVE_INET_NTOA_R_DECL 1)
set(NEED_REENTRANT 1)
endif()
# Check clock_gettime(CLOCK_MONOTONIC, x) support
curl_internal_test(HAVE_CLOCK_GETTIME_MONOTONIC)
# Check compiler support of __builtin_available()
curl_internal_test(HAVE_BUILTIN_AVAILABLE)
# Some other minor tests
if(NOT HAVE_IN_ADDR_T)
set(in_addr_t "unsigned long")
endif()
# Check for nonblocking
set(HAVE_DISABLED_NONBLOCKING 1)
if(HAVE_FIONBIO OR
HAVE_IOCTLSOCKET OR
HAVE_IOCTLSOCKET_CASE OR
HAVE_O_NONBLOCK)
set(HAVE_DISABLED_NONBLOCKING)
endif()
set(CURL_PULL_SYS_TYPES_H ${HAVE_SYS_TYPES_H})
set(CURL_PULL_SYS_SOCKET_H ${HAVE_SYS_SOCKET_H})
set(CURL_PULL_SYS_POLL_H ${HAVE_SYS_POLL_H})
set(CURL_PULL_STDINT_H ${HAVE_STDINT_H})
set(CURL_PULL_INTTYPES_H ${HAVE_INTTYPES_H})
include(CMake/OtherTests.cmake)
SET(LIB_VAUTH_CFILES
"${CURL_LIBRARY_DIR}/vauth/vauth.c" "${CURL_LIBRARY_DIR}/vauth/cleartext.c" "${CURL_LIBRARY_DIR}/vauth/cram.c"
"${CURL_LIBRARY_DIR}/vauth/digest.c" "${CURL_LIBRARY_DIR}/vauth/digest_sspi.c" "${CURL_LIBRARY_DIR}/vauth/krb5_gssapi.c"
"${CURL_LIBRARY_DIR}/vauth/krb5_sspi.c" "${CURL_LIBRARY_DIR}/vauth/ntlm.c" "${CURL_LIBRARY_DIR}/vauth/ntlm_sspi.c" "${CURL_LIBRARY_DIR}/vauth/oauth2.c"
"${CURL_LIBRARY_DIR}/vauth/spnego_gssapi.c" "${CURL_LIBRARY_DIR}/vauth/spnego_sspi.c")
SET(LIB_VAUTH_HFILES "${CURL_LIBRARY_DIR}/vauth/vauth.h" "${CURL_LIBRARY_DIR}/vauth/digest.h" "${CURL_LIBRARY_DIR}/vauth/ntlm.h")
SET(LIB_VTLS_CFILES "${CURL_LIBRARY_DIR}/vtls/openssl.c" "${CURL_LIBRARY_DIR}/vtls/gtls.c" "${CURL_LIBRARY_DIR}/vtls/vtls.c" "${CURL_LIBRARY_DIR}/vtls/nss.c"
"${CURL_LIBRARY_DIR}/vtls/polarssl.c" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.c"
"${CURL_LIBRARY_DIR}/vtls/wolfssl.c" "${CURL_LIBRARY_DIR}/vtls/schannel.c" "${CURL_LIBRARY_DIR}/vtls/schannel_verify.c"
"${CURL_LIBRARY_DIR}/vtls/sectransp.c" "${CURL_LIBRARY_DIR}/vtls/gskit.c" "${CURL_LIBRARY_DIR}/vtls/mbedtls.c" "${CURL_LIBRARY_DIR}/vtls/mesalink.c"
"${CURL_LIBRARY_DIR}/vtls/bearssl.c")
SET(LIB_VTLS_HFILES "${CURL_LIBRARY_DIR}/vtls/openssl.h" "${CURL_LIBRARY_DIR}/vtls/vtls.h" "${CURL_LIBRARY_DIR}/vtls/gtls.h"
"${CURL_LIBRARY_DIR}/vtls/nssg.h" "${CURL_LIBRARY_DIR}/vtls/polarssl.h" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.h"
"${CURL_LIBRARY_DIR}/vtls/wolfssl.h" "${CURL_LIBRARY_DIR}/vtls/schannel.h" "${CURL_LIBRARY_DIR}/vtls/sectransp.h" "${CURL_LIBRARY_DIR}/vtls/gskit.h"
"${CURL_LIBRARY_DIR}/vtls/mbedtls.h" "${CURL_LIBRARY_DIR}/vtls/mesalink.h" "${CURL_LIBRARY_DIR}/vtls/bearssl.h")
SET(LIB_VQUIC_CFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.c" "${CURL_LIBRARY_DIR}/vquic/quiche.c")
SET(LIB_VQUIC_HFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.h" "${CURL_LIBRARY_DIR}/vquic/quiche.h")
SET(LIB_VSSH_CFILES "${CURL_LIBRARY_DIR}/vssh/libssh2.c" "${CURL_LIBRARY_DIR}/vssh/libssh.c")
SET(LIB_VSSH_HFILES "${CURL_LIBRARY_DIR}/vssh/ssh.h")
SET(LIB_CFILES "${CURL_LIBRARY_DIR}/file.c"
"${CURL_LIBRARY_DIR}/timeval.c" "${CURL_LIBRARY_DIR}/base64.c" "${CURL_LIBRARY_DIR}/hostip.c" "${CURL_LIBRARY_DIR}/progress.c" "${CURL_LIBRARY_DIR}/formdata.c"
"${CURL_LIBRARY_DIR}/cookie.c" "${CURL_LIBRARY_DIR}/http.c" "${CURL_LIBRARY_DIR}/sendf.c" "${CURL_LIBRARY_DIR}/url.c" "${CURL_LIBRARY_DIR}/dict.c" "${CURL_LIBRARY_DIR}/if2ip.c" "${CURL_LIBRARY_DIR}/speedcheck.c"
"${CURL_LIBRARY_DIR}/ldap.c" "${CURL_LIBRARY_DIR}/version.c" "${CURL_LIBRARY_DIR}/getenv.c" "${CURL_LIBRARY_DIR}/escape.c" "${CURL_LIBRARY_DIR}/mprintf.c" "${CURL_LIBRARY_DIR}/telnet.c" "${CURL_LIBRARY_DIR}/netrc.c"
"${CURL_LIBRARY_DIR}/getinfo.c" "${CURL_LIBRARY_DIR}/transfer.c" "${CURL_LIBRARY_DIR}/strcase.c" "${CURL_LIBRARY_DIR}/easy.c" "${CURL_LIBRARY_DIR}/security.c" "${CURL_LIBRARY_DIR}/curl_fnmatch.c"
"${CURL_LIBRARY_DIR}/fileinfo.c" "${CURL_LIBRARY_DIR}/wildcard.c" "${CURL_LIBRARY_DIR}/krb5.c" "${CURL_LIBRARY_DIR}/memdebug.c" "${CURL_LIBRARY_DIR}/http_chunks.c"
"${CURL_LIBRARY_DIR}/strtok.c" "${CURL_LIBRARY_DIR}/connect.c" "${CURL_LIBRARY_DIR}/llist.c" "${CURL_LIBRARY_DIR}/hash.c" "${CURL_LIBRARY_DIR}/multi.c" "${CURL_LIBRARY_DIR}/content_encoding.c" "${CURL_LIBRARY_DIR}/share.c"
"${CURL_LIBRARY_DIR}/http_digest.c" "${CURL_LIBRARY_DIR}/md4.c" "${CURL_LIBRARY_DIR}/md5.c" "${CURL_LIBRARY_DIR}/http_negotiate.c" "${CURL_LIBRARY_DIR}/inet_pton.c" "${CURL_LIBRARY_DIR}/strtoofft.c"
"${CURL_LIBRARY_DIR}/strerror.c" "${CURL_LIBRARY_DIR}/amigaos.c" "${CURL_LIBRARY_DIR}/hostasyn.c" "${CURL_LIBRARY_DIR}/hostip4.c" "${CURL_LIBRARY_DIR}/hostip6.c" "${CURL_LIBRARY_DIR}/hostsyn.c"
"${CURL_LIBRARY_DIR}/inet_ntop.c" "${CURL_LIBRARY_DIR}/parsedate.c" "${CURL_LIBRARY_DIR}/select.c" "${CURL_LIBRARY_DIR}/splay.c" "${CURL_LIBRARY_DIR}/strdup.c" "${CURL_LIBRARY_DIR}/socks.c"
"${CURL_LIBRARY_DIR}/curl_addrinfo.c" "${CURL_LIBRARY_DIR}/socks_gssapi.c" "${CURL_LIBRARY_DIR}/socks_sspi.c"
"${CURL_LIBRARY_DIR}/curl_sspi.c" "${CURL_LIBRARY_DIR}/slist.c" "${CURL_LIBRARY_DIR}/nonblock.c" "${CURL_LIBRARY_DIR}/curl_memrchr.c" "${CURL_LIBRARY_DIR}/imap.c" "${CURL_LIBRARY_DIR}/pop3.c" "${CURL_LIBRARY_DIR}/smtp.c"
"${CURL_LIBRARY_DIR}/pingpong.c" "${CURL_LIBRARY_DIR}/rtsp.c" "${CURL_LIBRARY_DIR}/curl_threads.c" "${CURL_LIBRARY_DIR}/warnless.c" "${CURL_LIBRARY_DIR}/hmac.c" "${CURL_LIBRARY_DIR}/curl_rtmp.c"
"${CURL_LIBRARY_DIR}/openldap.c" "${CURL_LIBRARY_DIR}/curl_gethostname.c" "${CURL_LIBRARY_DIR}/gopher.c" "${CURL_LIBRARY_DIR}/idn_win32.c"
"${CURL_LIBRARY_DIR}/http_proxy.c" "${CURL_LIBRARY_DIR}/non-ascii.c" "${CURL_LIBRARY_DIR}/asyn-ares.c" "${CURL_LIBRARY_DIR}/asyn-thread.c" "${CURL_LIBRARY_DIR}/curl_gssapi.c"
"${CURL_LIBRARY_DIR}/http_ntlm.c" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.c" "${CURL_LIBRARY_DIR}/curl_ntlm_core.c" "${CURL_LIBRARY_DIR}/curl_sasl.c" "${CURL_LIBRARY_DIR}/rand.c"
"${CURL_LIBRARY_DIR}/curl_multibyte.c" "${CURL_LIBRARY_DIR}/hostcheck.c" "${CURL_LIBRARY_DIR}/conncache.c" "${CURL_LIBRARY_DIR}/dotdot.c"
"${CURL_LIBRARY_DIR}/x509asn1.c" "${CURL_LIBRARY_DIR}/http2.c" "${CURL_LIBRARY_DIR}/smb.c" "${CURL_LIBRARY_DIR}/curl_endian.c" "${CURL_LIBRARY_DIR}/curl_des.c" "${CURL_LIBRARY_DIR}/system_win32.c"
"${CURL_LIBRARY_DIR}/mime.c" "${CURL_LIBRARY_DIR}/sha256.c" "${CURL_LIBRARY_DIR}/setopt.c" "${CURL_LIBRARY_DIR}/curl_path.c" "${CURL_LIBRARY_DIR}/curl_ctype.c" "${CURL_LIBRARY_DIR}/curl_range.c" "${CURL_LIBRARY_DIR}/psl.c"
"${CURL_LIBRARY_DIR}/doh.c" "${CURL_LIBRARY_DIR}/urlapi.c" "${CURL_LIBRARY_DIR}/curl_get_line.c" "${CURL_LIBRARY_DIR}/altsvc.c" "${CURL_LIBRARY_DIR}/socketpair.c")
SET(LIB_HFILES "${CURL_LIBRARY_DIR}/arpa_telnet.h" "${CURL_LIBRARY_DIR}/netrc.h" "${CURL_LIBRARY_DIR}/file.h" "${CURL_LIBRARY_DIR}/timeval.h" "${CURL_LIBRARY_DIR}/hostip.h" "${CURL_LIBRARY_DIR}/progress.h"
"${CURL_LIBRARY_DIR}/formdata.h" "${CURL_LIBRARY_DIR}/cookie.h" "${CURL_LIBRARY_DIR}/http.h" "${CURL_LIBRARY_DIR}/sendf.h" "${CURL_LIBRARY_DIR}/url.h" "${CURL_LIBRARY_DIR}/dict.h" "${CURL_LIBRARY_DIR}/if2ip.h"
"${CURL_LIBRARY_DIR}/speedcheck.h" "${CURL_LIBRARY_DIR}/urldata.h" "${CURL_LIBRARY_DIR}/curl_ldap.h" "${CURL_LIBRARY_DIR}/escape.h" "${CURL_LIBRARY_DIR}/telnet.h" "${CURL_LIBRARY_DIR}/getinfo.h"
"${CURL_LIBRARY_DIR}/strcase.h" "${CURL_LIBRARY_DIR}/curl_sec.h" "${CURL_LIBRARY_DIR}/memdebug.h" "${CURL_LIBRARY_DIR}/http_chunks.h" "${CURL_LIBRARY_DIR}/curl_fnmatch.h"
"${CURL_LIBRARY_DIR}/wildcard.h" "${CURL_LIBRARY_DIR}/fileinfo.h" "${CURL_LIBRARY_DIR}/strtok.h" "${CURL_LIBRARY_DIR}/connect.h" "${CURL_LIBRARY_DIR}/llist.h"
"${CURL_LIBRARY_DIR}/hash.h" "${CURL_LIBRARY_DIR}/content_encoding.h" "${CURL_LIBRARY_DIR}/share.h" "${CURL_LIBRARY_DIR}/curl_md4.h" "${CURL_LIBRARY_DIR}/curl_md5.h" "${CURL_LIBRARY_DIR}/http_digest.h"
"${CURL_LIBRARY_DIR}/http_negotiate.h" "${CURL_LIBRARY_DIR}/inet_pton.h" "${CURL_LIBRARY_DIR}/amigaos.h" "${CURL_LIBRARY_DIR}/strtoofft.h" "${CURL_LIBRARY_DIR}/strerror.h"
"${CURL_LIBRARY_DIR}/inet_ntop.h" "${CURL_LIBRARY_DIR}/curlx.h" "${CURL_LIBRARY_DIR}/curl_memory.h" "${CURL_LIBRARY_DIR}/curl_setup.h" "${CURL_LIBRARY_DIR}/transfer.h" "${CURL_LIBRARY_DIR}/select.h"
"${CURL_LIBRARY_DIR}/easyif.h" "${CURL_LIBRARY_DIR}/multiif.h" "${CURL_LIBRARY_DIR}/parsedate.h" "${CURL_LIBRARY_DIR}/sockaddr.h" "${CURL_LIBRARY_DIR}/splay.h" "${CURL_LIBRARY_DIR}/strdup.h"
"${CURL_LIBRARY_DIR}/socks.h" "${CURL_LIBRARY_DIR}/curl_base64.h" "${CURL_LIBRARY_DIR}/curl_addrinfo.h" "${CURL_LIBRARY_DIR}/curl_sspi.h"
"${CURL_LIBRARY_DIR}/slist.h" "${CURL_LIBRARY_DIR}/nonblock.h" "${CURL_LIBRARY_DIR}/curl_memrchr.h" "${CURL_LIBRARY_DIR}/imap.h" "${CURL_LIBRARY_DIR}/pop3.h" "${CURL_LIBRARY_DIR}/smtp.h" "${CURL_LIBRARY_DIR}/pingpong.h"
"${CURL_LIBRARY_DIR}/rtsp.h" "${CURL_LIBRARY_DIR}/curl_threads.h" "${CURL_LIBRARY_DIR}/warnless.h" "${CURL_LIBRARY_DIR}/curl_hmac.h" "${CURL_LIBRARY_DIR}/curl_rtmp.h"
"${CURL_LIBRARY_DIR}/curl_gethostname.h" "${CURL_LIBRARY_DIR}/gopher.h" "${CURL_LIBRARY_DIR}/http_proxy.h" "${CURL_LIBRARY_DIR}/non-ascii.h" "${CURL_LIBRARY_DIR}/asyn.h"
"${CURL_LIBRARY_DIR}/http_ntlm.h" "${CURL_LIBRARY_DIR}/curl_gssapi.h" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.h" "${CURL_LIBRARY_DIR}/curl_ntlm_core.h"
"${CURL_LIBRARY_DIR}/curl_sasl.h" "${CURL_LIBRARY_DIR}/curl_multibyte.h" "${CURL_LIBRARY_DIR}/hostcheck.h" "${CURL_LIBRARY_DIR}/conncache.h"
"${CURL_LIBRARY_DIR}/multihandle.h" "${CURL_LIBRARY_DIR}/setup-vms.h" "${CURL_LIBRARY_DIR}/dotdot.h"
"${CURL_LIBRARY_DIR}/x509asn1.h" "${CURL_LIBRARY_DIR}/http2.h" "${CURL_LIBRARY_DIR}/sigpipe.h" "${CURL_LIBRARY_DIR}/smb.h" "${CURL_LIBRARY_DIR}/curl_endian.h" "${CURL_LIBRARY_DIR}/curl_des.h"
"${CURL_LIBRARY_DIR}/curl_printf.h" "${CURL_LIBRARY_DIR}/system_win32.h" "${CURL_LIBRARY_DIR}/rand.h" "${CURL_LIBRARY_DIR}/mime.h" "${CURL_LIBRARY_DIR}/curl_sha256.h" "${CURL_LIBRARY_DIR}/setopt.h"
"${CURL_LIBRARY_DIR}/curl_path.h" "${CURL_LIBRARY_DIR}/curl_ctype.h" "${CURL_LIBRARY_DIR}/curl_range.h" "${CURL_LIBRARY_DIR}/psl.h" "${CURL_LIBRARY_DIR}/doh.h" "${CURL_LIBRARY_DIR}/urlapi-int.h"
"${CURL_LIBRARY_DIR}/curl_get_line.h" "${CURL_LIBRARY_DIR}/altsvc.h" "${CURL_LIBRARY_DIR}/quic.h" "${CURL_LIBRARY_DIR}/socketpair.h")
SET(LIB_RCFILES "${CURL_LIBRARY_DIR}/libcurl.rc")
SET(CSOURCES ${LIB_CFILES} ${LIB_VAUTH_CFILES} ${LIB_VTLS_CFILES}
${LIB_VQUIC_CFILES} ${LIB_VSSH_CFILES})
SET(HHEADERS ${LIB_HFILES} ${LIB_VAUTH_HFILES} ${LIB_VTLS_HFILES}
${LIB_VQUIC_HFILES} ${LIB_VSSH_HFILES})
configure_file(${CURL_SOURCE_DIR}/lib/curl_config.h.cmake
${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h)
list(APPEND HHEADERS
${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h
)
add_library(libcurl ${HHEADERS} ${CSOURCES})
if(NOT BUILD_SHARED_LIBS)
set_target_properties(libcurl PROPERTIES INTERFACE_COMPILE_DEFINITIONS CURL_STATICLIB)
endif()
if(HIDES_CURL_PRIVATE_SYMBOLS)
set_property(TARGET libcurl APPEND PROPERTY COMPILE_DEFINITIONS "CURL_HIDDEN_SYMBOLS")
set_property(TARGET libcurl APPEND PROPERTY COMPILE_FLAGS ${CURL_CFLAG_SYMBOLS_HIDE})
endif()
if(OPENSSL_FOUND)
target_include_directories(libcurl PUBLIC ${OPENSSL_INCLUDE_DIR})
message("-- Including openssl ${OPENSSL_INCLUDE_DIR} to curl")
endif()
if(ZLIB_FOUND)
target_include_directories(libcurl PUBLIC ${ZLIB_INCLUDE_DIRS}})
message("-- Including zlib ${ZLIB_INCLUDE_DIRS} to curl")
endif()
target_compile_definitions(libcurl PUBLIC -DHAVE_CONFIG_H)
target_compile_definitions(libcurl PUBLIC -DBUILDING_LIBCURL)
target_include_directories(libcurl PUBLIC "${CURL_SOURCE_DIR}/include" "${CURL_LIBRARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/curl")
target_link_libraries(libcurl ${CURL_LIBS})
set (CURL_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl)
set (SRCS
${CURL_DIR}/lib/file.c
${CURL_DIR}/lib/timeval.c
${CURL_DIR}/lib/base64.c
${CURL_DIR}/lib/hostip.c
${CURL_DIR}/lib/progress.c
${CURL_DIR}/lib/formdata.c
${CURL_DIR}/lib/cookie.c
${CURL_DIR}/lib/http.c
${CURL_DIR}/lib/sendf.c
${CURL_DIR}/lib/url.c
${CURL_DIR}/lib/dict.c
${CURL_DIR}/lib/if2ip.c
${CURL_DIR}/lib/speedcheck.c
${CURL_DIR}/lib/ldap.c
${CURL_DIR}/lib/version.c
${CURL_DIR}/lib/getenv.c
${CURL_DIR}/lib/escape.c
${CURL_DIR}/lib/mprintf.c
${CURL_DIR}/lib/telnet.c
${CURL_DIR}/lib/netrc.c
${CURL_DIR}/lib/getinfo.c
${CURL_DIR}/lib/transfer.c
${CURL_DIR}/lib/strcase.c
${CURL_DIR}/lib/easy.c
${CURL_DIR}/lib/security.c
${CURL_DIR}/lib/curl_fnmatch.c
${CURL_DIR}/lib/fileinfo.c
${CURL_DIR}/lib/wildcard.c
${CURL_DIR}/lib/krb5.c
${CURL_DIR}/lib/memdebug.c
${CURL_DIR}/lib/http_chunks.c
${CURL_DIR}/lib/strtok.c
${CURL_DIR}/lib/connect.c
${CURL_DIR}/lib/llist.c
${CURL_DIR}/lib/hash.c
${CURL_DIR}/lib/multi.c
${CURL_DIR}/lib/content_encoding.c
${CURL_DIR}/lib/share.c
${CURL_DIR}/lib/http_digest.c
${CURL_DIR}/lib/md4.c
${CURL_DIR}/lib/md5.c
${CURL_DIR}/lib/http_negotiate.c
${CURL_DIR}/lib/inet_pton.c
${CURL_DIR}/lib/strtoofft.c
${CURL_DIR}/lib/strerror.c
${CURL_DIR}/lib/amigaos.c
${CURL_DIR}/lib/hostasyn.c
${CURL_DIR}/lib/hostip4.c
${CURL_DIR}/lib/hostip6.c
${CURL_DIR}/lib/hostsyn.c
${CURL_DIR}/lib/inet_ntop.c
${CURL_DIR}/lib/parsedate.c
${CURL_DIR}/lib/select.c
${CURL_DIR}/lib/splay.c
${CURL_DIR}/lib/strdup.c
${CURL_DIR}/lib/socks.c
${CURL_DIR}/lib/curl_addrinfo.c
${CURL_DIR}/lib/socks_gssapi.c
${CURL_DIR}/lib/socks_sspi.c
${CURL_DIR}/lib/curl_sspi.c
${CURL_DIR}/lib/slist.c
${CURL_DIR}/lib/nonblock.c
${CURL_DIR}/lib/curl_memrchr.c
${CURL_DIR}/lib/imap.c
${CURL_DIR}/lib/pop3.c
${CURL_DIR}/lib/smtp.c
${CURL_DIR}/lib/pingpong.c
${CURL_DIR}/lib/rtsp.c
${CURL_DIR}/lib/curl_threads.c
${CURL_DIR}/lib/warnless.c
${CURL_DIR}/lib/hmac.c
${CURL_DIR}/lib/curl_rtmp.c
${CURL_DIR}/lib/openldap.c
${CURL_DIR}/lib/curl_gethostname.c
${CURL_DIR}/lib/gopher.c
${CURL_DIR}/lib/idn_win32.c
${CURL_DIR}/lib/http_proxy.c
${CURL_DIR}/lib/non-ascii.c
${CURL_DIR}/lib/asyn-thread.c
${CURL_DIR}/lib/curl_gssapi.c
${CURL_DIR}/lib/http_ntlm.c
${CURL_DIR}/lib/curl_ntlm_wb.c
${CURL_DIR}/lib/curl_ntlm_core.c
${CURL_DIR}/lib/curl_sasl.c
${CURL_DIR}/lib/rand.c
${CURL_DIR}/lib/curl_multibyte.c
${CURL_DIR}/lib/hostcheck.c
${CURL_DIR}/lib/conncache.c
${CURL_DIR}/lib/dotdot.c
${CURL_DIR}/lib/x509asn1.c
${CURL_DIR}/lib/http2.c
${CURL_DIR}/lib/smb.c
${CURL_DIR}/lib/curl_endian.c
${CURL_DIR}/lib/curl_des.c
${CURL_DIR}/lib/system_win32.c
${CURL_DIR}/lib/mime.c
${CURL_DIR}/lib/sha256.c
${CURL_DIR}/lib/setopt.c
${CURL_DIR}/lib/curl_path.c
${CURL_DIR}/lib/curl_ctype.c
${CURL_DIR}/lib/curl_range.c
${CURL_DIR}/lib/psl.c
${CURL_DIR}/lib/doh.c
${CURL_DIR}/lib/urlapi.c
${CURL_DIR}/lib/curl_get_line.c
${CURL_DIR}/lib/altsvc.c
${CURL_DIR}/lib/socketpair.c
${CURL_DIR}/lib/vauth/vauth.c
${CURL_DIR}/lib/vauth/cleartext.c
${CURL_DIR}/lib/vauth/cram.c
${CURL_DIR}/lib/vauth/digest.c
${CURL_DIR}/lib/vauth/digest_sspi.c
${CURL_DIR}/lib/vauth/krb5_gssapi.c
${CURL_DIR}/lib/vauth/krb5_sspi.c
${CURL_DIR}/lib/vauth/ntlm.c
${CURL_DIR}/lib/vauth/ntlm_sspi.c
${CURL_DIR}/lib/vauth/oauth2.c
${CURL_DIR}/lib/vauth/spnego_gssapi.c
${CURL_DIR}/lib/vauth/spnego_sspi.c
${CURL_DIR}/lib/vtls/openssl.c
${CURL_DIR}/lib/vtls/gtls.c
${CURL_DIR}/lib/vtls/vtls.c
${CURL_DIR}/lib/vtls/nss.c
${CURL_DIR}/lib/vtls/polarssl.c
${CURL_DIR}/lib/vtls/polarssl_threadlock.c
${CURL_DIR}/lib/vtls/wolfssl.c
${CURL_DIR}/lib/vtls/schannel.c
${CURL_DIR}/lib/vtls/schannel_verify.c
${CURL_DIR}/lib/vtls/sectransp.c
${CURL_DIR}/lib/vtls/gskit.c
${CURL_DIR}/lib/vtls/mbedtls.c
${CURL_DIR}/lib/vtls/mesalink.c
${CURL_DIR}/lib/vtls/bearssl.c
${CURL_DIR}/lib/vquic/ngtcp2.c
${CURL_DIR}/lib/vquic/quiche.c
${CURL_DIR}/lib/vssh/libssh2.c
${CURL_DIR}/lib/vssh/libssh.c
)
add_library (curl ${SRCS})
target_compile_definitions(curl PRIVATE HAVE_CONFIG_H BUILDING_LIBCURL CURL_HIDDEN_SYMBOLS libcurl_EXPORTS)
target_include_directories(curl PUBLIC ${CURL_DIR}/include ${CURL_DIR}/lib .)
target_compile_definitions(curl PRIVATE OS="${CMAKE_SYSTEM_NAME}")

38
contrib/curl-cmake/curl_config.h Normal file
View File

@ -0,0 +1,38 @@
#define CURL_DISABLE_FTP
#define CURL_DISABLE_TFTP
#define CURL_DISABLE_LDAP
#define CURL_EXTERN_SYMBOL __attribute__ ((__visibility__ ("default")))
#define SIZEOF_SHORT 2
#define SIZEOF_INT 4
#define SIZEOF_LONG 8
#define SIZEOF_CURL_OFF_T 8
#define SIZEOF_SIZE_T 8
#define HAVE_FCNTL_O_NONBLOCK
#define HAVE_LONGLONG
#define HAVE_POLL_FINE
#define HAVE_SOCKET
#define HAVE_STRUCT_TIMEVAL
#define HAVE_RECV
#define RECV_TYPE_ARG1 int
#define RECV_TYPE_ARG2 void*
#define RECV_TYPE_ARG3 size_t
#define RECV_TYPE_ARG4 int
#define RECV_TYPE_RETV ssize_t
#define HAVE_SEND
#define SEND_TYPE_ARG1 int
#define SEND_TYPE_ARG2 void*
#define SEND_QUAL_ARG2 const
#define SEND_TYPE_ARG3 size_t
#define SEND_TYPE_ARG4 int
#define SEND_TYPE_RETV ssize_t
#define HAVE_ARPA_INET_H
#define HAVE_ERRNO_H
#define HAVE_FCNTL_H
#define HAVE_NETDB_H
#define HAVE_SYS_STAT_H
#define HAVE_UNISTD_H

2
contrib/googletest vendored

@ -1 +1 @@
Subproject commit d175c8bf823e709d570772b038757fadf63bc632
Subproject commit 703bd9caab50b139428cea1aaff9974ebee5742e

10
contrib/icu-cmake/CMakeLists.txt
View File

@ -437,8 +437,12 @@ ${ICU_SOURCE_DIR}/i18n/formattedval_iterimpl.cpp
${ICU_SOURCE_DIR}/i18n/formattedval_sbimpl.cpp
${ICU_SOURCE_DIR}/i18n/formatted_string_builder.cpp)
file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/empty.cpp CONTENT " ")
enable_language(ASM)
set(ICUDATA_SOURCES ${ICUDATA_SOURCE_DIR}/icudt66l_dat.S)
set(ICUDATA_SOURCES
${ICUDATA_SOURCE_DIR}/icudt66l_dat.S
${CMAKE_CURRENT_BINARY_DIR}/empty.cpp # Without this cmake can incorrectly detects library type (OBJECT) instead of SHARED/STATIC
)
# Note that we don't like any kind of binary plugins (because of runtime dependencies, vulnerabilities, ABI incompatibilities).
add_definitions(-D_REENTRANT -DU_HAVE_ELF_H=1 -DU_HAVE_STRTOD_L=1 -DU_HAVE_XLOCALE_H=0 -DDEFAULT_ICU_PLUGINS="/dev/null")
@ -447,8 +451,8 @@ add_library(icuuc ${ICUUC_SOURCES})
add_library(icui18n ${ICUI18N_SOURCES})
add_library(icudata ${ICUDATA_SOURCES})
target_link_libraries(icuuc icudata)
target_link_libraries(icui18n icuuc)
target_link_libraries(icuuc PRIVATE icudata)
target_link_libraries(icui18n PRIVATE icuuc)
target_include_directories(icuuc SYSTEM PUBLIC ${ICU_SOURCE_DIR}/common/)
target_include_directories(icui18n SYSTEM PUBLIC ${ICU_SOURCE_DIR}/i18n/)

7
contrib/libcxx-cmake/CMakeLists.txt
View File

@ -54,13 +54,12 @@ endif ()
target_compile_options(cxx PUBLIC $<$<COMPILE_LANGUAGE:CXX>:-nostdinc++>)
check_cxx_compiler_flag(-Wreserved-id-macro HAVE_WARNING_RESERVED_ID_MACRO)
if (HAVE_WARNING_RESERVED_ID_MACRO)
if (SUPPORTS_FLAG_no_reserved_id_macro)
target_compile_options(cxx PUBLIC -Wno-reserved-id-macro)
endif ()
check_cxx_compiler_flag(-Wctad-maybe-unsupported HAVE_WARNING_CTAD_MAYBE_UNSUPPORTED)
if (HAVE_WARNING_CTAD_MAYBE_UNSUPPORTED)
if (SUPPORTS_FLAG_no_ctad_maybe_unsupported)
target_compile_options(cxx PUBLIC -Wno-ctad-maybe-unsupported)
endif ()

8
contrib/murmurhash/include/murmurhash3.h
View File

@ -23,6 +23,10 @@ typedef unsigned __int64 uint64_t;
#endif // !defined(_MSC_VER)
#ifdef __cplusplus
extern "C" {
#endif
//-----------------------------------------------------------------------------
void MurmurHash3_x86_32 ( const void * key, int len, uint32_t seed, void * out );
@ -32,3 +36,7 @@ void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out
void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out );
//-----------------------------------------------------------------------------
#ifdef __cplusplus
}
#endif

1
contrib/replxx vendored Submodule

@ -0,0 +1 @@
Subproject commit 37582f0bb8c52513c6c6b76797c02d852d701dad

57
contrib/replxx-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,57 @@
option (ENABLE_REPLXX "Enable replxx support" ${ENABLE_LIBRARIES})
if (ENABLE_REPLXX)
option (USE_INTERNAL_REPLXX "Use internal replxx library" ${NOT_UNBUNDLED})
if (USE_INTERNAL_REPLXX)
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx")
set(SRCS
${LIBRARY_DIR}/src/conversion.cxx
${LIBRARY_DIR}/src/ConvertUTF.cpp
${LIBRARY_DIR}/src/escape.cxx
${LIBRARY_DIR}/src/history.cxx
${LIBRARY_DIR}/src/io.cxx
${LIBRARY_DIR}/src/prompt.cxx
${LIBRARY_DIR}/src/replxx_impl.cxx
${LIBRARY_DIR}/src/replxx.cxx
${LIBRARY_DIR}/src/util.cxx
${LIBRARY_DIR}/src/wcwidth.cpp
)
add_library (replxx ${SRCS})
target_include_directories(replxx PUBLIC ${LIBRARY_DIR}/include)
else ()
find_library(LIBRARY_REPLXX NAMES replxx replxx-static)
find_path(INCLUDE_REPLXX replxx.hxx)
add_library(replxx UNKNOWN IMPORTED)
set_property(TARGET replxx PROPERTY IMPORTED_LOCATION ${LIBRARY_REPLXX})
target_include_directories(replxx PUBLIC ${INCLUDE_REPLXX})
set(CMAKE_REQUIRED_LIBRARIES replxx)
check_cxx_source_compiles(
"
#include <replxx.hxx>
int main() {
replxx::Replxx rx;
}
"
EXTERNAL_REPLXX_WORKS
)
if (NOT EXTERNAL_REPLXX_WORKS)
message (FATAL_ERROR "replxx is unusable: ${LIBRARY_REPLXX} ${INCLUDE_REPLXX}")
endif ()
endif ()
target_compile_options(replxx PUBLIC -Wno-documentation)
target_compile_definitions(replxx PUBLIC USE_REPLXX=1)
message (STATUS "Using replxx")
else ()
add_library(replxx INTERFACE)
target_compile_definitions(replxx INTERFACE USE_REPLXX=0)
message (STATUS "Not using replxx (Beware! Runtime fallback to readline is possible!)")
endif ()

115
dbms/CMakeLists.txt
View File

@ -45,36 +45,75 @@ endif ()
option (WEVERYTHING "Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang." ON)
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wpedantic -Wno-vla-extension -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types")
if (COMPILER_CLANG)
add_warning(pedantic)
no_warning(gnu-anonymous-struct)
no_warning(nested-anon-types)
no_warning(vla-extension)
no_warning(zero-length-array)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow -Wshadow-uncaptured-local -Wextra-semi -Wcomma -Winconsistent-missing-destructor-override -Wunused-exception-parameter -Wcovered-switch-default -Wold-style-cast -Wrange-loop-analysis -Wunused-member-function -Wunreachable-code -Wunreachable-code-return -Wnewline-eof -Wembedded-directive -Wgnu-case-range -Wunused-macros -Wconditional-uninitialized -Wdeprecated -Wundef -Wreserved-id-macro -Wredundant-parens -Wzero-as-null-pointer-constant")
add_warning(comma)
add_warning(conditional-uninitialized)
add_warning(covered-switch-default)
add_warning(deprecated)
add_warning(embedded-directive)
add_warning(empty-init-stmt) # linux-only
add_warning(extra-semi-stmt) # linux-only
add_warning(extra-semi)
add_warning(gnu-case-range)
add_warning(inconsistent-missing-destructor-override)
add_warning(newline-eof)
add_warning(old-style-cast)
add_warning(range-loop-analysis)
add_warning(redundant-parens)
add_warning(reserved-id-macro)
add_warning(shadow-field) # clang 8+
add_warning(shadow-uncaptured-local)
add_warning(shadow)
add_warning(string-plus-int) # clang 8+
add_warning(undef)
add_warning(unreachable-code-return)
add_warning(unreachable-code)
add_warning(unused-exception-parameter)
add_warning(unused-macros)
add_warning(unused-member-function)
add_warning(zero-as-null-pointer-constant)
if (WEVERYTHING)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-padded -Wno-switch-enum -Wno-deprecated-dynamic-exception-spec -Wno-float-equal -Wno-weak-vtables -Wno-shift-sign-overflow -Wno-sign-conversion -Wno-conversion -Wno-exit-time-destructors -Wno-undefined-func-template -Wno-documentation-unknown-command -Wno-missing-variable-declarations -Wno-unused-template -Wno-global-constructors -Wno-c99-extensions -Wno-missing-prototypes -Wno-weak-template-vtables -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types -Wno-double-promotion -Wno-disabled-macro-expansion -Wno-vla-extension -Wno-vla -Wno-packed")
add_warning(everything)
no_warning(c++98-compat-pedantic)
no_warning(c++98-compat)
no_warning(c99-extensions)
no_warning(conversion)
no_warning(ctad-maybe-unsupported) # clang 9+, linux-only
no_warning(deprecated-dynamic-exception-spec)
no_warning(disabled-macro-expansion)
no_warning(documentation-unknown-command)
no_warning(double-promotion)
no_warning(exit-time-destructors)
no_warning(float-equal)
no_warning(global-constructors)
no_warning(gnu-anonymous-struct)
no_warning(missing-prototypes)
no_warning(missing-variable-declarations)
no_warning(nested-anon-types)
no_warning(packed)
no_warning(padded)
no_warning(return-std-move-in-c++11) # clang 7+
no_warning(shift-sign-overflow)
no_warning(sign-conversion)
no_warning(switch-enum)
no_warning(undefined-func-template)
no_warning(unused-template)
no_warning(vla-extension)
no_warning(vla)
no_warning(weak-template-vtables)
no_warning(weak-vtables)
no_warning(zero-length-array)
# TODO Enable conversion, sign-conversion, double-promotion warnings.
endif ()
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7)
if (WEVERYTHING)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-std-move-in-c++11")
endif ()
endif ()
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow-field -Wstring-plus-int")
if(NOT APPLE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra-semi-stmt -Wempty-init-stmt")
endif()
endif ()
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9)
if (WEVERYTHING AND NOT APPLE)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-ctad-maybe-unsupported")
endif ()
endif ()
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
elseif (COMPILER_GCC)
# Add compiler options only to c++ compiler
function(add_cxx_compile_options option)
add_compile_options("$<$<STREQUAL:$<TARGET_PROPERTY:LINKER_LANGUAGE>,CXX>:${option}>")
@ -103,10 +142,10 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
add_cxx_compile_options(-Wmaybe-uninitialized)
# Warn when the indentation of the code does not reflect the block structure
add_cxx_compile_options(-Wmisleading-indentation)
# Warn if a global function is defined without a previous declaration
# Warn if a global function is defined without a previous declaration - disabled because of build times
# add_cxx_compile_options(-Wmissing-declarations)
# Warn if a user-supplied include directory does not exist
# add_cxx_compile_options(-Wmissing-include-dirs)
add_cxx_compile_options(-Wmissing-include-dirs)
# Obvious
add_cxx_compile_options(-Wnon-virtual-dtor)
# Obvious
@ -138,7 +177,7 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
# Warn for suspicious length parameters to certain string and memory built-in functions if the argument uses sizeof
add_cxx_compile_options(-Wsizeof-pointer-memaccess)
# Warn about overriding virtual functions that are not marked with the override keyword
# add_cxx_compile_options(-Wsuggest-override)
add_cxx_compile_options(-Wsuggest-override)
# Warn whenever a switch statement has an index of boolean type and the case values are outside the range of a boolean type
add_cxx_compile_options(-Wswitch-bool)
# Warn if a self-comparison always evaluates to true or false
@ -156,7 +195,7 @@ if (USE_DEBUG_HELPERS)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${INCLUDE_DEBUG_HELPERS}")
endif ()
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
if (COMPILER_GCC)
# If we leave this optimization enabled, gcc-7 replaces a pair of SSE intrinsics (16 byte load, store) with a call to memcpy.
# It leads to slow code. This is compiler bug. It looks like this:
#
@ -465,6 +504,10 @@ if (USE_POCO_NETSSL)
dbms_target_link_libraries (PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
endif()
if (USE_POCO_JSON)
dbms_target_link_libraries (PRIVATE ${Poco_JSON_LIBRARY})
endif()
dbms_target_link_libraries (PRIVATE ${Poco_Foundation_LIBRARY})
if (USE_ICU)
@ -483,6 +526,11 @@ if (USE_PARQUET)
endif ()
endif ()
if (USE_AVRO)
dbms_target_link_libraries(PRIVATE ${AVROCPP_LIBRARY})
dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${AVROCPP_INCLUDE_DIR})
endif ()
if (OPENSSL_CRYPTO_LIBRARY)
dbms_target_link_libraries (PRIVATE ${OPENSSL_CRYPTO_LIBRARY})
target_link_libraries (clickhouse_common_io PRIVATE ${OPENSSL_CRYPTO_LIBRARY})
@ -524,7 +572,7 @@ if (USE_JEMALLOC)
endif()
endif ()
dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/Formats/include)
dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR})
target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR})
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR})
@ -546,7 +594,14 @@ if (ENABLE_TESTS AND USE_GTEST)
add_executable(unit_tests_dbms ${dbms_gtest_sources})
# gtest framework has substandard code
target_compile_options(unit_tests_dbms PRIVATE -Wno-zero-as-null-pointer-constant -Wno-undef -Wno-sign-compare -Wno-used-but-marked-unused -Wno-missing-noreturn)
target_compile_options(unit_tests_dbms PRIVATE
-Wno-zero-as-null-pointer-constant
-Wno-undef
-Wno-sign-compare
-Wno-used-but-marked-unused
-Wno-missing-noreturn
-Wno-gnu-zero-variadic-macro-arguments
)
target_link_libraries(unit_tests_dbms PRIVATE ${GTEST_BOTH_LIBRARIES} clickhouse_functions clickhouse_parsers dbms clickhouse_common_zookeeper string_utils)
add_check(unit_tests_dbms)

22
dbms/benchmark/clickhouse/benchmark-chyt.sh Executable file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env bash
QUERIES_FILE="queries.sql"
TABLE=$1
TRIES=3
cat "$QUERIES_FILE" | sed "s|{table}|\"${TABLE}\"|g" | while read query; do
echo -n "["
for i in $(seq 1 $TRIES); do
while true; do
RES=$(command time -f %e -o /dev/stdout curl -sS -G --data-urlencode "query=$query" --data "default_format=Null&max_memory_usage=100000000000&max_memory_usage_for_all_queries=100000000000&max_concurrent_queries_for_user=100&database=*$YT_CLIQUE_ID" --location-trusted -H "Authorization: OAuth $YT_TOKEN" "$YT_PROXY.yt.yandex.net/query" 2>/dev/null);
if [[ $? == 0 ]]; then
[[ $RES =~ 'fail|Exception' ]] || break;
fi
done
[[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null"
[[ "$i" != $TRIES ]] && echo -n ", "
done
echo "],"
done

19
dbms/benchmark/clickhouse/benchmark-yql.sh Executable file
View File

@ -0,0 +1,19 @@
#!/usr/bin/env bash
QUERIES_FILE="queries.sql"
TABLE=$1
TRIES=3
cat "$QUERIES_FILE" | sed "s|{table}|\"${TABLE}\"|g" | while read query; do
echo -n "["
for i in $(seq 1 $TRIES); do
while true; do
RES=$(command time -f %e -o time ./yql --clickhouse --syntax-version 1 -f empty <<< "USE chyt.hume; PRAGMA max_memory_usage = 100000000000; PRAGMA max_memory_usage_for_all_queries = 100000000000; $query" >/dev/null 2>&1 && cat time) && break;
done
[[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null"
[[ "$i" != $TRIES ]] && echo -n ", "
done
echo "],"
done

14
dbms/benchmark/clickhouse/queries.sql
View File

@ -34,10 +34,10 @@ SELECT WatchID, ClientIP, count() AS c, sum(Refresh), avg(ResolutionWidth) FROM
SELECT URL, count() AS c FROM {table} GROUP BY URL ORDER BY c DESC LIMIT 10;
SELECT 1, URL, count() AS c FROM {table} GROUP BY 1, URL ORDER BY c DESC LIMIT 10;
SELECT ClientIP AS x, x - 1, x - 2, x - 3, count() AS c FROM {table} GROUP BY x, x - 1, x - 2, x - 3 ORDER BY c DESC LIMIT 10;
SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND notEmpty(URL) GROUP BY URL ORDER BY PageViews DESC LIMIT 10;
SELECT Title, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT DontCountHits AND NOT Refresh AND notEmpty(Title) GROUP BY Title ORDER BY PageViews DESC LIMIT 10;
SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000;
SELECT TraficSourceID, SearchEngineID, AdvEngineID, ((SearchEngineID = 0 AND AdvEngineID = 0) ? Referer : '') AS Src, URL AS Dst, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000;
SELECT URLHash, EventDate, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = halfMD5('http://example.ru/') GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100;
SELECT WindowClientWidth, WindowClientHeight, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-31') AND NOT Refresh AND NOT DontCountHits AND URLHash = halfMD5('http://example.ru/') GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000;
SELECT toStartOfMinute(EventTime) AS Minute, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= toDate('2013-07-01') AND EventDate <= toDate('2013-07-02') AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute;
SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND notEmpty(URL) GROUP BY URL ORDER BY PageViews DESC LIMIT 10;
SELECT Title, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT DontCountHits AND NOT Refresh AND notEmpty(Title) GROUP BY Title ORDER BY PageViews DESC LIMIT 10;
SELECT URL, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND IsLink AND NOT IsDownload GROUP BY URL ORDER BY PageViews DESC LIMIT 1000;
SELECT TraficSourceID, SearchEngineID, AdvEngineID, ((SearchEngineID = 0 AND AdvEngineID = 0) ? Referer : '') AS Src, URL AS Dst, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 1000;
SELECT URLHash, EventDate, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND TraficSourceID IN (-1, 6) AND RefererHash = halfMD5('http://example.ru/') GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 100;
SELECT WindowClientWidth, WindowClientHeight, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND NOT Refresh AND NOT DontCountHits AND URLHash = halfMD5('http://example.ru/') GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10000;
SELECT toStartOfMinute(EventTime) AS Minute, count() AS PageViews FROM {table} WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-02' AND NOT Refresh AND NOT DontCountHits GROUP BY Minute ORDER BY Minute;

10
dbms/cmake/version.cmake
View File

@ -1,11 +1,11 @@
# This strings autochanged from release_lib.sh:
set(VERSION_REVISION 54431)
set(VERSION_REVISION 54432)
set(VERSION_MAJOR 20)
set(VERSION_MINOR 1)
set(VERSION_MINOR 2)
set(VERSION_PATCH 1)
set(VERSION_GITHASH 51d4c8a53be94504e3607b2232e12e5ef7a8ec28)
set(VERSION_DESCRIBE v20.1.1.1-prestable)
set(VERSION_STRING 20.1.1.1)
set(VERSION_GITHASH 4b9acaaa9099e71c36e5c818031149c5cba2bbdb)
set(VERSION_DESCRIBE v20.2.1.1-prestable)
set(VERSION_STRING 20.2.1.1)
# end of autochange
set(VERSION_EXTRA "" CACHE STRING "")

12
dbms/programs/benchmark/Benchmark.cpp
View File

@ -101,7 +101,7 @@ public:
}
void initialize(Poco::Util::Application & self [[maybe_unused]])
void initialize(Poco::Util::Application & self [[maybe_unused]]) override
{
std::string home_path;
const char * home_path_cstr = getenv("HOME");
@ -111,7 +111,7 @@ public:
configReadClient(config(), home_path);
}
int main(const std::vector<std::string> &)
int main(const std::vector<std::string> &) override
{
if (!json_path.empty() && Poco::File(json_path).exists()) /// Clear file with previous results
Poco::File(json_path).remove();
@ -254,7 +254,7 @@ private:
if (interrupt_listener.check())
{
std::cout << "Stopping launch of queries. SIGINT recieved.\n";
std::cout << "Stopping launch of queries. SIGINT received.\n";
return false;
}
@ -418,7 +418,7 @@ private:
std::cerr << percent << "%\t\t";
for (const auto & info : infos)
{
std::cerr << info->sampler.quantileInterpolated(percent / 100.0) << " sec." << "\t";
std::cerr << info->sampler.quantileNearest(percent / 100.0) << " sec." << "\t";
}
std::cerr << "\n";
};
@ -453,7 +453,7 @@ private:
auto print_percentile = [&json_out](Stats & info, auto percent, bool with_comma = true)
{
json_out << "\"" << percent << "\"" << ": " << info.sampler.quantileInterpolated(percent / 100.0) << (with_comma ? ",\n" : "\n");
json_out << "\"" << percent << "\"" << ": " << info.sampler.quantileNearest(percent / 100.0) << (with_comma ? ",\n" : "\n");
};
json_out << "{\n";
@ -492,7 +492,7 @@ private:
public:
~Benchmark()
~Benchmark() override
{
shutdown = true;
}

8
dbms/programs/client/CMakeLists.txt
View File

@ -1,14 +1,10 @@
set(CLICKHOUSE_CLIENT_SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/Client.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ConnectionParameters.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Suggest.cpp
)
set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
set(CLICKHOUSE_CLIENT_INCLUDE PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include)
if (READLINE_INCLUDE_DIR)
set(CLICKHOUSE_CLIENT_INCLUDE ${CLICKHOUSE_CLIENT_INCLUDE} SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
endif ()
set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${Boost_PROGRAM_OPTIONS_LIBRARY})
include(CheckSymbolExists)
check_symbol_exists(readpassphrase readpassphrase.h HAVE_READPASSPHRASE)

299
dbms/programs/client/Client.cpp
View File

@ -1,7 +1,13 @@
#include "TestHint.h"
#include "ConnectionParameters.h"
#include "Suggest.h"
#if USE_REPLXX
# include <common/ReplxxLineReader.h>
#else
# include <common/LineReader.h>
#endif
#include <port/unistd.h>
#include <stdlib.h>
#include <fcntl.h>
#include <signal.h>
@ -18,8 +24,8 @@
#include <Poco/String.h>
#include <Poco/File.h>
#include <Poco/Util/Application.h>
#include <common/readline_use.h>
#include <common/find_symbols.h>
#include <common/LineReader.h>
#include <Common/ClickHouseRevision.h>
#include <Common/Stopwatch.h>
#include <Common/Exception.h>
@ -69,10 +75,6 @@
#include <common/argsToConfig.h>
#include <Common/TerminalSize.h>
#if USE_READLINE
#include "Suggest.h"
#endif
#ifndef __clang__
#pragma GCC optimize("-fno-var-tracking-assignments")
#endif
@ -89,39 +91,6 @@
#define DISABLE_LINE_WRAPPING "\033[?7l"
#define ENABLE_LINE_WRAPPING "\033[?7h"
#if USE_READLINE && RL_VERSION_MAJOR >= 7
#define BRACK_PASTE_PREF "\033[200~"
#define BRACK_PASTE_SUFF "\033[201~"
#define BRACK_PASTE_LAST '~'
#define BRACK_PASTE_SLEN 6
/// This handler bypasses some unused macro/event checkings.
static int clickhouse_rl_bracketed_paste_begin(int /* count */, int /* key */)
{
std::string buf;
buf.reserve(128);
RL_SETSTATE(RL_STATE_MOREINPUT);
SCOPE_EXIT(RL_UNSETSTATE(RL_STATE_MOREINPUT));
int c;
while ((c = rl_read_key()) >= 0)
{
if (c == '\r')
c = '\n';
buf.push_back(c);
if (buf.size() >= BRACK_PASTE_SLEN && c == BRACK_PASTE_LAST && buf.substr(buf.size() - BRACK_PASTE_SLEN) == BRACK_PASTE_SUFF)
{
buf.resize(buf.size() - BRACK_PASTE_SLEN);
break;
}
}
return static_cast<size_t>(rl_insert_text(buf.c_str())) == buf.size() ? 0 : 1;
}
#endif
namespace DB
{
@ -135,9 +104,7 @@ namespace ErrorCodes
extern const int UNKNOWN_PACKET_FROM_SERVER;
extern const int UNEXPECTED_PACKET_FROM_SERVER;
extern const int CLIENT_OUTPUT_FORMAT_SPECIFIED;
extern const int LOGICAL_ERROR;
extern const int CANNOT_SET_SIGNAL_HANDLER;
extern const int CANNOT_READLINE;
extern const int SYSTEM_ERROR;
extern const int INVALID_USAGE_OF_INPUT;
}
@ -158,7 +125,7 @@ private:
"учшеж", "йгшеж", "дщпщгеж",
"q", "й", "\\q", "\\Q", "\\й", "\\Й", ":q", "Жй"
};
bool is_interactive = true; /// Use either readline interface or batch mode.
bool is_interactive = true; /// Use either interactive line editing interface or batch mode.
bool need_render_progress = true; /// Render query execution progress.
bool echo_queries = false; /// Print queries before execution in batch mode.
bool ignore_error = false; /// In case of errors, don't print error message, continue to next query. Only applicable for non-interactive mode.
@ -243,7 +210,7 @@ private:
ConnectionParameters connection_parameters;
void initialize(Poco::Util::Application & self)
void initialize(Poco::Util::Application & self) override
{
Poco::Util::Application::initialize(self);
@ -271,7 +238,7 @@ private:
}
int main(const std::vector<std::string> & /*args*/)
int main(const std::vector<std::string> & /*args*/) override
{
try
{
@ -515,26 +482,10 @@ private:
if (print_time_to_stderr)
throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS);
#if USE_READLINE
SCOPE_EXIT({ Suggest::instance().finalize(); });
if (server_revision >= Suggest::MIN_SERVER_REVISION
&& !config().getBool("disable_suggestion", false))
{
if (server_revision >= Suggest::MIN_SERVER_REVISION && !config().getBool("disable_suggestion", false))
/// Load suggestion data from the server.
Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit"));
/// Added '.' to the default list. Because it is used to separate database and table.
rl_basic_word_break_characters = " \t\n\r\"\\'`@$><=;|&{(.";
/// Not append whitespace after single suggestion. Because whitespace after function name is meaningless.
rl_completion_append_character = '\0';
rl_completion_entry_function = Suggest::generator;
}
else
/// Turn tab completion off.
rl_bind_key('\t', rl_insert);
#endif
/// Load command history if present.
if (config().has("history_file"))
history_file = config().getString("history_file");
@ -547,70 +498,56 @@ private:
history_file = home_path + "/.clickhouse-client-history";
}
if (!history_file.empty())
{
if (Poco::File(history_file).exists())
{
#if USE_READLINE
int res = read_history(history_file.c_str());
if (res)
std::cerr << "Cannot read history from file " + history_file + ": "+ errnoToString(ErrorCodes::CANNOT_READ_HISTORY);
if (!history_file.empty() && !Poco::File(history_file).exists())
Poco::File(history_file).createFile();
#if USE_REPLXX
ReplxxLineReader lr(Suggest::instance(), history_file, '\\', config().has("multiline") ? ';' : 0);
#else
LineReader lr(history_file, '\\', config().has("multiline") ? ';' : 0);
#endif
do
{
auto input = lr.readLine(prompt(), ":-] ");
if (input.empty())
break;
has_vertical_output_suffix = false;
if (input.ends_with("\\G"))
{
input.resize(input.size() - 2);
has_vertical_output_suffix = true;
}
try
{
if (!process(input))
break;
}
catch (const Exception & e)
{
actual_client_error = e.code();
if (!actual_client_error || actual_client_error != expected_client_error)
{
std::cerr << std::endl
<< "Exception on client:" << std::endl
<< "Code: " << e.code() << ". " << e.displayText() << std::endl;
if (config().getBool("stacktrace", false))
std::cerr << "Stack trace:" << std::endl << e.getStackTraceString() << std::endl;
std::cerr << std::endl;
}
/// Client-side exception during query execution can result in the loss of
/// sync in the connection protocol.
/// So we reconnect and allow to enter the next query.
connect();
}
else /// Create history file.
Poco::File(history_file).createFile();
}
#if USE_READLINE
/// Install Ctrl+C signal handler that will be used in interactive mode.
if (rl_initialize())
throw Exception("Cannot initialize readline", ErrorCodes::CANNOT_READLINE);
#if RL_VERSION_MAJOR >= 7
/// Enable bracketed-paste-mode only when multiquery is enabled and multiline is
/// disabled, so that we are able to paste and execute multiline queries in a whole
/// instead of erroring out, while be less intrusive.
if (config().has("multiquery") && !config().has("multiline"))
{
/// When bracketed paste mode is set, pasted text is bracketed with control sequences so
/// that the program can differentiate pasted text from typed-in text. This helps
/// clickhouse-client so that without -m flag, one can still paste multiline queries, and
/// possibly get better pasting performance. See https://cirw.in/blog/bracketed-paste for
/// more details.
rl_variable_bind("enable-bracketed-paste", "on");
/// Use our bracketed paste handler to get better user experience. See comments above.
rl_bind_keyseq(BRACK_PASTE_PREF, clickhouse_rl_bracketed_paste_begin);
}
#endif
auto clear_prompt_or_exit = [](int)
{
/// This is signal safe.
ssize_t res = write(STDOUT_FILENO, "\n", 1);
/// Allow to quit client while query is in progress by pressing Ctrl+C twice.
/// (First press to Ctrl+C will try to cancel query by InterruptListener).
if (res == 1 && rl_line_buffer[0] && !RL_ISSTATE(RL_STATE_DONE))
{
rl_replace_line("", 0);
if (rl_forced_update_display())
_exit(0);
}
else
{
/// A little dirty, but we struggle to find better way to correctly
/// force readline to exit after returning from the signal handler.
_exit(0);
}
};
if (signal(SIGINT, clear_prompt_or_exit) == SIG_ERR)
throwFromErrno("Cannot set signal handler.", ErrorCodes::CANNOT_SET_SIGNAL_HANDLER);
#endif
loop();
while (true);
if (isNewYearMode())
std::cout << "Happy new year." << std::endl;
@ -622,17 +559,6 @@ private:
}
else
{
/// This is intended for testing purposes.
if (config().getBool("always_load_suggestion_data", false))
{
#if USE_READLINE
SCOPE_EXIT({ Suggest::instance().finalize(); });
Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit"));
#else
throw Exception("Command line suggestions cannot work without readline", ErrorCodes::BAD_ARGUMENTS);
#endif
}
query_id = config().getString("query_id", "");
nonInteractive();
@ -707,111 +633,11 @@ private:
}
/// Check if multi-line query is inserted from the paste buffer.
/// Allows delaying the start of query execution until the entirety of query is inserted.
static bool hasDataInSTDIN()
{
timeval timeout = { 0, 0 };
fd_set fds;
FD_ZERO(&fds);
FD_SET(STDIN_FILENO, &fds);
return select(1, &fds, nullptr, nullptr, &timeout) == 1;
}
inline const String prompt() const
{
return boost::replace_all_copy(prompt_by_server_display_name, "{database}", config().getString("database", "default"));
}
void loop()
{
String input;
String prev_input;
while (char * line_ = readline(input.empty() ? prompt().c_str() : ":-] "))
{
String line = line_;
free(line_);
size_t ws = line.size();
while (ws > 0 && isWhitespaceASCII(line[ws - 1]))
--ws;
if (ws == 0 || line.empty())
continue;
bool ends_with_semicolon = line[ws - 1] == ';';
bool ends_with_backslash = line[ws - 1] == '\\';
has_vertical_output_suffix = (ws >= 2) && (line[ws - 2] == '\\') && (line[ws - 1] == 'G');
if (ends_with_backslash)
line = line.substr(0, ws - 1);
input += line;
if (!ends_with_backslash && (ends_with_semicolon || has_vertical_output_suffix || (!config().has("multiline") && !hasDataInSTDIN())))
{
// TODO: should we do sensitive data masking on client too? History file can be source of secret leaks.
if (input != prev_input)
{
/// Replace line breaks with spaces to prevent the following problem.
/// Every line of multi-line query is saved to history file as a separate line.
/// If the user restarts the client then after pressing the "up" button
/// every line of the query will be displayed separately.
std::string logged_query = input;
if (config().has("multiline"))
std::replace(logged_query.begin(), logged_query.end(), '\n', ' ');
add_history(logged_query.c_str());
#if USE_READLINE && HAVE_READLINE_HISTORY
if (!history_file.empty() && append_history(1, history_file.c_str()))
std::cerr << "Cannot append history to file " + history_file + ": " + errnoToString(ErrorCodes::CANNOT_APPEND_HISTORY);
#endif
prev_input = input;
}
if (has_vertical_output_suffix)
input = input.substr(0, input.length() - 2);
try
{
if (!process(input))
break;
}
catch (const Exception & e)
{
actual_client_error = e.code();
if (!actual_client_error || actual_client_error != expected_client_error)
{
std::cerr << std::endl
<< "Exception on client:" << std::endl
<< "Code: " << e.code() << ". " << e.displayText() << std::endl;
if (config().getBool("stacktrace", false))
std::cerr << "Stack trace:" << std::endl
<< e.getStackTraceString() << std::endl;
std::cerr << std::endl;
}
/// Client-side exception during query execution can result in the loss of
/// sync in the connection protocol.
/// So we reconnect and allow to enter the next query.
connect();
}
input = "";
}
else
{
input += '\n';
}
}
}
void nonInteractive()
{
@ -2002,13 +1828,6 @@ public:
server_logs_file = options["server_logs_file"].as<std::string>();
if (options.count("disable_suggestion"))
config().setBool("disable_suggestion", true);
if (options.count("always_load_suggestion_data"))
{
if (options.count("disable_suggestion"))
throw Exception("Command line parameters disable_suggestion (-A) and always_load_suggestion_data cannot be specified simultaneously",
ErrorCodes::BAD_ARGUMENTS);
config().setBool("always_load_suggestion_data", true);
}
if (options.count("suggestion_limit"))
config().setInt("suggestion_limit", options["suggestion_limit"].as<int>());

146
dbms/programs/client/Suggest.cpp Normal file
View File

@ -0,0 +1,146 @@
#include "Suggest.h"
#include <Columns/ColumnString.h>
#include <Common/typeid_cast.h>
namespace DB
{
void Suggest::load(const ConnectionParameters & connection_parameters, size_t suggestion_limit)
{
loading_thread = std::thread([connection_parameters, suggestion_limit, this]
{
try
{
Connection connection(
connection_parameters.host,
connection_parameters.port,
connection_parameters.default_database,
connection_parameters.user,
connection_parameters.password,
"client",
connection_parameters.compression,
connection_parameters.security);
loadImpl(connection, connection_parameters.timeouts, suggestion_limit);
}
catch (...)
{
std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n";
}
/// Note that keyword suggestions are available even if we cannot load data from server.
std::sort(words.begin(), words.end());
ready = true;
});
}
Suggest::Suggest()
{
/// Keywords may be not up to date with ClickHouse parser.
words = {"CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT",
"MATERIALIZED", "ALIAS", "ENGINE", "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP",
"RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", "COPY", "PROJECT",
"PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO",
"OUTFILE", "FORMAT", "TABLES", "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE",
"END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", "INSERT", "VALUES",
"SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER",
"LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY",
"WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC",
"IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE"};
}
void Suggest::loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit)
{
std::stringstream query;
query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM ("
"SELECT name FROM system.functions"
" UNION ALL "
"SELECT name FROM system.table_engines"
" UNION ALL "
"SELECT name FROM system.formats"
" UNION ALL "
"SELECT name FROM system.table_functions"
" UNION ALL "
"SELECT name FROM system.data_type_families"
" UNION ALL "
"SELECT name FROM system.merge_tree_settings"
" UNION ALL "
"SELECT name FROM system.settings"
" UNION ALL "
"SELECT cluster FROM system.clusters"
" UNION ALL "
"SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate";
/// The user may disable loading of databases, tables, columns by setting suggestion_limit to zero.
if (suggestion_limit > 0)
{
String limit_str = toString(suggestion_limit);
query <<
" UNION ALL "
"SELECT name FROM system.databases LIMIT " << limit_str
<< " UNION ALL "
"SELECT DISTINCT name FROM system.tables LIMIT " << limit_str
<< " UNION ALL "
"SELECT DISTINCT name FROM system.columns LIMIT " << limit_str;
}
query << ") WHERE notEmpty(res)";
fetch(connection, timeouts, query.str());
}
void Suggest::fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query)
{
connection.sendQuery(timeouts, query);
while (true)
{
Packet packet = connection.receivePacket();
switch (packet.type)
{
case Protocol::Server::Data:
fillWordsFromBlock(packet.block);
continue;
case Protocol::Server::Progress:
continue;
case Protocol::Server::ProfileInfo:
continue;
case Protocol::Server::Totals:
continue;
case Protocol::Server::Extremes:
continue;
case Protocol::Server::Log:
continue;
case Protocol::Server::Exception:
packet.exception->rethrow();
return;
case Protocol::Server::EndOfStream:
return;
default:
throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
}
}
}
void Suggest::fillWordsFromBlock(const Block & block)
{
if (!block)
return;
if (block.columns() != 1)
throw Exception("Wrong number of columns received for query to read words for suggestion", ErrorCodes::LOGICAL_ERROR);
const ColumnString & column = typeid_cast<const ColumnString &>(*block.getByPosition(0).column);
size_t rows = block.rows();
for (size_t i = 0; i < rows; ++i)
words.emplace_back(column.getDataAt(i).toString());
}
}

207
dbms/programs/client/Suggest.h
View File

@ -2,18 +2,9 @@
#include "ConnectionParameters.h"
#include <string>
#include <sstream>
#include <string.h>
#include <vector>
#include <algorithm>
#include <common/readline_use.h>
#include <Common/typeid_cast.h>
#include <Columns/ColumnString.h>
#include <Client/Connection.h>
#include <IO/ConnectionTimeouts.h>
#include <common/LineReader.h>
namespace DB
@ -24,141 +15,8 @@ namespace ErrorCodes
extern const int UNKNOWN_PACKET_FROM_SERVER;
}
class Suggest : private boost::noncopyable
class Suggest : public LineReader::Suggest, boost::noncopyable
{
private:
/// The vector will be filled with completion words from the server and sorted.
using Words = std::vector<std::string>;
/// Keywords may be not up to date with ClickHouse parser.
Words words
{
"CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", "MATERIALIZED", "ALIAS", "ENGINE",
"AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP", "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER",
"COPY", "PROJECT", "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO", "OUTFILE", "FORMAT", "TABLES",
"DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE", "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE",
"INSERT", "VALUES", "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER", "LEFT", "RIGHT", "FULL", "OUTER",
"CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY", "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", "IN",
"KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE"
};
/// Words are fetched asynchronously.
std::thread loading_thread;
std::atomic<bool> ready{false};
/// Points to current word to suggest.
Words::const_iterator pos;
/// Points after the last possible match.
Words::const_iterator end;
/// Set iterators to the matched range of words if any.
void findRange(const char * prefix, size_t prefix_length)
{
std::string prefix_str(prefix);
std::tie(pos, end) = std::equal_range(words.begin(), words.end(), prefix_str,
[prefix_length](const std::string & s, const std::string & prefix_searched) { return strncmp(s.c_str(), prefix_searched.c_str(), prefix_length) < 0; });
}
/// Iterates through matched range.
char * nextMatch()
{
if (pos >= end)
return nullptr;
/// readline will free memory by itself.
char * word = strdup(pos->c_str());
++pos;
return word;
}
void loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit)
{
std::stringstream query;
query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM ("
"SELECT name FROM system.functions"
" UNION ALL "
"SELECT name FROM system.table_engines"
" UNION ALL "
"SELECT name FROM system.formats"
" UNION ALL "
"SELECT name FROM system.table_functions"
" UNION ALL "
"SELECT name FROM system.data_type_families"
" UNION ALL "
"SELECT name FROM system.settings"
" UNION ALL "
"SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate";
/// The user may disable loading of databases, tables, columns by setting suggestion_limit to zero.
if (suggestion_limit > 0)
{
String limit_str = toString(suggestion_limit);
query <<
" UNION ALL "
"SELECT name FROM system.databases LIMIT " << limit_str
<< " UNION ALL "
"SELECT DISTINCT name FROM system.tables LIMIT " << limit_str
<< " UNION ALL "
"SELECT DISTINCT name FROM system.columns LIMIT " << limit_str;
}
query << ") WHERE notEmpty(res)";
fetch(connection, timeouts, query.str());
}
void fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query)
{
connection.sendQuery(timeouts, query);
while (true)
{
Packet packet = connection.receivePacket();
switch (packet.type)
{
case Protocol::Server::Data:
fillWordsFromBlock(packet.block);
continue;
case Protocol::Server::Progress:
continue;
case Protocol::Server::ProfileInfo:
continue;
case Protocol::Server::Totals:
continue;
case Protocol::Server::Extremes:
continue;
case Protocol::Server::Log:
continue;
case Protocol::Server::Exception:
packet.exception->rethrow();
return;
case Protocol::Server::EndOfStream:
return;
default:
throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
}
}
}
void fillWordsFromBlock(const Block & block)
{
if (!block)
return;
if (block.columns() != 1)
throw Exception("Wrong number of columns received for query to read words for suggestion", ErrorCodes::LOGICAL_ERROR);
const ColumnString & column = typeid_cast<const ColumnString &>(*block.getByPosition(0).column);
size_t rows = block.rows();
for (size_t i = 0; i < rows; ++i)
words.emplace_back(column.getDataAt(i).toString());
}
public:
static Suggest & instance()
{
@ -166,64 +24,25 @@ public:
return instance;
}
/// More old server versions cannot execute the query above.
void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit);
/// Older server versions cannot execute the query above.
static constexpr int MIN_SERVER_REVISION = 54406;
void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit)
{
loading_thread = std::thread([connection_parameters, suggestion_limit, this]
{
try
{
Connection connection(
connection_parameters.host,
connection_parameters.port,
connection_parameters.default_database,
connection_parameters.user,
connection_parameters.password,
"client",
connection_parameters.compression,
connection_parameters.security);
loadImpl(connection, connection_parameters.timeouts, suggestion_limit);
}
catch (...)
{
std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n";
}
/// Note that keyword suggestions are available even if we cannot load data from server.
std::sort(words.begin(), words.end());
ready = true;
});
}
void finalize()
private:
Suggest();
~Suggest()
{
if (loading_thread.joinable())
loading_thread.join();
}
/// A function for readline.
static char * generator(const char * text, int state)
{
Suggest & suggest = Suggest::instance();
if (!suggest.ready)
return nullptr;
if (state == 0)
suggest.findRange(text, strlen(text));
void loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit);
void fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query);
void fillWordsFromBlock(const Block & block);
/// Do not append whitespace after word. For unknown reason, rl_completion_append_character = '\0' does not work.
rl_completion_suppress_append = 1;
return suggest.nextMatch();
}
~Suggest()
{
finalize();
}
/// Words are fetched asynchronously.
std::thread loading_thread;
};
}

4
dbms/programs/client/TestHint.h
View File

@ -13,7 +13,7 @@ namespace DB
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int UNEXPECTED_ERROR_CODE;
}
@ -74,7 +74,7 @@ public:
std::cerr << "Success when error expected. It expects server error "
<< server_error << ", client error " << client_error << "." << std::endl;
got_exception = true;
last_exception = std::make_unique<Exception>("Success when error expected", ErrorCodes::LOGICAL_ERROR); /// return error to OS
last_exception = std::make_unique<Exception>("Success when error expected", ErrorCodes::UNEXPECTED_ERROR_CODE); /// return error to OS
return false;
}

2
dbms/programs/client/readpassphrase/CMakeLists.txt
View File

@ -10,4 +10,4 @@ set_target_properties(readpassphrase
PROPERTIES LINKER_LANGUAGE C
)
# . to allow #include <readpassphrase.h>
target_include_directories(readpassphrase PUBLIC . ${CMAKE_CURRENT_BINARY_DIR}/include ${CMAKE_CURRENT_BINARY_DIR}/../include)
target_include_directories(readpassphrase PUBLIC . ${CMAKE_CURRENT_BINARY_DIR}/include)

2
dbms/programs/compressor/Compressor.cpp
View File

@ -70,7 +70,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
("hc", "use LZ4HC instead of LZ4")
("zstd", "use ZSTD instead of LZ4")
("codec", boost::program_options::value<std::vector<std::string>>()->multitoken(), "use codecs combination instead of LZ4")
("level", boost::program_options::value<int>(), "compression level for codecs spicified via flags")
("level", boost::program_options::value<int>(), "compression level for codecs specified via flags")
("none", "use no compression instead of LZ4")
("stat", "print block statistics of compressed data")
;

2
dbms/programs/copier/ClusterCopier.cpp
View File

@ -2430,7 +2430,7 @@ void ClusterCopierApp::defineOptions(Poco::Util::OptionSet & options)
.argument("copy-fault-probability").binding("copy-fault-probability"));
options.addOption(Poco::Util::Option("log-level", "", "sets log level")
.argument("log-level").binding("log-level"));
options.addOption(Poco::Util::Option("base-dir", "", "base directory for copiers, consequitive copier launches will populate /base-dir/launch_id/* directories")
options.addOption(Poco::Util::Option("base-dir", "", "base directory for copiers, consecutive copier launches will populate /base-dir/launch_id/* directories")
.argument("base-dir").binding("base-dir"));
using Me = std::decay_t<decltype(*this)>;

6
dbms/programs/local/LocalServer.cpp
View File

@ -111,7 +111,7 @@ void LocalServer::tryInitPath()
/// In case of empty path set paths to helpful directories
std::string cd = Poco::Path::current();
context->setTemporaryPath(cd + "tmp");
context->setTemporaryStorage(cd + "tmp");
context->setFlagsPath(cd + "flags");
context->setUserFilesPath(""); // user's files are everywhere
}
@ -164,7 +164,7 @@ try
setupUsers();
/// Limit on total number of concurrently executing queries.
/// Threre are no need for concurrent threads, override max_concurrent_queries.
/// There is no need for concurrent threads, override max_concurrent_queries.
context->getProcessList().setMaxSize(0);
/// Size of cache for uncompressed blocks. Zero means disabled.
@ -182,7 +182,7 @@ try
context->setDefaultProfiles(config());
/** Init dummy default DB
* NOTE: We force using isolated default database to avoid conflicts with default database from server enviroment
* NOTE: We force using isolated default database to avoid conflicts with default database from server environment
* Otherwise, metadata of temporary File(format, EXPLICIT_PATH) tables will pollute metadata/ directory;
* if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons.
*/

6
dbms/programs/obfuscator/Obfuscator.cpp
View File

@ -40,7 +40,7 @@
#include <Common/TerminalSize.h>
static const char * documantation = R"(
static const char * documentation = R"(
Simple tool for table data obfuscation.
It reads input table and produces output table, that retain some properties of input, but contains different data.
@ -979,7 +979,7 @@ try
|| !options.count("input-format")
|| !options.count("output-format"))
{
std::cout << documantation << "\n"
std::cout << documentation << "\n"
<< "\nUsage: " << argv[0] << " [options] < in > out\n"
<< "\nInput must be seekable file (it will be read twice).\n"
<< "\n" << description << "\n"
@ -1070,7 +1070,7 @@ try
if (!silent)
std::cerr << "Generating data\n";
file_in.seek(0);
file_in.seek(0, SEEK_SET);
BlockInputStreamPtr input = context.getInputFormat(input_format, file_in, header, max_block_size);
BlockOutputStreamPtr output = context.getOutputFormat(output_format, file_out, header);

12
dbms/programs/odbc-bridge/CMakeLists.txt
View File

@ -15,20 +15,24 @@ set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemo
if (USE_POCO_SQLODBC)
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQLODBC_LIBRARY})
set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_SQLODBC_INCLUDE_DIR})
# Wouldnt work anyway because of the way list variable got expanded in `target_include_directories`
# set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_SQLODBC_INCLUDE_DIR})
endif ()
if (Poco_SQL_FOUND)
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQL_LIBRARY})
set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR})
# Wouldnt work anyway because of the way list variable got expanded in `target_include_directories`
# set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR})
endif ()
if (USE_POCO_DATAODBC)
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_DataODBC_LIBRARY})
set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_DataODBC_INCLUDE_DIR})
# Wouldnt work anyway because of the way list variable got expanded in `target_include_directories`
# set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_DataODBC_INCLUDE_DIR})
endif()
if (Poco_Data_FOUND)
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_Data_LIBRARY})
set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR})
# Wouldnt work anyway because of the way list variable got expanded in `target_include_directories`
# set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR})
endif ()
clickhouse_program_add_library(odbc-bridge)

2
dbms/programs/odbc-bridge/MainHandler.cpp
View File

@ -138,7 +138,7 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne
{
auto message = getCurrentExceptionMessage(true);
response.setStatusAndReason(
Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); // can't call process_error, bacause of too soon response sending
Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); // can't call process_error, because of too soon response sending
writeStringBinary(message, out);
tryLogCurrentException(log);
}

2
dbms/programs/odbc-bridge/ODBCBridge.cpp
View File

@ -88,7 +88,7 @@ void ODBCBridge::defineOptions(Poco::Util::OptionSet & options)
options.addOption(
Poco::Util::Option("listen-host", "", "hostname to listen, default localhost").argument("listen-host").binding("listen-host"));
options.addOption(
Poco::Util::Option("http-timeout", "", "http timout for socket, default 1800").argument("http-timeout").binding("http-timeout"));
Poco::Util::Option("http-timeout", "", "http timeout for socket, default 1800").argument("http-timeout").binding("http-timeout"));
options.addOption(Poco::Util::Option("max-server-connections", "", "max connections to server, default 1024")
.argument("max-server-connections")

2
dbms/programs/performance-test/PerformanceTest.cpp
View File

@ -315,7 +315,7 @@ void PerformanceTest::runQueries(
stop_conditions.reportIterations(iteration);
if (stop_conditions.areFulfilled())
{
LOG_INFO(log, "Stop conditions fullfilled");
LOG_INFO(log, "Stop conditions fulfilled");
break;
}

8
dbms/programs/performance-test/PerformanceTestSuite.cpp
View File

@ -200,7 +200,7 @@ private:
if (current.checkPreconditions())
{
LOG_INFO(log, "Preconditions for test '" << info.test_name << "' are fullfilled");
LOG_INFO(log, "Preconditions for test '" << info.test_name << "' are fulfilled");
LOG_INFO(
log,
"Preparing for run, have " << info.create_and_fill_queries.size() << " create and fill queries");
@ -219,7 +219,7 @@ private:
return {report_builder->buildFullReport(info, result, query_indexes[info.path]), current.checkSIGINT()};
}
else
LOG_INFO(log, "Preconditions for test '" << info.test_name << "' are not fullfilled, skip run");
LOG_INFO(log, "Preconditions for test '" << info.test_name << "' are not fulfilled, skip run");
return {"", current.checkSIGINT()};
}
@ -361,8 +361,8 @@ try
po::notify(options);
Poco::AutoPtr<Poco::PatternFormatter> formatter(new Poco::PatternFormatter("%Y.%m.%d %H:%M:%S.%F <%p> %s: %t"));
Poco::AutoPtr<Poco::ConsoleChannel> console_chanel(new Poco::ConsoleChannel);
Poco::AutoPtr<Poco::FormattingChannel> channel(new Poco::FormattingChannel(formatter, console_chanel));
Poco::AutoPtr<Poco::ConsoleChannel> console_channel(new Poco::ConsoleChannel);
Poco::AutoPtr<Poco::FormattingChannel> channel(new Poco::FormattingChannel(formatter, console_channel));
Poco::Logger::root().setLevel(options["log-level"].as<std::string>());
Poco::Logger::root().setChannel(channel);

2
dbms/programs/performance-test/ReportBuilder.cpp
View File

@ -117,7 +117,7 @@ std::string ReportBuilder::buildFullReport(
if (isASCIIString(statistics.exception))
runJSON.set("exception", jsonString(statistics.exception, settings), false);
else
runJSON.set("exception", "Some exception occured with non ASCII message. This may produce invalid JSON. Try reproduce locally.");
runJSON.set("exception", "Some exception occurred with non ASCII message. This may produce invalid JSON. Try reproduce locally.");
}
if (test_info.exec_type == ExecutionType::Loop)

12
dbms/programs/server/HTTPHandler.cpp
View File

@ -17,6 +17,7 @@
#include <Common/setThreadName.h>
#include <Common/config.h>
#include <Common/SettingsChanges.h>
#include <Disks/DiskSpaceMonitor.h>
#include <Compression/CompressedReadBuffer.h>
#include <Compression/CompressedWriteBuffer.h>
#include <IO/ReadBufferFromIStream.h>
@ -351,7 +352,8 @@ void HTTPHandler::processQuery(
if (buffer_until_eof)
{
std::string tmp_path_template = context.getTemporaryPath() + "http_buffers/";
const std::string tmp_path(context.getTemporaryVolume()->getNextDisk()->getPath());
const std::string tmp_path_template(tmp_path + "http_buffers/");
auto create_tmp_disk_buffer = [tmp_path_template] (const WriteBufferPtr &)
{
@ -590,7 +592,11 @@ void HTTPHandler::processQuery(
customizeContext(context);
executeQuery(*in, *used_output.out_maybe_delayed_and_compressed, /* allow_into_outfile = */ false, context,
[&response] (const String & content_type) { response.setContentType(content_type); },
[&response] (const String & content_type, const String & format)
{
response.setContentType(content_type);
response.add("X-ClickHouse-Format", format);
},
[&response] (const String & current_query_id) { response.add("X-ClickHouse-Query-Id", current_query_id); });
if (used_output.hasDelayed())
@ -610,6 +616,8 @@ void HTTPHandler::trySendExceptionToClient(const std::string & s, int exception_
{
try
{
response.set("X-ClickHouse-Exception-Code", toString<int>(exception_code));
/// If HTTP method is POST and Keep-Alive is turned on, we should read the whole request body
/// to avoid reading part of the current request body in the next request.
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST

14
dbms/programs/server/InterserverIOHTTPHandler.cpp
View File

@ -28,23 +28,23 @@ std::pair<String, bool> InterserverIOHTTPHandler::checkAuthentication(Poco::Net:
if (config.has("interserver_http_credentials.user"))
{
if (!request.hasCredentials())
return {"Server requires HTTP Basic authentification, but client doesn't provide it", false};
return {"Server requires HTTP Basic authentication, but client doesn't provide it", false};
String scheme, info;
request.getCredentials(scheme, info);
if (scheme != "Basic")
return {"Server requires HTTP Basic authentification but client provides another method", false};
return {"Server requires HTTP Basic authentication but client provides another method", false};
String user = config.getString("interserver_http_credentials.user");
String password = config.getString("interserver_http_credentials.password", "");
Poco::Net::HTTPBasicCredentials credentials(info);
if (std::make_pair(user, password) != std::make_pair(credentials.getUsername(), credentials.getPassword()))
return {"Incorrect user or password in HTTP Basic authentification", false};
return {"Incorrect user or password in HTTP Basic authentication", false};
}
else if (request.hasCredentials())
{
return {"Client requires HTTP Basic authentification, but server doesn't provide it", false};
return {"Client requires HTTP Basic authentication, but server doesn't provide it", false};
}
return {"", true};
}
@ -61,6 +61,10 @@ void InterserverIOHTTPHandler::processQuery(Poco::Net::HTTPServerRequest & reque
ReadBufferFromIStream body(request.stream());
auto endpoint = server.context().getInterserverIOHandler().getEndpoint(endpoint_name);
/// Locked for read while query processing
std::shared_lock lock(endpoint->rwlock);
if (endpoint->blocker.isCancelled())
throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED);
if (compress)
{
@ -99,7 +103,7 @@ void InterserverIOHTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & requ
response.setStatusAndReason(Poco::Net::HTTPServerResponse::HTTP_UNAUTHORIZED);
if (!response.sent())
writeString(message, *used_output.out);
LOG_WARNING(log, "Query processing failed request: '" << request.getURI() << "' authentification failed");
LOG_WARNING(log, "Query processing failed request: '" << request.getURI() << "' authentication failed");
}
}
catch (Exception & e)

25
dbms/programs/server/MySQLHandler.cpp
View File

@ -79,21 +79,19 @@ void MySQLHandler::run()
if (!connection_context.mysql.max_packet_size)
connection_context.mysql.max_packet_size = MAX_PACKET_LENGTH;
/* LOG_TRACE(log, "Capabilities: " << handshake_response.capability_flags
<< "\nmax_packet_size: "
LOG_TRACE(log, "Capabilities: " << handshake_response.capability_flags
<< ", max_packet_size: "
<< handshake_response.max_packet_size
<< "\ncharacter_set: "
<< handshake_response.character_set
<< "\nuser: "
<< ", character_set: "
<< static_cast<int>(handshake_response.character_set)
<< ", user: "
<< handshake_response.username
<< "\nauth_response length: "
<< ", auth_response length: "
<< handshake_response.auth_response.length()
<< "\nauth_response: "
<< handshake_response.auth_response
<< "\ndatabase: "
<< ", database: "
<< handshake_response.database
<< "\nauth_plugin_name: "
<< handshake_response.auth_plugin_name);*/
<< ", auth_plugin_name: "
<< handshake_response.auth_plugin_name);
client_capability_flags = handshake_response.capability_flags;
if (!(client_capability_flags & CLIENT_PROTOCOL_41))
@ -284,7 +282,8 @@ void MySQLHandler::comQuery(ReadBuffer & payload)
else
{
bool with_output = false;
std::function<void(const String &)> set_content_type = [&with_output](const String &) -> void {
std::function<void(const String &, const String &)> set_content_type_and_format = [&with_output](const String &, const String &) -> void
{
with_output = true;
};
@ -307,7 +306,7 @@ void MySQLHandler::comQuery(ReadBuffer & payload)
ReadBufferFromString replacement(replacement_query);
Context query_context = connection_context;
executeQuery(should_replace ? replacement : payload, *out, true, query_context, set_content_type, nullptr);
executeQuery(should_replace ? replacement : payload, *out, true, query_context, set_content_type_and_format, {});
if (!with_output)
packet_sender->sendPacket(OK_Packet(0x00, client_capability_flags, 0, 0, 0), true);

6
dbms/programs/server/MySQLHandlerFactory.cpp
View File

@ -34,7 +34,7 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_)
}
catch (...)
{
LOG_INFO(log, "Failed to create SSL context. SSL will be disabled. Error: " << getCurrentExceptionMessage(false));
LOG_TRACE(log, "Failed to create SSL context. SSL will be disabled. Error: " << getCurrentExceptionMessage(false));
ssl_enabled = false;
}
#endif
@ -47,7 +47,7 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_)
}
catch (...)
{
LOG_WARNING(log, "Failed to read RSA keys. Error: " << getCurrentExceptionMessage(false));
LOG_TRACE(log, "Failed to read RSA key pair from server certificate. Error: " << getCurrentExceptionMessage(false));
generateRSAKeys();
}
#endif
@ -104,7 +104,7 @@ void MySQLHandlerFactory::readRSAKeys()
void MySQLHandlerFactory::generateRSAKeys()
{
LOG_INFO(log, "Generating new RSA key.");
LOG_TRACE(log, "Generating new RSA key pair.");
public_key.reset(RSA_new());
if (!public_key)
throw Exception("Failed to allocate RSA key. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);

6
dbms/programs/server/PrometheusRequestHandler.h
View File

@ -31,7 +31,7 @@ public:
template <typename HandlerType>
class PrometeusRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory
class PrometheusRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory
{
private:
IServer & server;
@ -39,7 +39,7 @@ private:
PrometheusMetricsWriter metrics_writer;
public:
PrometeusRequestHandlerFactory(IServer & server_, const AsynchronousMetrics & async_metrics_)
PrometheusRequestHandlerFactory(IServer & server_, const AsynchronousMetrics & async_metrics_)
: server(server_)
, endpoint_path(server_.config().getString("prometheus.endpoint", "/metrics"))
, metrics_writer(server_.config(), "prometheus", async_metrics_)
@ -56,6 +56,6 @@ public:
}
};
using PrometeusHandlerFactory = PrometeusRequestHandlerFactory<PrometheusRequestHandler>;
using PrometheusHandlerFactory = PrometheusRequestHandlerFactory<PrometheusRequestHandler>;
}

75
dbms/programs/server/Server.cpp
View File

@ -77,6 +77,31 @@ namespace CurrentMetrics
extern const Metric VersionInteger;
}
namespace
{
void setupTmpPath(Logger * log, const std::string & path)
{
LOG_DEBUG(log, "Setting up " << path << " to store temporary data in it");
Poco::File(path).createDirectories();
/// Clearing old temporary files.
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator it(path); it != dir_end; ++it)
{
if (it->isFile() && startsWith(it.name(), "tmp"))
{
LOG_DEBUG(log, "Removing old temporary file " << it->path());
it->remove();
}
else
LOG_DEBUG(log, "Skipped file in temporary path " << it->path());
}
}
}
namespace DB
{
@ -277,15 +302,6 @@ int Server::main(const std::vector<std::string> & /*args*/)
Poco::File(path + "data/" + default_database).createDirectories();
Poco::File(path + "metadata/" + default_database).createDirectories();
/// Check that we have read and write access to all data paths
auto disk_selector = global_context->getDiskSelector();
for (const auto & [name, disk] : disk_selector.getDisksMap())
{
Poco::File disk_path(disk->getPath());
if (!disk_path.canRead() || !disk_path.canWrite())
throw Exception("There is no RW access to disk " + name + " (" + disk->getPath() + ")", ErrorCodes::PATH_ACCESS_DENIED);
}
StatusFile status{path + "status"};
SCOPE_EXIT({
@ -340,22 +356,14 @@ int Server::main(const std::vector<std::string> & /*args*/)
DateLUT::instance();
LOG_TRACE(log, "Initialized DateLUT with time zone '" << DateLUT::instance().getTimeZone() << "'.");
/// Directory with temporary data for processing of heavy queries.
/// Storage with temporary data for processing of heavy queries.
{
std::string tmp_path = config().getString("tmp_path", path + "tmp/");
global_context->setTemporaryPath(tmp_path);
Poco::File(tmp_path).createDirectories();
/// Clearing old temporary files.
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator it(tmp_path); it != dir_end; ++it)
{
if (it->isFile() && startsWith(it.name(), "tmp"))
{
LOG_DEBUG(log, "Removing old temporary file " << it->path());
it->remove();
}
}
std::string tmp_policy = config().getString("tmp_policy", "");
const VolumePtr & volume = global_context->setTemporaryStorage(tmp_path, tmp_policy);
for (const DiskPtr & disk : volume->disks)
setupTmpPath(log, disk->getPath());
}
/** Directory with 'flags': files indicating temporary settings for the server set by system administrator.
@ -445,8 +453,10 @@ int Server::main(const std::vector<std::string> & /*args*/)
main_config_zk_changed_event,
[&](ConfigurationPtr config)
{
setTextLog(global_context->getTextLog());
buildLoggers(*config, logger());
// FIXME logging-related things need synchronization -- see the 'Logger * log' saved
// in a lot of places. For now, disable updating log configuration without server restart.
//setTextLog(global_context->getTextLog());
//buildLoggers(*config, logger());
global_context->setClustersConfig(config);
global_context->setMacros(std::make_unique<Macros>(*config, "macros"));
@ -554,8 +564,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
///
/// It also cannot work with sanitizers.
/// Sanitizers are using quick "frame walking" stack unwinding (this implies -fno-omit-frame-pointer)
/// And they do unwiding frequently (on every malloc/free, thread/mutex operations, etc).
/// They change %rbp during unwinding and it confuses libunwind if signal comes during sanitizer unwiding
/// And they do unwinding frequently (on every malloc/free, thread/mutex operations, etc).
/// They change %rbp during unwinding and it confuses libunwind if signal comes during sanitizer unwinding
/// and query profiler decide to unwind stack with libunwind at this moment.
///
/// Symptoms: you'll get silent Segmentation Fault - without sanitizer message and without usual ClickHouse diagnostics.
@ -724,7 +734,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
socket.setSendTimeout(settings.http_send_timeout);
auto handler_factory = createDefaultHandlerFatory<HTTPHandler>(*this, "HTTPHandler-factory");
if (config().has("prometheus") && config().getInt("prometheus.port", 0) == 0)
handler_factory->addHandler<PrometeusHandlerFactory>(async_metrics);
handler_factory->addHandler<PrometheusHandlerFactory>(async_metrics);
servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>(
handler_factory,
@ -854,7 +864,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
socket.setReceiveTimeout(settings.http_receive_timeout);
socket.setSendTimeout(settings.http_send_timeout);
auto handler_factory = new HTTPRequestHandlerFactoryMain(*this, "PrometheusHandler-factory");
handler_factory->addHandler<PrometeusHandlerFactory>(async_metrics);
handler_factory->addHandler<PrometheusHandlerFactory>(async_metrics);
servers.emplace_back(std::make_unique<Poco::Net::HTTPServer>(
handler_factory,
server_pool,
@ -871,6 +881,13 @@ int Server::main(const std::vector<std::string> & /*args*/)
for (auto & server : servers)
server->start();
{
String level_str = config().getString("text_log.level", "");
int level = level_str.empty() ? INT_MAX : Poco::Logger::parseLevel(level_str);
setTextLog(global_context->getTextLog(), level);
}
buildLoggers(config(), logger());
main_config_reloader->start();
users_config_reloader->start();
if (dns_cache_updater)

110
dbms/programs/server/TCPHandler.cpp
View File

@ -591,11 +591,9 @@ void TCPHandler::processOrdinaryQueryWithProcessors(size_t num_threads)
}
});
/// Wait in case of exception. Delete pipeline to release memory.
/// Wait in case of exception happened outside of pool.
SCOPE_EXIT(
/// Clear queue in case if somebody is waiting lazy_format to push.
lazy_format->finish();
lazy_format->clearQueue();
try
{
@ -604,72 +602,58 @@ void TCPHandler::processOrdinaryQueryWithProcessors(size_t num_threads)
catch (...)
{
/// If exception was thrown during pipeline execution, skip it while processing other exception.
tryLogCurrentException(log);
}
pipeline = QueryPipeline()
);
while (true)
while (!lazy_format->isFinished() && !exception)
{
Block block;
while (true)
if (isQueryCancelled())
{
if (isQueryCancelled())
{
/// A packet was received requesting to stop execution of the request.
executor->cancel();
break;
}
else
{
if (after_send_progress.elapsed() / 1000 >= query_context->getSettingsRef().interactive_delay)
{
/// Some time passed and there is a progress.
after_send_progress.restart();
sendProgress();
}
sendLogs();
if ((block = lazy_format->getBlock(query_context->getSettingsRef().interactive_delay / 1000)))
break;
if (lazy_format->isFinished())
break;
if (exception)
{
pool.wait();
break;
}
}
}
/** If data has run out, we will send the profiling data and total values to
* the last zero block to be able to use
* this information in the suffix output of stream.
* If the request was interrupted, then `sendTotals` and other methods could not be called,
* because we have not read all the data yet,
* and there could be ongoing calculations in other threads at the same time.
*/
if (!block && !isQueryCancelled())
{
pool.wait();
pipeline.finalize();
sendTotals(lazy_format->getTotals());
sendExtremes(lazy_format->getExtremes());
sendProfileInfo(lazy_format->getProfileInfo());
sendProgress();
sendLogs();
}
sendData(block);
if (!block)
/// A packet was received requesting to stop execution of the request.
executor->cancel();
break;
}
if (after_send_progress.elapsed() / 1000 >= query_context->getSettingsRef().interactive_delay)
{
/// Some time passed and there is a progress.
after_send_progress.restart();
sendProgress();
}
sendLogs();
if (auto block = lazy_format->getBlock(query_context->getSettingsRef().interactive_delay / 1000))
{
if (!state.io.null_format)
sendData(block);
}
}
/// Finish lazy_format before waiting. Otherwise some thread may write into it, and waiting will lock.
lazy_format->finish();
pool.wait();
/** If data has run out, we will send the profiling data and total values to
* the last zero block to be able to use
* this information in the suffix output of stream.
* If the request was interrupted, then `sendTotals` and other methods could not be called,
* because we have not read all the data yet,
* and there could be ongoing calculations in other threads at the same time.
*/
if (!isQueryCancelled())
{
pipeline.finalize();
sendTotals(lazy_format->getTotals());
sendExtremes(lazy_format->getExtremes());
sendProfileInfo(lazy_format->getProfileInfo());
sendProgress();
sendLogs();
}
sendData({});
}
state.io.onFinish();
@ -993,7 +977,7 @@ bool TCPHandler::receiveData(bool scalar)
if (!(storage = query_context->tryGetExternalTable(name)))
{
NamesAndTypesList columns = block.getNamesAndTypesList();
storage = StorageMemory::create("_external", name, ColumnsDescription{columns}, ConstraintsDescription{});
storage = StorageMemory::create(StorageID("_external", name), ColumnsDescription{columns}, ConstraintsDescription{});
storage->startup();
query_context->addExternalTable(name, storage);
}

2
dbms/programs/server/TCPHandler.h
View File

@ -111,7 +111,7 @@ public:
server_display_name = server.config().getString("display_name", getFQDNOrHostName());
}
void run();
void run() override;
/// This method is called right before the query execution.
virtual void customizeContext(DB::Context & /*context*/) {}

7
dbms/programs/server/config.d/part_log.xml Normal file
View File

@ -0,0 +1,7 @@
<yandex>
<part_log>
<database>system</database>
<table>part_log</table>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</part_log>
</yandex>

55
dbms/programs/server/config.xml
View File

@ -3,27 +3,27 @@
NOTE: User and query level settings are set up in "users.xml" file.
-->
<yandex>
<!-- The list of hosts allowed to use in URL-related storage engines and table functions.
If this section is not present in configuration, all hosts are allowed.
-->
<remote_url_allow_hosts>
<!-- Host should be specified exactly as in URL. The name is checked before DNS resolution.
Example: "yandex.ru", "yandex.ru." and "www.yandex.ru" are different hosts.
If port is explicitly specified in URL, the host:port is checked as a whole.
If host specified here without port, any port with this host allowed.
"yandex.ru" -> "yandex.ru:443", "yandex.ru:80" etc. is allowed, but "yandex.ru:80" -> only "yandex.ru:80" is allowed.
If the host is specified as IP address, it is checked as specified in URL. Example: "[2a02:6b8:a::a]".
If there are redirects and support for redirects is enabled, every redirect (the Location field) is checked.
-->
<!-- The list of hosts allowed to use in URL-related storage engines and table functions.
If this section is not present in configuration, all hosts are allowed.
-->
<remote_url_allow_hosts>
<!-- Host should be specified exactly as in URL. The name is checked before DNS resolution.
Example: "yandex.ru", "yandex.ru." and "www.yandex.ru" are different hosts.
If port is explicitly specified in URL, the host:port is checked as a whole.
If host specified here without port, any port with this host allowed.
"yandex.ru" -> "yandex.ru:443", "yandex.ru:80" etc. is allowed, but "yandex.ru:80" -> only "yandex.ru:80" is allowed.
If the host is specified as IP address, it is checked as specified in URL. Example: "[2a02:6b8:a::a]".
If there are redirects and support for redirects is enabled, every redirect (the Location field) is checked.
-->
<!-- Regular expression can be specified. RE2 engine is used for regexps.
Regexps are not aligned: don't forget to add ^ and $. Also don't forget to escape dot (.) metacharacter
(forgetting to do so is a common source of error).
-->
</remote_url_allow_hosts>
<!-- Regular expression can be specified. RE2 engine is used for regexps.
Regexps are not aligned: don't forget to add ^ and $. Also don't forget to escape dot (.) metacharacter
(forgetting to do so is a common source of error).
-->
</remote_url_allow_hosts>
<logger>
<!-- Possible levels: https://github.com/pocoproject/poco/blob/develop/Foundation/include/Poco/Logger.h#L105 -->
<!-- Possible levels: https://github.com/pocoproject/poco/blob/poco-1.9.4-release/Foundation/include/Poco/Logger.h#L105 -->
<level>trace</level>
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
@ -34,6 +34,7 @@
<!--display_name>production</display_name--> <!-- It is the name that will be shown in the client -->
<http_port>8123</http_port>
<tcp_port>9000</tcp_port>
<mysql_port>9004</mysql_port>
<!-- For HTTPS and SSL over native protocol. -->
<!--
<https_port>8443</https_port>
@ -132,6 +133,17 @@
<!-- Path to temporary data for processing hard queries. -->
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
<!-- Policy from the <storage_configuration> for the temporary files.
If not set <tmp_path> is used, otherwise <tmp_path> is ignored.
Notes:
- move_factor is ignored
- keep_free_space_bytes is ignored
- max_data_part_size_bytes is ignored
- you must have exactly one volume in that policy
-->
<!-- <tmp_policy>tmp</tmp_policy> -->
<!-- Directory with user provided files that are accessible by 'file' table function. -->
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
@ -343,6 +355,11 @@
toStartOfHour(event_time)
-->
<partition_by>toYYYYMM(event_date)</partition_by>
<!-- Instead of partition_by, you can provide full engine expression (starting with ENGINE = ) with parameters,
Example: <engine>ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024</engine>
-->
<!-- Interval of flushing data. -->
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</query_log>
@ -377,10 +394,12 @@
<!-- Uncomment to write text log into table.
Text log contains all information from usual server log but stores it in structured and efficient way.
The level of the messages that goes to the table can be limited (<level>), if not specified all messages will go to the table.
<text_log>
<database>system</database>
<table>text_log</table>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
<level></level>
</text_log>
-->

25
dbms/programs/server/users.xml
View File

@ -49,7 +49,7 @@
In first line will be password and in second - corresponding SHA256.
How to generate double SHA1:
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | openssl dgst -sha1 -binary | openssl dgst -sha1
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-'
In first line will be password and in second - corresponding double SHA1.
-->
<password></password>
@ -83,30 +83,7 @@
<!-- Quota for user. -->
<quota>default</quota>
<!-- Example of row level security policy. -->
<!-- <databases>
<test>
<filtered_table1>
<filter>a = 1</filter>
</filtered_table1>
<filtered_table2>
<filter>a + b &lt; 1 or c - d &gt; 5</filter>
</filtered_table2>
</test>
</databases> -->
</default>
<!-- Example of user with readonly access. -->
<!-- <readonly>
<password></password>
<networks incl="networks" replace="replace">
<ip>::1</ip>
<ip>127.0.0.1</ip>
</networks>
<profile>readonly</profile>
<quota>default</quota>
</readonly> -->
</users>
<!-- Quotas. -->

51
dbms/src/Access/AccessControlManager.cpp
View File

@ -2,8 +2,10 @@
#include <Access/MultipleAccessStorage.h>
#include <Access/MemoryAccessStorage.h>
#include <Access/UsersConfigAccessStorage.h>
#include <Access/User.h>
#include <Access/QuotaContextFactory.h>
#include <Access/RowPolicyContextFactory.h>
#include <Access/AccessRightsContext.h>
namespace DB
@ -33,6 +35,49 @@ AccessControlManager::~AccessControlManager()
}
UserPtr AccessControlManager::getUser(const String & user_name) const
{
return getUser(user_name, {}, nullptr);
}
UserPtr AccessControlManager::getUser(
const String & user_name, const std::function<void(const UserPtr &)> & on_change, ext::scope_guard * subscription) const
{
UUID id = getID<User>(user_name);
if (on_change && subscription)
{
*subscription = subscribeForChanges(id, [on_change](const UUID &, const AccessEntityPtr & user)
{
if (user)
on_change(typeid_cast<UserPtr>(user));
});
}
return read<User>(id);
}
UserPtr AccessControlManager::authorizeAndGetUser(
const String & user_name,
const String & password,
const Poco::Net::IPAddress & address) const
{
return authorizeAndGetUser(user_name, password, address, {}, nullptr);
}
UserPtr AccessControlManager::authorizeAndGetUser(
const String & user_name,
const String & password,
const Poco::Net::IPAddress & address,
const std::function<void(const UserPtr &)> & on_change,
ext::scope_guard * subscription) const
{
auto user = getUser(user_name, on_change, subscription);
user->allowed_client_hosts.checkContains(address, user_name);
user->authentication.checkPassword(password, user_name);
return user;
}
void AccessControlManager::loadFromConfig(const Poco::Util::AbstractConfiguration & users_config)
{
auto & users_config_access_storage = dynamic_cast<UsersConfigAccessStorage &>(getStorageByIndex(1));
@ -40,6 +85,12 @@ void AccessControlManager::loadFromConfig(const Poco::Util::AbstractConfiguratio
}
std::shared_ptr<const AccessRightsContext> AccessControlManager::getAccessRightsContext(const ClientInfo & client_info, const AccessRights & granted_to_user, const Settings & settings, const String & current_database)
{
return std::make_shared<AccessRightsContext>(client_info, granted_to_user, settings, current_database);
}
std::shared_ptr<QuotaContext> AccessControlManager::createQuotaContext(
const String & user_name, const Poco::Net::IPAddress & address, const String & custom_quota_key)
{

14
dbms/src/Access/AccessControlManager.h
View File

@ -2,6 +2,7 @@
#include <Access/MultipleAccessStorage.h>
#include <Poco/AutoPtr.h>
#include <ext/scope_guard.h>
#include <memory>
@ -19,11 +20,17 @@ namespace Poco
namespace DB
{
struct User;
using UserPtr = std::shared_ptr<const User>;
class QuotaContext;
class QuotaContextFactory;
struct QuotaUsageInfo;
class RowPolicyContext;
class RowPolicyContextFactory;
class AccessRights;
class AccessRightsContext;
class ClientInfo;
struct Settings;
/// Manages access control entities.
@ -35,6 +42,13 @@ public:
void loadFromConfig(const Poco::Util::AbstractConfiguration & users_config);
UserPtr getUser(const String & user_name) const;
UserPtr getUser(const String & user_name, const std::function<void(const UserPtr &)> & on_change, ext::scope_guard * subscription) const;
UserPtr authorizeAndGetUser(const String & user_name, const String & password, const Poco::Net::IPAddress & address) const;
UserPtr authorizeAndGetUser(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const std::function<void(const UserPtr &)> & on_change, ext::scope_guard * subscription) const;
std::shared_ptr<const AccessRightsContext> getAccessRightsContext(const ClientInfo & client_info, const AccessRights & granted_to_user, const Settings & settings, const String & current_database);
std::shared_ptr<QuotaContext>
createQuotaContext(const String & user_name, const Poco::Net::IPAddress & address, const String & custom_quota_key);

498
dbms/src/Access/AccessFlags.h Normal file
View File

@ -0,0 +1,498 @@
#pragma once
#include <Access/AccessType.h>
#include <Core/Types.h>
#include <Common/Exception.h>
#include <ext/range.h>
#include <ext/push_back.h>
#include <bitset>
#include <unordered_map>
namespace DB
{
/// Represents a combination of access types which can be granted globally, on databases, tables, columns, etc.
/// For example "SELECT, CREATE USER" is an access type.
class AccessFlags
{
public:
AccessFlags(AccessType type);
/// The same as AccessFlags(AccessType::NONE).
AccessFlags() = default;
/// Constructs from a string like "SELECT".
AccessFlags(const std::string_view & keyword);
/// Constructs from a list of strings like "SELECT, UPDATE, INSERT".
AccessFlags(const std::vector<std::string_view> & keywords);
AccessFlags(const Strings & keywords);
AccessFlags(const AccessFlags & src) = default;
AccessFlags(AccessFlags && src) = default;
AccessFlags & operator =(const AccessFlags & src) = default;
AccessFlags & operator =(AccessFlags && src) = default;
/// Returns the access type which contains two specified access types.
AccessFlags & operator |=(const AccessFlags & other) { flags |= other.flags; return *this; }
friend AccessFlags operator |(const AccessFlags & left, const AccessFlags & right) { return AccessFlags(left) |= right; }
/// Returns the access type which contains the common part of two access types.
AccessFlags & operator &=(const AccessFlags & other) { flags &= other.flags; return *this; }
friend AccessFlags operator &(const AccessFlags & left, const AccessFlags & right) { return AccessFlags(left) &= right; }
/// Returns the access type which contains only the part of the first access type which is not the part of the second access type.
/// (lhs - rhs) is the same as (lhs & ~rhs).
AccessFlags & operator -=(const AccessFlags & other) { flags &= ~other.flags; return *this; }
friend AccessFlags operator -(const AccessFlags & left, const AccessFlags & right) { return AccessFlags(left) -= right; }
AccessFlags operator ~() const { AccessFlags res; res.flags = ~flags; return res; }
bool isEmpty() const { return flags.none(); }
explicit operator bool() const { return !isEmpty(); }
bool contains(const AccessFlags & other) const { return (flags & other.flags) == other.flags; }
friend bool operator ==(const AccessFlags & left, const AccessFlags & right) { return left.flags == right.flags; }
friend bool operator !=(const AccessFlags & left, const AccessFlags & right) { return !(left == right); }
void clear() { flags.reset(); }
/// Returns a comma-separated list of keywords, like "SELECT, CREATE USER, UPDATE".
String toString() const;
/// Returns a list of keywords.
std::vector<std::string_view> toKeywords() const;
/// Returns the access types which could be granted on the database level.
/// For example, SELECT can be granted on the database level, but CREATE_USER cannot.
static AccessFlags databaseLevel();
/// Returns the access types which could be granted on the table/dictionary level.
static AccessFlags tableLevel();
/// Returns the access types which could be granted on the column/attribute level.
static AccessFlags columnLevel();
private:
static constexpr size_t NUM_FLAGS = 128;
using Flags = std::bitset<NUM_FLAGS>;
Flags flags;
AccessFlags(const Flags & flags_) : flags(flags_) {}
template <typename = void>
class Impl;
};
namespace ErrorCodes
{
extern const int UNKNOWN_ACCESS_TYPE;
}
template <typename>
class AccessFlags::Impl
{
public:
static const Impl & instance()
{
static const Impl res;
return res;
}
Flags accessTypeToFlags(AccessType type) const
{
return access_type_to_flags_mapping[static_cast<size_t>(type)];
}
Flags keywordToFlags(const std::string_view & keyword) const
{
auto it = keyword_to_flags_map.find(keyword);
if (it == keyword_to_flags_map.end())
{
String uppercased_keyword{keyword};
boost::to_upper(uppercased_keyword);
it = keyword_to_flags_map.find(uppercased_keyword);
if (it == keyword_to_flags_map.end())
throw Exception("Unknown access type: " + String(keyword), ErrorCodes::UNKNOWN_ACCESS_TYPE);
}
return it->second;
}
Flags keywordsToFlags(const std::vector<std::string_view> & keywords) const
{
Flags res;
for (const auto & keyword : keywords)
res |= keywordToFlags(keyword);
return res;
}
Flags keywordsToFlags(const Strings & keywords) const
{
Flags res;
for (const auto & keyword : keywords)
res |= keywordToFlags(keyword);
return res;
}
std::vector<std::string_view> flagsToKeywords(const Flags & flags_) const
{
std::vector<std::string_view> keywords;
flagsToKeywordsRec(flags_, keywords, *flags_to_keyword_tree);
if (keywords.empty())
keywords.push_back("USAGE");
return keywords;
}
String flagsToString(const Flags & flags_) const
{
String str;
for (const auto & keyword : flagsToKeywords(flags_))
{
if (!str.empty())
str += ", ";
str += keyword;
}
return str;
}
const Flags & getDatabaseLevelFlags() const { return all_grantable_on_level[DATABASE_LEVEL]; }
const Flags & getTableLevelFlags() const { return all_grantable_on_level[TABLE_LEVEL]; }
const Flags & getColumnLevelFlags() const { return all_grantable_on_level[COLUMN_LEVEL]; }
private:
enum Level
{
UNKNOWN_LEVEL = -1,
GLOBAL_LEVEL = 0,
DATABASE_LEVEL = 1,
TABLE_LEVEL = 2,
VIEW_LEVEL = 2,
DICTIONARY_LEVEL = 2,
COLUMN_LEVEL = 3,
};
struct Node;
using NodePtr = std::unique_ptr<Node>;
using Nodes = std::vector<NodePtr>;
template <typename... Args>
static Nodes nodes(Args&& ... args)
{
Nodes res;
ext::push_back(res, std::move(args)...);
return res;
}
struct Node
{
std::string_view keyword;
std::vector<String> aliases;
Flags flags;
Level level = UNKNOWN_LEVEL;
Nodes children;
Node(std::string_view keyword_, size_t flag_, Level level_)
: keyword(keyword_), level(level_)
{
flags.set(flag_);
}
Node(std::string_view keyword_, Nodes children_)
: keyword(keyword_), children(std::move(children_))
{
for (const auto & child : children)
flags |= child->flags;
}
template <typename... Args>
Node(std::string_view keyword_, NodePtr first_child, Args &&... other_children)
: Node(keyword_, nodes(std::move(first_child), std::move(other_children)...)) {}
};
static void flagsToKeywordsRec(const Flags & flags_, std::vector<std::string_view> & keywords, const Node & start_node)
{
Flags matching_flags = (flags_ & start_node.flags);
if (matching_flags.any())
{
if (matching_flags == start_node.flags)
{
keywords.push_back(start_node.keyword);
}
else
{
for (const auto & child : start_node.children)
flagsToKeywordsRec(flags_, keywords, *child);
}
}
}
static void makeFlagsToKeywordTree(NodePtr & flags_to_keyword_tree_)
{
size_t next_flag = 0;
Nodes all;
auto show = std::make_unique<Node>("SHOW", next_flag++, COLUMN_LEVEL);
auto exists = std::make_unique<Node>("EXISTS", next_flag++, COLUMN_LEVEL);
ext::push_back(all, std::move(show), std::move(exists));
auto select = std::make_unique<Node>("SELECT", next_flag++, COLUMN_LEVEL);
auto insert = std::make_unique<Node>("INSERT", next_flag++, COLUMN_LEVEL);
ext::push_back(all, std::move(select), std::move(insert));
auto update = std::make_unique<Node>("UPDATE", next_flag++, COLUMN_LEVEL);
ext::push_back(update->aliases, "ALTER UPDATE");
auto delet = std::make_unique<Node>("DELETE", next_flag++, TABLE_LEVEL);
ext::push_back(delet->aliases, "ALTER DELETE");
auto add_column = std::make_unique<Node>("ADD COLUMN", next_flag++, COLUMN_LEVEL);
add_column->aliases.push_back("ALTER ADD COLUMN");
auto modify_column = std::make_unique<Node>("MODIFY COLUMN", next_flag++, COLUMN_LEVEL);
modify_column->aliases.push_back("ALTER MODIFY COLUMN");
auto drop_column = std::make_unique<Node>("DROP COLUMN", next_flag++, COLUMN_LEVEL);
drop_column->aliases.push_back("ALTER DROP COLUMN");
auto comment_column = std::make_unique<Node>("COMMENT COLUMN", next_flag++, COLUMN_LEVEL);
comment_column->aliases.push_back("ALTER COMMENT COLUMN");
auto clear_column = std::make_unique<Node>("CLEAR COLUMN", next_flag++, COLUMN_LEVEL);
clear_column->aliases.push_back("ALTER CLEAR COLUMN");
auto alter_column = std::make_unique<Node>("ALTER COLUMN", std::move(add_column), std::move(modify_column), std::move(drop_column), std::move(comment_column), std::move(clear_column));
auto alter_order_by = std::make_unique<Node>("ALTER ORDER BY", next_flag++, TABLE_LEVEL);
alter_order_by->aliases.push_back("MODIFY ORDER BY");
alter_order_by->aliases.push_back("ALTER MODIFY ORDER BY");
auto add_index = std::make_unique<Node>("ADD INDEX", next_flag++, TABLE_LEVEL);
add_index->aliases.push_back("ALTER ADD INDEX");
auto drop_index = std::make_unique<Node>("DROP INDEX", next_flag++, TABLE_LEVEL);
drop_index->aliases.push_back("ALTER DROP INDEX");
auto materialize_index = std::make_unique<Node>("MATERIALIZE INDEX", next_flag++, TABLE_LEVEL);
materialize_index->aliases.push_back("ALTER MATERIALIZE INDEX");
auto clear_index = std::make_unique<Node>("CLEAR INDEX", next_flag++, TABLE_LEVEL);
clear_index->aliases.push_back("ALTER CLEAR INDEX");
auto index = std::make_unique<Node>("INDEX", std::move(alter_order_by), std::move(add_index), std::move(drop_index), std::move(materialize_index), std::move(clear_index));
index->aliases.push_back("ALTER INDEX");
auto add_constraint = std::make_unique<Node>("ADD CONSTRAINT", next_flag++, TABLE_LEVEL);
add_constraint->aliases.push_back("ALTER ADD CONSTRAINT");
auto drop_constraint = std::make_unique<Node>("DROP CONSTRAINT", next_flag++, TABLE_LEVEL);
drop_constraint->aliases.push_back("ALTER DROP CONSTRAINT");
auto alter_constraint = std::make_unique<Node>("CONSTRAINT", std::move(add_constraint), std::move(drop_constraint));
alter_constraint->aliases.push_back("ALTER CONSTRAINT");
auto modify_ttl = std::make_unique<Node>("MODIFY TTL", next_flag++, TABLE_LEVEL);
modify_ttl->aliases.push_back("ALTER MODIFY TTL");
auto modify_setting = std::make_unique<Node>("MODIFY SETTING", next_flag++, TABLE_LEVEL);
modify_setting->aliases.push_back("ALTER MODIFY SETTING");
auto move_partition = std::make_unique<Node>("MOVE PARTITION", next_flag++, TABLE_LEVEL);
ext::push_back(move_partition->aliases, "ALTER MOVE PARTITION", "MOVE PART", "ALTER MOVE PART");
auto fetch_partition = std::make_unique<Node>("FETCH PARTITION", next_flag++, TABLE_LEVEL);
ext::push_back(fetch_partition->aliases, "ALTER FETCH PARTITION");
auto freeze_partition = std::make_unique<Node>("FREEZE PARTITION", next_flag++, TABLE_LEVEL);
ext::push_back(freeze_partition->aliases, "ALTER FREEZE PARTITION");
auto alter_table = std::make_unique<Node>("ALTER TABLE", std::move(update), std::move(delet), std::move(alter_column), std::move(index), std::move(alter_constraint), std::move(modify_ttl), std::move(modify_setting), std::move(move_partition), std::move(fetch_partition), std::move(freeze_partition));
auto refresh_view = std::make_unique<Node>("REFRESH VIEW", next_flag++, VIEW_LEVEL);
ext::push_back(refresh_view->aliases, "ALTER LIVE VIEW REFRESH");
auto modify_view_query = std::make_unique<Node>("MODIFY VIEW QUERY", next_flag++, VIEW_LEVEL);
auto alter_view = std::make_unique<Node>("ALTER VIEW", std::move(refresh_view), std::move(modify_view_query));
auto alter = std::make_unique<Node>("ALTER", std::move(alter_table), std::move(alter_view));
ext::push_back(all, std::move(alter));
auto create_database = std::make_unique<Node>("CREATE DATABASE", next_flag++, DATABASE_LEVEL);
ext::push_back(create_database->aliases, "ATTACH DATABASE");
auto create_table = std::make_unique<Node>("CREATE TABLE", next_flag++, TABLE_LEVEL);
ext::push_back(create_table->aliases, "ATTACH TABLE");
auto create_view = std::make_unique<Node>("CREATE VIEW", next_flag++, VIEW_LEVEL);
ext::push_back(create_view->aliases, "ATTACH VIEW");
auto create_dictionary = std::make_unique<Node>("CREATE DICTIONARY", next_flag++, DICTIONARY_LEVEL);
ext::push_back(create_dictionary->aliases, "ATTACH DICTIONARY");
auto create = std::make_unique<Node>("CREATE", std::move(create_database), std::move(create_table), std::move(create_view), std::move(create_dictionary));
ext::push_back(create->aliases, "ATTACH");
ext::push_back(all, std::move(create));
auto create_temporary_table = std::make_unique<Node>("CREATE TEMPORARY TABLE", next_flag++, GLOBAL_LEVEL);
ext::push_back(all, std::move(create_temporary_table));
auto drop_database = std::make_unique<Node>("DROP DATABASE", next_flag++, DATABASE_LEVEL);
auto drop_table = std::make_unique<Node>("DROP TABLE", next_flag++, TABLE_LEVEL);
auto drop_view = std::make_unique<Node>("DROP VIEW", next_flag++, VIEW_LEVEL);
auto drop_dictionary = std::make_unique<Node>("DROP DICTIONARY", next_flag++, DICTIONARY_LEVEL);
auto drop = std::make_unique<Node>("DROP", std::move(drop_database), std::move(drop_table), std::move(drop_view), std::move(drop_dictionary));
ext::push_back(all, std::move(drop));
auto detach_database = std::make_unique<Node>("DETACH DATABASE", next_flag++, DATABASE_LEVEL);
auto detach_table = std::make_unique<Node>("DETACH TABLE", next_flag++, TABLE_LEVEL);
auto detach_view = std::make_unique<Node>("DETACH VIEW", next_flag++, VIEW_LEVEL);
auto detach_dictionary = std::make_unique<Node>("DETACH DICTIONARY", next_flag++, DICTIONARY_LEVEL);
auto detach = std::make_unique<Node>("DETACH", std::move(detach_database), std::move(detach_table), std::move(detach_view), std::move(detach_dictionary));
ext::push_back(all, std::move(detach));
auto truncate_table = std::make_unique<Node>("TRUNCATE TABLE", next_flag++, TABLE_LEVEL);
auto truncate_view = std::make_unique<Node>("TRUNCATE VIEW", next_flag++, VIEW_LEVEL);
auto truncate = std::make_unique<Node>("TRUNCATE", std::move(truncate_table), std::move(truncate_view));
ext::push_back(all, std::move(truncate));
auto optimize = std::make_unique<Node>("OPTIMIZE", next_flag++, TABLE_LEVEL);
optimize->aliases.push_back("OPTIMIZE TABLE");
ext::push_back(all, std::move(optimize));
auto kill_query = std::make_unique<Node>("KILL QUERY", next_flag++, GLOBAL_LEVEL);
auto kill_mutation = std::make_unique<Node>("KILL MUTATION", next_flag++, TABLE_LEVEL);
auto kill = std::make_unique<Node>("KILL", std::move(kill_query), std::move(kill_mutation));
ext::push_back(all, std::move(kill));
auto create_user = std::make_unique<Node>("CREATE USER", next_flag++, GLOBAL_LEVEL);
ext::push_back(create_user->aliases, "ALTER USER", "DROP USER", "CREATE ROLE", "DROP ROLE", "CREATE POLICY", "ALTER POLICY", "DROP POLICY", "CREATE QUOTA", "ALTER QUOTA", "DROP QUOTA");
ext::push_back(all, std::move(create_user));
auto shutdown = std::make_unique<Node>("SHUTDOWN", next_flag++, GLOBAL_LEVEL);
ext::push_back(shutdown->aliases, "SYSTEM SHUTDOWN", "SYSTEM KILL");
auto drop_cache = std::make_unique<Node>("DROP CACHE", next_flag++, GLOBAL_LEVEL);
ext::push_back(drop_cache->aliases, "SYSTEM DROP CACHE", "DROP DNS CACHE", "SYSTEM DROP DNS CACHE", "DROP MARK CACHE", "SYSTEM DROP MARK CACHE", "DROP UNCOMPRESSED CACHE", "SYSTEM DROP UNCOMPRESSED CACHE", "DROP COMPILED EXPRESSION CACHE", "SYSTEM DROP COMPILED EXPRESSION CACHE");
auto reload_config = std::make_unique<Node>("RELOAD CONFIG", next_flag++, GLOBAL_LEVEL);
ext::push_back(reload_config->aliases, "SYSTEM RELOAD CONFIG");
auto reload_dictionary = std::make_unique<Node>("RELOAD DICTIONARY", next_flag++, GLOBAL_LEVEL);
ext::push_back(reload_dictionary->aliases, "SYSTEM RELOAD DICTIONARY", "RELOAD DICTIONARIES", "SYSTEM RELOAD DICTIONARIES", "RELOAD EMBEDDED DICTIONARIES", "SYSTEM RELOAD EMBEDDED DICTIONARIES");
auto stop_merges = std::make_unique<Node>("STOP MERGES", next_flag++, TABLE_LEVEL);
ext::push_back(stop_merges->aliases, "SYSTEM STOP MERGES", "START MERGES", "SYSTEM START MERGES");
auto stop_ttl_merges = std::make_unique<Node>("STOP TTL MERGES", next_flag++, TABLE_LEVEL);
ext::push_back(stop_ttl_merges->aliases, "SYSTEM STOP TTL MERGES", "START TTL MERGES", "SYSTEM START TTL MERGES");
auto stop_fetches = std::make_unique<Node>("STOP FETCHES", next_flag++, TABLE_LEVEL);
ext::push_back(stop_fetches->aliases, "SYSTEM STOP FETCHES", "START FETCHES", "SYSTEM START FETCHES");
auto stop_moves = std::make_unique<Node>("STOP MOVES", next_flag++, TABLE_LEVEL);
ext::push_back(stop_moves->aliases, "SYSTEM STOP MOVES", "START MOVES", "SYSTEM START MOVES");
auto stop_distributed_sends = std::make_unique<Node>("STOP DISTRIBUTED SENDS", next_flag++, TABLE_LEVEL);
ext::push_back(stop_distributed_sends->aliases, "SYSTEM STOP DISTRIBUTED SENDS", "START DISTRIBUTED SENDS", "SYSTEM START DISTRIBUTED SENDS");
auto stop_replicated_sends = std::make_unique<Node>("STOP REPLICATED SENDS", next_flag++, TABLE_LEVEL);
ext::push_back(stop_replicated_sends->aliases, "SYSTEM STOP REPLICATED SENDS", "START REPLICATED SENDS", "SYSTEM START REPLICATED SENDS");
auto stop_replication_queues = std::make_unique<Node>("STOP REPLICATION QUEUES", next_flag++, TABLE_LEVEL);
ext::push_back(stop_replication_queues->aliases, "SYSTEM STOP REPLICATION QUEUES", "START REPLICATION QUEUES", "SYSTEM START REPLICATION QUEUES");
auto sync_replica = std::make_unique<Node>("SYNC REPLICA", next_flag++, TABLE_LEVEL);
ext::push_back(sync_replica->aliases, "SYSTEM SYNC REPLICA");
auto restart_replica = std::make_unique<Node>("RESTART REPLICA", next_flag++, TABLE_LEVEL);
ext::push_back(restart_replica->aliases, "SYSTEM RESTART REPLICA");
auto flush_distributed = std::make_unique<Node>("FLUSH DISTRIBUTED", next_flag++, TABLE_LEVEL);
ext::push_back(flush_distributed->aliases, "SYSTEM FLUSH DISTRIBUTED");
auto flush_logs = std::make_unique<Node>("FLUSH LOGS", next_flag++, GLOBAL_LEVEL);
ext::push_back(flush_logs->aliases, "SYSTEM FLUSH LOGS");
auto system = std::make_unique<Node>("SYSTEM", std::move(shutdown), std::move(drop_cache), std::move(reload_config), std::move(reload_dictionary), std::move(stop_merges), std::move(stop_ttl_merges), std::move(stop_fetches), std::move(stop_moves), std::move(stop_distributed_sends), std::move(stop_replicated_sends), std::move(stop_replication_queues), std::move(sync_replica), std::move(restart_replica), std::move(flush_distributed), std::move(flush_logs));
ext::push_back(all, std::move(system));
auto dict_get = std::make_unique<Node>("dictGet()", next_flag++, DICTIONARY_LEVEL);
dict_get->aliases.push_back("dictHas()");
dict_get->aliases.push_back("dictGetHierarchy()");
dict_get->aliases.push_back("dictIsIn()");
ext::push_back(all, std::move(dict_get));
auto address_to_line = std::make_unique<Node>("addressToLine()", next_flag++, GLOBAL_LEVEL);
auto address_to_symbol = std::make_unique<Node>("addressToSymbol()", next_flag++, GLOBAL_LEVEL);
auto demangle = std::make_unique<Node>("demangle()", next_flag++, GLOBAL_LEVEL);
auto introspection = std::make_unique<Node>("INTROSPECTION", std::move(address_to_line), std::move(address_to_symbol), std::move(demangle));
ext::push_back(introspection->aliases, "INTROSPECTION FUNCTIONS");
ext::push_back(all, std::move(introspection));
auto file = std::make_unique<Node>("file()", next_flag++, GLOBAL_LEVEL);
auto url = std::make_unique<Node>("url()", next_flag++, GLOBAL_LEVEL);
auto input = std::make_unique<Node>("input()", next_flag++, GLOBAL_LEVEL);
auto values = std::make_unique<Node>("values()", next_flag++, GLOBAL_LEVEL);
auto numbers = std::make_unique<Node>("numbers()", next_flag++, GLOBAL_LEVEL);
auto merge = std::make_unique<Node>("merge()", next_flag++, DATABASE_LEVEL);
auto remote = std::make_unique<Node>("remote()", next_flag++, GLOBAL_LEVEL);
ext::push_back(remote->aliases, "remoteSecure()", "cluster()");
auto mysql = std::make_unique<Node>("mysql()", next_flag++, GLOBAL_LEVEL);
auto odbc = std::make_unique<Node>("odbc()", next_flag++, GLOBAL_LEVEL);
auto jdbc = std::make_unique<Node>("jdbc()", next_flag++, GLOBAL_LEVEL);
auto hdfs = std::make_unique<Node>("hdfs()", next_flag++, GLOBAL_LEVEL);
auto s3 = std::make_unique<Node>("s3()", next_flag++, GLOBAL_LEVEL);
auto table_functions = std::make_unique<Node>("TABLE FUNCTIONS", std::move(file), std::move(url), std::move(input), std::move(values), std::move(numbers), std::move(merge), std::move(remote), std::move(mysql), std::move(odbc), std::move(jdbc), std::move(hdfs), std::move(s3));
ext::push_back(all, std::move(table_functions));
flags_to_keyword_tree_ = std::make_unique<Node>("ALL", std::move(all));
flags_to_keyword_tree_->aliases.push_back("ALL PRIVILEGES");
}
void makeKeywordToFlagsMap(std::unordered_map<std::string_view, Flags> & keyword_to_flags_map_, Node * start_node = nullptr)
{
if (!start_node)
{
start_node = flags_to_keyword_tree.get();
keyword_to_flags_map_["USAGE"] = {};
keyword_to_flags_map_["NONE"] = {};
keyword_to_flags_map_["NO PRIVILEGES"] = {};
}
start_node->aliases.emplace_back(start_node->keyword);
for (auto & alias : start_node->aliases)
{
boost::to_upper(alias);
keyword_to_flags_map_[alias] = start_node->flags;
}
for (auto & child : start_node->children)
makeKeywordToFlagsMap(keyword_to_flags_map_, child.get());
}
void makeAccessTypeToFlagsMapping(std::vector<Flags> & access_type_to_flags_mapping_)
{
access_type_to_flags_mapping_.resize(MAX_ACCESS_TYPE);
for (auto access_type : ext::range_with_static_cast<AccessType>(0, MAX_ACCESS_TYPE))
{
auto str = toKeyword(access_type);
auto it = keyword_to_flags_map.find(str);
if (it == keyword_to_flags_map.end())
{
String uppercased{str};
boost::to_upper(uppercased);
it = keyword_to_flags_map.find(uppercased);
}
access_type_to_flags_mapping_[static_cast<size_t>(access_type)] = it->second;
}
}
void collectAllGrantableOnLevel(std::vector<Flags> & all_grantable_on_level_, const Node * start_node = nullptr)
{
if (!start_node)
{
start_node = flags_to_keyword_tree.get();
all_grantable_on_level.resize(COLUMN_LEVEL + 1);
}
for (int i = 0; i <= start_node->level; ++i)
all_grantable_on_level_[i] |= start_node->flags;
for (const auto & child : start_node->children)
collectAllGrantableOnLevel(all_grantable_on_level_, child.get());
}
Impl()
{
makeFlagsToKeywordTree(flags_to_keyword_tree);
makeKeywordToFlagsMap(keyword_to_flags_map);
makeAccessTypeToFlagsMapping(access_type_to_flags_mapping);
collectAllGrantableOnLevel(all_grantable_on_level);
}
std::unique_ptr<Node> flags_to_keyword_tree;
std::unordered_map<std::string_view, Flags> keyword_to_flags_map;
std::vector<Flags> access_type_to_flags_mapping;
std::vector<Flags> all_grantable_on_level;
};
inline AccessFlags::AccessFlags(AccessType type) : flags(Impl<>::instance().accessTypeToFlags(type)) {}
inline AccessFlags::AccessFlags(const std::string_view & keyword) : flags(Impl<>::instance().keywordToFlags(keyword)) {}
inline AccessFlags::AccessFlags(const std::vector<std::string_view> & keywords) : flags(Impl<>::instance().keywordsToFlags(keywords)) {}
inline AccessFlags::AccessFlags(const Strings & keywords) : flags(Impl<>::instance().keywordsToFlags(keywords)) {}
inline String AccessFlags::toString() const { return Impl<>::instance().flagsToString(flags); }
inline std::vector<std::string_view> AccessFlags::toKeywords() const { return Impl<>::instance().flagsToKeywords(flags); }
inline AccessFlags AccessFlags::databaseLevel() { return Impl<>::instance().getDatabaseLevelFlags(); }
inline AccessFlags AccessFlags::tableLevel() { return Impl<>::instance().getTableLevelFlags(); }
inline AccessFlags AccessFlags::columnLevel() { return Impl<>::instance().getColumnLevelFlags(); }
inline AccessFlags operator |(AccessType left, AccessType right) { return AccessFlags(left) | right; }
inline AccessFlags operator &(AccessType left, AccessType right) { return AccessFlags(left) & right; }
inline AccessFlags operator -(AccessType left, AccessType right) { return AccessFlags(left) - right; }
inline AccessFlags operator ~(AccessType x) { return ~AccessFlags(x); }
}

729
dbms/src/Access/AccessRights.cpp Normal file
View File

@ -0,0 +1,729 @@
#include <Access/AccessRights.h>
#include <Common/Exception.h>
#include <boost/range/adaptor/map.hpp>
#include <unordered_map>
namespace DB
{
namespace ErrorCodes
{
extern const int INVALID_GRANT;
extern const int LOGICAL_ERROR;
}
namespace
{
enum Level
{
GLOBAL_LEVEL,
DATABASE_LEVEL,
TABLE_LEVEL,
COLUMN_LEVEL,
};
enum RevokeMode
{
NORMAL_REVOKE_MODE, /// for AccessRights::revoke()
PARTIAL_REVOKE_MODE, /// for AccessRights::partialRevoke()
FULL_REVOKE_MODE, /// for AccessRights::fullRevoke()
};
struct Helper
{
static const Helper & instance()
{
static const Helper res;
return res;
}
const AccessFlags database_level_flags = AccessFlags::databaseLevel();
const AccessFlags table_level_flags = AccessFlags::tableLevel();
const AccessFlags column_level_flags = AccessFlags::columnLevel();
const AccessFlags show_flag = AccessType::SHOW;
const AccessFlags exists_flag = AccessType::EXISTS;
const AccessFlags create_table_flag = AccessType::CREATE_TABLE;
const AccessFlags create_temporary_table_flag = AccessType::CREATE_TEMPORARY_TABLE;
};
}
struct AccessRights::Node
{
public:
std::shared_ptr<const String> node_name;
Level level = GLOBAL_LEVEL;
AccessFlags explicit_grants;
AccessFlags partial_revokes;
AccessFlags inherited_access; /// the access inherited from the parent node
AccessFlags raw_access; /// raw_access = (inherited_access - partial_revokes) | explicit_grants
AccessFlags access; /// access = raw_access | implicit_access
AccessFlags min_access; /// min_access = access & child[0].access & ... | child[N-1].access
AccessFlags max_access; /// max_access = access | child[0].access | ... | child[N-1].access
std::unique_ptr<std::unordered_map<std::string_view, Node>> children;
Node() = default;
Node(const Node & src) { *this = src; }
Node & operator =(const Node & src)
{
node_name = src.node_name;
level = src.level;
inherited_access = src.inherited_access;
explicit_grants = src.explicit_grants;
partial_revokes = src.partial_revokes;
access = src.access;
min_access = src.min_access;
max_access = src.max_access;
if (src.children)
children = std::make_unique<std::unordered_map<std::string_view, Node>>(*src.children);
else
children = nullptr;
return *this;
}
void grant(AccessFlags access_to_grant, const Helper & helper)
{
if (!access_to_grant)
return;
if (level == GLOBAL_LEVEL)
{
/// Everything can be granted on the global level.
}
else if (level == DATABASE_LEVEL)
{
AccessFlags grantable = access_to_grant & helper.database_level_flags;
if (!grantable)
throw Exception(access_to_grant.toString() + " cannot be granted on the database level", ErrorCodes::INVALID_GRANT);
access_to_grant = grantable;
}
else if (level == TABLE_LEVEL)
{
AccessFlags grantable = access_to_grant & helper.table_level_flags;
if (!grantable)
throw Exception(access_to_grant.toString() + " cannot be granted on the table level", ErrorCodes::INVALID_GRANT);
access_to_grant = grantable;
}
else if (level == COLUMN_LEVEL)
{
AccessFlags grantable = access_to_grant & helper.column_level_flags;
if (!grantable)
throw Exception(access_to_grant.toString() + " cannot be granted on the column level", ErrorCodes::INVALID_GRANT);
access_to_grant = grantable;
}
explicit_grants |= access_to_grant - partial_revokes;
partial_revokes -= access_to_grant;
calculateAllAccessRec(helper);
}
template <typename ... Args>
void grant(const AccessFlags & access_to_grant, const Helper & helper, const std::string_view & name, const Args &... subnames)
{
auto & child = getChild(name);
child.grant(access_to_grant, helper, subnames...);
eraseChildIfEmpty(child);
calculateImplicitAccess(helper);
calculateMinAndMaxAccess();
}
template <typename StringT>
void grant(const AccessFlags & access_to_grant, const Helper & helper, const std::vector<StringT> & names)
{
for (const auto & name : names)
{
auto & child = getChild(name);
child.grant(access_to_grant, helper);
eraseChildIfEmpty(child);
}
calculateImplicitAccess(helper);
calculateMinAndMaxAccess();
}
template <int mode>
void revoke(const AccessFlags & access_to_revoke, const Helper & helper)
{
if constexpr (mode == NORMAL_REVOKE_MODE)
{
explicit_grants -= access_to_revoke;
}
else if constexpr (mode == PARTIAL_REVOKE_MODE)
{
partial_revokes |= access_to_revoke - explicit_grants;
explicit_grants -= access_to_revoke;
}
else /// mode == FULL_REVOKE_MODE
{
fullRevokeRec(access_to_revoke);
}
calculateAllAccessRec(helper);
}
template <int mode, typename... Args>
void revoke(const AccessFlags & access_to_revoke, const Helper & helper, const std::string_view & name, const Args &... subnames)
{
Node * child;
if (mode == NORMAL_REVOKE_MODE)
{
if (!(child = tryGetChild(name)))
return;
}
else
child = &getChild(name);
child->revoke<mode>(access_to_revoke, helper, subnames...);
eraseChildIfEmpty(*child);
calculateImplicitAccess(helper);
calculateMinAndMaxAccess();
}
template <int mode, typename StringT>
void revoke(const AccessFlags & access_to_revoke, const Helper & helper, const std::vector<StringT> & names)
{
Node * child;
for (const auto & name : names)
{
if (mode == NORMAL_REVOKE_MODE)
{
if (!(child = tryGetChild(name)))
continue;
}
else
child = &getChild(name);
child->revoke<mode>(access_to_revoke, helper);
eraseChildIfEmpty(*child);
}
calculateImplicitAccess(helper);
calculateMinAndMaxAccess();
}
bool isGranted(const AccessFlags & flags) const
{
return min_access.contains(flags);
}
template <typename... Args>
bool isGranted(AccessFlags flags, const std::string_view & name, const Args &... subnames) const
{
if (min_access.contains(flags))
return true;
if (!max_access.contains(flags))
return false;
const Node * child = tryGetChild(name);
if (child)
return child->isGranted(flags, subnames...);
else
return access.contains(flags);
}
template <typename StringT>
bool isGranted(AccessFlags flags, const std::vector<StringT> & names) const
{
if (min_access.contains(flags))
return true;
if (!max_access.contains(flags))
return false;
for (const auto & name : names)
{
const Node * child = tryGetChild(name);
if (child)
{
if (!child->isGranted(flags, name))
return false;
}
else
{
if (!access.contains(flags))
return false;
}
}
return true;
}
friend bool operator ==(const Node & left, const Node & right)
{
if ((left.explicit_grants != right.explicit_grants) || (left.partial_revokes != right.partial_revokes))
return false;
if (!left.children)
return !right.children;
if (!right.children)
return false;
return *left.children == *right.children;
}
friend bool operator!=(const Node & left, const Node & right) { return !(left == right); }
bool isEmpty() const
{
return !explicit_grants && !partial_revokes && !children;
}
void merge(const Node & other, const Helper & helper)
{
mergeRawAccessRec(other);
calculateGrantsAndPartialRevokesRec();
calculateAllAccessRec(helper);
}
private:
Node * tryGetChild(const std::string_view & name)
{
if (!children)
return nullptr;
auto it = children->find(name);
if (it == children->end())
return nullptr;
return &it->second;
}
const Node * tryGetChild(const std::string_view & name) const
{
if (!children)
return nullptr;
auto it = children->find(name);
if (it == children->end())
return nullptr;
return &it->second;
}
Node & getChild(const std::string_view & name)
{
auto * child = tryGetChild(name);
if (child)
return *child;
if (!children)
children = std::make_unique<std::unordered_map<std::string_view, Node>>();
auto new_child_name = std::make_shared<const String>(name);
Node & new_child = (*children)[*new_child_name];
new_child.node_name = std::move(new_child_name);
new_child.level = static_cast<Level>(level + 1);
new_child.inherited_access = raw_access;
new_child.raw_access = raw_access;
return new_child;
}
void eraseChildIfEmpty(Node & child)
{
if (!child.isEmpty())
return;
auto it = children->find(*child.node_name);
children->erase(it);
if (children->empty())
children = nullptr;
}
void calculateImplicitAccess(const Helper & helper)
{
access = raw_access;
if (access & helper.database_level_flags)
access |= helper.show_flag | helper.exists_flag;
else if ((level >= DATABASE_LEVEL) && children)
access |= helper.exists_flag;
if ((level == GLOBAL_LEVEL) && (access & helper.create_table_flag))
access |= helper.create_temporary_table_flag;
}
void calculateMinAndMaxAccess()
{
min_access = access;
max_access = access;
if (children)
{
for (const auto & child : *children | boost::adaptors::map_values)
{
min_access &= child.min_access;
max_access |= child.max_access;
}
}
}
void calculateAllAccessRec(const Helper & helper)
{
partial_revokes &= inherited_access;
raw_access = (inherited_access - partial_revokes) | explicit_grants;
/// Traverse tree.
if (children)
{
for (auto it = children->begin(); it != children->end();)
{
auto & child = it->second;
child.inherited_access = raw_access;
child.calculateAllAccessRec(helper);
if (child.isEmpty())
it = children->erase(it);
else
++it;
}
if (children->empty())
children = nullptr;
}
calculateImplicitAccess(helper);
calculateMinAndMaxAccess();
}
void fullRevokeRec(const AccessFlags & access_to_revoke)
{
explicit_grants -= access_to_revoke;
partial_revokes |= access_to_revoke;
if (children)
{
for (auto & child : *children | boost::adaptors::map_values)
child.fullRevokeRec(access_to_revoke);
}
}
void mergeRawAccessRec(const Node & rhs)
{
if (rhs.children)
{
for (const auto & [rhs_childname, rhs_child] : *rhs.children)
getChild(rhs_childname).mergeRawAccessRec(rhs_child);
}
raw_access |= rhs.raw_access;
if (children)
{
for (auto & [lhs_childname, lhs_child] : *children)
{
lhs_child.inherited_access = raw_access;
if (!rhs.tryGetChild(lhs_childname))
lhs_child.raw_access |= rhs.raw_access;
}
}
}
void calculateGrantsAndPartialRevokesRec()
{
explicit_grants = raw_access - inherited_access;
partial_revokes = inherited_access - raw_access;
if (children)
{
for (auto & child : *children | boost::adaptors::map_values)
child.calculateGrantsAndPartialRevokesRec();
}
}
};
AccessRights::AccessRights() = default;
AccessRights::~AccessRights() = default;
AccessRights::AccessRights(AccessRights && src) = default;
AccessRights & AccessRights::operator =(AccessRights && src) = default;
AccessRights::AccessRights(const AccessRights & src)
{
*this = src;
}
AccessRights & AccessRights::operator =(const AccessRights & src)
{
if (src.root)
root = std::make_unique<Node>(*src.root);
else
root = nullptr;
return *this;
}
AccessRights::AccessRights(const AccessFlags & access)
{
grant(access);
}
bool AccessRights::isEmpty() const
{
return !root;
}
void AccessRights::clear()
{
root = nullptr;
}
template <typename... Args>
void AccessRights::grantImpl(const AccessFlags & access, const Args &... args)
{
if (!root)
root = std::make_unique<Node>();
root->grant(access, Helper::instance(), args...);
if (root->isEmpty())
root = nullptr;
}
void AccessRights::grantImpl(const AccessRightsElement & element, std::string_view current_database)
{
if (element.any_database)
{
grantImpl(element.access_flags);
}
else if (element.any_table)
{
if (element.database.empty())
grantImpl(element.access_flags, current_database);
else
grantImpl(element.access_flags, element.database);
}
else if (element.any_column)
{
if (element.database.empty())
grantImpl(element.access_flags, current_database, element.table);
else
grantImpl(element.access_flags, element.database, element.table);
}
else
{
if (element.database.empty())
grantImpl(element.access_flags, current_database, element.table, element.columns);
else
grantImpl(element.access_flags, element.database, element.table, element.columns);
}
}
void AccessRights::grantImpl(const AccessRightsElements & elements, std::string_view current_database)
{
for (const auto & element : elements)
grantImpl(element, current_database);
}
void AccessRights::grant(const AccessFlags & access) { grantImpl(access); }
void AccessRights::grant(const AccessFlags & access, const std::string_view & database) { grantImpl(access, database); }
void AccessRights::grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table) { grantImpl(access, database, table); }
void AccessRights::grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl(access, database, table, column); }
void AccessRights::grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl(access, database, table, columns); }
void AccessRights::grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl(access, database, table, columns); }
void AccessRights::grant(const AccessRightsElement & element, std::string_view current_database) { grantImpl(element, current_database); }
void AccessRights::grant(const AccessRightsElements & elements, std::string_view current_database) { grantImpl(elements, current_database); }
template <int mode, typename... Args>
void AccessRights::revokeImpl(const AccessFlags & access, const Args &... args)
{
if (!root)
return;
root->revoke<mode>(access, Helper::instance(), args...);
if (root->isEmpty())
root = nullptr;
}
template <int mode>
void AccessRights::revokeImpl(const AccessRightsElement & element, std::string_view current_database)
{
if (element.any_database)
{
revokeImpl<mode>(element.access_flags);
}
else if (element.any_table)
{
if (element.database.empty())
revokeImpl<mode>(element.access_flags, current_database);
else
revokeImpl<mode>(element.access_flags, element.database);
}
else if (element.any_column)
{
if (element.database.empty())
revokeImpl<mode>(element.access_flags, current_database, element.table);
else
revokeImpl<mode>(element.access_flags, element.database, element.table);
}
else
{
if (element.database.empty())
revokeImpl<mode>(element.access_flags, current_database, element.table, element.columns);
else
revokeImpl<mode>(element.access_flags, element.database, element.table, element.columns);
}
}
template <int mode>
void AccessRights::revokeImpl(const AccessRightsElements & elements, std::string_view current_database)
{
for (const auto & element : elements)
revokeImpl<mode>(element, current_database);
}
void AccessRights::revoke(const AccessFlags & access) { revokeImpl<NORMAL_REVOKE_MODE>(access); }
void AccessRights::revoke(const AccessFlags & access, const std::string_view & database) { revokeImpl<NORMAL_REVOKE_MODE>(access, database); }
void AccessRights::revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table) { revokeImpl<NORMAL_REVOKE_MODE>(access, database, table); }
void AccessRights::revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<NORMAL_REVOKE_MODE>(access, database, table, column); }
void AccessRights::revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<NORMAL_REVOKE_MODE>(access, database, table, columns); }
void AccessRights::revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<NORMAL_REVOKE_MODE>(access, database, table, columns); }
void AccessRights::revoke(const AccessRightsElement & element, std::string_view current_database) { revokeImpl<NORMAL_REVOKE_MODE>(element, current_database); }
void AccessRights::revoke(const AccessRightsElements & elements, std::string_view current_database) { revokeImpl<NORMAL_REVOKE_MODE>(elements, current_database); }
void AccessRights::partialRevoke(const AccessFlags & access) { revokeImpl<PARTIAL_REVOKE_MODE>(access); }
void AccessRights::partialRevoke(const AccessFlags & access, const std::string_view & database) { revokeImpl<PARTIAL_REVOKE_MODE>(access, database); }
void AccessRights::partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table) { revokeImpl<PARTIAL_REVOKE_MODE>(access, database, table); }
void AccessRights::partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<PARTIAL_REVOKE_MODE>(access, database, table, column); }
void AccessRights::partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<PARTIAL_REVOKE_MODE>(access, database, table, columns); }
void AccessRights::partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<PARTIAL_REVOKE_MODE>(access, database, table, columns); }
void AccessRights::partialRevoke(const AccessRightsElement & element, std::string_view current_database) { revokeImpl<PARTIAL_REVOKE_MODE>(element, current_database); }
void AccessRights::partialRevoke(const AccessRightsElements & elements, std::string_view current_database) { revokeImpl<PARTIAL_REVOKE_MODE>(elements, current_database); }
void AccessRights::fullRevoke(const AccessFlags & access) { revokeImpl<FULL_REVOKE_MODE>(access); }
void AccessRights::fullRevoke(const AccessFlags & access, const std::string_view & database) { revokeImpl<FULL_REVOKE_MODE>(access, database); }
void AccessRights::fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table) { revokeImpl<FULL_REVOKE_MODE>(access, database, table); }
void AccessRights::fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<FULL_REVOKE_MODE>(access, database, table, column); }
void AccessRights::fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<FULL_REVOKE_MODE>(access, database, table, columns); }
void AccessRights::fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<FULL_REVOKE_MODE>(access, database, table, columns); }
void AccessRights::fullRevoke(const AccessRightsElement & element, std::string_view current_database) { revokeImpl<FULL_REVOKE_MODE>(element, current_database); }
void AccessRights::fullRevoke(const AccessRightsElements & elements, std::string_view current_database) { revokeImpl<FULL_REVOKE_MODE>(elements, current_database); }
AccessRights::Elements AccessRights::getElements() const
{
if (!root)
return {};
Elements res;
if (root->explicit_grants)
res.grants.push_back({root->explicit_grants});
if (root->children)
{
for (const auto & [db_name, db_node] : *root->children)
{
if (db_node.partial_revokes)
res.partial_revokes.push_back({db_node.partial_revokes, db_name});
if (db_node.explicit_grants)
res.grants.push_back({db_node.explicit_grants, db_name});
if (db_node.children)
{
for (const auto & [table_name, table_node] : *db_node.children)
{
if (table_node.partial_revokes)
res.partial_revokes.push_back({table_node.partial_revokes, db_name, table_name});
if (table_node.explicit_grants)
res.grants.push_back({table_node.explicit_grants, db_name, table_name});
if (table_node.children)
{
for (const auto & [column_name, column_node] : *table_node.children)
{
if (column_node.partial_revokes)
res.partial_revokes.push_back({column_node.partial_revokes, db_name, table_name, column_name});
if (column_node.explicit_grants)
res.grants.push_back({column_node.explicit_grants, db_name, table_name, column_name});
}
}
}
}
}
}
return res;
}
String AccessRights::toString() const
{
auto elements = getElements();
String res;
if (!elements.grants.empty())
{
res += "GRANT ";
res += elements.grants.toString();
}
if (!elements.partial_revokes.empty())
{
if (!res.empty())
res += ", ";
res += "REVOKE ";
res += elements.partial_revokes.toString();
}
if (res.empty())
res = "GRANT USAGE ON *.*";
return res;
}
template <typename... Args>
bool AccessRights::isGrantedImpl(const AccessFlags & access, const Args &... args) const
{
if (!root)
return access.isEmpty();
return root->isGranted(access, args...);
}
bool AccessRights::isGrantedImpl(const AccessRightsElement & element, std::string_view current_database) const
{
if (element.any_database)
{
return isGrantedImpl(element.access_flags);
}
else if (element.any_table)
{
if (element.database.empty())
return isGrantedImpl(element.access_flags, current_database);
else
return isGrantedImpl(element.access_flags, element.database);
}
else if (element.any_column)
{
if (element.database.empty())
return isGrantedImpl(element.access_flags, current_database, element.table);
else
return isGrantedImpl(element.access_flags, element.database, element.table);
}
else
{
if (element.database.empty())
return isGrantedImpl(element.access_flags, current_database, element.table, element.columns);
else
return isGrantedImpl(element.access_flags, element.database, element.table, element.columns);
}
}
bool AccessRights::isGrantedImpl(const AccessRightsElements & elements, std::string_view current_database) const
{
for (const auto & element : elements)
if (!isGrantedImpl(element, current_database))
return false;
return true;
}
bool AccessRights::isGranted(const AccessFlags & access) const { return isGrantedImpl(access); }
bool AccessRights::isGranted(const AccessFlags & access, const std::string_view & database) const { return isGrantedImpl(access, database); }
bool AccessRights::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl(access, database, table); }
bool AccessRights::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl(access, database, table, column); }
bool AccessRights::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl(access, database, table, columns); }
bool AccessRights::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl(access, database, table, columns); }
bool AccessRights::isGranted(const AccessRightsElement & element, std::string_view current_database) const { return isGrantedImpl(element, current_database); }
bool AccessRights::isGranted(const AccessRightsElements & elements, std::string_view current_database) const { return isGrantedImpl(elements, current_database); }
bool operator ==(const AccessRights & left, const AccessRights & right)
{
if (!left.root)
return !right.root;
if (!right.root)
return false;
return *left.root == *right.root;
}
void AccessRights::merge(const AccessRights & other)
{
if (!root)
{
*this = other;
return;
}
if (other.root)
{
root->merge(*other.root, Helper::instance());
if (root->isEmpty())
root = nullptr;
}
}
}

137
dbms/src/Access/AccessRights.h Normal file
View File

@ -0,0 +1,137 @@
#pragma once
#include <Core/Types.h>
#include <Access/AccessRightsElement.h>
#include <memory>
#include <vector>
namespace DB
{
/// Represents a set of access types granted on databases, tables, columns, etc.
/// For example, "GRANT SELECT, UPDATE ON db.*, GRANT INSERT ON db2.mytbl2" are access rights.
class AccessRights
{
public:
AccessRights();
AccessRights(const AccessFlags & access);
~AccessRights();
AccessRights(const AccessRights & src);
AccessRights & operator =(const AccessRights & src);
AccessRights(AccessRights && src);
AccessRights & operator =(AccessRights && src);
bool isEmpty() const;
/// Revokes everything. It's the same as fullRevoke(AccessType::ALL).
void clear();
/// Grants access on a specified database/table/column.
/// Does nothing if the specified access has been already granted.
void grant(const AccessFlags & access);
void grant(const AccessFlags & access, const std::string_view & database);
void grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table);
void grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column);
void grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
void grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns);
void grant(const AccessRightsElement & access, std::string_view current_database = {});
void grant(const AccessRightsElements & access, std::string_view current_database = {});
/// Revokes a specified access granted earlier on a specified database/table/column.
/// Does nothing if the specified access is not granted.
/// If the specified access is granted but on upper level (e.g. database for table, table for columns)
/// or lower level, the function also does nothing.
/// This function implements the standard SQL REVOKE behaviour.
void revoke(const AccessFlags & access);
void revoke(const AccessFlags & access, const std::string_view & database);
void revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table);
void revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column);
void revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
void revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns);
void revoke(const AccessRightsElement & access, std::string_view current_database = {});
void revoke(const AccessRightsElements & access, std::string_view current_database = {});
/// Revokes a specified access granted earlier on a specified database/table/column or on lower levels.
/// The function also restricts access if it's granted on upper level.
/// For example, an access could be granted on a database and then revoked on a table in this database.
/// This function implements the MySQL REVOKE behaviour with partial_revokes is ON.
void partialRevoke(const AccessFlags & access);
void partialRevoke(const AccessFlags & access, const std::string_view & database);
void partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table);
void partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column);
void partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
void partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns);
void partialRevoke(const AccessRightsElement & access, std::string_view current_database = {});
void partialRevoke(const AccessRightsElements & access, std::string_view current_database = {});
/// Revokes a specified access granted earlier on a specified database/table/column or on lower levels.
/// The function also restricts access if it's granted on upper level.
/// For example, fullRevoke(AccessType::ALL) revokes all grants at all, just like clear();
/// fullRevoke(AccessType::SELECT, db) means it's not allowed to execute SELECT in that database anymore (from any table).
void fullRevoke(const AccessFlags & access);
void fullRevoke(const AccessFlags & access, const std::string_view & database);
void fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table);
void fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column);
void fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
void fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns);
void fullRevoke(const AccessRightsElement & access, std::string_view current_database = {});
void fullRevoke(const AccessRightsElements & access, std::string_view current_database = {});
/// Returns the information about all the access granted.
struct Elements
{
AccessRightsElements grants;
AccessRightsElements partial_revokes;
};
Elements getElements() const;
/// Returns the information about all the access granted as a string.
String toString() const;
/// Whether a specified access granted.
bool isGranted(const AccessFlags & access) const;
bool isGranted(const AccessFlags & access, const std::string_view & database) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
bool isGranted(const AccessRightsElement & access, std::string_view current_database = {}) const;
bool isGranted(const AccessRightsElements & access, std::string_view current_database = {}) const;
friend bool operator ==(const AccessRights & left, const AccessRights & right);
friend bool operator !=(const AccessRights & left, const AccessRights & right) { return !(left == right); }
/// Merges two sets of access rights together.
/// It's used to combine access rights from multiple roles.
void merge(const AccessRights & other);
private:
template <typename... Args>
void grantImpl(const AccessFlags & access, const Args &... args);
void grantImpl(const AccessRightsElement & access, std::string_view current_database);
void grantImpl(const AccessRightsElements & access, std::string_view current_database);
template <int mode, typename... Args>
void revokeImpl(const AccessFlags & access, const Args &... args);
template <int mode>
void revokeImpl(const AccessRightsElement & access, std::string_view current_database);
template <int mode>
void revokeImpl(const AccessRightsElements & access, std::string_view current_database);
template <typename... Args>
bool isGrantedImpl(const AccessFlags & access, const Args &... args) const;
bool isGrantedImpl(const AccessRightsElement & access, std::string_view current_database) const;
bool isGrantedImpl(const AccessRightsElements & access, std::string_view current_database) const;
template <typename... Args>
AccessFlags getAccessImpl(const Args &... args) const;
struct Node;
std::unique_ptr<Node> root;
};
}

288
dbms/src/Access/AccessRightsContext.cpp Normal file
View File

@ -0,0 +1,288 @@
#include <Access/AccessRightsContext.h>
#include <Common/Exception.h>
#include <Common/quoteString.h>
#include <Core/Settings.h>
#include <Poco/Logger.h>
#include <common/logger_useful.h>
#include <boost/smart_ptr/make_shared_object.hpp>
#include <assert.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ACCESS_DENIED;
extern const int READONLY;
extern const int QUERY_IS_PROHIBITED;
extern const int FUNCTION_NOT_ALLOWED;
}
namespace
{
enum CheckAccessRightsMode
{
RETURN_FALSE_IF_ACCESS_DENIED,
LOG_WARNING_IF_ACCESS_DENIED,
THROW_IF_ACCESS_DENIED,
};
String formatSkippedMessage()
{
return "";
}
String formatSkippedMessage(const std::string_view & database)
{
return ". Skipped database " + backQuoteIfNeed(database);
}
String formatSkippedMessage(const std::string_view & database, const std::string_view & table)
{
String str = ". Skipped table ";
if (!database.empty())
str += backQuoteIfNeed(database) + ".";
str += backQuoteIfNeed(table);
return str;
}
String formatSkippedMessage(const std::string_view & database, const std::string_view & table, const std::string_view & column)
{
String str = ". Skipped column " + backQuoteIfNeed(column) + " ON ";
if (!database.empty())
str += backQuoteIfNeed(database) + ".";
str += backQuoteIfNeed(table);
return str;
}
template <typename StringT>
String formatSkippedMessage(const std::string_view & database, const std::string_view & table, const std::vector<StringT> & columns)
{
if (columns.size() == 1)
return formatSkippedMessage(database, table, columns[0]);
String str = ". Skipped columns ";
bool need_comma = false;
for (const auto & column : columns)
{
if (std::exchange(need_comma, true))
str += ", ";
str += backQuoteIfNeed(column);
}
str += " ON ";
if (!database.empty())
str += backQuoteIfNeed(database) + ".";
str += backQuoteIfNeed(table);
return str;
}
}
AccessRightsContext::AccessRightsContext()
{
auto everything_granted = boost::make_shared<AccessRights>();
everything_granted->grant(AccessType::ALL);
result_access_cache[0] = std::move(everything_granted);
}
AccessRightsContext::AccessRightsContext(const ClientInfo & client_info_, const AccessRights & granted_to_user_, const Settings & settings, const String & current_database_)
: user_name(client_info_.current_user)
, granted_to_user(granted_to_user_)
, readonly(settings.readonly)
, allow_ddl(settings.allow_ddl)
, allow_introspection(settings.allow_introspection_functions)
, current_database(current_database_)
, interface(client_info_.interface)
, http_method(client_info_.http_method)
, trace_log(&Poco::Logger::get("AccessRightsContext (" + user_name + ")"))
{
}
template <int mode, typename... Args>
bool AccessRightsContext::checkImpl(Poco::Logger * log_, const AccessFlags & access, const Args &... args) const
{
auto result_access = calculateResultAccess();
bool is_granted = result_access->isGranted(access, args...);
if (trace_log)
LOG_TRACE(trace_log, "Access " << (is_granted ? "granted" : "denied") << ": " << (AccessRightsElement{access, args...}.toString()));
if (is_granted)
return true;
if constexpr (mode == RETURN_FALSE_IF_ACCESS_DENIED)
return false;
if constexpr (mode == LOG_WARNING_IF_ACCESS_DENIED)
{
if (!log_)
return false;
}
auto show_error = [&](const String & msg, [[maybe_unused]] int error_code)
{
if constexpr (mode == THROW_IF_ACCESS_DENIED)
throw Exception(msg, error_code);
else if constexpr (mode == LOG_WARNING_IF_ACCESS_DENIED)
LOG_WARNING(log_, msg + formatSkippedMessage(args...));
};
if (readonly && calculateResultAccess(false, allow_ddl, allow_introspection)->isGranted(access, args...))
{
if (interface == ClientInfo::Interface::HTTP && http_method == ClientInfo::HTTPMethod::GET)
show_error(
"Cannot execute query in readonly mode. "
"For queries over HTTP, method GET implies readonly. You should use method POST for modifying queries",
ErrorCodes::READONLY);
else
show_error("Cannot execute query in readonly mode", ErrorCodes::READONLY);
}
else if (!allow_ddl && calculateResultAccess(readonly, true, allow_introspection)->isGranted(access, args...))
{
show_error("Cannot execute query. DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED);
}
else if (!allow_introspection && calculateResultAccess(readonly, allow_ddl, true)->isGranted(access, args...))
{
show_error("Introspection functions are disabled, because setting 'allow_introspection_functions' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED);
}
else
{
show_error(
user_name + ": Not enough privileges. To perform this operation you should have grant "
+ AccessRightsElement{access, args...}.toString(),
ErrorCodes::ACCESS_DENIED);
}
return false;
}
template <int mode>
bool AccessRightsContext::checkImpl(Poco::Logger * log_, const AccessRightsElement & element) const
{
if (element.any_database)
{
return checkImpl<mode>(log_, element.access_flags);
}
else if (element.any_table)
{
if (element.database.empty())
return checkImpl<mode>(log_, element.access_flags, current_database);
else
return checkImpl<mode>(log_, element.access_flags, element.database);
}
else if (element.any_column)
{
if (element.database.empty())
return checkImpl<mode>(log_, element.access_flags, current_database, element.table);
else
return checkImpl<mode>(log_, element.access_flags, element.database, element.table);
}
else
{
if (element.database.empty())
return checkImpl<mode>(log_, element.access_flags, current_database, element.table, element.columns);
else
return checkImpl<mode>(log_, element.access_flags, element.database, element.table, element.columns);
}
}
template <int mode>
bool AccessRightsContext::checkImpl(Poco::Logger * log_, const AccessRightsElements & elements) const
{
for (const auto & element : elements)
if (!checkImpl<mode>(log_, element))
return false;
return true;
}
void AccessRightsContext::check(const AccessFlags & access) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access); }
void AccessRightsContext::check(const AccessFlags & access, const std::string_view & database) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access, database); }
void AccessRightsContext::check(const AccessFlags & access, const std::string_view & database, const std::string_view & table) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access, database, table); }
void AccessRightsContext::check(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access, database, table, column); }
void AccessRightsContext::check(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access, database, table, columns); }
void AccessRightsContext::check(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access, database, table, columns); }
void AccessRightsContext::check(const AccessRightsElement & access) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access); }
void AccessRightsContext::check(const AccessRightsElements & access) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access); }
bool AccessRightsContext::isGranted(const AccessFlags & access) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access); }
bool AccessRightsContext::isGranted(const AccessFlags & access, const std::string_view & database) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access, database); }
bool AccessRightsContext::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access, database, table); }
bool AccessRightsContext::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access, database, table, column); }
bool AccessRightsContext::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access, database, table, columns); }
bool AccessRightsContext::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access, database, table, columns); }
bool AccessRightsContext::isGranted(const AccessRightsElement & access) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access); }
bool AccessRightsContext::isGranted(const AccessRightsElements & access) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessFlags & access) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access, database); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access, database, table); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access, database, table, column); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access, database, table, columns); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access, database, table, columns); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessRightsElement & access) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessRightsElements & access) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access); }
boost::shared_ptr<const AccessRights> AccessRightsContext::calculateResultAccess() const
{
auto res = result_access_cache[0].load();
if (res)
return res;
return calculateResultAccess(readonly, allow_ddl, allow_introspection);
}
boost::shared_ptr<const AccessRights> AccessRightsContext::calculateResultAccess(UInt64 readonly_, bool allow_ddl_, bool allow_introspection_) const
{
size_t cache_index = static_cast<size_t>(readonly_ != readonly)
+ static_cast<size_t>(allow_ddl_ != allow_ddl) * 2 +
+ static_cast<size_t>(allow_introspection_ != allow_introspection) * 3;
assert(cache_index < std::size(result_access_cache));
auto cached = result_access_cache[cache_index].load();
if (cached)
return cached;
std::lock_guard lock{mutex};
cached = result_access_cache[cache_index].load();
if (cached)
return cached;
auto result_ptr = boost::make_shared<AccessRights>();
auto & result = *result_ptr;
result = granted_to_user;
static const AccessFlags table_ddl = AccessType::CREATE_DATABASE | AccessType::CREATE_TABLE | AccessType::CREATE_VIEW
| AccessType::ALTER_TABLE | AccessType::ALTER_VIEW | AccessType::DROP_DATABASE | AccessType::DROP_TABLE | AccessType::DROP_VIEW
| AccessType::DETACH_DATABASE | AccessType::DETACH_TABLE | AccessType::DETACH_VIEW | AccessType::TRUNCATE;
static const AccessFlags dictionary_ddl = AccessType::CREATE_DICTIONARY | AccessType::DROP_DICTIONARY | AccessType::DETACH_DICTIONARY;
static const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl;
static const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE;
if (readonly_)
result.fullRevoke(write_table_access | AccessType::SYSTEM);
if (readonly_ || !allow_ddl_)
result.fullRevoke(table_and_dictionary_ddl);
if (readonly_ == 1)
{
/// Table functions are forbidden in readonly mode.
/// For example, for readonly = 2 - allowed.
result.fullRevoke(AccessType::CREATE_TEMPORARY_TABLE | AccessType::TABLE_FUNCTIONS);
}
if (!allow_introspection_)
result.fullRevoke(AccessType::INTROSPECTION);
result_access_cache[cache_index].store(result_ptr);
return std::move(result_ptr);
}
}

81
dbms/src/Access/AccessRightsContext.h Normal file
View File

@ -0,0 +1,81 @@
#pragma once
#include <Access/AccessRights.h>
#include <Interpreters/ClientInfo.h>
#include <boost/smart_ptr/atomic_shared_ptr.hpp>
#include <mutex>
namespace Poco { class Logger; }
namespace DB
{
struct Settings;
class AccessRightsContext
{
public:
/// Default constructor creates access rights' context which allows everything.
AccessRightsContext();
AccessRightsContext(const ClientInfo & client_info_, const AccessRights & granted_to_user, const Settings & settings, const String & current_database_);
/// Checks if a specified access granted, and throws an exception if not.
/// Empty database means the current database.
void check(const AccessFlags & access) const;
void check(const AccessFlags & access, const std::string_view & database) const;
void check(const AccessFlags & access, const std::string_view & database, const std::string_view & table) const;
void check(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
void check(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
void check(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
void check(const AccessRightsElement & access) const;
void check(const AccessRightsElements & access) const;
/// Checks if a specified access granted.
bool isGranted(const AccessFlags & access) const;
bool isGranted(const AccessFlags & access, const std::string_view & database) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
bool isGranted(const AccessRightsElement & access) const;
bool isGranted(const AccessRightsElements & access) const;
/// Checks if a specified access granted, and logs a warning if not.
bool isGranted(Poco::Logger * log_, const AccessFlags & access) const;
bool isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database) const;
bool isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table) const;
bool isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
bool isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
bool isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
bool isGranted(Poco::Logger * log_, const AccessRightsElement & access) const;
bool isGranted(Poco::Logger * log_, const AccessRightsElements & access) const;
private:
template <int mode, typename... Args>
bool checkImpl(Poco::Logger * log_, const AccessFlags & access, const Args &... args) const;
template <int mode>
bool checkImpl(Poco::Logger * log_, const AccessRightsElement & access) const;
template <int mode>
bool checkImpl(Poco::Logger * log_, const AccessRightsElements & access) const;
boost::shared_ptr<const AccessRights> calculateResultAccess() const;
boost::shared_ptr<const AccessRights> calculateResultAccess(UInt64 readonly_, bool allow_ddl_, bool allow_introspection_) const;
const String user_name;
const AccessRights granted_to_user;
const UInt64 readonly = 0;
const bool allow_ddl = true;
const bool allow_introspection = true;
const String current_database;
const ClientInfo::Interface interface = ClientInfo::Interface::TCP;
const ClientInfo::HTTPMethod http_method = ClientInfo::HTTPMethod::UNKNOWN;
Poco::Logger * const trace_log = nullptr;
mutable boost::atomic_shared_ptr<const AccessRights> result_access_cache[4];
mutable std::mutex mutex;
};
}

86
dbms/src/Access/AccessRightsElement.cpp Normal file
View File

@ -0,0 +1,86 @@
#include <Access/AccessRightsElement.h>
#include <Dictionaries/IDictionary.h>
#include <Common/quoteString.h>
namespace DB
{
void AccessRightsElement::setDatabase(const String & new_database)
{
database = new_database;
any_database = false;
}
void AccessRightsElement::replaceEmptyDatabase(const String & new_database)
{
if (isEmptyDatabase())
setDatabase(new_database);
}
bool AccessRightsElement::isEmptyDatabase() const
{
return !any_database && database.empty();
}
String AccessRightsElement::toString() const
{
String columns_in_parentheses;
if (!any_column)
{
for (const auto & column : columns)
{
columns_in_parentheses += columns_in_parentheses.empty() ? "(" : ", ";
columns_in_parentheses += backQuoteIfNeed(column);
}
columns_in_parentheses += ")";
}
String msg;
for (const std::string_view & keyword : access_flags.toKeywords())
{
if (!msg.empty())
msg += ", ";
msg += String{keyword} + columns_in_parentheses;
}
msg += " ON ";
if (any_database)
msg += "*.";
else if (!database.empty() && (database != IDictionary::NO_DATABASE_TAG))
msg += backQuoteIfNeed(database) + ".";
if (any_table)
msg += "*";
else
msg += backQuoteIfNeed(table);
return msg;
}
void AccessRightsElements::replaceEmptyDatabase(const String & new_database)
{
for (auto & element : *this)
element.replaceEmptyDatabase(new_database);
}
String AccessRightsElements::toString() const
{
String res;
bool need_comma = false;
for (auto & element : *this)
{
if (std::exchange(need_comma, true))
res += ", ";
res += element.toString();
}
if (res.empty())
res = "USAGE ON *.*";
return res;
}
}

100
dbms/src/Access/AccessRightsElement.h Normal file
View File

@ -0,0 +1,100 @@
#pragma once
#include <Access/AccessFlags.h>
namespace DB
{
/// An element of access rights which can be represented by single line
/// GRANT ... ON ...
struct AccessRightsElement
{
AccessFlags access_flags;
String database;
String table;
Strings columns;
bool any_database = true;
bool any_table = true;
bool any_column = true;
AccessRightsElement() = default;
AccessRightsElement(const AccessRightsElement &) = default;
AccessRightsElement & operator=(const AccessRightsElement &) = default;
AccessRightsElement(AccessRightsElement &&) = default;
AccessRightsElement & operator=(AccessRightsElement &&) = default;
AccessRightsElement(AccessFlags access_flags_) : access_flags(access_flags_) {}
AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_)
: access_flags(access_flags_), database(database_), any_database(false)
{
}
AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_)
: access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false)
{
}
AccessRightsElement(
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_)
: access_flags(access_flags_)
, database(database_)
, table(table_)
, columns({String{column_}})
, any_database(false)
, any_table(false)
, any_column(false)
{
}
AccessRightsElement(
AccessFlags access_flags_,
const std::string_view & database_,
const std::string_view & table_,
const std::vector<std::string_view> & columns_)
: access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false), any_column(false)
{
columns.resize(columns_.size());
for (size_t i = 0; i != columns_.size(); ++i)
columns[i] = String{columns_[i]};
}
AccessRightsElement(
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_)
: access_flags(access_flags_)
, database(database_)
, table(table_)
, columns(columns_)
, any_database(false)
, any_table(false)
, any_column(false)
{
}
/// Sets the database.
void setDatabase(const String & new_database);
/// If the database is empty, replaces it with `new_database`. Otherwise does nothing.
void replaceEmptyDatabase(const String & new_database);
bool isEmptyDatabase() const;
/// Returns a human-readable representation like "SELECT, UPDATE(x, y) ON db.table".
/// The returned string isn't prefixed with the "GRANT" keyword.
String toString() const;
};
/// Multiple elements of access rights.
class AccessRightsElements : public std::vector<AccessRightsElement>
{
public:
/// Replaces the empty database with `new_database`.
void replaceEmptyDatabase(const String & new_database);
/// Returns a human-readable representation like "SELECT, UPDATE(x, y) ON db.table".
/// The returned string isn't prefixed with the "GRANT" keyword.
String toString() const;
};
}

328
dbms/src/Access/AccessType.h Normal file
View File

@ -0,0 +1,328 @@
#pragma once
#include <Core/Types.h>
#include <boost/algorithm/string/case_conv.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <array>
namespace DB
{
/// Represents an access type which can be granted on databases, tables, columns, etc.
enum class AccessType
{
NONE, /// no access
ALL, /// full access
SHOW, /// allows to execute SHOW TABLES, SHOW CREATE TABLE, SHOW DATABASES and so on
/// (granted implicitly with any other grant)
EXISTS, /// allows to execute EXISTS, USE, i.e. to check existence
/// (granted implicitly on the database level with any other grant on the database and lower levels,
/// e.g. "GRANT SELECT(x) ON db.table" also grants EXISTS on db.*)
SELECT,
INSERT,
UPDATE, /// allows to execute ALTER UPDATE
DELETE, /// allows to execute ALTER DELETE
ADD_COLUMN,
DROP_COLUMN,
MODIFY_COLUMN,
COMMENT_COLUMN,
CLEAR_COLUMN,
ALTER_COLUMN, /// allow to execute ALTER {ADD|DROP|MODIFY...} COLUMN
ALTER_ORDER_BY,
ADD_INDEX,
DROP_INDEX,
MATERIALIZE_INDEX,
CLEAR_INDEX,
INDEX, /// allows to execute ALTER ORDER BY or ALTER {ADD|DROP...} INDEX
ADD_CONSTRAINT,
DROP_CONSTRAINT,
ALTER_CONSTRAINT, /// allows to execute ALTER {ADD|DROP} CONSTRAINT
MODIFY_TTL, /// allows to execute ALTER MODIFY TTL
MODIFY_SETTING, /// allows to execute ALTER MODIFY SETTING
MOVE_PARTITION,
FETCH_PARTITION,
FREEZE_PARTITION,
ALTER_TABLE, /// allows to execute ALTER TABLE ...
REFRESH_VIEW, /// allows to execute ALTER LIVE VIEW REFRESH
MODIFY_VIEW_QUERY, /// allows to execute ALTER TABLE MODIFY QUERY
ALTER_VIEW, /// allows to execute ALTER LIVE VIEW REFRESH, ALTER TABLE MODIFY QUERY
ALTER, /// allows to execute ALTER {TABLE|LIVE VIEW} ...
CREATE_DATABASE, /// allows to execute {CREATE|ATTACH} DATABASE
CREATE_TABLE, /// allows to execute {CREATE|ATTACH} TABLE
CREATE_VIEW, /// allows to execute {CREATE|ATTACH} VIEW
CREATE_DICTIONARY, /// allows to execute {CREATE|ATTACH} DICTIONARY
CREATE_TEMPORARY_TABLE, /// allows to create and manipulate temporary tables and views.
CREATE, /// allows to execute {CREATE|ATTACH} [TEMPORARY] {DATABASE|TABLE|VIEW|DICTIONARY}
ATTACH_DATABASE, /// allows to execute {CREATE|ATTACH} DATABASE
ATTACH_TABLE, /// allows to execute {CREATE|ATTACH} TABLE
ATTACH_VIEW, /// allows to execute {CREATE|ATTACH} VIEW
ATTACH_DICTIONARY, /// allows to execute {CREATE|ATTACH} DICTIONARY
ATTACH, /// allows to execute {CREATE|ATTACH} {DATABASE|TABLE|VIEW|DICTIONARY}
DROP_DATABASE,
DROP_TABLE,
DROP_VIEW,
DROP_DICTIONARY,
DROP, /// allows to execute DROP {DATABASE|TABLE|VIEW|DICTIONARY}
DETACH_DATABASE,
DETACH_TABLE,
DETACH_VIEW,
DETACH_DICTIONARY,
DETACH, /// allows to execute DETACH {DATABASE|TABLE|VIEW|DICTIONARY}
TRUNCATE_TABLE,
TRUNCATE_VIEW,
TRUNCATE, /// allows to execute TRUNCATE {TABLE|VIEW}
OPTIMIZE, /// allows to execute OPTIMIZE TABLE
KILL_QUERY, /// allows to kill a query started by another user (anyone can kill his own queries)
KILL_MUTATION, /// allows to kill a mutation
KILL, /// allows to execute KILL {MUTATION|QUERY}
CREATE_USER, /// allows to create, alter and drop users, roles, quotas, row policies.
ALTER_USER,
DROP_USER,
CREATE_ROLE,
DROP_ROLE,
CREATE_POLICY,
ALTER_POLICY,
DROP_POLICY,
CREATE_QUOTA,
ALTER_QUOTA,
DROP_QUOTA,
SHUTDOWN,
DROP_CACHE,
RELOAD_CONFIG,
RELOAD_DICTIONARY,
STOP_MERGES,
STOP_TTL_MERGES,
STOP_FETCHES,
STOP_MOVES,
STOP_DISTRIBUTED_SENDS,
STOP_REPLICATED_SENDS,
STOP_REPLICATION_QUEUES,
SYNC_REPLICA,
RESTART_REPLICA,
FLUSH_DISTRIBUTED,
FLUSH_LOGS,
SYSTEM, /// allows to execute SYSTEM {SHUTDOWN|RELOAD CONFIG|...}
dictGet, /// allows to execute functions dictGet, dictHas, dictGetHierarchy, dictIsIn
dictHas, /// allows to execute functions dictGet, dictHas, dictGetHierarchy, dictIsIn
dictGetHierarchy, /// allows to execute functions dictGet, dictHas, dictGetHierarchy, dictIsIn
dictIsIn, /// allows to execute functions dictGet, dictHas, dictGetHierarchy, dictIsIn
addressToLine, /// allows to execute function addressToLine
addressToSymbol, /// allows to execute function addressToSymbol
demangle, /// allows to execute function demangle
INTROSPECTION, /// allows to execute functions addressToLine, addressToSymbol, demangle
file,
url,
input,
values,
numbers,
merge,
remote,
mysql,
odbc,
jdbc,
hdfs,
s3,
TABLE_FUNCTIONS, /// allows to execute any table function
};
constexpr size_t MAX_ACCESS_TYPE = static_cast<size_t>(AccessType::TABLE_FUNCTIONS) + 1;
std::string_view toString(AccessType type);
namespace impl
{
template <typename = void>
class AccessTypeToKeywordConverter
{
public:
static const AccessTypeToKeywordConverter & instance()
{
static const AccessTypeToKeywordConverter res;
return res;
}
std::string_view convert(AccessType type) const
{
return access_type_to_keyword_mapping[static_cast<size_t>(type)];
}
private:
void addToMapping(AccessType type, const std::string_view & str)
{
String str2{str};
boost::replace_all(str2, "_", " ");
if (islower(str2[0]))
str2 += "()";
access_type_to_keyword_mapping[static_cast<size_t>(type)] = str2;
}
AccessTypeToKeywordConverter()
{
#define ACCESS_TYPE_TO_KEYWORD_CASE(type) \
addToMapping(AccessType::type, #type)
ACCESS_TYPE_TO_KEYWORD_CASE(NONE);
ACCESS_TYPE_TO_KEYWORD_CASE(ALL);
ACCESS_TYPE_TO_KEYWORD_CASE(SHOW);
ACCESS_TYPE_TO_KEYWORD_CASE(EXISTS);
ACCESS_TYPE_TO_KEYWORD_CASE(SELECT);
ACCESS_TYPE_TO_KEYWORD_CASE(INSERT);
ACCESS_TYPE_TO_KEYWORD_CASE(UPDATE);
ACCESS_TYPE_TO_KEYWORD_CASE(DELETE);
ACCESS_TYPE_TO_KEYWORD_CASE(ADD_COLUMN);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_COLUMN);
ACCESS_TYPE_TO_KEYWORD_CASE(MODIFY_COLUMN);
ACCESS_TYPE_TO_KEYWORD_CASE(COMMENT_COLUMN);
ACCESS_TYPE_TO_KEYWORD_CASE(CLEAR_COLUMN);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_COLUMN);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_ORDER_BY);
ACCESS_TYPE_TO_KEYWORD_CASE(ADD_INDEX);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_INDEX);
ACCESS_TYPE_TO_KEYWORD_CASE(MATERIALIZE_INDEX);
ACCESS_TYPE_TO_KEYWORD_CASE(CLEAR_INDEX);
ACCESS_TYPE_TO_KEYWORD_CASE(INDEX);
ACCESS_TYPE_TO_KEYWORD_CASE(ADD_CONSTRAINT);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_CONSTRAINT);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_CONSTRAINT);
ACCESS_TYPE_TO_KEYWORD_CASE(MODIFY_TTL);
ACCESS_TYPE_TO_KEYWORD_CASE(MODIFY_SETTING);
ACCESS_TYPE_TO_KEYWORD_CASE(MOVE_PARTITION);
ACCESS_TYPE_TO_KEYWORD_CASE(FETCH_PARTITION);
ACCESS_TYPE_TO_KEYWORD_CASE(FREEZE_PARTITION);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_TABLE);
ACCESS_TYPE_TO_KEYWORD_CASE(REFRESH_VIEW);
ACCESS_TYPE_TO_KEYWORD_CASE(MODIFY_VIEW_QUERY);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_VIEW);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_DATABASE);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_TABLE);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_VIEW);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_DICTIONARY);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_TEMPORARY_TABLE);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE);
ACCESS_TYPE_TO_KEYWORD_CASE(ATTACH_DATABASE);
ACCESS_TYPE_TO_KEYWORD_CASE(ATTACH_TABLE);
ACCESS_TYPE_TO_KEYWORD_CASE(ATTACH_VIEW);
ACCESS_TYPE_TO_KEYWORD_CASE(ATTACH_DICTIONARY);
ACCESS_TYPE_TO_KEYWORD_CASE(ATTACH);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_DATABASE);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_TABLE);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_VIEW);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_DICTIONARY);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP);
ACCESS_TYPE_TO_KEYWORD_CASE(DETACH_DATABASE);
ACCESS_TYPE_TO_KEYWORD_CASE(DETACH_TABLE);
ACCESS_TYPE_TO_KEYWORD_CASE(DETACH_VIEW);
ACCESS_TYPE_TO_KEYWORD_CASE(DETACH_DICTIONARY);
ACCESS_TYPE_TO_KEYWORD_CASE(DETACH);
ACCESS_TYPE_TO_KEYWORD_CASE(TRUNCATE_TABLE);
ACCESS_TYPE_TO_KEYWORD_CASE(TRUNCATE_VIEW);
ACCESS_TYPE_TO_KEYWORD_CASE(TRUNCATE);
ACCESS_TYPE_TO_KEYWORD_CASE(OPTIMIZE);
ACCESS_TYPE_TO_KEYWORD_CASE(KILL_QUERY);
ACCESS_TYPE_TO_KEYWORD_CASE(KILL_MUTATION);
ACCESS_TYPE_TO_KEYWORD_CASE(KILL);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_USER);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_USER);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_USER);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_ROLE);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_ROLE);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_POLICY);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_POLICY);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_POLICY);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_QUOTA);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_QUOTA);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_QUOTA);
ACCESS_TYPE_TO_KEYWORD_CASE(SHUTDOWN);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_CACHE);
ACCESS_TYPE_TO_KEYWORD_CASE(RELOAD_CONFIG);
ACCESS_TYPE_TO_KEYWORD_CASE(RELOAD_DICTIONARY);
ACCESS_TYPE_TO_KEYWORD_CASE(STOP_MERGES);
ACCESS_TYPE_TO_KEYWORD_CASE(STOP_TTL_MERGES);
ACCESS_TYPE_TO_KEYWORD_CASE(STOP_FETCHES);
ACCESS_TYPE_TO_KEYWORD_CASE(STOP_MOVES);
ACCESS_TYPE_TO_KEYWORD_CASE(STOP_DISTRIBUTED_SENDS);
ACCESS_TYPE_TO_KEYWORD_CASE(STOP_REPLICATED_SENDS);
ACCESS_TYPE_TO_KEYWORD_CASE(STOP_REPLICATION_QUEUES);
ACCESS_TYPE_TO_KEYWORD_CASE(SYNC_REPLICA);
ACCESS_TYPE_TO_KEYWORD_CASE(RESTART_REPLICA);
ACCESS_TYPE_TO_KEYWORD_CASE(FLUSH_DISTRIBUTED);
ACCESS_TYPE_TO_KEYWORD_CASE(FLUSH_LOGS);
ACCESS_TYPE_TO_KEYWORD_CASE(SYSTEM);
ACCESS_TYPE_TO_KEYWORD_CASE(dictGet);
ACCESS_TYPE_TO_KEYWORD_CASE(dictHas);
ACCESS_TYPE_TO_KEYWORD_CASE(dictGetHierarchy);
ACCESS_TYPE_TO_KEYWORD_CASE(dictIsIn);
ACCESS_TYPE_TO_KEYWORD_CASE(addressToLine);
ACCESS_TYPE_TO_KEYWORD_CASE(addressToSymbol);
ACCESS_TYPE_TO_KEYWORD_CASE(demangle);
ACCESS_TYPE_TO_KEYWORD_CASE(INTROSPECTION);
ACCESS_TYPE_TO_KEYWORD_CASE(file);
ACCESS_TYPE_TO_KEYWORD_CASE(url);
ACCESS_TYPE_TO_KEYWORD_CASE(input);
ACCESS_TYPE_TO_KEYWORD_CASE(values);
ACCESS_TYPE_TO_KEYWORD_CASE(numbers);
ACCESS_TYPE_TO_KEYWORD_CASE(merge);
ACCESS_TYPE_TO_KEYWORD_CASE(remote);
ACCESS_TYPE_TO_KEYWORD_CASE(mysql);
ACCESS_TYPE_TO_KEYWORD_CASE(odbc);
ACCESS_TYPE_TO_KEYWORD_CASE(jdbc);
ACCESS_TYPE_TO_KEYWORD_CASE(hdfs);
ACCESS_TYPE_TO_KEYWORD_CASE(s3);
ACCESS_TYPE_TO_KEYWORD_CASE(TABLE_FUNCTIONS);
#undef ACCESS_TYPE_TO_KEYWORD_CASE
}
std::array<String, MAX_ACCESS_TYPE> access_type_to_keyword_mapping;
};
}
inline std::string_view toKeyword(AccessType type) { return impl::AccessTypeToKeywordConverter<>::instance().convert(type); }
}

3
dbms/src/Access/IAccessEntity.cpp
View File

@ -1,6 +1,7 @@
#include <Access/IAccessEntity.h>
#include <Access/Quota.h>
#include <Access/RowPolicy.h>
#include <Access/User.h>
#include <common/demangle.h>
@ -8,6 +9,8 @@ namespace DB
{
String IAccessEntity::getTypeName(std::type_index type)
{
if (type == typeid(User))
return "User";
if (type == typeid(Quota))
return "Quota";
if (type == typeid(RowPolicy))

32
dbms/src/Access/IAccessStorage.cpp
View File

@ -299,44 +299,24 @@ std::vector<UUID> IAccessStorage::tryUpdate(const std::vector<UUID> & ids, const
}
IAccessStorage::SubscriptionPtr IAccessStorage::subscribeForChanges(std::type_index type, const OnChangedHandler & handler) const
ext::scope_guard IAccessStorage::subscribeForChanges(std::type_index type, const OnChangedHandler & handler) const
{
return subscribeForChangesImpl(type, handler);
}
IAccessStorage::SubscriptionPtr IAccessStorage::subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const
ext::scope_guard IAccessStorage::subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const
{
return subscribeForChangesImpl(id, handler);
}
IAccessStorage::SubscriptionPtr IAccessStorage::subscribeForChanges(const std::vector<UUID> & ids, const OnChangedHandler & handler) const
ext::scope_guard IAccessStorage::subscribeForChanges(const std::vector<UUID> & ids, const OnChangedHandler & handler) const
{
if (ids.empty())
return nullptr;
if (ids.size() == 1)
return subscribeForChangesImpl(ids[0], handler);
std::vector<SubscriptionPtr> subscriptions;
subscriptions.reserve(ids.size());
ext::scope_guard subscriptions;
for (const auto & id : ids)
{
auto subscription = subscribeForChangesImpl(id, handler);
if (subscription)
subscriptions.push_back(std::move(subscription));
}
class SubscriptionImpl : public Subscription
{
public:
SubscriptionImpl(std::vector<SubscriptionPtr> subscriptions_)
: subscriptions(std::move(subscriptions_)) {}
private:
std::vector<SubscriptionPtr> subscriptions;
};
return std::make_unique<SubscriptionImpl>(std::move(subscriptions));
subscriptions.join(subscribeForChangesImpl(id, handler));
return subscriptions;
}

20
dbms/src/Access/IAccessStorage.h
View File

@ -3,6 +3,7 @@
#include <Access/IAccessEntity.h>
#include <Core/Types.h>
#include <Core/UUID.h>
#include <ext/scope_guard.h>
#include <functional>
#include <optional>
#include <vector>
@ -109,26 +110,19 @@ public:
/// Updates multiple entities in the storage. Returns the list of successfully updated.
std::vector<UUID> tryUpdate(const std::vector<UUID> & ids, const UpdateFunc & update_func);
class Subscription
{
public:
virtual ~Subscription() {}
};
using SubscriptionPtr = std::unique_ptr<Subscription>;
using OnChangedHandler = std::function<void(const UUID & /* id */, const AccessEntityPtr & /* new or changed entity, null if removed */)>;
/// Subscribes for all changes.
/// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only).
SubscriptionPtr subscribeForChanges(std::type_index type, const OnChangedHandler & handler) const;
ext::scope_guard subscribeForChanges(std::type_index type, const OnChangedHandler & handler) const;
template <typename EntityType>
SubscriptionPtr subscribeForChanges(OnChangedHandler handler) const { return subscribeForChanges(typeid(EntityType), handler); }
ext::scope_guard subscribeForChanges(OnChangedHandler handler) const { return subscribeForChanges(typeid(EntityType), handler); }
/// Subscribes for changes of a specific entry.
/// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only).
SubscriptionPtr subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const;
SubscriptionPtr subscribeForChanges(const std::vector<UUID> & ids, const OnChangedHandler & handler) const;
ext::scope_guard subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const;
ext::scope_guard subscribeForChanges(const std::vector<UUID> & ids, const OnChangedHandler & handler) const;
bool hasSubscription(std::type_index type) const;
bool hasSubscription(const UUID & id) const;
@ -142,8 +136,8 @@ protected:
virtual UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) = 0;
virtual void removeImpl(const UUID & id) = 0;
virtual void updateImpl(const UUID & id, const UpdateFunc & update_func) = 0;
virtual SubscriptionPtr subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const = 0;
virtual SubscriptionPtr subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const = 0;
virtual ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const = 0;
virtual ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const = 0;
virtual bool hasSubscriptionImpl(const UUID & id) const = 0;
virtual bool hasSubscriptionImpl(std::type_index type) const = 0;

91
dbms/src/Access/MemoryAccessStorage.cpp
View File

@ -6,7 +6,7 @@
namespace DB
{
MemoryAccessStorage::MemoryAccessStorage(const String & storage_name_)
: IAccessStorage(storage_name_), shared_ptr_to_this{std::make_shared<const MemoryAccessStorage *>(this)}
: IAccessStorage(storage_name_)
{
}
@ -256,85 +256,38 @@ void MemoryAccessStorage::prepareNotifications(const Entry & entry, bool remove,
}
IAccessStorage::SubscriptionPtr MemoryAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const
ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const
{
class SubscriptionImpl : public Subscription
std::lock_guard lock{mutex};
auto handler_it = handlers_by_type.emplace(type, handler);
return [this, handler_it]
{
public:
SubscriptionImpl(
const MemoryAccessStorage & storage_,
std::type_index type_,
const OnChangedHandler & handler_)
: storage_weak(storage_.shared_ptr_to_this)
{
std::lock_guard lock{storage_.mutex};
handler_it = storage_.handlers_by_type.emplace(type_, handler_);
}
~SubscriptionImpl() override
{
auto storage = storage_weak.lock();
if (storage)
{
std::lock_guard lock{(*storage)->mutex};
(*storage)->handlers_by_type.erase(handler_it);
}
}
private:
std::weak_ptr<const MemoryAccessStorage *> storage_weak;
std::unordered_multimap<std::type_index, OnChangedHandler>::iterator handler_it;
std::lock_guard lock2{mutex};
handlers_by_type.erase(handler_it);
};
return std::make_unique<SubscriptionImpl>(*this, type, handler);
}
IAccessStorage::SubscriptionPtr MemoryAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const
ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const
{
class SubscriptionImpl : public Subscription
std::lock_guard lock{mutex};
auto it = entries.find(id);
if (it == entries.end())
return {};
const Entry & entry = it->second;
auto handler_it = entry.handlers_by_id.insert(entry.handlers_by_id.end(), handler);
return [this, id, handler_it]
{
public:
SubscriptionImpl(
const MemoryAccessStorage & storage_,
const UUID & id_,
const OnChangedHandler & handler_)
: storage_weak(storage_.shared_ptr_to_this),
id(id_)
std::lock_guard lock2{mutex};
auto it2 = entries.find(id);
if (it2 != entries.end())
{
std::lock_guard lock{storage_.mutex};
auto it = storage_.entries.find(id);
if (it == storage_.entries.end())
{
storage_weak.reset();
return;
}
const Entry & entry = it->second;
handler_it = entry.handlers_by_id.insert(entry.handlers_by_id.end(), handler_);
const Entry & entry2 = it2->second;
entry2.handlers_by_id.erase(handler_it);
}
~SubscriptionImpl() override
{
auto storage = storage_weak.lock();
if (storage)
{
std::lock_guard lock{(*storage)->mutex};
auto it = (*storage)->entries.find(id);
if (it != (*storage)->entries.end())
{
const Entry & entry = it->second;
entry.handlers_by_id.erase(handler_it);
}
}
}
private:
std::weak_ptr<const MemoryAccessStorage *> storage_weak;
UUID id;
std::list<OnChangedHandler>::iterator handler_it;
};
return std::make_unique<SubscriptionImpl>(*this, id, handler);
}

5
dbms/src/Access/MemoryAccessStorage.h
View File

@ -29,8 +29,8 @@ private:
UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override;
void removeImpl(const UUID & id) override;
void updateImpl(const UUID & id, const UpdateFunc & update_func) override;
SubscriptionPtr subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
SubscriptionPtr subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override;
ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override;
bool hasSubscriptionImpl(const UUID & id) const override;
bool hasSubscriptionImpl(std::type_index type) const override;
@ -60,6 +60,5 @@ private:
std::unordered_map<UUID, Entry> entries; /// We want to search entries both by ID and by the pair of name and type.
std::unordered_map<NameTypePair, Entry *, Hash> names; /// and by the pair of name and type.
mutable std::unordered_multimap<std::type_index, OnChangedHandler> handlers_by_type;
std::shared_ptr<const MemoryAccessStorage *> shared_ptr_to_this; /// We need weak pointers to `this` to implement subscriptions.
};
}

32
dbms/src/Access/MultipleAccessStorage.cpp
View File

@ -185,41 +185,21 @@ void MultipleAccessStorage::updateImpl(const UUID & id, const UpdateFunc & updat
}
IAccessStorage::SubscriptionPtr MultipleAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const
ext::scope_guard MultipleAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const
{
auto storage = findStorage(id);
if (!storage)
return nullptr;
return {};
return storage->subscribeForChanges(id, handler);
}
IAccessStorage::SubscriptionPtr MultipleAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const
ext::scope_guard MultipleAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const
{
std::vector<SubscriptionPtr> subscriptions;
ext::scope_guard subscriptions;
for (const auto & nested_storage : nested_storages)
{
auto subscription = nested_storage->subscribeForChanges(type, handler);
if (subscription)
subscriptions.emplace_back(std::move(subscription));
}
if (subscriptions.empty())
return nullptr;
if (subscriptions.size() == 1)
return std::move(subscriptions[0]);
class SubscriptionImpl : public Subscription
{
public:
SubscriptionImpl(std::vector<SubscriptionPtr> subscriptions_)
: subscriptions(std::move(subscriptions_)) {}
private:
std::vector<SubscriptionPtr> subscriptions;
};
return std::make_unique<SubscriptionImpl>(std::move(subscriptions));
subscriptions.join(nested_storage->subscribeForChanges(type, handler));
return subscriptions;
}

4
dbms/src/Access/MultipleAccessStorage.h
View File

@ -38,8 +38,8 @@ protected:
UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override;
void removeImpl(const UUID & id) override;
void updateImpl(const UUID & id, const UpdateFunc & update_func) override;
SubscriptionPtr subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
SubscriptionPtr subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override;
ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override;
bool hasSubscriptionImpl(const UUID & id) const override;
bool hasSubscriptionImpl(std::type_index type) const override;

4
dbms/src/Access/QuotaContextFactory.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <Access/QuotaContext.h>
#include <Access/IAccessStorage.h>
#include <ext/scope_guard.h>
#include <memory>
#include <mutex>
#include <unordered_map>
@ -56,7 +56,7 @@ private:
mutable std::mutex mutex;
std::unordered_map<UUID /* quota id */, QuotaInfo> all_quotas;
bool all_quotas_read = false;
IAccessStorage::SubscriptionPtr subscription;
ext::scope_guard subscription;
std::vector<std::weak_ptr<QuotaContext>> contexts;
};
}

4
dbms/src/Access/RowPolicyContextFactory.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <Access/RowPolicyContext.h>
#include <Access/IAccessStorage.h>
#include <ext/scope_guard.h>
#include <mutex>
#include <unordered_map>
#include <unordered_set>
@ -46,7 +46,7 @@ private:
const AccessControlManager & access_control_manager;
std::unordered_map<UUID, PolicyInfo> all_policies;
bool all_policies_read = false;
IAccessStorage::SubscriptionPtr subscription;
ext::scope_guard subscription;
std::vector<std::weak_ptr<RowPolicyContext>> contexts;
std::mutex mutex;
};

16
dbms/src/Access/User.cpp Normal file
View File

@ -0,0 +1,16 @@
#include <Access/User.h>
namespace DB
{
bool User::equal(const IAccessEntity & other) const
{
if (!IAccessEntity::equal(other))
return false;
const auto & other_user = typeid_cast<const User &>(other);
return (authentication == other_user.authentication) && (allowed_client_hosts == other_user.allowed_client_hosts)
&& (access == other_user.access) && (profile == other_user.profile);
}
}

26
dbms/src/Access/User.h Normal file
View File

@ -0,0 +1,26 @@
#pragma once
#include <Access/IAccessEntity.h>
#include <Access/Authentication.h>
#include <Access/AllowedClientHosts.h>
#include <Access/AccessRights.h>
#include <Core/Types.h>
namespace DB
{
/** User and ACL.
*/
struct User : public IAccessEntity
{
Authentication authentication;
AllowedClientHosts allowed_client_hosts;
AccessRights access;
String profile;
bool equal(const IAccessEntity & other) const override;
std::shared_ptr<IAccessEntity> clone() const override { return cloneImpl<User>(); }
};
using UserPtr = std::shared_ptr<const User>;
}

170
dbms/src/Access/UsersConfigAccessStorage.cpp
View File

@ -1,6 +1,8 @@
#include <Access/UsersConfigAccessStorage.h>
#include <Access/Quota.h>
#include <Access/RowPolicy.h>
#include <Access/User.h>
#include <Dictionaries/IDictionary.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/quoteString.h>
#include <Poco/Util/AbstractConfiguration.h>
@ -10,10 +12,19 @@
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int UNKNOWN_ADDRESS_PATTERN_TYPE;
}
namespace
{
char getTypeChar(std::type_index type)
{
if (type == typeid(User))
return 'U';
if (type == typeid(Quota))
return 'Q';
if (type == typeid(RowPolicy))
@ -37,6 +48,139 @@ namespace
UUID generateID(const IAccessEntity & entity) { return generateID(entity.getType(), entity.getFullName()); }
UserPtr parseUser(const Poco::Util::AbstractConfiguration & config, const String & user_name)
{
auto user = std::make_shared<User>();
user->setName(user_name);
String user_config = "users." + user_name;
bool has_password = config.has(user_config + ".password");
bool has_password_sha256_hex = config.has(user_config + ".password_sha256_hex");
bool has_password_double_sha1_hex = config.has(user_config + ".password_double_sha1_hex");
if (has_password + has_password_sha256_hex + has_password_double_sha1_hex > 1)
throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex' is used to specify password for user " + user_name + ". Must be only one of them.",
ErrorCodes::BAD_ARGUMENTS);
if (!has_password && !has_password_sha256_hex && !has_password_double_sha1_hex)
throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS);
if (has_password)
{
user->authentication = Authentication{Authentication::PLAINTEXT_PASSWORD};
user->authentication.setPassword(config.getString(user_config + ".password"));
}
else if (has_password_sha256_hex)
{
user->authentication = Authentication{Authentication::SHA256_PASSWORD};
user->authentication.setPasswordHashHex(config.getString(user_config + ".password_sha256_hex"));
}
else if (has_password_double_sha1_hex)
{
user->authentication = Authentication{Authentication::DOUBLE_SHA1_PASSWORD};
user->authentication.setPasswordHashHex(config.getString(user_config + ".password_double_sha1_hex"));
}
user->profile = config.getString(user_config + ".profile");
/// Fill list of allowed hosts.
const auto networks_config = user_config + ".networks";
if (config.has(networks_config))
{
Poco::Util::AbstractConfiguration::Keys keys;
config.keys(networks_config, keys);
for (const String & key : keys)
{
String value = config.getString(networks_config + "." + key);
if (key.starts_with("ip"))
user->allowed_client_hosts.addSubnet(value);
else if (key.starts_with("host_regexp"))
user->allowed_client_hosts.addHostRegexp(value);
else if (key.starts_with("host"))
user->allowed_client_hosts.addHostName(value);
else
throw Exception("Unknown address pattern type: " + key, ErrorCodes::UNKNOWN_ADDRESS_PATTERN_TYPE);
}
}
/// Fill list of allowed databases.
const auto databases_config = user_config + ".allow_databases";
std::optional<Strings> databases;
if (config.has(databases_config))
{
Poco::Util::AbstractConfiguration::Keys keys;
config.keys(databases_config, keys);
databases.emplace();
databases->reserve(keys.size());
for (const auto & key : keys)
{
const auto database_name = config.getString(databases_config + "." + key);
databases->push_back(database_name);
}
}
/// Fill list of allowed dictionaries.
const auto dictionaries_config = user_config + ".allow_dictionaries";
std::optional<Strings> dictionaries;
if (config.has(dictionaries_config))
{
Poco::Util::AbstractConfiguration::Keys keys;
config.keys(dictionaries_config, keys);
dictionaries.emplace();
dictionaries->reserve(keys.size());
for (const auto & key : keys)
{
const auto dictionary_name = config.getString(dictionaries_config + "." + key);
dictionaries->push_back(dictionary_name);
}
}
user->access.grant(AccessType::ALL); /// By default all databases are accessible.
if (databases)
{
user->access.fullRevoke(AccessFlags::databaseLevel());
for (const String & database : *databases)
user->access.grant(AccessFlags::databaseLevel(), database);
user->access.grant(AccessFlags::databaseLevel(), "system"); /// Anyone has access to the "system" database.
}
if (dictionaries)
{
user->access.fullRevoke(AccessType::dictGet, IDictionary::NO_DATABASE_TAG);
for (const String & dictionary : *dictionaries)
user->access.grant(AccessType::dictGet, IDictionary::NO_DATABASE_TAG, dictionary);
}
else if (databases)
user->access.grant(AccessType::dictGet, IDictionary::NO_DATABASE_TAG);
return user;
}
std::vector<AccessEntityPtr> parseUsers(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log)
{
Poco::Util::AbstractConfiguration::Keys user_names;
config.keys("users", user_names);
std::vector<AccessEntityPtr> users;
users.reserve(user_names.size());
for (const auto & user_name : user_names)
{
try
{
users.push_back(parseUser(config, user_name));
}
catch (...)
{
tryLogCurrentException(log, "Could not parse user " + backQuote(user_name));
}
}
return users;
}
QuotaPtr parseQuota(const Poco::Util::AbstractConfiguration & config, const String & quota_name, const Strings & user_names)
{
auto quota = std::make_shared<Quota>();
@ -135,13 +279,25 @@ namespace
for (const String & database : databases)
{
const String database_config = databases_config + "." + database;
Poco::Util::AbstractConfiguration::Keys table_names;
config.keys(database_config, table_names);
Poco::Util::AbstractConfiguration::Keys keys_in_database_config;
config.keys(database_config, keys_in_database_config);
/// Read table properties
for (const String & table_name : table_names)
for (const String & key_in_database_config : keys_in_database_config)
{
const auto filter_config = database_config + "." + table_name + ".filter";
String table_name = key_in_database_config;
String filter_config = database_config + "." + table_name + ".filter";
if (key_in_database_config.starts_with("table["))
{
const auto table_name_config = database_config + "." + table_name + "[@name]";
if (config.has(table_name_config))
{
table_name = config.getString(table_name_config);
filter_config = database_config + ".table[@name='" + table_name + "']";
}
}
if (config.has(filter_config))
{
try
@ -180,6 +336,8 @@ UsersConfigAccessStorage::~UsersConfigAccessStorage() {}
void UsersConfigAccessStorage::loadFromConfig(const Poco::Util::AbstractConfiguration & config)
{
std::vector<std::pair<UUID, AccessEntityPtr>> all_entities;
for (const auto & entity : parseUsers(config, getLogger()))
all_entities.emplace_back(generateID(*entity), entity);
for (const auto & entity : parseQuotas(config, getLogger()))
all_entities.emplace_back(generateID(*entity), entity);
for (const auto & entity : parseRowPolicies(config, getLogger()))
@ -238,13 +396,13 @@ void UsersConfigAccessStorage::updateImpl(const UUID & id, const UpdateFunc &)
}
IAccessStorage::SubscriptionPtr UsersConfigAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const
ext::scope_guard UsersConfigAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const
{
return memory_storage.subscribeForChanges(id, handler);
}
IAccessStorage::SubscriptionPtr UsersConfigAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const
ext::scope_guard UsersConfigAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const
{
return memory_storage.subscribeForChanges(type, handler);
}

Some files were not shown because too many files have changed in this diff Show More