thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-23 16:12:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ClickHouse/ClickHouse into secure_password

Browse Source
This commit is contained in:
Mikhail Korotov 2020-02-03 13:08:20 +03:00
commit bf3caa0373
869 changed files with 11475 additions and 6170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@ -13,12 +13,14 @@
/build_*
/build-*
/docs/build
/docs/publish
/docs/edit
/docs/tools/venv/
/docs/en/development/build/
/docs/ru/development/build/
/docs/en/single.md
/docs/ru/single.md
/docs/zh/single.md
/docs/ja/single.md
/docs/fa/single.md
# callgrind files
callgrind.out.*

1
README.md
View File

@ -15,3 +15,4 @@ ClickHouse is an open-source column-oriented database management system that all
## Upcoming Events
* [ClickHouse Meetup in San Francisco](https://www.eventbrite.com/e/clickhouse-february-meetup-registration-88496227599) on February 5.
* [ClickHouse Meetup in New York](https://www.meetup.com/Uber-Engineering-Events-New-York/events/268328663/) on February 11.

2
contrib/CMakeLists.txt vendored
View File

@ -306,7 +306,7 @@ if (USE_INTERNAL_AWS_S3_LIBRARY)
# The library is large - avoid bloat.
target_compile_options (aws_s3 PRIVATE -g0)
target_compile_options (aws_s3_checksums PRIVATE -g0)
target_compile_options (libcurl PRIVATE -g0)
target_compile_options (curl PRIVATE -g0)
endif ()
if (USE_BASE64)

2
contrib/aws-s3-cmake/CMakeLists.txt
View File

@ -102,4 +102,4 @@ if (OPENSSL_FOUND)
target_link_libraries(aws_s3 PRIVATE ${OPENSSL_LIBRARIES})
endif()
target_link_libraries(aws_s3 PRIVATE aws_s3_checksums libcurl)
target_link_libraries(aws_s3 PRIVATE aws_s3_checksums curl)

61
contrib/curl-cmake/CMake/CurlSymbolHiding.cmake
View File

@ -1,61 +0,0 @@
include(CheckCSourceCompiles)
option(CURL_HIDDEN_SYMBOLS "Set to ON to hide libcurl internal symbols (=hide all symbols that aren't officially external)." ON)
mark_as_advanced(CURL_HIDDEN_SYMBOLS)
if(CURL_HIDDEN_SYMBOLS)
set(SUPPORTS_SYMBOL_HIDING FALSE)
if(CMAKE_C_COMPILER_ID MATCHES "Clang")
set(SUPPORTS_SYMBOL_HIDING TRUE)
set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))")
set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden")
elseif(CMAKE_COMPILER_IS_GNUCC)
if(NOT CMAKE_VERSION VERSION_LESS 2.8.10)
set(GCC_VERSION ${CMAKE_C_COMPILER_VERSION})
else()
execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion
OUTPUT_VARIABLE GCC_VERSION)
endif()
if(NOT GCC_VERSION VERSION_LESS 3.4)
# note: this is considered buggy prior to 4.0 but the autotools don't care, so let's ignore that fact
set(SUPPORTS_SYMBOL_HIDING TRUE)
set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))")
set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden")
endif()
elseif(CMAKE_C_COMPILER_ID MATCHES "SunPro" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 8.0)
set(SUPPORTS_SYMBOL_HIDING TRUE)
set(_SYMBOL_EXTERN "__global")
set(_CFLAG_SYMBOLS_HIDE "-xldscope=hidden")
elseif(CMAKE_C_COMPILER_ID MATCHES "Intel" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.0)
# note: this should probably just check for version 9.1.045 but I'm not 100% sure
# so let's do it the same way autotools do.
set(SUPPORTS_SYMBOL_HIDING TRUE)
set(_SYMBOL_EXTERN "__attribute__ ((__visibility__ (\"default\")))")
set(_CFLAG_SYMBOLS_HIDE "-fvisibility=hidden")
check_c_source_compiles("#include <stdio.h>
int main (void) { printf(\"icc fvisibility bug test\"); return 0; }" _no_bug)
if(NOT _no_bug)
set(SUPPORTS_SYMBOL_HIDING FALSE)
set(_SYMBOL_EXTERN "")
set(_CFLAG_SYMBOLS_HIDE "")
endif()
elseif(MSVC)
set(SUPPORTS_SYMBOL_HIDING TRUE)
endif()
set(HIDES_CURL_PRIVATE_SYMBOLS ${SUPPORTS_SYMBOL_HIDING})
elseif(MSVC)
if(NOT CMAKE_VERSION VERSION_LESS 3.7)
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) #present since 3.4.3 but broken
set(HIDES_CURL_PRIVATE_SYMBOLS FALSE)
else()
message(WARNING "Hiding private symbols regardless CURL_HIDDEN_SYMBOLS being disabled.")
set(HIDES_CURL_PRIVATE_SYMBOLS TRUE)
endif()
else()
set(HIDES_CURL_PRIVATE_SYMBOLS FALSE)
endif()
set(CURL_CFLAG_SYMBOLS_HIDE ${_CFLAG_SYMBOLS_HIDE})
set(CURL_EXTERN_SYMBOL ${_SYMBOL_EXTERN})

617
contrib/curl-cmake/CMake/CurlTests.c
View File

@ -1,617 +0,0 @@
/***************************************************************************
* _ _ ____ _
* Project ___| | | | _ \| |
* / __| | | | |_) | |
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
* Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
* are also available at https://curl.haxx.se/docs/copyright.html.
*
* You may opt to use, copy, modify, merge, publish, distribute and/or sell
* copies of the Software, and permit persons to whom the Software is
* furnished to do so, under the terms of the COPYING file.
*
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
* KIND, either express or implied.
*
***************************************************************************/
#ifdef TIME_WITH_SYS_TIME
/* Time with sys/time test */
#include <sys/types.h>
#include <sys/time.h>
#include <time.h>
int
main ()
{
if ((struct tm *) 0)
return 0;
;
return 0;
}
#endif
#ifdef HAVE_FCNTL_O_NONBLOCK
/* headers for FCNTL_O_NONBLOCK test */
#include <sys/types.h>
#include <unistd.h>
#include <fcntl.h>
/* */
#if defined(sun) || defined(__sun__) || \
defined(__SUNPRO_C) || defined(__SUNPRO_CC)
# if defined(__SVR4) || defined(__srv4__)
# define PLATFORM_SOLARIS
# else
# define PLATFORM_SUNOS4
# endif
#endif
#if (defined(_AIX) || defined(__xlC__)) && !defined(_AIX41)
# define PLATFORM_AIX_V3
#endif
/* */
#if defined(PLATFORM_SUNOS4) || defined(PLATFORM_AIX_V3) || defined(__BEOS__)
#error "O_NONBLOCK does not work on this platform"
#endif
int
main ()
{
/* O_NONBLOCK source test */
int flags = 0;
if(0 != fcntl(0, F_SETFL, flags | O_NONBLOCK))
return 1;
return 0;
}
#endif
/* tests for gethostbyaddr_r or gethostbyname_r */
#if defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT) || \
defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT) || \
defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT) || \
defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT) || \
defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT) || \
defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT)
# define _REENTRANT
/* no idea whether _REENTRANT is always set, just invent a new flag */
# define TEST_GETHOSTBYFOO_REENTRANT
#endif
#if defined(HAVE_GETHOSTBYADDR_R_5) || \
defined(HAVE_GETHOSTBYADDR_R_7) || \
defined(HAVE_GETHOSTBYADDR_R_8) || \
defined(HAVE_GETHOSTBYNAME_R_3) || \
defined(HAVE_GETHOSTBYNAME_R_5) || \
defined(HAVE_GETHOSTBYNAME_R_6) || \
defined(TEST_GETHOSTBYFOO_REENTRANT)
#include <sys/types.h>
#include <netdb.h>
int main(void)
{
char *address = "example.com";
int length = 0;
int type = 0;
struct hostent h;
int rc = 0;
#if defined(HAVE_GETHOSTBYADDR_R_5) || \
defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT) || \
\
defined(HAVE_GETHOSTBYNAME_R_3) || \
defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT)
struct hostent_data hdata;
#elif defined(HAVE_GETHOSTBYADDR_R_7) || \
defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT) || \
defined(HAVE_GETHOSTBYADDR_R_8) || \
defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT) || \
\
defined(HAVE_GETHOSTBYNAME_R_5) || \
defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT) || \
defined(HAVE_GETHOSTBYNAME_R_6) || \
defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT)
char buffer[8192];
int h_errnop;
struct hostent *hp;
#endif
#ifndef gethostbyaddr_r
(void)gethostbyaddr_r;
#endif
#if defined(HAVE_GETHOSTBYADDR_R_5) || \
defined(HAVE_GETHOSTBYADDR_R_5_REENTRANT)
rc = gethostbyaddr_r(address, length, type, &h, &hdata);
(void)rc;
#elif defined(HAVE_GETHOSTBYADDR_R_7) || \
defined(HAVE_GETHOSTBYADDR_R_7_REENTRANT)
hp = gethostbyaddr_r(address, length, type, &h, buffer, 8192, &h_errnop);
(void)hp;
#elif defined(HAVE_GETHOSTBYADDR_R_8) || \
defined(HAVE_GETHOSTBYADDR_R_8_REENTRANT)
rc = gethostbyaddr_r(address, length, type, &h, buffer, 8192, &hp, &h_errnop);
(void)rc;
#endif
#if defined(HAVE_GETHOSTBYNAME_R_3) || \
defined(HAVE_GETHOSTBYNAME_R_3_REENTRANT)
rc = gethostbyname_r(address, &h, &hdata);
#elif defined(HAVE_GETHOSTBYNAME_R_5) || \
defined(HAVE_GETHOSTBYNAME_R_5_REENTRANT)
rc = gethostbyname_r(address, &h, buffer, 8192, &h_errnop);
(void)hp; /* not used for test */
#elif defined(HAVE_GETHOSTBYNAME_R_6) || \
defined(HAVE_GETHOSTBYNAME_R_6_REENTRANT)
rc = gethostbyname_r(address, &h, buffer, 8192, &hp, &h_errnop);
#endif
(void)length;
(void)type;
(void)rc;
return 0;
}
#endif
#ifdef HAVE_SOCKLEN_T
#ifdef _WIN32
#include <ws2tcpip.h>
#else
#include <sys/types.h>
#include <sys/socket.h>
#endif
int
main ()
{
if ((socklen_t *) 0)
return 0;
if (sizeof (socklen_t))
return 0;
;
return 0;
}
#endif
#ifdef HAVE_IN_ADDR_T
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
int
main ()
{
if ((in_addr_t *) 0)
return 0;
if (sizeof (in_addr_t))
return 0;
;
return 0;
}
#endif
#ifdef HAVE_BOOL_T
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_STDBOOL_H
#include <stdbool.h>
#endif
int
main ()
{
if (sizeof (bool *) )
return 0;
;
return 0;
}
#endif
#ifdef STDC_HEADERS
#include <stdlib.h>
#include <stdarg.h>
#include <string.h>
#include <float.h>
int main() { return 0; }
#endif
#ifdef RETSIGTYPE_TEST
#include <sys/types.h>
#include <signal.h>
#ifdef signal
# undef signal
#endif
#ifdef __cplusplus
extern "C" void (*signal (int, void (*)(int)))(int);
#else
void (*signal ()) ();
#endif
int
main ()
{
return 0;
}
#endif
#ifdef HAVE_INET_NTOA_R_DECL
#include <arpa/inet.h>
typedef void (*func_type)();
int main()
{
#ifndef inet_ntoa_r
func_type func;
func = (func_type)inet_ntoa_r;
(void)func;
#endif
return 0;
}
#endif
#ifdef HAVE_INET_NTOA_R_DECL_REENTRANT
#define _REENTRANT
#include <arpa/inet.h>
typedef void (*func_type)();
int main()
{
#ifndef inet_ntoa_r
func_type func;
func = (func_type)&inet_ntoa_r;
(void)func;
#endif
return 0;
}
#endif
#ifdef HAVE_GETADDRINFO
#include <netdb.h>
#include <sys/types.h>
#include <sys/socket.h>
int main(void) {
struct addrinfo hints, *ai;
int error;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
#ifndef getaddrinfo
(void)getaddrinfo;
#endif
error = getaddrinfo("127.0.0.1", "8080", &hints, &ai);
if (error) {
return 1;
}
return 0;
}
#endif
#ifdef HAVE_FILE_OFFSET_BITS
#ifdef _FILE_OFFSET_BITS
#undef _FILE_OFFSET_BITS
#endif
#define _FILE_OFFSET_BITS 64
#include <sys/types.h>
/* Check that off_t can represent 2**63 - 1 correctly.
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
int main () { ; return 0; }
#endif
#ifdef HAVE_IOCTLSOCKET
/* includes start */
#ifdef HAVE_WINDOWS_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
# else
# ifdef HAVE_WINSOCK_H
# include <winsock.h>
# endif
# endif
#endif
int
main ()
{
/* ioctlsocket source code */
int socket;
unsigned long flags = ioctlsocket(socket, FIONBIO, &flags);
;
return 0;
}
#endif
#ifdef HAVE_IOCTLSOCKET_CAMEL
/* includes start */
#ifdef HAVE_WINDOWS_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
# else
# ifdef HAVE_WINSOCK_H
# include <winsock.h>
# endif
# endif
#endif
int
main ()
{
/* IoctlSocket source code */
if(0 != IoctlSocket(0, 0, 0))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_IOCTLSOCKET_CAMEL_FIONBIO
/* includes start */
#ifdef HAVE_WINDOWS_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
# else
# ifdef HAVE_WINSOCK_H
# include <winsock.h>
# endif
# endif
#endif
int
main ()
{
/* IoctlSocket source code */
long flags = 0;
if(0 != ioctlsocket(0, FIONBIO, &flags))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_IOCTLSOCKET_FIONBIO
/* includes start */
#ifdef HAVE_WINDOWS_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
# else
# ifdef HAVE_WINSOCK_H
# include <winsock.h>
# endif
# endif
#endif
int
main ()
{
int flags = 0;
if(0 != ioctlsocket(0, FIONBIO, &flags))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_IOCTL_FIONBIO
/* headers for FIONBIO test */
/* includes start */
#ifdef HAVE_SYS_TYPES_H
# include <sys/types.h>
#endif
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
# include <sys/socket.h>
#endif
#ifdef HAVE_SYS_IOCTL_H
# include <sys/ioctl.h>
#endif
#ifdef HAVE_STROPTS_H
# include <stropts.h>
#endif
int
main ()
{
int flags = 0;
if(0 != ioctl(0, FIONBIO, &flags))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_IOCTL_SIOCGIFADDR
/* headers for FIONBIO test */
/* includes start */
#ifdef HAVE_SYS_TYPES_H
# include <sys/types.h>
#endif
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
# include <sys/socket.h>
#endif
#ifdef HAVE_SYS_IOCTL_H
# include <sys/ioctl.h>
#endif
#ifdef HAVE_STROPTS_H
# include <stropts.h>
#endif
#include <net/if.h>
int
main ()
{
struct ifreq ifr;
if(0 != ioctl(0, SIOCGIFADDR, &ifr))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_SETSOCKOPT_SO_NONBLOCK
/* includes start */
#ifdef HAVE_WINDOWS_H
# ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
# endif
# include <windows.h>
# ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
# else
# ifdef HAVE_WINSOCK_H
# include <winsock.h>
# endif
# endif
#endif
/* includes start */
#ifdef HAVE_SYS_TYPES_H
# include <sys/types.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
# include <sys/socket.h>
#endif
/* includes end */
int
main ()
{
if(0 != setsockopt(0, SOL_SOCKET, SO_NONBLOCK, 0, 0))
return 1;
;
return 0;
}
#endif
#ifdef HAVE_GLIBC_STRERROR_R
#include <string.h>
#include <errno.h>
void check(char c) {}
int
main () {
char buffer[1024];
/* This will not compile if strerror_r does not return a char* */
check(strerror_r(EACCES, buffer, sizeof(buffer))[0]);
return 0;
}
#endif
#ifdef HAVE_POSIX_STRERROR_R
#include <string.h>
#include <errno.h>
/* float, because a pointer can't be implicitly cast to float */
void check(float f) {}
int
main () {
char buffer[1024];
/* This will not compile if strerror_r does not return an int */
check(strerror_r(EACCES, buffer, sizeof(buffer)));
return 0;
}
#endif
#ifdef HAVE_FSETXATTR_6
#include <sys/xattr.h> /* header from libc, not from libattr */
int
main() {
fsetxattr(0, 0, 0, 0, 0, 0);
return 0;
}
#endif
#ifdef HAVE_FSETXATTR_5
#include <sys/xattr.h> /* header from libc, not from libattr */
int
main() {
fsetxattr(0, 0, 0, 0, 0);
return 0;
}
#endif
#ifdef HAVE_CLOCK_GETTIME_MONOTONIC
#include <time.h>
int
main() {
struct timespec ts = {0, 0};
clock_gettime(CLOCK_MONOTONIC, &ts);
return 0;
}
#endif
#ifdef HAVE_BUILTIN_AVAILABLE
int
main() {
if(__builtin_available(macOS 10.12, *)) {}
return 0;
}
#endif
#ifdef HAVE_VARIADIC_MACROS_C99
#define c99_vmacro3(first, ...) fun3(first, __VA_ARGS__)
#define c99_vmacro2(first, ...) fun2(first, __VA_ARGS__)
int fun3(int arg1, int arg2, int arg3);
int fun2(int arg1, int arg2);
int fun3(int arg1, int arg2, int arg3) {
return arg1 + arg2 + arg3;
}
int fun2(int arg1, int arg2) {
return arg1 + arg2;
}
int
main() {
int res3 = c99_vmacro3(1, 2, 3);
int res2 = c99_vmacro2(1, 2);
(void)res3;
(void)res2;
return 0;
}
#endif
#ifdef HAVE_VARIADIC_MACROS_GCC
#define gcc_vmacro3(first, args...) fun3(first, args)
#define gcc_vmacro2(first, args...) fun2(first, args)
int fun3(int arg1, int arg2, int arg3);
int fun2(int arg1, int arg2);
int fun3(int arg1, int arg2, int arg3) {
return arg1 + arg2 + arg3;
}
int fun2(int arg1, int arg2) {
return arg1 + arg2;
}
int
main() {
int res3 = gcc_vmacro3(1, 2, 3);
int res2 = gcc_vmacro2(1, 2);
(void)res3;
(void)res2;
return 0;
}
#endif

84
contrib/curl-cmake/CMake/Macros.cmake
View File

@ -1,84 +0,0 @@
#File defines convenience macros for available feature testing
# This macro checks if the symbol exists in the library and if it
# does, it prepends library to the list. It is intended to be called
# multiple times with a sequence of possibly dependent libraries in
# order of least-to-most-dependent. Some libraries depend on others
# to link correctly.
macro(check_library_exists_concat LIBRARY SYMBOL VARIABLE)
check_library_exists("${LIBRARY};${CURL_LIBS}" ${SYMBOL} "${CMAKE_LIBRARY_PATH}"
${VARIABLE})
if(${VARIABLE})
set(CURL_LIBS ${LIBRARY} ${CURL_LIBS})
endif()
endmacro()
# Check if header file exists and add it to the list.
# This macro is intended to be called multiple times with a sequence of
# possibly dependent header files. Some headers depend on others to be
# compiled correctly.
macro(check_include_file_concat FILE VARIABLE)
check_include_files("${CURL_INCLUDES};${FILE}" ${VARIABLE})
if(${VARIABLE})
set(CURL_INCLUDES ${CURL_INCLUDES} ${FILE})
set(CURL_TEST_DEFINES "${CURL_TEST_DEFINES} -D${VARIABLE}")
endif()
endmacro()
# For other curl specific tests, use this macro.
macro(curl_internal_test CURL_TEST)
if(NOT DEFINED "${CURL_TEST}")
set(MACRO_CHECK_FUNCTION_DEFINITIONS
"-D${CURL_TEST} ${CURL_TEST_DEFINES} ${CMAKE_REQUIRED_FLAGS}")
if(CMAKE_REQUIRED_LIBRARIES)
set(CURL_TEST_ADD_LIBRARIES
"-DLINK_LIBRARIES:STRING=${CMAKE_REQUIRED_LIBRARIES}")
endif()
try_compile(${CURL_TEST}
${CMAKE_BINARY_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/CMake/CurlTests.c
CMAKE_FLAGS -DCOMPILE_DEFINITIONS:STRING=${MACRO_CHECK_FUNCTION_DEFINITIONS}
"${CURL_TEST_ADD_LIBRARIES}"
OUTPUT_VARIABLE OUTPUT)
if(${CURL_TEST})
set(${CURL_TEST} 1 CACHE INTERNAL "Curl test ${FUNCTION}")
file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log
"Performing Curl Test ${CURL_TEST} passed with the following output:\n"
"${OUTPUT}\n")
else()
set(${CURL_TEST} "" CACHE INTERNAL "Curl test ${FUNCTION}")
file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log
"Performing Curl Test ${CURL_TEST} failed with the following output:\n"
"${OUTPUT}\n")
endif()
endif()
endmacro()
macro(curl_nroff_check)
find_program(NROFF NAMES gnroff nroff)
if(NROFF)
# Need a way to write to stdin, this will do
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt" "test")
# Tests for a valid nroff option to generate a manpage
foreach(_MANOPT "-man" "-mandoc")
execute_process(COMMAND "${NROFF}" ${_MANOPT}
OUTPUT_VARIABLE NROFF_MANOPT_OUTPUT
INPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt"
ERROR_QUIET)
# Save the option if it was valid
if(NROFF_MANOPT_OUTPUT)
set(NROFF_MANOPT ${_MANOPT})
set(NROFF_USEFUL ON)
break()
endif()
endforeach()
# No need for the temporary file
file(REMOVE "${CMAKE_CURRENT_BINARY_DIR}/nroff-input.txt")
if(NOT NROFF_USEFUL)
message(WARNING "Found no *nroff option to get plaintext from man pages")
endif()
else()
message(WARNING "Found no *nroff program")
endif()
endmacro()

260
contrib/curl-cmake/CMake/OtherTests.cmake
View File

@ -1,260 +0,0 @@
include(CheckCSourceCompiles)
# The begin of the sources (macros and includes)
set(_source_epilogue "#undef inline")
macro(add_header_include check header)
if(${check})
set(_source_epilogue "${_source_epilogue}\n#include <${header}>")
endif()
endmacro()
set(signature_call_conv)
if(HAVE_WINDOWS_H)
add_header_include(HAVE_WINSOCK2_H "winsock2.h")
add_header_include(HAVE_WINDOWS_H "windows.h")
add_header_include(HAVE_WINSOCK_H "winsock.h")
set(_source_epilogue
"${_source_epilogue}\n#ifndef WIN32_LEAN_AND_MEAN\n#define WIN32_LEAN_AND_MEAN\n#endif")
set(signature_call_conv "PASCAL")
if(HAVE_LIBWS2_32)
set(CMAKE_REQUIRED_LIBRARIES ws2_32)
endif()
else()
add_header_include(HAVE_SYS_TYPES_H "sys/types.h")
add_header_include(HAVE_SYS_SOCKET_H "sys/socket.h")
endif()
set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
check_c_source_compiles("${_source_epilogue}
int main(void) {
recv(0, 0, 0, 0);
return 0;
}" curl_cv_recv)
if(curl_cv_recv)
if(NOT DEFINED curl_cv_func_recv_args OR "${curl_cv_func_recv_args}" STREQUAL "unknown")
foreach(recv_retv "int" "ssize_t" )
foreach(recv_arg1 "SOCKET" "int" )
foreach(recv_arg2 "char *" "void *" )
foreach(recv_arg3 "int" "size_t" "socklen_t" "unsigned int")
foreach(recv_arg4 "int" "unsigned int")
if(NOT curl_cv_func_recv_done)
unset(curl_cv_func_recv_test CACHE)
check_c_source_compiles("
${_source_epilogue}
extern ${recv_retv} ${signature_call_conv}
recv(${recv_arg1}, ${recv_arg2}, ${recv_arg3}, ${recv_arg4});
int main(void) {
${recv_arg1} s=0;
${recv_arg2} buf=0;
${recv_arg3} len=0;
${recv_arg4} flags=0;
${recv_retv} res = recv(s, buf, len, flags);
(void) res;
return 0;
}"
curl_cv_func_recv_test)
if(curl_cv_func_recv_test)
set(curl_cv_func_recv_args
"${recv_arg1},${recv_arg2},${recv_arg3},${recv_arg4},${recv_retv}")
set(RECV_TYPE_ARG1 "${recv_arg1}")
set(RECV_TYPE_ARG2 "${recv_arg2}")
set(RECV_TYPE_ARG3 "${recv_arg3}")
set(RECV_TYPE_ARG4 "${recv_arg4}")
set(RECV_TYPE_RETV "${recv_retv}")
set(HAVE_RECV 1)
set(curl_cv_func_recv_done 1)
endif()
endif()
endforeach()
endforeach()
endforeach()
endforeach()
endforeach()
else()
string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG1 "${curl_cv_func_recv_args}")
string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG2 "${curl_cv_func_recv_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,([^,]*),[^,]*,[^,]*$" "\\1" RECV_TYPE_ARG3 "${curl_cv_func_recv_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,([^,]*),[^,]*$" "\\1" RECV_TYPE_ARG4 "${curl_cv_func_recv_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,([^,]*)$" "\\1" RECV_TYPE_RETV "${curl_cv_func_recv_args}")
endif()
if("${curl_cv_func_recv_args}" STREQUAL "unknown")
message(FATAL_ERROR "Cannot find proper types to use for recv args")
endif()
else()
message(FATAL_ERROR "Unable to link function recv")
endif()
set(curl_cv_func_recv_args "${curl_cv_func_recv_args}" CACHE INTERNAL "Arguments for recv")
set(HAVE_RECV 1)
check_c_source_compiles("${_source_epilogue}
int main(void) {
send(0, 0, 0, 0);
return 0;
}" curl_cv_send)
if(curl_cv_send)
if(NOT DEFINED curl_cv_func_send_args OR "${curl_cv_func_send_args}" STREQUAL "unknown")
foreach(send_retv "int" "ssize_t" )
foreach(send_arg1 "SOCKET" "int" "ssize_t" )
foreach(send_arg2 "const char *" "const void *" "void *" "char *")
foreach(send_arg3 "int" "size_t" "socklen_t" "unsigned int")
foreach(send_arg4 "int" "unsigned int")
if(NOT curl_cv_func_send_done)
unset(curl_cv_func_send_test CACHE)
check_c_source_compiles("
${_source_epilogue}
extern ${send_retv} ${signature_call_conv}
send(${send_arg1}, ${send_arg2}, ${send_arg3}, ${send_arg4});
int main(void) {
${send_arg1} s=0;
${send_arg2} buf=0;
${send_arg3} len=0;
${send_arg4} flags=0;
${send_retv} res = send(s, buf, len, flags);
(void) res;
return 0;
}"
curl_cv_func_send_test)
if(curl_cv_func_send_test)
string(REGEX REPLACE "(const) .*" "\\1" send_qual_arg2 "${send_arg2}")
string(REGEX REPLACE "const (.*)" "\\1" send_arg2 "${send_arg2}")
set(curl_cv_func_send_args
"${send_arg1},${send_arg2},${send_arg3},${send_arg4},${send_retv},${send_qual_arg2}")
set(SEND_TYPE_ARG1 "${send_arg1}")
set(SEND_TYPE_ARG2 "${send_arg2}")
set(SEND_TYPE_ARG3 "${send_arg3}")
set(SEND_TYPE_ARG4 "${send_arg4}")
set(SEND_TYPE_RETV "${send_retv}")
set(HAVE_SEND 1)
set(curl_cv_func_send_done 1)
endif()
endif()
endforeach()
endforeach()
endforeach()
endforeach()
endforeach()
else()
string(REGEX REPLACE "^([^,]*),[^,]*,[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG1 "${curl_cv_func_send_args}")
string(REGEX REPLACE "^[^,]*,([^,]*),[^,]*,[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG2 "${curl_cv_func_send_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,([^,]*),[^,]*,[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG3 "${curl_cv_func_send_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,([^,]*),[^,]*,[^,]*$" "\\1" SEND_TYPE_ARG4 "${curl_cv_func_send_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,([^,]*),[^,]*$" "\\1" SEND_TYPE_RETV "${curl_cv_func_send_args}")
string(REGEX REPLACE "^[^,]*,[^,]*,[^,]*,[^,]*,[^,]*,([^,]*)$" "\\1" SEND_QUAL_ARG2 "${curl_cv_func_send_args}")
endif()
if("${curl_cv_func_send_args}" STREQUAL "unknown")
message(FATAL_ERROR "Cannot find proper types to use for send args")
endif()
set(SEND_QUAL_ARG2 "const")
else()
message(FATAL_ERROR "Unable to link function send")
endif()
set(curl_cv_func_send_args "${curl_cv_func_send_args}" CACHE INTERNAL "Arguments for send")
set(HAVE_SEND 1)
check_c_source_compiles("${_source_epilogue}
int main(void) {
int flag = MSG_NOSIGNAL;
(void)flag;
return 0;
}" HAVE_MSG_NOSIGNAL)
if(NOT HAVE_WINDOWS_H)
add_header_include(HAVE_SYS_TIME_H "sys/time.h")
add_header_include(TIME_WITH_SYS_TIME "time.h")
add_header_include(HAVE_TIME_H "time.h")
endif()
check_c_source_compiles("${_source_epilogue}
int main(void) {
struct timeval ts;
ts.tv_sec = 0;
ts.tv_usec = 0;
(void)ts;
return 0;
}" HAVE_STRUCT_TIMEVAL)
set(HAVE_SIG_ATOMIC_T 1)
set(CMAKE_REQUIRED_FLAGS)
if(HAVE_SIGNAL_H)
set(CMAKE_REQUIRED_FLAGS "-DHAVE_SIGNAL_H")
set(CMAKE_EXTRA_INCLUDE_FILES "signal.h")
endif()
check_type_size("sig_atomic_t" SIZEOF_SIG_ATOMIC_T)
if(HAVE_SIZEOF_SIG_ATOMIC_T)
check_c_source_compiles("
#ifdef HAVE_SIGNAL_H
# include <signal.h>
#endif
int main(void) {
static volatile sig_atomic_t dummy = 0;
(void)dummy;
return 0;
}" HAVE_SIG_ATOMIC_T_NOT_VOLATILE)
if(NOT HAVE_SIG_ATOMIC_T_NOT_VOLATILE)
set(HAVE_SIG_ATOMIC_T_VOLATILE 1)
endif()
endif()
if(HAVE_WINDOWS_H)
set(CMAKE_EXTRA_INCLUDE_FILES winsock2.h)
else()
set(CMAKE_EXTRA_INCLUDE_FILES)
if(HAVE_SYS_SOCKET_H)
set(CMAKE_EXTRA_INCLUDE_FILES sys/socket.h)
endif()
endif()
check_type_size("struct sockaddr_storage" SIZEOF_STRUCT_SOCKADDR_STORAGE)
if(HAVE_SIZEOF_STRUCT_SOCKADDR_STORAGE)
set(HAVE_STRUCT_SOCKADDR_STORAGE 1)
endif()
unset(CMAKE_TRY_COMPILE_TARGET_TYPE)
if(NOT DEFINED CMAKE_TOOLCHAIN_FILE)
# if not cross-compilation...
include(CheckCSourceRuns)
set(CMAKE_REQUIRED_FLAGS "")
if(HAVE_SYS_POLL_H)
set(CMAKE_REQUIRED_FLAGS "-DHAVE_SYS_POLL_H")
elseif(HAVE_POLL_H)
set(CMAKE_REQUIRED_FLAGS "-DHAVE_POLL_H")
endif()
check_c_source_runs("
#include <stdlib.h>
#include <sys/time.h>
#ifdef HAVE_SYS_POLL_H
# include <sys/poll.h>
#elif HAVE_POLL_H
# include <poll.h>
#endif
int main(void)
{
if(0 != poll(0, 0, 10)) {
return 1; /* fail */
}
else {
/* detect the 10.12 poll() breakage */
struct timeval before, after;
int rc;
size_t us;
gettimeofday(&before, NULL);
rc = poll(NULL, 0, 500);
gettimeofday(&after, NULL);
us = (after.tv_sec - before.tv_sec) * 1000000 +
(after.tv_usec - before.tv_usec);
if(us < 400000) {
return 1;
}
}
return 0;
}" HAVE_POLL_FINE)
endif()

28
contrib/curl-cmake/CMake/README.md
View File

@ -1,28 +0,0 @@
#***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
# | (__| |_| | _ <| |___
# \___|\___/|_| \_\_____|
#
# Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at https://curl.haxx.se/docs/copyright.html.
#
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
# copies of the Software, and permit persons to whom the Software is
# furnished to do so, under the terms of the COPYING file.
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
# KIND, either express or implied.
#
###########################################################################
CMake files under this directory were reused from project curl.
Here are links to original source files:
https://github.com/curl/curl/blob/master/CMake/CurlSymbolHiding.cmake
https://github.com/curl/curl/blob/master/CMake/CurlTests,c
https://github.com/curl/curl/blob/master/CMake/Macros.cmake
https://github.com/curl/curl/blob/master/CMake/OtherTests.cmake

1509
contrib/curl-cmake/CMakeLists.patch
View File

File diff suppressed because it is too large Load Diff

753
contrib/curl-cmake/CMakeLists.txt
View File

@ -1,605 +1,148 @@
#***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
# | (__| |_| | _ <| |___
# \___|\___/|_| \_\_____|
#
# Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at https://curl.haxx.se/docs/copyright.html.
#
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
# copies of the Software, and permit persons to whom the Software is
# furnished to do so, under the terms of the COPYING file.
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
# KIND, either express or implied.
#
###########################################################################
# NOTE:
# This file is shrinked and reworked version of original curl CMakeLists.txt
# Original file link https://github.com/curl/curl/blob/3b8bbbbd1609c638a3d3d0acb148a33dedb67be3/CMakeLists.txt
# If you need to update curl building you can find patch file in this directory
# and apply it to fresh original CMakeLists.txt file.
cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
SET(CURL_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl)
SET(CURL_LIBRARY_DIR ${CURL_SOURCE_DIR}/lib)
set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake;${CMAKE_MODULE_PATH}")
# Disable status messages when perform checks
set(CMAKE_REQUIRED_QUIET TRUE)
include(Macros)
include(CMakeDependentOption)
include(CheckCCompilerFlag)
file(READ ${CURL_SOURCE_DIR}/include/curl/curlver.h CURL_VERSION_H_CONTENTS)
string(REGEX MATCH "#define LIBCURL_VERSION \"[^\"]*"
CURL_VERSION ${CURL_VERSION_H_CONTENTS})
string(REGEX REPLACE "[^\"]+\"" "" CURL_VERSION ${CURL_VERSION})
string(REGEX MATCH "#define LIBCURL_VERSION_NUM 0x[0-9a-fA-F]+"
CURL_VERSION_NUM ${CURL_VERSION_H_CONTENTS})
string(REGEX REPLACE "[^0]+0x" "" CURL_VERSION_NUM ${CURL_VERSION_NUM})
message(STATUS "Use curl version=[${CURL_VERSION}]")
set(OPERATING_SYSTEM "${CMAKE_SYSTEM_NAME}")
set(OS "\"${CMAKE_SYSTEM_NAME}\"")
option(PICKY_COMPILER "Enable picky compiler options" ON)
option(ENABLE_THREADED_RESOLVER "Set to ON to enable threaded DNS lookup" ON)
if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)
if(PICKY_COMPILER)
foreach(_CCOPT -pedantic -Wall -W -Wpointer-arith -Wwrite-strings -Wunused -Wshadow -Winline -Wnested-externs -Wmissing-declarations -Wmissing-prototypes -Wno-long-long -Wfloat-equal -Wno-multichar -Wsign-compare -Wundef -Wno-format-nonliteral -Wendif-labels -Wstrict-prototypes -Wdeclaration-after-statement -Wstrict-aliasing=3 -Wcast-align -Wtype-limits -Wold-style-declaration -Wmissing-parameter-type -Wempty-body -Wclobbered -Wignored-qualifiers -Wconversion -Wno-sign-conversion -Wvla -Wdouble-promotion -Wno-system-headers -Wno-pedantic-ms-format)
# surprisingly, CHECK_C_COMPILER_FLAG needs a new variable to store each new
# test result in.
check_c_compiler_flag(${_CCOPT} OPT${_CCOPT})
if(OPT${_CCOPT})
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_CCOPT}")
endif()
endforeach()
endif()
endif()
# initialize CURL_LIBS
set(CURL_LIBS "")
include(CurlSymbolHiding)
# Http only
set(CURL_DISABLE_FTP ON)
set(CURL_DISABLE_LDAP ON)
set(CURL_DISABLE_LDAPS ON)
set(CURL_DISABLE_TELNET ON)
set(CURL_DISABLE_DICT ON)
set(CURL_DISABLE_FILE ON)
set(CURL_DISABLE_TFTP ON)
set(CURL_DISABLE_RTSP ON)
set(CURL_DISABLE_POP3 ON)
set(CURL_DISABLE_IMAP ON)
set(CURL_DISABLE_SMTP ON)
set(CURL_DISABLE_GOPHER ON)
option(CURL_DISABLE_COOKIES "to disable cookies support" OFF)
mark_as_advanced(CURL_DISABLE_COOKIES)
option(CURL_DISABLE_CRYPTO_AUTH "to disable cryptographic authentication" OFF)
mark_as_advanced(CURL_DISABLE_CRYPTO_AUTH)
option(CURL_DISABLE_VERBOSE_STRINGS "to disable verbose strings" OFF)
mark_as_advanced(CURL_DISABLE_VERBOSE_STRINGS)
option(ENABLE_IPV6 "Define if you want to enable IPv6 support" ON)
mark_as_advanced(ENABLE_IPV6)
if(ENABLE_IPV6 AND NOT WIN32)
include(CheckStructHasMember)
check_struct_has_member("struct sockaddr_in6" sin6_addr "netinet/in.h"
HAVE_SOCKADDR_IN6_SIN6_ADDR)
check_struct_has_member("struct sockaddr_in6" sin6_scope_id "netinet/in.h"
HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID)
if(NOT HAVE_SOCKADDR_IN6_SIN6_ADDR)
message(WARNING "struct sockaddr_in6 not available, disabling IPv6 support")
# Force the feature off as this name is used as guard macro...
set(ENABLE_IPV6 OFF
CACHE BOOL "Define if you want to enable IPv6 support" FORCE)
endif()
endif()
# We need ansi c-flags, especially on HP
set(CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS}")
set(CMAKE_REQUIRED_FLAGS ${CMAKE_ANSI_CFLAGS})
# Include all the necessary files for macros
include(CheckFunctionExists)
include(CheckIncludeFile)
include(CheckIncludeFiles)
include(CheckLibraryExists)
include(CheckSymbolExists)
include(CheckTypeSize)
include(CheckCSourceCompiles)
if(ENABLE_THREADED_RESOLVER)
find_package(Threads REQUIRED)
set(USE_THREADS_POSIX ${CMAKE_USE_PTHREADS_INIT})
set(HAVE_PTHREAD_H ${CMAKE_USE_PTHREADS_INIT})
set(CURL_LIBS ${CURL_LIBS} ${CMAKE_THREAD_LIBS_INIT})
endif()
# Check for all needed libraries
# We don't want any plugin loading at runtime. It is harmful.
#check_library_exists_concat("${CMAKE_DL_LIBS}" dlopen HAVE_LIBDL)
# This is unneeded.
#check_library_exists_concat("socket" connect HAVE_LIBSOCKET)
set (NOT_NEED_LIBNSL 1)
set (gethostname HAVE_GETHOSTNAME 1)
# From cmake/find/ssl.cmake
if (OPENSSL_FOUND)
set(SSL_ENABLED ON)
set(USE_OPENSSL ON)
list(APPEND CURL_LIBS ${OPENSSL_LIBRARIES})
check_include_file("openssl/crypto.h" HAVE_OPENSSL_CRYPTO_H)
check_include_file("openssl/err.h" HAVE_OPENSSL_ERR_H)
check_include_file("openssl/pem.h" HAVE_OPENSSL_PEM_H)
check_include_file("openssl/rsa.h" HAVE_OPENSSL_RSA_H)
check_include_file("openssl/ssl.h" HAVE_OPENSSL_SSL_H)
check_include_file("openssl/x509.h" HAVE_OPENSSL_X509_H)
check_include_file("openssl/rand.h" HAVE_OPENSSL_RAND_H)
check_symbol_exists(RAND_status "${CURL_INCLUDES}" HAVE_RAND_STATUS)
check_symbol_exists(RAND_screen "${CURL_INCLUDES}" HAVE_RAND_SCREEN)
check_symbol_exists(RAND_egd "${CURL_INCLUDES}" HAVE_RAND_EGD)
endif()
# Check for idn
# No, we don't need that.
# check_library_exists_concat("idn2" idn2_lookup_ul HAVE_LIBIDN2)
# Check for symbol dlopen (same as HAVE_LIBDL)
# We don't want any plugin loading at runtime. It is harmful.
# check_library_exists("${CURL_LIBS}" dlopen "" HAVE_DLOPEN)
# From /cmake/find/zlib.cmake
if (ZLIB_FOUND)
set(HAVE_ZLIB_H ON)
set(HAVE_LIBZ ON)
set(USE_ZLIB ON)
list(APPEND CURL_LIBS ${ZLIB_LIBRARIES})
endif()
option(ENABLE_UNIX_SOCKETS "Define if you want Unix domain sockets support" OFF)
if(ENABLE_UNIX_SOCKETS)
include(CheckStructHasMember)
check_struct_has_member("struct sockaddr_un" sun_path "sys/un.h" USE_UNIX_SOCKETS)
else()
unset(USE_UNIX_SOCKETS CACHE)
endif()
# CA handling
# Explicitly set to most common case
if (OPENSSL_FOUND)
set(CURL_CA_BUNDLE "/etc/ssl/certs/ca-certificates.crt")
set(CURL_CA_BUNDLE_SET TRUE CACHE BOOL "Path to the CA bundle has been set")
set(CURL_CA_PATH "/etc/ssl/certs")
set(CURL_CA_PATH_SET TRUE CACHE BOOL "Path to the CA bundle has been set")
endif()
check_include_file_concat("stdio.h" HAVE_STDIO_H)
check_include_file_concat("inttypes.h" HAVE_INTTYPES_H)
check_include_file_concat("sys/filio.h" HAVE_SYS_FILIO_H)
check_include_file_concat("sys/ioctl.h" HAVE_SYS_IOCTL_H)
check_include_file_concat("sys/param.h" HAVE_SYS_PARAM_H)
check_include_file_concat("sys/poll.h" HAVE_SYS_POLL_H)
check_include_file_concat("sys/resource.h" HAVE_SYS_RESOURCE_H)
check_include_file_concat("sys/select.h" HAVE_SYS_SELECT_H)
check_include_file_concat("sys/socket.h" HAVE_SYS_SOCKET_H)
check_include_file_concat("sys/sockio.h" HAVE_SYS_SOCKIO_H)
check_include_file_concat("sys/stat.h" HAVE_SYS_STAT_H)
check_include_file_concat("sys/time.h" HAVE_SYS_TIME_H)
check_include_file_concat("sys/types.h" HAVE_SYS_TYPES_H)
check_include_file_concat("sys/uio.h" HAVE_SYS_UIO_H)
check_include_file_concat("sys/un.h" HAVE_SYS_UN_H)
check_include_file_concat("sys/utime.h" HAVE_SYS_UTIME_H)
check_include_file_concat("sys/xattr.h" HAVE_SYS_XATTR_H)
check_include_file_concat("alloca.h" HAVE_ALLOCA_H)
check_include_file_concat("arpa/inet.h" HAVE_ARPA_INET_H)
#check_include_file_concat("arpa/tftp.h" HAVE_ARPA_TFTP_H)
check_include_file_concat("assert.h" HAVE_ASSERT_H)
check_include_file_concat("crypto.h" HAVE_CRYPTO_H)
check_include_file_concat("des.h" HAVE_DES_H)
check_include_file_concat("err.h" HAVE_ERR_H)
check_include_file_concat("errno.h" HAVE_ERRNO_H)
check_include_file_concat("fcntl.h" HAVE_FCNTL_H)
#check_include_file_concat("idn2.h" HAVE_IDN2_H)
check_include_file_concat("ifaddrs.h" HAVE_IFADDRS_H)
check_include_file_concat("io.h" HAVE_IO_H)
check_include_file_concat("krb.h" HAVE_KRB_H)
check_include_file_concat("libgen.h" HAVE_LIBGEN_H)
check_include_file_concat("locale.h" HAVE_LOCALE_H)
check_include_file_concat("net/if.h" HAVE_NET_IF_H)
check_include_file_concat("netdb.h" HAVE_NETDB_H)
check_include_file_concat("netinet/in.h" HAVE_NETINET_IN_H)
check_include_file_concat("netinet/tcp.h" HAVE_NETINET_TCP_H)
check_include_file_concat("pem.h" HAVE_PEM_H)
check_include_file_concat("poll.h" HAVE_POLL_H)
check_include_file_concat("pwd.h" HAVE_PWD_H)
check_include_file_concat("rsa.h" HAVE_RSA_H)
check_include_file_concat("setjmp.h" HAVE_SETJMP_H)
check_include_file_concat("sgtty.h" HAVE_SGTTY_H)
check_include_file_concat("signal.h" HAVE_SIGNAL_H)
check_include_file_concat("ssl.h" HAVE_SSL_H)
check_include_file_concat("stdbool.h" HAVE_STDBOOL_H)
check_include_file_concat("stdint.h" HAVE_STDINT_H)
check_include_file_concat("stdio.h" HAVE_STDIO_H)
check_include_file_concat("stdlib.h" HAVE_STDLIB_H)
check_include_file_concat("string.h" HAVE_STRING_H)
check_include_file_concat("strings.h" HAVE_STRINGS_H)
check_include_file_concat("stropts.h" HAVE_STROPTS_H)
check_include_file_concat("termio.h" HAVE_TERMIO_H)
check_include_file_concat("termios.h" HAVE_TERMIOS_H)
check_include_file_concat("time.h" HAVE_TIME_H)
check_include_file_concat("unistd.h" HAVE_UNISTD_H)
check_include_file_concat("utime.h" HAVE_UTIME_H)
check_include_file_concat("x509.h" HAVE_X509_H)
check_include_file_concat("process.h" HAVE_PROCESS_H)
check_include_file_concat("stddef.h" HAVE_STDDEF_H)
#check_include_file_concat("dlfcn.h" HAVE_DLFCN_H)
check_include_file_concat("malloc.h" HAVE_MALLOC_H)
check_include_file_concat("memory.h" HAVE_MEMORY_H)
check_include_file_concat("netinet/if_ether.h" HAVE_NETINET_IF_ETHER_H)
check_include_file_concat("stdint.h" HAVE_STDINT_H)
check_include_file_concat("sockio.h" HAVE_SOCKIO_H)
check_include_file_concat("sys/utsname.h" HAVE_SYS_UTSNAME_H)
check_type_size(size_t SIZEOF_SIZE_T)
check_type_size(ssize_t SIZEOF_SSIZE_T)
check_type_size("long long" SIZEOF_LONG_LONG)
check_type_size("long" SIZEOF_LONG)
check_type_size("short" SIZEOF_SHORT)
check_type_size("int" SIZEOF_INT)
check_type_size("__int64" SIZEOF___INT64)
check_type_size("long double" SIZEOF_LONG_DOUBLE)
check_type_size("time_t" SIZEOF_TIME_T)
set(HAVE_LONGLONG 1)
set(HAVE_LL 1)
set(RANDOM_FILE /dev/urandom)
check_symbol_exists(basename "${CURL_INCLUDES}" HAVE_BASENAME)
check_symbol_exists(socket "${CURL_INCLUDES}" HAVE_SOCKET)
check_symbol_exists(select "${CURL_INCLUDES}" HAVE_SELECT)
check_symbol_exists(poll "${CURL_INCLUDES}" HAVE_POLL)
check_symbol_exists(strdup "${CURL_INCLUDES}" HAVE_STRDUP)
check_symbol_exists(strstr "${CURL_INCLUDES}" HAVE_STRSTR)
check_symbol_exists(strtok_r "${CURL_INCLUDES}" HAVE_STRTOK_R)
check_symbol_exists(strftime "${CURL_INCLUDES}" HAVE_STRFTIME)
check_symbol_exists(uname "${CURL_INCLUDES}" HAVE_UNAME)
check_symbol_exists(strcasecmp "${CURL_INCLUDES}" HAVE_STRCASECMP)
#check_symbol_exists(stricmp "${CURL_INCLUDES}" HAVE_STRICMP)
#check_symbol_exists(strcmpi "${CURL_INCLUDES}" HAVE_STRCMPI)
#check_symbol_exists(strncmpi "${CURL_INCLUDES}" HAVE_STRNCMPI)
check_symbol_exists(alarm "${CURL_INCLUDES}" HAVE_ALARM)
#check_symbol_exists(gethostbyaddr "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR)
check_symbol_exists(gethostbyaddr_r "${CURL_INCLUDES}" HAVE_GETHOSTBYADDR_R)
check_symbol_exists(gettimeofday "${CURL_INCLUDES}" HAVE_GETTIMEOFDAY)
check_symbol_exists(inet_addr "${CURL_INCLUDES}" HAVE_INET_ADDR)
#check_symbol_exists(inet_ntoa "${CURL_INCLUDES}" HAVE_INET_NTOA)
check_symbol_exists(inet_ntoa_r "${CURL_INCLUDES}" HAVE_INET_NTOA_R)
check_symbol_exists(tcsetattr "${CURL_INCLUDES}" HAVE_TCSETATTR)
check_symbol_exists(tcgetattr "${CURL_INCLUDES}" HAVE_TCGETATTR)
check_symbol_exists(perror "${CURL_INCLUDES}" HAVE_PERROR)
check_symbol_exists(closesocket "${CURL_INCLUDES}" HAVE_CLOSESOCKET)
check_symbol_exists(setvbuf "${CURL_INCLUDES}" HAVE_SETVBUF)
check_symbol_exists(sigsetjmp "${CURL_INCLUDES}" HAVE_SIGSETJMP)
check_symbol_exists(getpass_r "${CURL_INCLUDES}" HAVE_GETPASS_R)
#check_symbol_exists(strlcat "${CURL_INCLUDES}" HAVE_STRLCAT)
#check_symbol_exists(getpwuid "${CURL_INCLUDES}" HAVE_GETPWUID)
check_symbol_exists(getpwuid_r "${CURL_INCLUDES}" HAVE_GETPWUID_R)
check_symbol_exists(geteuid "${CURL_INCLUDES}" HAVE_GETEUID)
check_symbol_exists(usleep "${CURL_INCLUDES}" HAVE_USLEEP)
check_symbol_exists(utime "${CURL_INCLUDES}" HAVE_UTIME)
check_symbol_exists(gmtime_r "${CURL_INCLUDES}" HAVE_GMTIME_R)
check_symbol_exists(localtime_r "${CURL_INCLUDES}" HAVE_LOCALTIME_R)
#check_symbol_exists(gethostbyname "${CURL_INCLUDES}" HAVE_GETHOSTBYNAME)
check_symbol_exists(gethostbyname_r "${CURL_INCLUDES}" HAVE_GETHOSTBYNAME_R)
check_symbol_exists(signal "${CURL_INCLUDES}" HAVE_SIGNAL_FUNC)
check_symbol_exists(SIGALRM "${CURL_INCLUDES}" HAVE_SIGNAL_MACRO)
set(HAVE_SIGNAL 1)
check_symbol_exists(uname "${CURL_INCLUDES}" HAVE_UNAME)
check_symbol_exists(strtoll "${CURL_INCLUDES}" HAVE_STRTOLL)
#check_symbol_exists(_strtoi64 "${CURL_INCLUDES}" HAVE__STRTOI64)
check_symbol_exists(strerror_r "${CURL_INCLUDES}" HAVE_STRERROR_R)
check_symbol_exists(siginterrupt "${CURL_INCLUDES}" HAVE_SIGINTERRUPT)
check_symbol_exists(perror "${CURL_INCLUDES}" HAVE_PERROR)
check_symbol_exists(fork "${CURL_INCLUDES}" HAVE_FORK)
check_symbol_exists(getaddrinfo "${CURL_INCLUDES}" HAVE_GETADDRINFO)
check_symbol_exists(freeaddrinfo "${CURL_INCLUDES}" HAVE_FREEADDRINFO)
check_symbol_exists(freeifaddrs "${CURL_INCLUDES}" HAVE_FREEIFADDRS)
check_symbol_exists(pipe "${CURL_INCLUDES}" HAVE_PIPE)
check_symbol_exists(ftruncate "${CURL_INCLUDES}" HAVE_FTRUNCATE)
check_symbol_exists(getprotobyname "${CURL_INCLUDES}" HAVE_GETPROTOBYNAME)
check_symbol_exists(getpeername "${CURL_INCLUDES}" HAVE_GETPEERNAME)
check_symbol_exists(getsockname "${CURL_INCLUDES}" HAVE_GETSOCKNAME)
check_symbol_exists(if_nametoindex "${CURL_INCLUDES}" HAVE_IF_NAMETOINDEX)
check_symbol_exists(getrlimit "${CURL_INCLUDES}" HAVE_GETRLIMIT)
check_symbol_exists(setlocale "${CURL_INCLUDES}" HAVE_SETLOCALE)
check_symbol_exists(setmode "${CURL_INCLUDES}" HAVE_SETMODE)
check_symbol_exists(setrlimit "${CURL_INCLUDES}" HAVE_SETRLIMIT)
check_symbol_exists(fcntl "${CURL_INCLUDES}" HAVE_FCNTL)
check_symbol_exists(ioctl "${CURL_INCLUDES}" HAVE_IOCTL)
check_symbol_exists(setsockopt "${CURL_INCLUDES}" HAVE_SETSOCKOPT)
check_function_exists(mach_absolute_time HAVE_MACH_ABSOLUTE_TIME)
check_symbol_exists(fsetxattr "${CURL_INCLUDES}" HAVE_FSETXATTR)
if(HAVE_FSETXATTR)
foreach(CURL_TEST HAVE_FSETXATTR_5 HAVE_FSETXATTR_6)
curl_internal_test(${CURL_TEST})
endforeach()
endif()
# sigaction and sigsetjmp are special. Use special mechanism for
# detecting those, but only if previous attempt failed.
if(HAVE_SIGNAL_H)
check_symbol_exists(sigaction "signal.h" HAVE_SIGACTION)
endif()
if(NOT HAVE_SIGSETJMP)
if(HAVE_SETJMP_H)
check_symbol_exists(sigsetjmp "setjmp.h" HAVE_MACRO_SIGSETJMP)
if(HAVE_MACRO_SIGSETJMP)
set(HAVE_SIGSETJMP 1)
endif()
endif()
endif()
# If there is no stricmp(), do not allow LDAP to parse URLs
if(NOT HAVE_STRICMP)
set(HAVE_LDAP_URL_PARSE 1)
endif()
# Do curl specific tests
foreach(CURL_TEST
HAVE_FCNTL_O_NONBLOCK
HAVE_IOCTLSOCKET
HAVE_IOCTLSOCKET_CAMEL
HAVE_IOCTLSOCKET_CAMEL_FIONBIO
HAVE_IOCTLSOCKET_FIONBIO
HAVE_IOCTL_FIONBIO
HAVE_IOCTL_SIOCGIFADDR
HAVE_SETSOCKOPT_SO_NONBLOCK
HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID
TIME_WITH_SYS_TIME
HAVE_O_NONBLOCK
HAVE_GETHOSTBYADDR_R_5
HAVE_GETHOSTBYADDR_R_7
HAVE_GETHOSTBYADDR_R_8
HAVE_GETHOSTBYADDR_R_5_REENTRANT
HAVE_GETHOSTBYADDR_R_7_REENTRANT
HAVE_GETHOSTBYADDR_R_8_REENTRANT
HAVE_GETHOSTBYNAME_R_3
HAVE_GETHOSTBYNAME_R_5
HAVE_GETHOSTBYNAME_R_6
HAVE_GETHOSTBYNAME_R_3_REENTRANT
HAVE_GETHOSTBYNAME_R_5_REENTRANT
HAVE_GETHOSTBYNAME_R_6_REENTRANT
HAVE_IN_ADDR_T
HAVE_BOOL_T
STDC_HEADERS
RETSIGTYPE_TEST
HAVE_INET_NTOA_R_DECL
HAVE_INET_NTOA_R_DECL_REENTRANT
HAVE_GETADDRINFO
HAVE_FILE_OFFSET_BITS
HAVE_VARIADIC_MACROS_C99
HAVE_VARIADIC_MACROS_GCC
)
curl_internal_test(${CURL_TEST})
endforeach()
if(HAVE_FILE_OFFSET_BITS)
set(_FILE_OFFSET_BITS 64)
set(CMAKE_REQUIRED_FLAGS "-D_FILE_OFFSET_BITS=64")
endif()
check_type_size("off_t" SIZEOF_OFF_T)
# include this header to get the type
set(CMAKE_REQUIRED_INCLUDES "${CURL_SOURCE_DIR}/include")
set(CMAKE_EXTRA_INCLUDE_FILES "curl/system.h")
check_type_size("curl_off_t" SIZEOF_CURL_OFF_T)
set(CMAKE_EXTRA_INCLUDE_FILES "")
foreach(CURL_TEST
HAVE_GLIBC_STRERROR_R
HAVE_POSIX_STRERROR_R
)
curl_internal_test(${CURL_TEST})
endforeach()
# Check for reentrant
foreach(CURL_TEST
HAVE_GETHOSTBYADDR_R_5
HAVE_GETHOSTBYADDR_R_7
HAVE_GETHOSTBYADDR_R_8
HAVE_GETHOSTBYNAME_R_3
HAVE_GETHOSTBYNAME_R_5
HAVE_GETHOSTBYNAME_R_6
HAVE_INET_NTOA_R_DECL_REENTRANT)
if(NOT ${CURL_TEST})
if(${CURL_TEST}_REENTRANT)
set(NEED_REENTRANT 1)
endif()
endif()
endforeach()
if(NEED_REENTRANT)
foreach(CURL_TEST
HAVE_GETHOSTBYADDR_R_5
HAVE_GETHOSTBYADDR_R_7
HAVE_GETHOSTBYADDR_R_8
HAVE_GETHOSTBYNAME_R_3
HAVE_GETHOSTBYNAME_R_5
HAVE_GETHOSTBYNAME_R_6)
set(${CURL_TEST} 0)
if(${CURL_TEST}_REENTRANT)
set(${CURL_TEST} 1)
endif()
endforeach()
endif()
if(HAVE_INET_NTOA_R_DECL_REENTRANT)
set(HAVE_INET_NTOA_R_DECL 1)
set(NEED_REENTRANT 1)
endif()
# Check clock_gettime(CLOCK_MONOTONIC, x) support
curl_internal_test(HAVE_CLOCK_GETTIME_MONOTONIC)
# Check compiler support of __builtin_available()
curl_internal_test(HAVE_BUILTIN_AVAILABLE)
# Some other minor tests
if(NOT HAVE_IN_ADDR_T)
set(in_addr_t "unsigned long")
endif()
# Check for nonblocking
set(HAVE_DISABLED_NONBLOCKING 1)
if(HAVE_FIONBIO OR
HAVE_IOCTLSOCKET OR
HAVE_IOCTLSOCKET_CASE OR
HAVE_O_NONBLOCK)
set(HAVE_DISABLED_NONBLOCKING)
endif()
set(CURL_PULL_SYS_TYPES_H ${HAVE_SYS_TYPES_H})
set(CURL_PULL_SYS_SOCKET_H ${HAVE_SYS_SOCKET_H})
set(CURL_PULL_SYS_POLL_H ${HAVE_SYS_POLL_H})
set(CURL_PULL_STDINT_H ${HAVE_STDINT_H})
set(CURL_PULL_INTTYPES_H ${HAVE_INTTYPES_H})
include(CMake/OtherTests.cmake)
SET(LIB_VAUTH_CFILES
"${CURL_LIBRARY_DIR}/vauth/vauth.c" "${CURL_LIBRARY_DIR}/vauth/cleartext.c" "${CURL_LIBRARY_DIR}/vauth/cram.c"
"${CURL_LIBRARY_DIR}/vauth/digest.c" "${CURL_LIBRARY_DIR}/vauth/digest_sspi.c" "${CURL_LIBRARY_DIR}/vauth/krb5_gssapi.c"
"${CURL_LIBRARY_DIR}/vauth/krb5_sspi.c" "${CURL_LIBRARY_DIR}/vauth/ntlm.c" "${CURL_LIBRARY_DIR}/vauth/ntlm_sspi.c" "${CURL_LIBRARY_DIR}/vauth/oauth2.c"
"${CURL_LIBRARY_DIR}/vauth/spnego_gssapi.c" "${CURL_LIBRARY_DIR}/vauth/spnego_sspi.c")
SET(LIB_VAUTH_HFILES "${CURL_LIBRARY_DIR}/vauth/vauth.h" "${CURL_LIBRARY_DIR}/vauth/digest.h" "${CURL_LIBRARY_DIR}/vauth/ntlm.h")
SET(LIB_VTLS_CFILES "${CURL_LIBRARY_DIR}/vtls/openssl.c" "${CURL_LIBRARY_DIR}/vtls/gtls.c" "${CURL_LIBRARY_DIR}/vtls/vtls.c" "${CURL_LIBRARY_DIR}/vtls/nss.c"
"${CURL_LIBRARY_DIR}/vtls/polarssl.c" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.c"
"${CURL_LIBRARY_DIR}/vtls/wolfssl.c" "${CURL_LIBRARY_DIR}/vtls/schannel.c" "${CURL_LIBRARY_DIR}/vtls/schannel_verify.c"
"${CURL_LIBRARY_DIR}/vtls/sectransp.c" "${CURL_LIBRARY_DIR}/vtls/gskit.c" "${CURL_LIBRARY_DIR}/vtls/mbedtls.c" "${CURL_LIBRARY_DIR}/vtls/mesalink.c"
"${CURL_LIBRARY_DIR}/vtls/bearssl.c")
SET(LIB_VTLS_HFILES "${CURL_LIBRARY_DIR}/vtls/openssl.h" "${CURL_LIBRARY_DIR}/vtls/vtls.h" "${CURL_LIBRARY_DIR}/vtls/gtls.h"
"${CURL_LIBRARY_DIR}/vtls/nssg.h" "${CURL_LIBRARY_DIR}/vtls/polarssl.h" "${CURL_LIBRARY_DIR}/vtls/polarssl_threadlock.h"
"${CURL_LIBRARY_DIR}/vtls/wolfssl.h" "${CURL_LIBRARY_DIR}/vtls/schannel.h" "${CURL_LIBRARY_DIR}/vtls/sectransp.h" "${CURL_LIBRARY_DIR}/vtls/gskit.h"
"${CURL_LIBRARY_DIR}/vtls/mbedtls.h" "${CURL_LIBRARY_DIR}/vtls/mesalink.h" "${CURL_LIBRARY_DIR}/vtls/bearssl.h")
SET(LIB_VQUIC_CFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.c" "${CURL_LIBRARY_DIR}/vquic/quiche.c")
SET(LIB_VQUIC_HFILES "${CURL_LIBRARY_DIR}/vquic/ngtcp2.h" "${CURL_LIBRARY_DIR}/vquic/quiche.h")
SET(LIB_VSSH_CFILES "${CURL_LIBRARY_DIR}/vssh/libssh2.c" "${CURL_LIBRARY_DIR}/vssh/libssh.c")
SET(LIB_VSSH_HFILES "${CURL_LIBRARY_DIR}/vssh/ssh.h")
SET(LIB_CFILES "${CURL_LIBRARY_DIR}/file.c"
"${CURL_LIBRARY_DIR}/timeval.c" "${CURL_LIBRARY_DIR}/base64.c" "${CURL_LIBRARY_DIR}/hostip.c" "${CURL_LIBRARY_DIR}/progress.c" "${CURL_LIBRARY_DIR}/formdata.c"
"${CURL_LIBRARY_DIR}/cookie.c" "${CURL_LIBRARY_DIR}/http.c" "${CURL_LIBRARY_DIR}/sendf.c" "${CURL_LIBRARY_DIR}/url.c" "${CURL_LIBRARY_DIR}/dict.c" "${CURL_LIBRARY_DIR}/if2ip.c" "${CURL_LIBRARY_DIR}/speedcheck.c"
"${CURL_LIBRARY_DIR}/ldap.c" "${CURL_LIBRARY_DIR}/version.c" "${CURL_LIBRARY_DIR}/getenv.c" "${CURL_LIBRARY_DIR}/escape.c" "${CURL_LIBRARY_DIR}/mprintf.c" "${CURL_LIBRARY_DIR}/telnet.c" "${CURL_LIBRARY_DIR}/netrc.c"
"${CURL_LIBRARY_DIR}/getinfo.c" "${CURL_LIBRARY_DIR}/transfer.c" "${CURL_LIBRARY_DIR}/strcase.c" "${CURL_LIBRARY_DIR}/easy.c" "${CURL_LIBRARY_DIR}/security.c" "${CURL_LIBRARY_DIR}/curl_fnmatch.c"
"${CURL_LIBRARY_DIR}/fileinfo.c" "${CURL_LIBRARY_DIR}/wildcard.c" "${CURL_LIBRARY_DIR}/krb5.c" "${CURL_LIBRARY_DIR}/memdebug.c" "${CURL_LIBRARY_DIR}/http_chunks.c"
"${CURL_LIBRARY_DIR}/strtok.c" "${CURL_LIBRARY_DIR}/connect.c" "${CURL_LIBRARY_DIR}/llist.c" "${CURL_LIBRARY_DIR}/hash.c" "${CURL_LIBRARY_DIR}/multi.c" "${CURL_LIBRARY_DIR}/content_encoding.c" "${CURL_LIBRARY_DIR}/share.c"
"${CURL_LIBRARY_DIR}/http_digest.c" "${CURL_LIBRARY_DIR}/md4.c" "${CURL_LIBRARY_DIR}/md5.c" "${CURL_LIBRARY_DIR}/http_negotiate.c" "${CURL_LIBRARY_DIR}/inet_pton.c" "${CURL_LIBRARY_DIR}/strtoofft.c"
"${CURL_LIBRARY_DIR}/strerror.c" "${CURL_LIBRARY_DIR}/amigaos.c" "${CURL_LIBRARY_DIR}/hostasyn.c" "${CURL_LIBRARY_DIR}/hostip4.c" "${CURL_LIBRARY_DIR}/hostip6.c" "${CURL_LIBRARY_DIR}/hostsyn.c"
"${CURL_LIBRARY_DIR}/inet_ntop.c" "${CURL_LIBRARY_DIR}/parsedate.c" "${CURL_LIBRARY_DIR}/select.c" "${CURL_LIBRARY_DIR}/splay.c" "${CURL_LIBRARY_DIR}/strdup.c" "${CURL_LIBRARY_DIR}/socks.c"
"${CURL_LIBRARY_DIR}/curl_addrinfo.c" "${CURL_LIBRARY_DIR}/socks_gssapi.c" "${CURL_LIBRARY_DIR}/socks_sspi.c"
"${CURL_LIBRARY_DIR}/curl_sspi.c" "${CURL_LIBRARY_DIR}/slist.c" "${CURL_LIBRARY_DIR}/nonblock.c" "${CURL_LIBRARY_DIR}/curl_memrchr.c" "${CURL_LIBRARY_DIR}/imap.c" "${CURL_LIBRARY_DIR}/pop3.c" "${CURL_LIBRARY_DIR}/smtp.c"
"${CURL_LIBRARY_DIR}/pingpong.c" "${CURL_LIBRARY_DIR}/rtsp.c" "${CURL_LIBRARY_DIR}/curl_threads.c" "${CURL_LIBRARY_DIR}/warnless.c" "${CURL_LIBRARY_DIR}/hmac.c" "${CURL_LIBRARY_DIR}/curl_rtmp.c"
"${CURL_LIBRARY_DIR}/openldap.c" "${CURL_LIBRARY_DIR}/curl_gethostname.c" "${CURL_LIBRARY_DIR}/gopher.c" "${CURL_LIBRARY_DIR}/idn_win32.c"
"${CURL_LIBRARY_DIR}/http_proxy.c" "${CURL_LIBRARY_DIR}/non-ascii.c" "${CURL_LIBRARY_DIR}/asyn-ares.c" "${CURL_LIBRARY_DIR}/asyn-thread.c" "${CURL_LIBRARY_DIR}/curl_gssapi.c"
"${CURL_LIBRARY_DIR}/http_ntlm.c" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.c" "${CURL_LIBRARY_DIR}/curl_ntlm_core.c" "${CURL_LIBRARY_DIR}/curl_sasl.c" "${CURL_LIBRARY_DIR}/rand.c"
"${CURL_LIBRARY_DIR}/curl_multibyte.c" "${CURL_LIBRARY_DIR}/hostcheck.c" "${CURL_LIBRARY_DIR}/conncache.c" "${CURL_LIBRARY_DIR}/dotdot.c"
"${CURL_LIBRARY_DIR}/x509asn1.c" "${CURL_LIBRARY_DIR}/http2.c" "${CURL_LIBRARY_DIR}/smb.c" "${CURL_LIBRARY_DIR}/curl_endian.c" "${CURL_LIBRARY_DIR}/curl_des.c" "${CURL_LIBRARY_DIR}/system_win32.c"
"${CURL_LIBRARY_DIR}/mime.c" "${CURL_LIBRARY_DIR}/sha256.c" "${CURL_LIBRARY_DIR}/setopt.c" "${CURL_LIBRARY_DIR}/curl_path.c" "${CURL_LIBRARY_DIR}/curl_ctype.c" "${CURL_LIBRARY_DIR}/curl_range.c" "${CURL_LIBRARY_DIR}/psl.c"
"${CURL_LIBRARY_DIR}/doh.c" "${CURL_LIBRARY_DIR}/urlapi.c" "${CURL_LIBRARY_DIR}/curl_get_line.c" "${CURL_LIBRARY_DIR}/altsvc.c" "${CURL_LIBRARY_DIR}/socketpair.c")
SET(LIB_HFILES "${CURL_LIBRARY_DIR}/arpa_telnet.h" "${CURL_LIBRARY_DIR}/netrc.h" "${CURL_LIBRARY_DIR}/file.h" "${CURL_LIBRARY_DIR}/timeval.h" "${CURL_LIBRARY_DIR}/hostip.h" "${CURL_LIBRARY_DIR}/progress.h"
"${CURL_LIBRARY_DIR}/formdata.h" "${CURL_LIBRARY_DIR}/cookie.h" "${CURL_LIBRARY_DIR}/http.h" "${CURL_LIBRARY_DIR}/sendf.h" "${CURL_LIBRARY_DIR}/url.h" "${CURL_LIBRARY_DIR}/dict.h" "${CURL_LIBRARY_DIR}/if2ip.h"
"${CURL_LIBRARY_DIR}/speedcheck.h" "${CURL_LIBRARY_DIR}/urldata.h" "${CURL_LIBRARY_DIR}/curl_ldap.h" "${CURL_LIBRARY_DIR}/escape.h" "${CURL_LIBRARY_DIR}/telnet.h" "${CURL_LIBRARY_DIR}/getinfo.h"
"${CURL_LIBRARY_DIR}/strcase.h" "${CURL_LIBRARY_DIR}/curl_sec.h" "${CURL_LIBRARY_DIR}/memdebug.h" "${CURL_LIBRARY_DIR}/http_chunks.h" "${CURL_LIBRARY_DIR}/curl_fnmatch.h"
"${CURL_LIBRARY_DIR}/wildcard.h" "${CURL_LIBRARY_DIR}/fileinfo.h" "${CURL_LIBRARY_DIR}/strtok.h" "${CURL_LIBRARY_DIR}/connect.h" "${CURL_LIBRARY_DIR}/llist.h"
"${CURL_LIBRARY_DIR}/hash.h" "${CURL_LIBRARY_DIR}/content_encoding.h" "${CURL_LIBRARY_DIR}/share.h" "${CURL_LIBRARY_DIR}/curl_md4.h" "${CURL_LIBRARY_DIR}/curl_md5.h" "${CURL_LIBRARY_DIR}/http_digest.h"
"${CURL_LIBRARY_DIR}/http_negotiate.h" "${CURL_LIBRARY_DIR}/inet_pton.h" "${CURL_LIBRARY_DIR}/amigaos.h" "${CURL_LIBRARY_DIR}/strtoofft.h" "${CURL_LIBRARY_DIR}/strerror.h"
"${CURL_LIBRARY_DIR}/inet_ntop.h" "${CURL_LIBRARY_DIR}/curlx.h" "${CURL_LIBRARY_DIR}/curl_memory.h" "${CURL_LIBRARY_DIR}/curl_setup.h" "${CURL_LIBRARY_DIR}/transfer.h" "${CURL_LIBRARY_DIR}/select.h"
"${CURL_LIBRARY_DIR}/easyif.h" "${CURL_LIBRARY_DIR}/multiif.h" "${CURL_LIBRARY_DIR}/parsedate.h" "${CURL_LIBRARY_DIR}/sockaddr.h" "${CURL_LIBRARY_DIR}/splay.h" "${CURL_LIBRARY_DIR}/strdup.h"
"${CURL_LIBRARY_DIR}/socks.h" "${CURL_LIBRARY_DIR}/curl_base64.h" "${CURL_LIBRARY_DIR}/curl_addrinfo.h" "${CURL_LIBRARY_DIR}/curl_sspi.h"
"${CURL_LIBRARY_DIR}/slist.h" "${CURL_LIBRARY_DIR}/nonblock.h" "${CURL_LIBRARY_DIR}/curl_memrchr.h" "${CURL_LIBRARY_DIR}/imap.h" "${CURL_LIBRARY_DIR}/pop3.h" "${CURL_LIBRARY_DIR}/smtp.h" "${CURL_LIBRARY_DIR}/pingpong.h"
"${CURL_LIBRARY_DIR}/rtsp.h" "${CURL_LIBRARY_DIR}/curl_threads.h" "${CURL_LIBRARY_DIR}/warnless.h" "${CURL_LIBRARY_DIR}/curl_hmac.h" "${CURL_LIBRARY_DIR}/curl_rtmp.h"
"${CURL_LIBRARY_DIR}/curl_gethostname.h" "${CURL_LIBRARY_DIR}/gopher.h" "${CURL_LIBRARY_DIR}/http_proxy.h" "${CURL_LIBRARY_DIR}/non-ascii.h" "${CURL_LIBRARY_DIR}/asyn.h"
"${CURL_LIBRARY_DIR}/http_ntlm.h" "${CURL_LIBRARY_DIR}/curl_gssapi.h" "${CURL_LIBRARY_DIR}/curl_ntlm_wb.h" "${CURL_LIBRARY_DIR}/curl_ntlm_core.h"
"${CURL_LIBRARY_DIR}/curl_sasl.h" "${CURL_LIBRARY_DIR}/curl_multibyte.h" "${CURL_LIBRARY_DIR}/hostcheck.h" "${CURL_LIBRARY_DIR}/conncache.h"
"${CURL_LIBRARY_DIR}/multihandle.h" "${CURL_LIBRARY_DIR}/setup-vms.h" "${CURL_LIBRARY_DIR}/dotdot.h"
"${CURL_LIBRARY_DIR}/x509asn1.h" "${CURL_LIBRARY_DIR}/http2.h" "${CURL_LIBRARY_DIR}/sigpipe.h" "${CURL_LIBRARY_DIR}/smb.h" "${CURL_LIBRARY_DIR}/curl_endian.h" "${CURL_LIBRARY_DIR}/curl_des.h"
"${CURL_LIBRARY_DIR}/curl_printf.h" "${CURL_LIBRARY_DIR}/system_win32.h" "${CURL_LIBRARY_DIR}/rand.h" "${CURL_LIBRARY_DIR}/mime.h" "${CURL_LIBRARY_DIR}/curl_sha256.h" "${CURL_LIBRARY_DIR}/setopt.h"
"${CURL_LIBRARY_DIR}/curl_path.h" "${CURL_LIBRARY_DIR}/curl_ctype.h" "${CURL_LIBRARY_DIR}/curl_range.h" "${CURL_LIBRARY_DIR}/psl.h" "${CURL_LIBRARY_DIR}/doh.h" "${CURL_LIBRARY_DIR}/urlapi-int.h"
"${CURL_LIBRARY_DIR}/curl_get_line.h" "${CURL_LIBRARY_DIR}/altsvc.h" "${CURL_LIBRARY_DIR}/quic.h" "${CURL_LIBRARY_DIR}/socketpair.h")
SET(LIB_RCFILES "${CURL_LIBRARY_DIR}/libcurl.rc")
SET(CSOURCES ${LIB_CFILES} ${LIB_VAUTH_CFILES} ${LIB_VTLS_CFILES}
${LIB_VQUIC_CFILES} ${LIB_VSSH_CFILES})
SET(HHEADERS ${LIB_HFILES} ${LIB_VAUTH_HFILES} ${LIB_VTLS_HFILES}
${LIB_VQUIC_HFILES} ${LIB_VSSH_HFILES})
configure_file(${CURL_SOURCE_DIR}/lib/curl_config.h.cmake
${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h)
list(APPEND HHEADERS
${CMAKE_CURRENT_BINARY_DIR}/curl/curl_config.h
)
add_library(libcurl ${HHEADERS} ${CSOURCES})
if(NOT BUILD_SHARED_LIBS)
set_target_properties(libcurl PROPERTIES INTERFACE_COMPILE_DEFINITIONS CURL_STATICLIB)
endif()
if(HIDES_CURL_PRIVATE_SYMBOLS)
set_property(TARGET libcurl APPEND PROPERTY COMPILE_DEFINITIONS "CURL_HIDDEN_SYMBOLS")
set_property(TARGET libcurl APPEND PROPERTY COMPILE_FLAGS ${CURL_CFLAG_SYMBOLS_HIDE})
endif()
if(OPENSSL_FOUND)
target_include_directories(libcurl PUBLIC ${OPENSSL_INCLUDE_DIR})
message("-- Including openssl ${OPENSSL_INCLUDE_DIR} to curl")
endif()
if(ZLIB_FOUND)
target_include_directories(libcurl PUBLIC ${ZLIB_INCLUDE_DIRS}})
message("-- Including zlib ${ZLIB_INCLUDE_DIRS} to curl")
endif()
target_compile_definitions(libcurl PUBLIC -DHAVE_CONFIG_H)
target_compile_definitions(libcurl PUBLIC -DBUILDING_LIBCURL)
target_include_directories(libcurl PUBLIC "${CURL_SOURCE_DIR}/include" "${CURL_LIBRARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/curl")
target_link_libraries(libcurl ${CURL_LIBS})
set (CURL_DIR ${ClickHouse_SOURCE_DIR}/contrib/curl)
set (SRCS
${CURL_DIR}/lib/file.c
${CURL_DIR}/lib/timeval.c
${CURL_DIR}/lib/base64.c
${CURL_DIR}/lib/hostip.c
${CURL_DIR}/lib/progress.c
${CURL_DIR}/lib/formdata.c
${CURL_DIR}/lib/cookie.c
${CURL_DIR}/lib/http.c
${CURL_DIR}/lib/sendf.c
${CURL_DIR}/lib/url.c
${CURL_DIR}/lib/dict.c
${CURL_DIR}/lib/if2ip.c
${CURL_DIR}/lib/speedcheck.c
${CURL_DIR}/lib/ldap.c
${CURL_DIR}/lib/version.c
${CURL_DIR}/lib/getenv.c
${CURL_DIR}/lib/escape.c
${CURL_DIR}/lib/mprintf.c
${CURL_DIR}/lib/telnet.c
${CURL_DIR}/lib/netrc.c
${CURL_DIR}/lib/getinfo.c
${CURL_DIR}/lib/transfer.c
${CURL_DIR}/lib/strcase.c
${CURL_DIR}/lib/easy.c
${CURL_DIR}/lib/security.c
${CURL_DIR}/lib/curl_fnmatch.c
${CURL_DIR}/lib/fileinfo.c
${CURL_DIR}/lib/wildcard.c
${CURL_DIR}/lib/krb5.c
${CURL_DIR}/lib/memdebug.c
${CURL_DIR}/lib/http_chunks.c
${CURL_DIR}/lib/strtok.c
${CURL_DIR}/lib/connect.c
${CURL_DIR}/lib/llist.c
${CURL_DIR}/lib/hash.c
${CURL_DIR}/lib/multi.c
${CURL_DIR}/lib/content_encoding.c
${CURL_DIR}/lib/share.c
${CURL_DIR}/lib/http_digest.c
${CURL_DIR}/lib/md4.c
${CURL_DIR}/lib/md5.c
${CURL_DIR}/lib/http_negotiate.c
${CURL_DIR}/lib/inet_pton.c
${CURL_DIR}/lib/strtoofft.c
${CURL_DIR}/lib/strerror.c
${CURL_DIR}/lib/amigaos.c
${CURL_DIR}/lib/hostasyn.c
${CURL_DIR}/lib/hostip4.c
${CURL_DIR}/lib/hostip6.c
${CURL_DIR}/lib/hostsyn.c
${CURL_DIR}/lib/inet_ntop.c
${CURL_DIR}/lib/parsedate.c
${CURL_DIR}/lib/select.c
${CURL_DIR}/lib/splay.c
${CURL_DIR}/lib/strdup.c
${CURL_DIR}/lib/socks.c
${CURL_DIR}/lib/curl_addrinfo.c
${CURL_DIR}/lib/socks_gssapi.c
${CURL_DIR}/lib/socks_sspi.c
${CURL_DIR}/lib/curl_sspi.c
${CURL_DIR}/lib/slist.c
${CURL_DIR}/lib/nonblock.c
${CURL_DIR}/lib/curl_memrchr.c
${CURL_DIR}/lib/imap.c
${CURL_DIR}/lib/pop3.c
${CURL_DIR}/lib/smtp.c
${CURL_DIR}/lib/pingpong.c
${CURL_DIR}/lib/rtsp.c
${CURL_DIR}/lib/curl_threads.c
${CURL_DIR}/lib/warnless.c
${CURL_DIR}/lib/hmac.c
${CURL_DIR}/lib/curl_rtmp.c
${CURL_DIR}/lib/openldap.c
${CURL_DIR}/lib/curl_gethostname.c
${CURL_DIR}/lib/gopher.c
${CURL_DIR}/lib/idn_win32.c
${CURL_DIR}/lib/http_proxy.c
${CURL_DIR}/lib/non-ascii.c
${CURL_DIR}/lib/asyn-thread.c
${CURL_DIR}/lib/curl_gssapi.c
${CURL_DIR}/lib/http_ntlm.c
${CURL_DIR}/lib/curl_ntlm_wb.c
${CURL_DIR}/lib/curl_ntlm_core.c
${CURL_DIR}/lib/curl_sasl.c
${CURL_DIR}/lib/rand.c
${CURL_DIR}/lib/curl_multibyte.c
${CURL_DIR}/lib/hostcheck.c
${CURL_DIR}/lib/conncache.c
${CURL_DIR}/lib/dotdot.c
${CURL_DIR}/lib/x509asn1.c
${CURL_DIR}/lib/http2.c
${CURL_DIR}/lib/smb.c
${CURL_DIR}/lib/curl_endian.c
${CURL_DIR}/lib/curl_des.c
${CURL_DIR}/lib/system_win32.c
${CURL_DIR}/lib/mime.c
${CURL_DIR}/lib/sha256.c
${CURL_DIR}/lib/setopt.c
${CURL_DIR}/lib/curl_path.c
${CURL_DIR}/lib/curl_ctype.c
${CURL_DIR}/lib/curl_range.c
${CURL_DIR}/lib/psl.c
${CURL_DIR}/lib/doh.c
${CURL_DIR}/lib/urlapi.c
${CURL_DIR}/lib/curl_get_line.c
${CURL_DIR}/lib/altsvc.c
${CURL_DIR}/lib/socketpair.c
${CURL_DIR}/lib/vauth/vauth.c
${CURL_DIR}/lib/vauth/cleartext.c
${CURL_DIR}/lib/vauth/cram.c
${CURL_DIR}/lib/vauth/digest.c
${CURL_DIR}/lib/vauth/digest_sspi.c
${CURL_DIR}/lib/vauth/krb5_gssapi.c
${CURL_DIR}/lib/vauth/krb5_sspi.c
${CURL_DIR}/lib/vauth/ntlm.c
${CURL_DIR}/lib/vauth/ntlm_sspi.c
${CURL_DIR}/lib/vauth/oauth2.c
${CURL_DIR}/lib/vauth/spnego_gssapi.c
${CURL_DIR}/lib/vauth/spnego_sspi.c
${CURL_DIR}/lib/vtls/openssl.c
${CURL_DIR}/lib/vtls/gtls.c
${CURL_DIR}/lib/vtls/vtls.c
${CURL_DIR}/lib/vtls/nss.c
${CURL_DIR}/lib/vtls/polarssl.c
${CURL_DIR}/lib/vtls/polarssl_threadlock.c
${CURL_DIR}/lib/vtls/wolfssl.c
${CURL_DIR}/lib/vtls/schannel.c
${CURL_DIR}/lib/vtls/schannel_verify.c
${CURL_DIR}/lib/vtls/sectransp.c
${CURL_DIR}/lib/vtls/gskit.c
${CURL_DIR}/lib/vtls/mbedtls.c
${CURL_DIR}/lib/vtls/mesalink.c
${CURL_DIR}/lib/vtls/bearssl.c
${CURL_DIR}/lib/vquic/ngtcp2.c
${CURL_DIR}/lib/vquic/quiche.c
${CURL_DIR}/lib/vssh/libssh2.c
${CURL_DIR}/lib/vssh/libssh.c
)
add_library (curl ${SRCS})
target_compile_definitions(curl PRIVATE HAVE_CONFIG_H BUILDING_LIBCURL CURL_HIDDEN_SYMBOLS libcurl_EXPORTS)
target_include_directories(curl PUBLIC ${CURL_DIR}/include ${CURL_DIR}/lib .)
target_compile_definitions(curl PRIVATE OS="${CMAKE_SYSTEM_NAME}")

38
contrib/curl-cmake/curl_config.h Normal file
View File

@ -0,0 +1,38 @@
#define CURL_DISABLE_FTP
#define CURL_DISABLE_TFTP
#define CURL_DISABLE_LDAP
#define CURL_EXTERN_SYMBOL __attribute__ ((__visibility__ ("default")))
#define SIZEOF_SHORT 2
#define SIZEOF_INT 4
#define SIZEOF_LONG 8
#define SIZEOF_CURL_OFF_T 8
#define SIZEOF_SIZE_T 8
#define HAVE_FCNTL_O_NONBLOCK
#define HAVE_LONGLONG
#define HAVE_POLL_FINE
#define HAVE_SOCKET
#define HAVE_STRUCT_TIMEVAL
#define HAVE_RECV
#define RECV_TYPE_ARG1 int
#define RECV_TYPE_ARG2 void*
#define RECV_TYPE_ARG3 size_t
#define RECV_TYPE_ARG4 int
#define RECV_TYPE_RETV ssize_t
#define HAVE_SEND
#define SEND_TYPE_ARG1 int
#define SEND_TYPE_ARG2 void*
#define SEND_QUAL_ARG2 const
#define SEND_TYPE_ARG3 size_t
#define SEND_TYPE_ARG4 int
#define SEND_TYPE_RETV ssize_t
#define HAVE_ARPA_INET_H
#define HAVE_ERRNO_H
#define HAVE_FCNTL_H
#define HAVE_NETDB_H
#define HAVE_SYS_STAT_H
#define HAVE_UNISTD_H

2
dbms/programs/obfuscator/Obfuscator.cpp
View File

@ -1070,7 +1070,7 @@ try
if (!silent)
std::cerr << "Generating data\n";
file_in.seek(0);
file_in.seek(0, SEEK_SET);
BlockInputStreamPtr input = context.getInputFormat(input_format, file_in, header, max_block_size);
BlockOutputStreamPtr output = context.getOutputFormat(output_format, file_out, header);

51
dbms/src/Access/AccessControlManager.cpp
View File

@ -2,8 +2,10 @@
#include <Access/MultipleAccessStorage.h>
#include <Access/MemoryAccessStorage.h>
#include <Access/UsersConfigAccessStorage.h>
#include <Access/User.h>
#include <Access/QuotaContextFactory.h>
#include <Access/RowPolicyContextFactory.h>
#include <Access/AccessRightsContext.h>
namespace DB
@ -33,6 +35,49 @@ AccessControlManager::~AccessControlManager()
}
UserPtr AccessControlManager::getUser(const String & user_name) const
{
return getUser(user_name, {}, nullptr);
}
UserPtr AccessControlManager::getUser(
const String & user_name, const std::function<void(const UserPtr &)> & on_change, ext::scope_guard * subscription) const
{
UUID id = getID<User>(user_name);
if (on_change && subscription)
{
*subscription = subscribeForChanges(id, [on_change](const UUID &, const AccessEntityPtr & user)
{
if (user)
on_change(typeid_cast<UserPtr>(user));
});
}
return read<User>(id);
}
UserPtr AccessControlManager::authorizeAndGetUser(
const String & user_name,
const String & password,
const Poco::Net::IPAddress & address) const
{
return authorizeAndGetUser(user_name, password, address, {}, nullptr);
}
UserPtr AccessControlManager::authorizeAndGetUser(
const String & user_name,
const String & password,
const Poco::Net::IPAddress & address,
const std::function<void(const UserPtr &)> & on_change,
ext::scope_guard * subscription) const
{
auto user = getUser(user_name, on_change, subscription);
user->allowed_client_hosts.checkContains(address, user_name);
user->authentication.checkPassword(password, user_name);
return user;
}
void AccessControlManager::loadFromConfig(const Poco::Util::AbstractConfiguration & users_config)
{
auto & users_config_access_storage = dynamic_cast<UsersConfigAccessStorage &>(getStorageByIndex(1));
@ -40,6 +85,12 @@ void AccessControlManager::loadFromConfig(const Poco::Util::AbstractConfiguratio
}
std::shared_ptr<const AccessRightsContext> AccessControlManager::getAccessRightsContext(const ClientInfo & client_info, const AccessRights & granted_to_user, const Settings & settings, const String & current_database)
{
return std::make_shared<AccessRightsContext>(client_info, granted_to_user, settings, current_database);
}
std::shared_ptr<QuotaContext> AccessControlManager::createQuotaContext(
const String & user_name, const Poco::Net::IPAddress & address, const String & custom_quota_key)
{

14
dbms/src/Access/AccessControlManager.h
View File

@ -2,6 +2,7 @@
#include <Access/MultipleAccessStorage.h>
#include <Poco/AutoPtr.h>
#include <ext/scope_guard.h>
#include <memory>
@ -19,11 +20,17 @@ namespace Poco
namespace DB
{
struct User;
using UserPtr = std::shared_ptr<const User>;
class QuotaContext;
class QuotaContextFactory;
struct QuotaUsageInfo;
class RowPolicyContext;
class RowPolicyContextFactory;
class AccessRights;
class AccessRightsContext;
class ClientInfo;
struct Settings;
/// Manages access control entities.
@ -35,6 +42,13 @@ public:
void loadFromConfig(const Poco::Util::AbstractConfiguration & users_config);
UserPtr getUser(const String & user_name) const;
UserPtr getUser(const String & user_name, const std::function<void(const UserPtr &)> & on_change, ext::scope_guard * subscription) const;
UserPtr authorizeAndGetUser(const String & user_name, const String & password, const Poco::Net::IPAddress & address) const;
UserPtr authorizeAndGetUser(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const std::function<void(const UserPtr &)> & on_change, ext::scope_guard * subscription) const;
std::shared_ptr<const AccessRightsContext> getAccessRightsContext(const ClientInfo & client_info, const AccessRights & granted_to_user, const Settings & settings, const String & current_database);
std::shared_ptr<QuotaContext>
createQuotaContext(const String & user_name, const Poco::Net::IPAddress & address, const String & custom_quota_key);

498
dbms/src/Access/AccessFlags.h Normal file
View File

@ -0,0 +1,498 @@
#pragma once
#include <Access/AccessType.h>
#include <Core/Types.h>
#include <Common/Exception.h>
#include <ext/range.h>
#include <ext/push_back.h>
#include <bitset>
#include <unordered_map>
namespace DB
{
/// Represents a combination of access types which can be granted globally, on databases, tables, columns, etc.
/// For example "SELECT, CREATE USER" is an access type.
class AccessFlags
{
public:
AccessFlags(AccessType type);
/// The same as AccessFlags(AccessType::NONE).
AccessFlags() = default;
/// Constructs from a string like "SELECT".
AccessFlags(const std::string_view & keyword);
/// Constructs from a list of strings like "SELECT, UPDATE, INSERT".
AccessFlags(const std::vector<std::string_view> & keywords);
AccessFlags(const Strings & keywords);
AccessFlags(const AccessFlags & src) = default;
AccessFlags(AccessFlags && src) = default;
AccessFlags & operator =(const AccessFlags & src) = default;
AccessFlags & operator =(AccessFlags && src) = default;
/// Returns the access type which contains two specified access types.
AccessFlags & operator |=(const AccessFlags & other) { flags |= other.flags; return *this; }
friend AccessFlags operator |(const AccessFlags & left, const AccessFlags & right) { return AccessFlags(left) |= right; }
/// Returns the access type which contains the common part of two access types.
AccessFlags & operator &=(const AccessFlags & other) { flags &= other.flags; return *this; }
friend AccessFlags operator &(const AccessFlags & left, const AccessFlags & right) { return AccessFlags(left) &= right; }
/// Returns the access type which contains only the part of the first access type which is not the part of the second access type.
/// (lhs - rhs) is the same as (lhs & ~rhs).
AccessFlags & operator -=(const AccessFlags & other) { flags &= ~other.flags; return *this; }
friend AccessFlags operator -(const AccessFlags & left, const AccessFlags & right) { return AccessFlags(left) -= right; }
AccessFlags operator ~() const { AccessFlags res; res.flags = ~flags; return res; }
bool isEmpty() const { return flags.none(); }
explicit operator bool() const { return !isEmpty(); }
bool contains(const AccessFlags & other) const { return (flags & other.flags) == other.flags; }
friend bool operator ==(const AccessFlags & left, const AccessFlags & right) { return left.flags == right.flags; }
friend bool operator !=(const AccessFlags & left, const AccessFlags & right) { return !(left == right); }
void clear() { flags.reset(); }
/// Returns a comma-separated list of keywords, like "SELECT, CREATE USER, UPDATE".
String toString() const;
/// Returns a list of keywords.
std::vector<std::string_view> toKeywords() const;
/// Returns the access types which could be granted on the database level.
/// For example, SELECT can be granted on the database level, but CREATE_USER cannot.
static AccessFlags databaseLevel();
/// Returns the access types which could be granted on the table/dictionary level.
static AccessFlags tableLevel();
/// Returns the access types which could be granted on the column/attribute level.
static AccessFlags columnLevel();
private:
static constexpr size_t NUM_FLAGS = 128;
using Flags = std::bitset<NUM_FLAGS>;
Flags flags;
AccessFlags(const Flags & flags_) : flags(flags_) {}
template <typename = void>
class Impl;
};
namespace ErrorCodes
{
extern const int UNKNOWN_ACCESS_TYPE;
}
template <typename>
class AccessFlags::Impl
{
public:
static const Impl & instance()
{
static const Impl res;
return res;
}
Flags accessTypeToFlags(AccessType type) const
{
return access_type_to_flags_mapping[static_cast<size_t>(type)];
}
Flags keywordToFlags(const std::string_view & keyword) const
{
auto it = keyword_to_flags_map.find(keyword);
if (it == keyword_to_flags_map.end())
{
String uppercased_keyword{keyword};
boost::to_upper(uppercased_keyword);
it = keyword_to_flags_map.find(uppercased_keyword);
if (it == keyword_to_flags_map.end())
throw Exception("Unknown access type: " + String(keyword), ErrorCodes::UNKNOWN_ACCESS_TYPE);
}
return it->second;
}
Flags keywordsToFlags(const std::vector<std::string_view> & keywords) const
{
Flags res;
for (const auto & keyword : keywords)
res |= keywordToFlags(keyword);
return res;
}
Flags keywordsToFlags(const Strings & keywords) const
{
Flags res;
for (const auto & keyword : keywords)
res |= keywordToFlags(keyword);
return res;
}
std::vector<std::string_view> flagsToKeywords(const Flags & flags_) const
{
std::vector<std::string_view> keywords;
flagsToKeywordsRec(flags_, keywords, *flags_to_keyword_tree);
if (keywords.empty())
keywords.push_back("USAGE");
return keywords;
}
String flagsToString(const Flags & flags_) const
{
String str;
for (const auto & keyword : flagsToKeywords(flags_))
{
if (!str.empty())
str += ", ";
str += keyword;
}
return str;
}
const Flags & getDatabaseLevelFlags() const { return all_grantable_on_level[DATABASE_LEVEL]; }
const Flags & getTableLevelFlags() const { return all_grantable_on_level[TABLE_LEVEL]; }
const Flags & getColumnLevelFlags() const { return all_grantable_on_level[COLUMN_LEVEL]; }
private:
enum Level
{
UNKNOWN_LEVEL = -1,
GLOBAL_LEVEL = 0,
DATABASE_LEVEL = 1,
TABLE_LEVEL = 2,
VIEW_LEVEL = 2,
DICTIONARY_LEVEL = 2,
COLUMN_LEVEL = 3,
};
struct Node;
using NodePtr = std::unique_ptr<Node>;
using Nodes = std::vector<NodePtr>;
template <typename... Args>
static Nodes nodes(Args&& ... args)
{
Nodes res;
ext::push_back(res, std::move(args)...);
return res;
}
struct Node
{
std::string_view keyword;
std::vector<String> aliases;
Flags flags;
Level level = UNKNOWN_LEVEL;
Nodes children;
Node(std::string_view keyword_, size_t flag_, Level level_)
: keyword(keyword_), level(level_)
{
flags.set(flag_);
}
Node(std::string_view keyword_, Nodes children_)
: keyword(keyword_), children(std::move(children_))
{
for (const auto & child : children)
flags |= child->flags;
}
template <typename... Args>
Node(std::string_view keyword_, NodePtr first_child, Args &&... other_children)
: Node(keyword_, nodes(std::move(first_child), std::move(other_children)...)) {}
};
static void flagsToKeywordsRec(const Flags & flags_, std::vector<std::string_view> & keywords, const Node & start_node)
{
Flags matching_flags = (flags_ & start_node.flags);
if (matching_flags.any())
{
if (matching_flags == start_node.flags)
{
keywords.push_back(start_node.keyword);
}
else
{
for (const auto & child : start_node.children)
flagsToKeywordsRec(flags_, keywords, *child);
}
}
}
static void makeFlagsToKeywordTree(NodePtr & flags_to_keyword_tree_)
{
size_t next_flag = 0;
Nodes all;
auto show = std::make_unique<Node>("SHOW", next_flag++, COLUMN_LEVEL);
auto exists = std::make_unique<Node>("EXISTS", next_flag++, COLUMN_LEVEL);
ext::push_back(all, std::move(show), std::move(exists));
auto select = std::make_unique<Node>("SELECT", next_flag++, COLUMN_LEVEL);
auto insert = std::make_unique<Node>("INSERT", next_flag++, COLUMN_LEVEL);
ext::push_back(all, std::move(select), std::move(insert));
auto update = std::make_unique<Node>("UPDATE", next_flag++, COLUMN_LEVEL);
ext::push_back(update->aliases, "ALTER UPDATE");
auto delet = std::make_unique<Node>("DELETE", next_flag++, TABLE_LEVEL);
ext::push_back(delet->aliases, "ALTER DELETE");
auto add_column = std::make_unique<Node>("ADD COLUMN", next_flag++, COLUMN_LEVEL);
add_column->aliases.push_back("ALTER ADD COLUMN");
auto modify_column = std::make_unique<Node>("MODIFY COLUMN", next_flag++, COLUMN_LEVEL);
modify_column->aliases.push_back("ALTER MODIFY COLUMN");
auto drop_column = std::make_unique<Node>("DROP COLUMN", next_flag++, COLUMN_LEVEL);
drop_column->aliases.push_back("ALTER DROP COLUMN");
auto comment_column = std::make_unique<Node>("COMMENT COLUMN", next_flag++, COLUMN_LEVEL);
comment_column->aliases.push_back("ALTER COMMENT COLUMN");
auto clear_column = std::make_unique<Node>("CLEAR COLUMN", next_flag++, COLUMN_LEVEL);
clear_column->aliases.push_back("ALTER CLEAR COLUMN");
auto alter_column = std::make_unique<Node>("ALTER COLUMN", std::move(add_column), std::move(modify_column), std::move(drop_column), std::move(comment_column), std::move(clear_column));
auto alter_order_by = std::make_unique<Node>("ALTER ORDER BY", next_flag++, TABLE_LEVEL);
alter_order_by->aliases.push_back("MODIFY ORDER BY");
alter_order_by->aliases.push_back("ALTER MODIFY ORDER BY");
auto add_index = std::make_unique<Node>("ADD INDEX", next_flag++, TABLE_LEVEL);
add_index->aliases.push_back("ALTER ADD INDEX");
auto drop_index = std::make_unique<Node>("DROP INDEX", next_flag++, TABLE_LEVEL);
drop_index->aliases.push_back("ALTER DROP INDEX");
auto materialize_index = std::make_unique<Node>("MATERIALIZE INDEX", next_flag++, TABLE_LEVEL);
materialize_index->aliases.push_back("ALTER MATERIALIZE INDEX");
auto clear_index = std::make_unique<Node>("CLEAR INDEX", next_flag++, TABLE_LEVEL);
clear_index->aliases.push_back("ALTER CLEAR INDEX");
auto index = std::make_unique<Node>("INDEX", std::move(alter_order_by), std::move(add_index), std::move(drop_index), std::move(materialize_index), std::move(clear_index));
index->aliases.push_back("ALTER INDEX");
auto add_constraint = std::make_unique<Node>("ADD CONSTRAINT", next_flag++, TABLE_LEVEL);
add_constraint->aliases.push_back("ALTER ADD CONSTRAINT");
auto drop_constraint = std::make_unique<Node>("DROP CONSTRAINT", next_flag++, TABLE_LEVEL);
drop_constraint->aliases.push_back("ALTER DROP CONSTRAINT");
auto alter_constraint = std::make_unique<Node>("CONSTRAINT", std::move(add_constraint), std::move(drop_constraint));
alter_constraint->aliases.push_back("ALTER CONSTRAINT");
auto modify_ttl = std::make_unique<Node>("MODIFY TTL", next_flag++, TABLE_LEVEL);
modify_ttl->aliases.push_back("ALTER MODIFY TTL");
auto modify_setting = std::make_unique<Node>("MODIFY SETTING", next_flag++, TABLE_LEVEL);
modify_setting->aliases.push_back("ALTER MODIFY SETTING");
auto move_partition = std::make_unique<Node>("MOVE PARTITION", next_flag++, TABLE_LEVEL);
ext::push_back(move_partition->aliases, "ALTER MOVE PARTITION", "MOVE PART", "ALTER MOVE PART");
auto fetch_partition = std::make_unique<Node>("FETCH PARTITION", next_flag++, TABLE_LEVEL);
ext::push_back(fetch_partition->aliases, "ALTER FETCH PARTITION");
auto freeze_partition = std::make_unique<Node>("FREEZE PARTITION", next_flag++, TABLE_LEVEL);
ext::push_back(freeze_partition->aliases, "ALTER FREEZE PARTITION");
auto alter_table = std::make_unique<Node>("ALTER TABLE", std::move(update), std::move(delet), std::move(alter_column), std::move(index), std::move(alter_constraint), std::move(modify_ttl), std::move(modify_setting), std::move(move_partition), std::move(fetch_partition), std::move(freeze_partition));
auto refresh_view = std::make_unique<Node>("REFRESH VIEW", next_flag++, VIEW_LEVEL);
ext::push_back(refresh_view->aliases, "ALTER LIVE VIEW REFRESH");
auto modify_view_query = std::make_unique<Node>("MODIFY VIEW QUERY", next_flag++, VIEW_LEVEL);
auto alter_view = std::make_unique<Node>("ALTER VIEW", std::move(refresh_view), std::move(modify_view_query));
auto alter = std::make_unique<Node>("ALTER", std::move(alter_table), std::move(alter_view));
ext::push_back(all, std::move(alter));
auto create_database = std::make_unique<Node>("CREATE DATABASE", next_flag++, DATABASE_LEVEL);
ext::push_back(create_database->aliases, "ATTACH DATABASE");
auto create_table = std::make_unique<Node>("CREATE TABLE", next_flag++, TABLE_LEVEL);
ext::push_back(create_table->aliases, "ATTACH TABLE");
auto create_view = std::make_unique<Node>("CREATE VIEW", next_flag++, VIEW_LEVEL);
ext::push_back(create_view->aliases, "ATTACH VIEW");
auto create_dictionary = std::make_unique<Node>("CREATE DICTIONARY", next_flag++, DICTIONARY_LEVEL);
ext::push_back(create_dictionary->aliases, "ATTACH DICTIONARY");
auto create = std::make_unique<Node>("CREATE", std::move(create_database), std::move(create_table), std::move(create_view), std::move(create_dictionary));
ext::push_back(create->aliases, "ATTACH");
ext::push_back(all, std::move(create));
auto create_temporary_table = std::make_unique<Node>("CREATE TEMPORARY TABLE", next_flag++, GLOBAL_LEVEL);
ext::push_back(all, std::move(create_temporary_table));
auto drop_database = std::make_unique<Node>("DROP DATABASE", next_flag++, DATABASE_LEVEL);
auto drop_table = std::make_unique<Node>("DROP TABLE", next_flag++, TABLE_LEVEL);
auto drop_view = std::make_unique<Node>("DROP VIEW", next_flag++, VIEW_LEVEL);
auto drop_dictionary = std::make_unique<Node>("DROP DICTIONARY", next_flag++, DICTIONARY_LEVEL);
auto drop = std::make_unique<Node>("DROP", std::move(drop_database), std::move(drop_table), std::move(drop_view), std::move(drop_dictionary));
ext::push_back(all, std::move(drop));
auto detach_database = std::make_unique<Node>("DETACH DATABASE", next_flag++, DATABASE_LEVEL);
auto detach_table = std::make_unique<Node>("DETACH TABLE", next_flag++, TABLE_LEVEL);
auto detach_view = std::make_unique<Node>("DETACH VIEW", next_flag++, VIEW_LEVEL);
auto detach_dictionary = std::make_unique<Node>("DETACH DICTIONARY", next_flag++, DICTIONARY_LEVEL);
auto detach = std::make_unique<Node>("DETACH", std::move(detach_database), std::move(detach_table), std::move(detach_view), std::move(detach_dictionary));
ext::push_back(all, std::move(detach));
auto truncate_table = std::make_unique<Node>("TRUNCATE TABLE", next_flag++, TABLE_LEVEL);
auto truncate_view = std::make_unique<Node>("TRUNCATE VIEW", next_flag++, VIEW_LEVEL);
auto truncate = std::make_unique<Node>("TRUNCATE", std::move(truncate_table), std::move(truncate_view));
ext::push_back(all, std::move(truncate));
auto optimize = std::make_unique<Node>("OPTIMIZE", next_flag++, TABLE_LEVEL);
optimize->aliases.push_back("OPTIMIZE TABLE");
ext::push_back(all, std::move(optimize));
auto kill_query = std::make_unique<Node>("KILL QUERY", next_flag++, GLOBAL_LEVEL);
auto kill_mutation = std::make_unique<Node>("KILL MUTATION", next_flag++, TABLE_LEVEL);
auto kill = std::make_unique<Node>("KILL", std::move(kill_query), std::move(kill_mutation));
ext::push_back(all, std::move(kill));
auto create_user = std::make_unique<Node>("CREATE USER", next_flag++, GLOBAL_LEVEL);
ext::push_back(create_user->aliases, "ALTER USER", "DROP USER", "CREATE ROLE", "DROP ROLE", "CREATE POLICY", "ALTER POLICY", "DROP POLICY", "CREATE QUOTA", "ALTER QUOTA", "DROP QUOTA");
ext::push_back(all, std::move(create_user));
auto shutdown = std::make_unique<Node>("SHUTDOWN", next_flag++, GLOBAL_LEVEL);
ext::push_back(shutdown->aliases, "SYSTEM SHUTDOWN", "SYSTEM KILL");
auto drop_cache = std::make_unique<Node>("DROP CACHE", next_flag++, GLOBAL_LEVEL);
ext::push_back(drop_cache->aliases, "SYSTEM DROP CACHE", "DROP DNS CACHE", "SYSTEM DROP DNS CACHE", "DROP MARK CACHE", "SYSTEM DROP MARK CACHE", "DROP UNCOMPRESSED CACHE", "SYSTEM DROP UNCOMPRESSED CACHE", "DROP COMPILED EXPRESSION CACHE", "SYSTEM DROP COMPILED EXPRESSION CACHE");
auto reload_config = std::make_unique<Node>("RELOAD CONFIG", next_flag++, GLOBAL_LEVEL);
ext::push_back(reload_config->aliases, "SYSTEM RELOAD CONFIG");
auto reload_dictionary = std::make_unique<Node>("RELOAD DICTIONARY", next_flag++, GLOBAL_LEVEL);
ext::push_back(reload_dictionary->aliases, "SYSTEM RELOAD DICTIONARY", "RELOAD DICTIONARIES", "SYSTEM RELOAD DICTIONARIES", "RELOAD EMBEDDED DICTIONARIES", "SYSTEM RELOAD EMBEDDED DICTIONARIES");
auto stop_merges = std::make_unique<Node>("STOP MERGES", next_flag++, TABLE_LEVEL);
ext::push_back(stop_merges->aliases, "SYSTEM STOP MERGES", "START MERGES", "SYSTEM START MERGES");
auto stop_ttl_merges = std::make_unique<Node>("STOP TTL MERGES", next_flag++, TABLE_LEVEL);
ext::push_back(stop_ttl_merges->aliases, "SYSTEM STOP TTL MERGES", "START TTL MERGES", "SYSTEM START TTL MERGES");
auto stop_fetches = std::make_unique<Node>("STOP FETCHES", next_flag++, TABLE_LEVEL);
ext::push_back(stop_fetches->aliases, "SYSTEM STOP FETCHES", "START FETCHES", "SYSTEM START FETCHES");
auto stop_moves = std::make_unique<Node>("STOP MOVES", next_flag++, TABLE_LEVEL);
ext::push_back(stop_moves->aliases, "SYSTEM STOP MOVES", "START MOVES", "SYSTEM START MOVES");
auto stop_distributed_sends = std::make_unique<Node>("STOP DISTRIBUTED SENDS", next_flag++, TABLE_LEVEL);
ext::push_back(stop_distributed_sends->aliases, "SYSTEM STOP DISTRIBUTED SENDS", "START DISTRIBUTED SENDS", "SYSTEM START DISTRIBUTED SENDS");
auto stop_replicated_sends = std::make_unique<Node>("STOP REPLICATED SENDS", next_flag++, TABLE_LEVEL);
ext::push_back(stop_replicated_sends->aliases, "SYSTEM STOP REPLICATED SENDS", "START REPLICATED SENDS", "SYSTEM START REPLICATED SENDS");
auto stop_replication_queues = std::make_unique<Node>("STOP REPLICATION QUEUES", next_flag++, TABLE_LEVEL);
ext::push_back(stop_replication_queues->aliases, "SYSTEM STOP REPLICATION QUEUES", "START REPLICATION QUEUES", "SYSTEM START REPLICATION QUEUES");
auto sync_replica = std::make_unique<Node>("SYNC REPLICA", next_flag++, TABLE_LEVEL);
ext::push_back(sync_replica->aliases, "SYSTEM SYNC REPLICA");
auto restart_replica = std::make_unique<Node>("RESTART REPLICA", next_flag++, TABLE_LEVEL);
ext::push_back(restart_replica->aliases, "SYSTEM RESTART REPLICA");
auto flush_distributed = std::make_unique<Node>("FLUSH DISTRIBUTED", next_flag++, TABLE_LEVEL);
ext::push_back(flush_distributed->aliases, "SYSTEM FLUSH DISTRIBUTED");
auto flush_logs = std::make_unique<Node>("FLUSH LOGS", next_flag++, GLOBAL_LEVEL);
ext::push_back(flush_logs->aliases, "SYSTEM FLUSH LOGS");
auto system = std::make_unique<Node>("SYSTEM", std::move(shutdown), std::move(drop_cache), std::move(reload_config), std::move(reload_dictionary), std::move(stop_merges), std::move(stop_ttl_merges), std::move(stop_fetches), std::move(stop_moves), std::move(stop_distributed_sends), std::move(stop_replicated_sends), std::move(stop_replication_queues), std::move(sync_replica), std::move(restart_replica), std::move(flush_distributed), std::move(flush_logs));
ext::push_back(all, std::move(system));
auto dict_get = std::make_unique<Node>("dictGet()", next_flag++, DICTIONARY_LEVEL);
dict_get->aliases.push_back("dictHas()");
dict_get->aliases.push_back("dictGetHierarchy()");
dict_get->aliases.push_back("dictIsIn()");
ext::push_back(all, std::move(dict_get));
auto address_to_line = std::make_unique<Node>("addressToLine()", next_flag++, GLOBAL_LEVEL);
auto address_to_symbol = std::make_unique<Node>("addressToSymbol()", next_flag++, GLOBAL_LEVEL);
auto demangle = std::make_unique<Node>("demangle()", next_flag++, GLOBAL_LEVEL);
auto introspection = std::make_unique<Node>("INTROSPECTION", std::move(address_to_line), std::move(address_to_symbol), std::move(demangle));
ext::push_back(introspection->aliases, "INTROSPECTION FUNCTIONS");
ext::push_back(all, std::move(introspection));
auto file = std::make_unique<Node>("file()", next_flag++, GLOBAL_LEVEL);
auto url = std::make_unique<Node>("url()", next_flag++, GLOBAL_LEVEL);
auto input = std::make_unique<Node>("input()", next_flag++, GLOBAL_LEVEL);
auto values = std::make_unique<Node>("values()", next_flag++, GLOBAL_LEVEL);
auto numbers = std::make_unique<Node>("numbers()", next_flag++, GLOBAL_LEVEL);
auto merge = std::make_unique<Node>("merge()", next_flag++, DATABASE_LEVEL);
auto remote = std::make_unique<Node>("remote()", next_flag++, GLOBAL_LEVEL);
ext::push_back(remote->aliases, "remoteSecure()", "cluster()");
auto mysql = std::make_unique<Node>("mysql()", next_flag++, GLOBAL_LEVEL);
auto odbc = std::make_unique<Node>("odbc()", next_flag++, GLOBAL_LEVEL);
auto jdbc = std::make_unique<Node>("jdbc()", next_flag++, GLOBAL_LEVEL);
auto hdfs = std::make_unique<Node>("hdfs()", next_flag++, GLOBAL_LEVEL);
auto s3 = std::make_unique<Node>("s3()", next_flag++, GLOBAL_LEVEL);
auto table_functions = std::make_unique<Node>("TABLE FUNCTIONS", std::move(file), std::move(url), std::move(input), std::move(values), std::move(numbers), std::move(merge), std::move(remote), std::move(mysql), std::move(odbc), std::move(jdbc), std::move(hdfs), std::move(s3));
ext::push_back(all, std::move(table_functions));
flags_to_keyword_tree_ = std::make_unique<Node>("ALL", std::move(all));
flags_to_keyword_tree_->aliases.push_back("ALL PRIVILEGES");
}
void makeKeywordToFlagsMap(std::unordered_map<std::string_view, Flags> & keyword_to_flags_map_, Node * start_node = nullptr)
{
if (!start_node)
{
start_node = flags_to_keyword_tree.get();
keyword_to_flags_map_["USAGE"] = {};
keyword_to_flags_map_["NONE"] = {};
keyword_to_flags_map_["NO PRIVILEGES"] = {};
}
start_node->aliases.emplace_back(start_node->keyword);
for (auto & alias : start_node->aliases)
{
boost::to_upper(alias);
keyword_to_flags_map_[alias] = start_node->flags;
}
for (auto & child : start_node->children)
makeKeywordToFlagsMap(keyword_to_flags_map_, child.get());
}
void makeAccessTypeToFlagsMapping(std::vector<Flags> & access_type_to_flags_mapping_)
{
access_type_to_flags_mapping_.resize(MAX_ACCESS_TYPE);
for (auto access_type : ext::range_with_static_cast<AccessType>(0, MAX_ACCESS_TYPE))
{
auto str = toKeyword(access_type);
auto it = keyword_to_flags_map.find(str);
if (it == keyword_to_flags_map.end())
{
String uppercased{str};
boost::to_upper(uppercased);
it = keyword_to_flags_map.find(uppercased);
}
access_type_to_flags_mapping_[static_cast<size_t>(access_type)] = it->second;
}
}
void collectAllGrantableOnLevel(std::vector<Flags> & all_grantable_on_level_, const Node * start_node = nullptr)
{
if (!start_node)
{
start_node = flags_to_keyword_tree.get();
all_grantable_on_level.resize(COLUMN_LEVEL + 1);
}
for (int i = 0; i <= start_node->level; ++i)
all_grantable_on_level_[i] |= start_node->flags;
for (const auto & child : start_node->children)
collectAllGrantableOnLevel(all_grantable_on_level_, child.get());
}
Impl()
{
makeFlagsToKeywordTree(flags_to_keyword_tree);
makeKeywordToFlagsMap(keyword_to_flags_map);
makeAccessTypeToFlagsMapping(access_type_to_flags_mapping);
collectAllGrantableOnLevel(all_grantable_on_level);
}
std::unique_ptr<Node> flags_to_keyword_tree;
std::unordered_map<std::string_view, Flags> keyword_to_flags_map;
std::vector<Flags> access_type_to_flags_mapping;
std::vector<Flags> all_grantable_on_level;
};
inline AccessFlags::AccessFlags(AccessType type) : flags(Impl<>::instance().accessTypeToFlags(type)) {}
inline AccessFlags::AccessFlags(const std::string_view & keyword) : flags(Impl<>::instance().keywordToFlags(keyword)) {}
inline AccessFlags::AccessFlags(const std::vector<std::string_view> & keywords) : flags(Impl<>::instance().keywordsToFlags(keywords)) {}
inline AccessFlags::AccessFlags(const Strings & keywords) : flags(Impl<>::instance().keywordsToFlags(keywords)) {}
inline String AccessFlags::toString() const { return Impl<>::instance().flagsToString(flags); }
inline std::vector<std::string_view> AccessFlags::toKeywords() const { return Impl<>::instance().flagsToKeywords(flags); }
inline AccessFlags AccessFlags::databaseLevel() { return Impl<>::instance().getDatabaseLevelFlags(); }
inline AccessFlags AccessFlags::tableLevel() { return Impl<>::instance().getTableLevelFlags(); }
inline AccessFlags AccessFlags::columnLevel() { return Impl<>::instance().getColumnLevelFlags(); }
inline AccessFlags operator |(AccessType left, AccessType right) { return AccessFlags(left) | right; }
inline AccessFlags operator &(AccessType left, AccessType right) { return AccessFlags(left) & right; }
inline AccessFlags operator -(AccessType left, AccessType right) { return AccessFlags(left) - right; }
inline AccessFlags operator ~(AccessType x) { return ~AccessFlags(x); }
}

729
dbms/src/Access/AccessRights.cpp Normal file
View File

@ -0,0 +1,729 @@
#include <Access/AccessRights.h>
#include <Common/Exception.h>
#include <boost/range/adaptor/map.hpp>
#include <unordered_map>
namespace DB
{
namespace ErrorCodes
{
extern const int INVALID_GRANT;
extern const int LOGICAL_ERROR;
}
namespace
{
enum Level
{
GLOBAL_LEVEL,
DATABASE_LEVEL,
TABLE_LEVEL,
COLUMN_LEVEL,
};
enum RevokeMode
{
NORMAL_REVOKE_MODE, /// for AccessRights::revoke()
PARTIAL_REVOKE_MODE, /// for AccessRights::partialRevoke()
FULL_REVOKE_MODE, /// for AccessRights::fullRevoke()
};
struct Helper
{
static const Helper & instance()
{
static const Helper res;
return res;
}
const AccessFlags database_level_flags = AccessFlags::databaseLevel();
const AccessFlags table_level_flags = AccessFlags::tableLevel();
const AccessFlags column_level_flags = AccessFlags::columnLevel();
const AccessFlags show_flag = AccessType::SHOW;
const AccessFlags exists_flag = AccessType::EXISTS;
const AccessFlags create_table_flag = AccessType::CREATE_TABLE;
const AccessFlags create_temporary_table_flag = AccessType::CREATE_TEMPORARY_TABLE;
};
}
struct AccessRights::Node
{
public:
std::shared_ptr<const String> node_name;
Level level = GLOBAL_LEVEL;
AccessFlags explicit_grants;
AccessFlags partial_revokes;
AccessFlags inherited_access; /// the access inherited from the parent node
AccessFlags raw_access; /// raw_access = (inherited_access - partial_revokes) | explicit_grants
AccessFlags access; /// access = raw_access | implicit_access
AccessFlags min_access; /// min_access = access & child[0].access & ... | child[N-1].access
AccessFlags max_access; /// max_access = access | child[0].access | ... | child[N-1].access
std::unique_ptr<std::unordered_map<std::string_view, Node>> children;
Node() = default;
Node(const Node & src) { *this = src; }
Node & operator =(const Node & src)
{
node_name = src.node_name;
level = src.level;
inherited_access = src.inherited_access;
explicit_grants = src.explicit_grants;
partial_revokes = src.partial_revokes;
access = src.access;
min_access = src.min_access;
max_access = src.max_access;
if (src.children)
children = std::make_unique<std::unordered_map<std::string_view, Node>>(*src.children);
else
children = nullptr;
return *this;
}
void grant(AccessFlags access_to_grant, const Helper & helper)
{
if (!access_to_grant)
return;
if (level == GLOBAL_LEVEL)
{
/// Everything can be granted on the global level.
}
else if (level == DATABASE_LEVEL)
{
AccessFlags grantable = access_to_grant & helper.database_level_flags;
if (!grantable)
throw Exception(access_to_grant.toString() + " cannot be granted on the database level", ErrorCodes::INVALID_GRANT);
access_to_grant = grantable;
}
else if (level == TABLE_LEVEL)
{
AccessFlags grantable = access_to_grant & helper.table_level_flags;
if (!grantable)
throw Exception(access_to_grant.toString() + " cannot be granted on the table level", ErrorCodes::INVALID_GRANT);
access_to_grant = grantable;
}
else if (level == COLUMN_LEVEL)
{
AccessFlags grantable = access_to_grant & helper.column_level_flags;
if (!grantable)
throw Exception(access_to_grant.toString() + " cannot be granted on the column level", ErrorCodes::INVALID_GRANT);
access_to_grant = grantable;
}
explicit_grants |= access_to_grant - partial_revokes;
partial_revokes -= access_to_grant;
calculateAllAccessRec(helper);
}
template <typename ... Args>
void grant(const AccessFlags & access_to_grant, const Helper & helper, const std::string_view & name, const Args &... subnames)
{
auto & child = getChild(name);
child.grant(access_to_grant, helper, subnames...);
eraseChildIfEmpty(child);
calculateImplicitAccess(helper);
calculateMinAndMaxAccess();
}
template <typename StringT>
void grant(const AccessFlags & access_to_grant, const Helper & helper, const std::vector<StringT> & names)
{
for (const auto & name : names)
{
auto & child = getChild(name);
child.grant(access_to_grant, helper);
eraseChildIfEmpty(child);
}
calculateImplicitAccess(helper);
calculateMinAndMaxAccess();
}
template <int mode>
void revoke(const AccessFlags & access_to_revoke, const Helper & helper)
{
if constexpr (mode == NORMAL_REVOKE_MODE)
{
explicit_grants -= access_to_revoke;
}
else if constexpr (mode == PARTIAL_REVOKE_MODE)
{
partial_revokes |= access_to_revoke - explicit_grants;
explicit_grants -= access_to_revoke;
}
else /// mode == FULL_REVOKE_MODE
{
fullRevokeRec(access_to_revoke);
}
calculateAllAccessRec(helper);
}
template <int mode, typename... Args>
void revoke(const AccessFlags & access_to_revoke, const Helper & helper, const std::string_view & name, const Args &... subnames)
{
Node * child;
if (mode == NORMAL_REVOKE_MODE)
{
if (!(child = tryGetChild(name)))
return;
}
else
child = &getChild(name);
child->revoke<mode>(access_to_revoke, helper, subnames...);
eraseChildIfEmpty(*child);
calculateImplicitAccess(helper);
calculateMinAndMaxAccess();
}
template <int mode, typename StringT>
void revoke(const AccessFlags & access_to_revoke, const Helper & helper, const std::vector<StringT> & names)
{
Node * child;
for (const auto & name : names)
{
if (mode == NORMAL_REVOKE_MODE)
{
if (!(child = tryGetChild(name)))
continue;
}
else
child = &getChild(name);
child->revoke<mode>(access_to_revoke, helper);
eraseChildIfEmpty(*child);
}
calculateImplicitAccess(helper);
calculateMinAndMaxAccess();
}
bool isGranted(const AccessFlags & flags) const
{
return min_access.contains(flags);
}
template <typename... Args>
bool isGranted(AccessFlags flags, const std::string_view & name, const Args &... subnames) const
{
if (min_access.contains(flags))
return true;
if (!max_access.contains(flags))
return false;
const Node * child = tryGetChild(name);
if (child)
return child->isGranted(flags, subnames...);
else
return access.contains(flags);
}
template <typename StringT>
bool isGranted(AccessFlags flags, const std::vector<StringT> & names) const
{
if (min_access.contains(flags))
return true;
if (!max_access.contains(flags))
return false;
for (const auto & name : names)
{
const Node * child = tryGetChild(name);
if (child)
{
if (!child->isGranted(flags, name))
return false;
}
else
{
if (!access.contains(flags))
return false;
}
}
return true;
}
friend bool operator ==(const Node & left, const Node & right)
{
if ((left.explicit_grants != right.explicit_grants) || (left.partial_revokes != right.partial_revokes))
return false;
if (!left.children)
return !right.children;
if (!right.children)
return false;
return *left.children == *right.children;
}
friend bool operator!=(const Node & left, const Node & right) { return !(left == right); }
bool isEmpty() const
{
return !explicit_grants && !partial_revokes && !children;
}
void merge(const Node & other, const Helper & helper)
{
mergeRawAccessRec(other);
calculateGrantsAndPartialRevokesRec();
calculateAllAccessRec(helper);
}
private:
Node * tryGetChild(const std::string_view & name)
{
if (!children)
return nullptr;
auto it = children->find(name);
if (it == children->end())
return nullptr;
return &it->second;
}
const Node * tryGetChild(const std::string_view & name) const
{
if (!children)
return nullptr;
auto it = children->find(name);
if (it == children->end())
return nullptr;
return &it->second;
}
Node & getChild(const std::string_view & name)
{
auto * child = tryGetChild(name);
if (child)
return *child;
if (!children)
children = std::make_unique<std::unordered_map<std::string_view, Node>>();
auto new_child_name = std::make_shared<const String>(name);
Node & new_child = (*children)[*new_child_name];
new_child.node_name = std::move(new_child_name);
new_child.level = static_cast<Level>(level + 1);
new_child.inherited_access = raw_access;
new_child.raw_access = raw_access;
return new_child;
}
void eraseChildIfEmpty(Node & child)
{
if (!child.isEmpty())
return;
auto it = children->find(*child.node_name);
children->erase(it);
if (children->empty())
children = nullptr;
}
void calculateImplicitAccess(const Helper & helper)
{
access = raw_access;
if (access & helper.database_level_flags)
access |= helper.show_flag | helper.exists_flag;
else if ((level >= DATABASE_LEVEL) && children)
access |= helper.exists_flag;
if ((level == GLOBAL_LEVEL) && (access & helper.create_table_flag))
access |= helper.create_temporary_table_flag;
}
void calculateMinAndMaxAccess()
{
min_access = access;
max_access = access;
if (children)
{
for (const auto & child : *children | boost::adaptors::map_values)
{
min_access &= child.min_access;
max_access |= child.max_access;
}
}
}
void calculateAllAccessRec(const Helper & helper)
{
partial_revokes &= inherited_access;
raw_access = (inherited_access - partial_revokes) | explicit_grants;
/// Traverse tree.
if (children)
{
for (auto it = children->begin(); it != children->end();)
{
auto & child = it->second;
child.inherited_access = raw_access;
child.calculateAllAccessRec(helper);
if (child.isEmpty())
it = children->erase(it);
else
++it;
}
if (children->empty())
children = nullptr;
}
calculateImplicitAccess(helper);
calculateMinAndMaxAccess();
}
void fullRevokeRec(const AccessFlags & access_to_revoke)
{
explicit_grants -= access_to_revoke;
partial_revokes |= access_to_revoke;
if (children)
{
for (auto & child : *children | boost::adaptors::map_values)
child.fullRevokeRec(access_to_revoke);
}
}
void mergeRawAccessRec(const Node & rhs)
{
if (rhs.children)
{
for (const auto & [rhs_childname, rhs_child] : *rhs.children)
getChild(rhs_childname).mergeRawAccessRec(rhs_child);
}
raw_access |= rhs.raw_access;
if (children)
{
for (auto & [lhs_childname, lhs_child] : *children)
{
lhs_child.inherited_access = raw_access;
if (!rhs.tryGetChild(lhs_childname))
lhs_child.raw_access |= rhs.raw_access;
}
}
}
void calculateGrantsAndPartialRevokesRec()
{
explicit_grants = raw_access - inherited_access;
partial_revokes = inherited_access - raw_access;
if (children)
{
for (auto & child : *children | boost::adaptors::map_values)
child.calculateGrantsAndPartialRevokesRec();
}
}
};
AccessRights::AccessRights() = default;
AccessRights::~AccessRights() = default;
AccessRights::AccessRights(AccessRights && src) = default;
AccessRights & AccessRights::operator =(AccessRights && src) = default;
AccessRights::AccessRights(const AccessRights & src)
{
*this = src;
}
AccessRights & AccessRights::operator =(const AccessRights & src)
{
if (src.root)
root = std::make_unique<Node>(*src.root);
else
root = nullptr;
return *this;
}
AccessRights::AccessRights(const AccessFlags & access)
{
grant(access);
}
bool AccessRights::isEmpty() const
{
return !root;
}
void AccessRights::clear()
{
root = nullptr;
}
template <typename... Args>
void AccessRights::grantImpl(const AccessFlags & access, const Args &... args)
{
if (!root)
root = std::make_unique<Node>();
root->grant(access, Helper::instance(), args...);
if (root->isEmpty())
root = nullptr;
}
void AccessRights::grantImpl(const AccessRightsElement & element, std::string_view current_database)
{
if (element.any_database)
{
grantImpl(element.access_flags);
}
else if (element.any_table)
{
if (element.database.empty())
grantImpl(element.access_flags, current_database);
else
grantImpl(element.access_flags, element.database);
}
else if (element.any_column)
{
if (element.database.empty())
grantImpl(element.access_flags, current_database, element.table);
else
grantImpl(element.access_flags, element.database, element.table);
}
else
{
if (element.database.empty())
grantImpl(element.access_flags, current_database, element.table, element.columns);
else
grantImpl(element.access_flags, element.database, element.table, element.columns);
}
}
void AccessRights::grantImpl(const AccessRightsElements & elements, std::string_view current_database)
{
for (const auto & element : elements)
grantImpl(element, current_database);
}
void AccessRights::grant(const AccessFlags & access) { grantImpl(access); }
void AccessRights::grant(const AccessFlags & access, const std::string_view & database) { grantImpl(access, database); }
void AccessRights::grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table) { grantImpl(access, database, table); }
void AccessRights::grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) { grantImpl(access, database, table, column); }
void AccessRights::grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { grantImpl(access, database, table, columns); }
void AccessRights::grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) { grantImpl(access, database, table, columns); }
void AccessRights::grant(const AccessRightsElement & element, std::string_view current_database) { grantImpl(element, current_database); }
void AccessRights::grant(const AccessRightsElements & elements, std::string_view current_database) { grantImpl(elements, current_database); }
template <int mode, typename... Args>
void AccessRights::revokeImpl(const AccessFlags & access, const Args &... args)
{
if (!root)
return;
root->revoke<mode>(access, Helper::instance(), args...);
if (root->isEmpty())
root = nullptr;
}
template <int mode>
void AccessRights::revokeImpl(const AccessRightsElement & element, std::string_view current_database)
{
if (element.any_database)
{
revokeImpl<mode>(element.access_flags);
}
else if (element.any_table)
{
if (element.database.empty())
revokeImpl<mode>(element.access_flags, current_database);
else
revokeImpl<mode>(element.access_flags, element.database);
}
else if (element.any_column)
{
if (element.database.empty())
revokeImpl<mode>(element.access_flags, current_database, element.table);
else
revokeImpl<mode>(element.access_flags, element.database, element.table);
}
else
{
if (element.database.empty())
revokeImpl<mode>(element.access_flags, current_database, element.table, element.columns);
else
revokeImpl<mode>(element.access_flags, element.database, element.table, element.columns);
}
}
template <int mode>
void AccessRights::revokeImpl(const AccessRightsElements & elements, std::string_view current_database)
{
for (const auto & element : elements)
revokeImpl<mode>(element, current_database);
}
void AccessRights::revoke(const AccessFlags & access) { revokeImpl<NORMAL_REVOKE_MODE>(access); }
void AccessRights::revoke(const AccessFlags & access, const std::string_view & database) { revokeImpl<NORMAL_REVOKE_MODE>(access, database); }
void AccessRights::revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table) { revokeImpl<NORMAL_REVOKE_MODE>(access, database, table); }
void AccessRights::revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<NORMAL_REVOKE_MODE>(access, database, table, column); }
void AccessRights::revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<NORMAL_REVOKE_MODE>(access, database, table, columns); }
void AccessRights::revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<NORMAL_REVOKE_MODE>(access, database, table, columns); }
void AccessRights::revoke(const AccessRightsElement & element, std::string_view current_database) { revokeImpl<NORMAL_REVOKE_MODE>(element, current_database); }
void AccessRights::revoke(const AccessRightsElements & elements, std::string_view current_database) { revokeImpl<NORMAL_REVOKE_MODE>(elements, current_database); }
void AccessRights::partialRevoke(const AccessFlags & access) { revokeImpl<PARTIAL_REVOKE_MODE>(access); }
void AccessRights::partialRevoke(const AccessFlags & access, const std::string_view & database) { revokeImpl<PARTIAL_REVOKE_MODE>(access, database); }
void AccessRights::partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table) { revokeImpl<PARTIAL_REVOKE_MODE>(access, database, table); }
void AccessRights::partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<PARTIAL_REVOKE_MODE>(access, database, table, column); }
void AccessRights::partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<PARTIAL_REVOKE_MODE>(access, database, table, columns); }
void AccessRights::partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<PARTIAL_REVOKE_MODE>(access, database, table, columns); }
void AccessRights::partialRevoke(const AccessRightsElement & element, std::string_view current_database) { revokeImpl<PARTIAL_REVOKE_MODE>(element, current_database); }
void AccessRights::partialRevoke(const AccessRightsElements & elements, std::string_view current_database) { revokeImpl<PARTIAL_REVOKE_MODE>(elements, current_database); }
void AccessRights::fullRevoke(const AccessFlags & access) { revokeImpl<FULL_REVOKE_MODE>(access); }
void AccessRights::fullRevoke(const AccessFlags & access, const std::string_view & database) { revokeImpl<FULL_REVOKE_MODE>(access, database); }
void AccessRights::fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table) { revokeImpl<FULL_REVOKE_MODE>(access, database, table); }
void AccessRights::fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) { revokeImpl<FULL_REVOKE_MODE>(access, database, table, column); }
void AccessRights::fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) { revokeImpl<FULL_REVOKE_MODE>(access, database, table, columns); }
void AccessRights::fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) { revokeImpl<FULL_REVOKE_MODE>(access, database, table, columns); }
void AccessRights::fullRevoke(const AccessRightsElement & element, std::string_view current_database) { revokeImpl<FULL_REVOKE_MODE>(element, current_database); }
void AccessRights::fullRevoke(const AccessRightsElements & elements, std::string_view current_database) { revokeImpl<FULL_REVOKE_MODE>(elements, current_database); }
AccessRights::Elements AccessRights::getElements() const
{
if (!root)
return {};
Elements res;
if (root->explicit_grants)
res.grants.push_back({root->explicit_grants});
if (root->children)
{
for (const auto & [db_name, db_node] : *root->children)
{
if (db_node.partial_revokes)
res.partial_revokes.push_back({db_node.partial_revokes, db_name});
if (db_node.explicit_grants)
res.grants.push_back({db_node.explicit_grants, db_name});
if (db_node.children)
{
for (const auto & [table_name, table_node] : *db_node.children)
{
if (table_node.partial_revokes)
res.partial_revokes.push_back({table_node.partial_revokes, db_name, table_name});
if (table_node.explicit_grants)
res.grants.push_back({table_node.explicit_grants, db_name, table_name});
if (table_node.children)
{
for (const auto & [column_name, column_node] : *table_node.children)
{
if (column_node.partial_revokes)
res.partial_revokes.push_back({column_node.partial_revokes, db_name, table_name, column_name});
if (column_node.explicit_grants)
res.grants.push_back({column_node.explicit_grants, db_name, table_name, column_name});
}
}
}
}
}
}
return res;
}
String AccessRights::toString() const
{
auto elements = getElements();
String res;
if (!elements.grants.empty())
{
res += "GRANT ";
res += elements.grants.toString();
}
if (!elements.partial_revokes.empty())
{
if (!res.empty())
res += ", ";
res += "REVOKE ";
res += elements.partial_revokes.toString();
}
if (res.empty())
res = "GRANT USAGE ON *.*";
return res;
}
template <typename... Args>
bool AccessRights::isGrantedImpl(const AccessFlags & access, const Args &... args) const
{
if (!root)
return access.isEmpty();
return root->isGranted(access, args...);
}
bool AccessRights::isGrantedImpl(const AccessRightsElement & element, std::string_view current_database) const
{
if (element.any_database)
{
return isGrantedImpl(element.access_flags);
}
else if (element.any_table)
{
if (element.database.empty())
return isGrantedImpl(element.access_flags, current_database);
else
return isGrantedImpl(element.access_flags, element.database);
}
else if (element.any_column)
{
if (element.database.empty())
return isGrantedImpl(element.access_flags, current_database, element.table);
else
return isGrantedImpl(element.access_flags, element.database, element.table);
}
else
{
if (element.database.empty())
return isGrantedImpl(element.access_flags, current_database, element.table, element.columns);
else
return isGrantedImpl(element.access_flags, element.database, element.table, element.columns);
}
}
bool AccessRights::isGrantedImpl(const AccessRightsElements & elements, std::string_view current_database) const
{
for (const auto & element : elements)
if (!isGrantedImpl(element, current_database))
return false;
return true;
}
bool AccessRights::isGranted(const AccessFlags & access) const { return isGrantedImpl(access); }
bool AccessRights::isGranted(const AccessFlags & access, const std::string_view & database) const { return isGrantedImpl(access, database); }
bool AccessRights::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table) const { return isGrantedImpl(access, database, table); }
bool AccessRights::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return isGrantedImpl(access, database, table, column); }
bool AccessRights::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return isGrantedImpl(access, database, table, columns); }
bool AccessRights::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return isGrantedImpl(access, database, table, columns); }
bool AccessRights::isGranted(const AccessRightsElement & element, std::string_view current_database) const { return isGrantedImpl(element, current_database); }
bool AccessRights::isGranted(const AccessRightsElements & elements, std::string_view current_database) const { return isGrantedImpl(elements, current_database); }
bool operator ==(const AccessRights & left, const AccessRights & right)
{
if (!left.root)
return !right.root;
if (!right.root)
return false;
return *left.root == *right.root;
}
void AccessRights::merge(const AccessRights & other)
{
if (!root)
{
*this = other;
return;
}
if (other.root)
{
root->merge(*other.root, Helper::instance());
if (root->isEmpty())
root = nullptr;
}
}
}

137
dbms/src/Access/AccessRights.h Normal file
View File

@ -0,0 +1,137 @@
#pragma once
#include <Core/Types.h>
#include <Access/AccessRightsElement.h>
#include <memory>
#include <vector>
namespace DB
{
/// Represents a set of access types granted on databases, tables, columns, etc.
/// For example, "GRANT SELECT, UPDATE ON db.*, GRANT INSERT ON db2.mytbl2" are access rights.
class AccessRights
{
public:
AccessRights();
AccessRights(const AccessFlags & access);
~AccessRights();
AccessRights(const AccessRights & src);
AccessRights & operator =(const AccessRights & src);
AccessRights(AccessRights && src);
AccessRights & operator =(AccessRights && src);
bool isEmpty() const;
/// Revokes everything. It's the same as fullRevoke(AccessType::ALL).
void clear();
/// Grants access on a specified database/table/column.
/// Does nothing if the specified access has been already granted.
void grant(const AccessFlags & access);
void grant(const AccessFlags & access, const std::string_view & database);
void grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table);
void grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column);
void grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
void grant(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns);
void grant(const AccessRightsElement & access, std::string_view current_database = {});
void grant(const AccessRightsElements & access, std::string_view current_database = {});
/// Revokes a specified access granted earlier on a specified database/table/column.
/// Does nothing if the specified access is not granted.
/// If the specified access is granted but on upper level (e.g. database for table, table for columns)
/// or lower level, the function also does nothing.
/// This function implements the standard SQL REVOKE behaviour.
void revoke(const AccessFlags & access);
void revoke(const AccessFlags & access, const std::string_view & database);
void revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table);
void revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column);
void revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
void revoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns);
void revoke(const AccessRightsElement & access, std::string_view current_database = {});
void revoke(const AccessRightsElements & access, std::string_view current_database = {});
/// Revokes a specified access granted earlier on a specified database/table/column or on lower levels.
/// The function also restricts access if it's granted on upper level.
/// For example, an access could be granted on a database and then revoked on a table in this database.
/// This function implements the MySQL REVOKE behaviour with partial_revokes is ON.
void partialRevoke(const AccessFlags & access);
void partialRevoke(const AccessFlags & access, const std::string_view & database);
void partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table);
void partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column);
void partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
void partialRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns);
void partialRevoke(const AccessRightsElement & access, std::string_view current_database = {});
void partialRevoke(const AccessRightsElements & access, std::string_view current_database = {});
/// Revokes a specified access granted earlier on a specified database/table/column or on lower levels.
/// The function also restricts access if it's granted on upper level.
/// For example, fullRevoke(AccessType::ALL) revokes all grants at all, just like clear();
/// fullRevoke(AccessType::SELECT, db) means it's not allowed to execute SELECT in that database anymore (from any table).
void fullRevoke(const AccessFlags & access);
void fullRevoke(const AccessFlags & access, const std::string_view & database);
void fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table);
void fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column);
void fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns);
void fullRevoke(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns);
void fullRevoke(const AccessRightsElement & access, std::string_view current_database = {});
void fullRevoke(const AccessRightsElements & access, std::string_view current_database = {});
/// Returns the information about all the access granted.
struct Elements
{
AccessRightsElements grants;
AccessRightsElements partial_revokes;
};
Elements getElements() const;
/// Returns the information about all the access granted as a string.
String toString() const;
/// Whether a specified access granted.
bool isGranted(const AccessFlags & access) const;
bool isGranted(const AccessFlags & access, const std::string_view & database) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
bool isGranted(const AccessRightsElement & access, std::string_view current_database = {}) const;
bool isGranted(const AccessRightsElements & access, std::string_view current_database = {}) const;
friend bool operator ==(const AccessRights & left, const AccessRights & right);
friend bool operator !=(const AccessRights & left, const AccessRights & right) { return !(left == right); }
/// Merges two sets of access rights together.
/// It's used to combine access rights from multiple roles.
void merge(const AccessRights & other);
private:
template <typename... Args>
void grantImpl(const AccessFlags & access, const Args &... args);
void grantImpl(const AccessRightsElement & access, std::string_view current_database);
void grantImpl(const AccessRightsElements & access, std::string_view current_database);
template <int mode, typename... Args>
void revokeImpl(const AccessFlags & access, const Args &... args);
template <int mode>
void revokeImpl(const AccessRightsElement & access, std::string_view current_database);
template <int mode>
void revokeImpl(const AccessRightsElements & access, std::string_view current_database);
template <typename... Args>
bool isGrantedImpl(const AccessFlags & access, const Args &... args) const;
bool isGrantedImpl(const AccessRightsElement & access, std::string_view current_database) const;
bool isGrantedImpl(const AccessRightsElements & access, std::string_view current_database) const;
template <typename... Args>
AccessFlags getAccessImpl(const Args &... args) const;
struct Node;
std::unique_ptr<Node> root;
};
}

288
dbms/src/Access/AccessRightsContext.cpp Normal file
View File

@ -0,0 +1,288 @@
#include <Access/AccessRightsContext.h>
#include <Common/Exception.h>
#include <Common/quoteString.h>
#include <Core/Settings.h>
#include <Poco/Logger.h>
#include <common/logger_useful.h>
#include <boost/smart_ptr/make_shared_object.hpp>
#include <assert.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ACCESS_DENIED;
extern const int READONLY;
extern const int QUERY_IS_PROHIBITED;
extern const int FUNCTION_NOT_ALLOWED;
}
namespace
{
enum CheckAccessRightsMode
{
RETURN_FALSE_IF_ACCESS_DENIED,
LOG_WARNING_IF_ACCESS_DENIED,
THROW_IF_ACCESS_DENIED,
};
String formatSkippedMessage()
{
return "";
}
String formatSkippedMessage(const std::string_view & database)
{
return ". Skipped database " + backQuoteIfNeed(database);
}
String formatSkippedMessage(const std::string_view & database, const std::string_view & table)
{
String str = ". Skipped table ";
if (!database.empty())
str += backQuoteIfNeed(database) + ".";
str += backQuoteIfNeed(table);
return str;
}
String formatSkippedMessage(const std::string_view & database, const std::string_view & table, const std::string_view & column)
{
String str = ". Skipped column " + backQuoteIfNeed(column) + " ON ";
if (!database.empty())
str += backQuoteIfNeed(database) + ".";
str += backQuoteIfNeed(table);
return str;
}
template <typename StringT>
String formatSkippedMessage(const std::string_view & database, const std::string_view & table, const std::vector<StringT> & columns)
{
if (columns.size() == 1)
return formatSkippedMessage(database, table, columns[0]);
String str = ". Skipped columns ";
bool need_comma = false;
for (const auto & column : columns)
{
if (std::exchange(need_comma, true))
str += ", ";
str += backQuoteIfNeed(column);
}
str += " ON ";
if (!database.empty())
str += backQuoteIfNeed(database) + ".";
str += backQuoteIfNeed(table);
return str;
}
}
AccessRightsContext::AccessRightsContext()
{
auto everything_granted = boost::make_shared<AccessRights>();
everything_granted->grant(AccessType::ALL);
result_access_cache[0] = std::move(everything_granted);
}
AccessRightsContext::AccessRightsContext(const ClientInfo & client_info_, const AccessRights & granted_to_user_, const Settings & settings, const String & current_database_)
: user_name(client_info_.current_user)
, granted_to_user(granted_to_user_)
, readonly(settings.readonly)
, allow_ddl(settings.allow_ddl)
, allow_introspection(settings.allow_introspection_functions)
, current_database(current_database_)
, interface(client_info_.interface)
, http_method(client_info_.http_method)
, trace_log(&Poco::Logger::get("AccessRightsContext (" + user_name + ")"))
{
}
template <int mode, typename... Args>
bool AccessRightsContext::checkImpl(Poco::Logger * log_, const AccessFlags & access, const Args &... args) const
{
auto result_access = calculateResultAccess();
bool is_granted = result_access->isGranted(access, args...);
if (trace_log)
LOG_TRACE(trace_log, "Access " << (is_granted ? "granted" : "denied") << ": " << (AccessRightsElement{access, args...}.toString()));
if (is_granted)
return true;
if constexpr (mode == RETURN_FALSE_IF_ACCESS_DENIED)
return false;
if constexpr (mode == LOG_WARNING_IF_ACCESS_DENIED)
{
if (!log_)
return false;
}
auto show_error = [&](const String & msg, [[maybe_unused]] int error_code)
{
if constexpr (mode == THROW_IF_ACCESS_DENIED)
throw Exception(msg, error_code);
else if constexpr (mode == LOG_WARNING_IF_ACCESS_DENIED)
LOG_WARNING(log_, msg + formatSkippedMessage(args...));
};
if (readonly && calculateResultAccess(false, allow_ddl, allow_introspection)->isGranted(access, args...))
{
if (interface == ClientInfo::Interface::HTTP && http_method == ClientInfo::HTTPMethod::GET)
show_error(
"Cannot execute query in readonly mode. "
"For queries over HTTP, method GET implies readonly. You should use method POST for modifying queries",
ErrorCodes::READONLY);
else
show_error("Cannot execute query in readonly mode", ErrorCodes::READONLY);
}
else if (!allow_ddl && calculateResultAccess(readonly, true, allow_introspection)->isGranted(access, args...))
{
show_error("Cannot execute query. DDL queries are prohibited for the user", ErrorCodes::QUERY_IS_PROHIBITED);
}
else if (!allow_introspection && calculateResultAccess(readonly, allow_ddl, true)->isGranted(access, args...))
{
show_error("Introspection functions are disabled, because setting 'allow_introspection_functions' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED);
}
else
{
show_error(
user_name + ": Not enough privileges. To perform this operation you should have grant "
+ AccessRightsElement{access, args...}.toString(),
ErrorCodes::ACCESS_DENIED);
}
return false;
}
template <int mode>
bool AccessRightsContext::checkImpl(Poco::Logger * log_, const AccessRightsElement & element) const
{
if (element.any_database)
{
return checkImpl<mode>(log_, element.access_flags);
}
else if (element.any_table)
{
if (element.database.empty())
return checkImpl<mode>(log_, element.access_flags, current_database);
else
return checkImpl<mode>(log_, element.access_flags, element.database);
}
else if (element.any_column)
{
if (element.database.empty())
return checkImpl<mode>(log_, element.access_flags, current_database, element.table);
else
return checkImpl<mode>(log_, element.access_flags, element.database, element.table);
}
else
{
if (element.database.empty())
return checkImpl<mode>(log_, element.access_flags, current_database, element.table, element.columns);
else
return checkImpl<mode>(log_, element.access_flags, element.database, element.table, element.columns);
}
}
template <int mode>
bool AccessRightsContext::checkImpl(Poco::Logger * log_, const AccessRightsElements & elements) const
{
for (const auto & element : elements)
if (!checkImpl<mode>(log_, element))
return false;
return true;
}
void AccessRightsContext::check(const AccessFlags & access) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access); }
void AccessRightsContext::check(const AccessFlags & access, const std::string_view & database) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access, database); }
void AccessRightsContext::check(const AccessFlags & access, const std::string_view & database, const std::string_view & table) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access, database, table); }
void AccessRightsContext::check(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access, database, table, column); }
void AccessRightsContext::check(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access, database, table, columns); }
void AccessRightsContext::check(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access, database, table, columns); }
void AccessRightsContext::check(const AccessRightsElement & access) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access); }
void AccessRightsContext::check(const AccessRightsElements & access) const { checkImpl<THROW_IF_ACCESS_DENIED>(nullptr, access); }
bool AccessRightsContext::isGranted(const AccessFlags & access) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access); }
bool AccessRightsContext::isGranted(const AccessFlags & access, const std::string_view & database) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access, database); }
bool AccessRightsContext::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access, database, table); }
bool AccessRightsContext::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access, database, table, column); }
bool AccessRightsContext::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access, database, table, columns); }
bool AccessRightsContext::isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access, database, table, columns); }
bool AccessRightsContext::isGranted(const AccessRightsElement & access) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access); }
bool AccessRightsContext::isGranted(const AccessRightsElements & access) const { return checkImpl<RETURN_FALSE_IF_ACCESS_DENIED>(nullptr, access); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessFlags & access) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access, database); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access, database, table); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access, database, table, column); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access, database, table, columns); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access, database, table, columns); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessRightsElement & access) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access); }
bool AccessRightsContext::isGranted(Poco::Logger * log_, const AccessRightsElements & access) const { return checkImpl<LOG_WARNING_IF_ACCESS_DENIED>(log_, access); }
boost::shared_ptr<const AccessRights> AccessRightsContext::calculateResultAccess() const
{
auto res = result_access_cache[0].load();
if (res)
return res;
return calculateResultAccess(readonly, allow_ddl, allow_introspection);
}
boost::shared_ptr<const AccessRights> AccessRightsContext::calculateResultAccess(UInt64 readonly_, bool allow_ddl_, bool allow_introspection_) const
{
size_t cache_index = static_cast<size_t>(readonly_ != readonly)
+ static_cast<size_t>(allow_ddl_ != allow_ddl) * 2 +
+ static_cast<size_t>(allow_introspection_ != allow_introspection) * 3;
assert(cache_index < std::size(result_access_cache));
auto cached = result_access_cache[cache_index].load();
if (cached)
return cached;
std::lock_guard lock{mutex};
cached = result_access_cache[cache_index].load();
if (cached)
return cached;
auto result_ptr = boost::make_shared<AccessRights>();
auto & result = *result_ptr;
result = granted_to_user;
static const AccessFlags table_ddl = AccessType::CREATE_DATABASE | AccessType::CREATE_TABLE | AccessType::CREATE_VIEW
| AccessType::ALTER_TABLE | AccessType::ALTER_VIEW | AccessType::DROP_DATABASE | AccessType::DROP_TABLE | AccessType::DROP_VIEW
| AccessType::DETACH_DATABASE | AccessType::DETACH_TABLE | AccessType::DETACH_VIEW | AccessType::TRUNCATE;
static const AccessFlags dictionary_ddl = AccessType::CREATE_DICTIONARY | AccessType::DROP_DICTIONARY | AccessType::DETACH_DICTIONARY;
static const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl;
static const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE;
if (readonly_)
result.fullRevoke(write_table_access | AccessType::SYSTEM);
if (readonly_ || !allow_ddl_)
result.fullRevoke(table_and_dictionary_ddl);
if (readonly_ == 1)
{
/// Table functions are forbidden in readonly mode.
/// For example, for readonly = 2 - allowed.
result.fullRevoke(AccessType::CREATE_TEMPORARY_TABLE | AccessType::TABLE_FUNCTIONS);
}
if (!allow_introspection_)
result.fullRevoke(AccessType::INTROSPECTION);
result_access_cache[cache_index].store(result_ptr);
return std::move(result_ptr);
}
}

81
dbms/src/Access/AccessRightsContext.h Normal file
View File

@ -0,0 +1,81 @@
#pragma once
#include <Access/AccessRights.h>
#include <Interpreters/ClientInfo.h>
#include <boost/smart_ptr/atomic_shared_ptr.hpp>
#include <mutex>
namespace Poco { class Logger; }
namespace DB
{
struct Settings;
class AccessRightsContext
{
public:
/// Default constructor creates access rights' context which allows everything.
AccessRightsContext();
AccessRightsContext(const ClientInfo & client_info_, const AccessRights & granted_to_user, const Settings & settings, const String & current_database_);
/// Checks if a specified access granted, and throws an exception if not.
/// Empty database means the current database.
void check(const AccessFlags & access) const;
void check(const AccessFlags & access, const std::string_view & database) const;
void check(const AccessFlags & access, const std::string_view & database, const std::string_view & table) const;
void check(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
void check(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
void check(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
void check(const AccessRightsElement & access) const;
void check(const AccessRightsElements & access) const;
/// Checks if a specified access granted.
bool isGranted(const AccessFlags & access) const;
bool isGranted(const AccessFlags & access, const std::string_view & database) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
bool isGranted(const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
bool isGranted(const AccessRightsElement & access) const;
bool isGranted(const AccessRightsElements & access) const;
/// Checks if a specified access granted, and logs a warning if not.
bool isGranted(Poco::Logger * log_, const AccessFlags & access) const;
bool isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database) const;
bool isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table) const;
bool isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::string_view & column) const;
bool isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table, const std::vector<std::string_view> & columns) const;
bool isGranted(Poco::Logger * log_, const AccessFlags & access, const std::string_view & database, const std::string_view & table, const Strings & columns) const;
bool isGranted(Poco::Logger * log_, const AccessRightsElement & access) const;
bool isGranted(Poco::Logger * log_, const AccessRightsElements & access) const;
private:
template <int mode, typename... Args>
bool checkImpl(Poco::Logger * log_, const AccessFlags & access, const Args &... args) const;
template <int mode>
bool checkImpl(Poco::Logger * log_, const AccessRightsElement & access) const;
template <int mode>
bool checkImpl(Poco::Logger * log_, const AccessRightsElements & access) const;
boost::shared_ptr<const AccessRights> calculateResultAccess() const;
boost::shared_ptr<const AccessRights> calculateResultAccess(UInt64 readonly_, bool allow_ddl_, bool allow_introspection_) const;
const String user_name;
const AccessRights granted_to_user;
const UInt64 readonly = 0;
const bool allow_ddl = true;
const bool allow_introspection = true;
const String current_database;
const ClientInfo::Interface interface = ClientInfo::Interface::TCP;
const ClientInfo::HTTPMethod http_method = ClientInfo::HTTPMethod::UNKNOWN;
Poco::Logger * const trace_log = nullptr;
mutable boost::atomic_shared_ptr<const AccessRights> result_access_cache[4];
mutable std::mutex mutex;
};
}

86
dbms/src/Access/AccessRightsElement.cpp Normal file
View File

@ -0,0 +1,86 @@
#include <Access/AccessRightsElement.h>
#include <Dictionaries/IDictionary.h>
#include <Common/quoteString.h>
namespace DB
{
void AccessRightsElement::setDatabase(const String & new_database)
{
database = new_database;
any_database = false;
}
void AccessRightsElement::replaceEmptyDatabase(const String & new_database)
{
if (isEmptyDatabase())
setDatabase(new_database);
}
bool AccessRightsElement::isEmptyDatabase() const
{
return !any_database && database.empty();
}
String AccessRightsElement::toString() const
{
String columns_in_parentheses;
if (!any_column)
{
for (const auto & column : columns)
{
columns_in_parentheses += columns_in_parentheses.empty() ? "(" : ", ";
columns_in_parentheses += backQuoteIfNeed(column);
}
columns_in_parentheses += ")";
}
String msg;
for (const std::string_view & keyword : access_flags.toKeywords())
{
if (!msg.empty())
msg += ", ";
msg += String{keyword} + columns_in_parentheses;
}
msg += " ON ";
if (any_database)
msg += "*.";
else if (!database.empty() && (database != IDictionary::NO_DATABASE_TAG))
msg += backQuoteIfNeed(database) + ".";
if (any_table)
msg += "*";
else
msg += backQuoteIfNeed(table);
return msg;
}
void AccessRightsElements::replaceEmptyDatabase(const String & new_database)
{
for (auto & element : *this)
element.replaceEmptyDatabase(new_database);
}
String AccessRightsElements::toString() const
{
String res;
bool need_comma = false;
for (auto & element : *this)
{
if (std::exchange(need_comma, true))
res += ", ";
res += element.toString();
}
if (res.empty())
res = "USAGE ON *.*";
return res;
}
}

100
dbms/src/Access/AccessRightsElement.h Normal file
View File

@ -0,0 +1,100 @@
#pragma once
#include <Access/AccessFlags.h>
namespace DB
{
/// An element of access rights which can be represented by single line
/// GRANT ... ON ...
struct AccessRightsElement
{
AccessFlags access_flags;
String database;
String table;
Strings columns;
bool any_database = true;
bool any_table = true;
bool any_column = true;
AccessRightsElement() = default;
AccessRightsElement(const AccessRightsElement &) = default;
AccessRightsElement & operator=(const AccessRightsElement &) = default;
AccessRightsElement(AccessRightsElement &&) = default;
AccessRightsElement & operator=(AccessRightsElement &&) = default;
AccessRightsElement(AccessFlags access_flags_) : access_flags(access_flags_) {}
AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_)
: access_flags(access_flags_), database(database_), any_database(false)
{
}
AccessRightsElement(AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_)
: access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false)
{
}
AccessRightsElement(
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const std::string_view & column_)
: access_flags(access_flags_)
, database(database_)
, table(table_)
, columns({String{column_}})
, any_database(false)
, any_table(false)
, any_column(false)
{
}
AccessRightsElement(
AccessFlags access_flags_,
const std::string_view & database_,
const std::string_view & table_,
const std::vector<std::string_view> & columns_)
: access_flags(access_flags_), database(database_), table(table_), any_database(false), any_table(false), any_column(false)
{
columns.resize(columns_.size());
for (size_t i = 0; i != columns_.size(); ++i)
columns[i] = String{columns_[i]};
}
AccessRightsElement(
AccessFlags access_flags_, const std::string_view & database_, const std::string_view & table_, const Strings & columns_)
: access_flags(access_flags_)
, database(database_)
, table(table_)
, columns(columns_)
, any_database(false)
, any_table(false)
, any_column(false)
{
}
/// Sets the database.
void setDatabase(const String & new_database);
/// If the database is empty, replaces it with `new_database`. Otherwise does nothing.
void replaceEmptyDatabase(const String & new_database);
bool isEmptyDatabase() const;
/// Returns a human-readable representation like "SELECT, UPDATE(x, y) ON db.table".
/// The returned string isn't prefixed with the "GRANT" keyword.
String toString() const;
};
/// Multiple elements of access rights.
class AccessRightsElements : public std::vector<AccessRightsElement>
{
public:
/// Replaces the empty database with `new_database`.
void replaceEmptyDatabase(const String & new_database);
/// Returns a human-readable representation like "SELECT, UPDATE(x, y) ON db.table".
/// The returned string isn't prefixed with the "GRANT" keyword.
String toString() const;
};
}

328
dbms/src/Access/AccessType.h Normal file
View File

@ -0,0 +1,328 @@
#pragma once
#include <Core/Types.h>
#include <boost/algorithm/string/case_conv.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <array>
namespace DB
{
/// Represents an access type which can be granted on databases, tables, columns, etc.
enum class AccessType
{
NONE, /// no access
ALL, /// full access
SHOW, /// allows to execute SHOW TABLES, SHOW CREATE TABLE, SHOW DATABASES and so on
/// (granted implicitly with any other grant)
EXISTS, /// allows to execute EXISTS, USE, i.e. to check existence
/// (granted implicitly on the database level with any other grant on the database and lower levels,
/// e.g. "GRANT SELECT(x) ON db.table" also grants EXISTS on db.*)
SELECT,
INSERT,
UPDATE, /// allows to execute ALTER UPDATE
DELETE, /// allows to execute ALTER DELETE
ADD_COLUMN,
DROP_COLUMN,
MODIFY_COLUMN,
COMMENT_COLUMN,
CLEAR_COLUMN,
ALTER_COLUMN, /// allow to execute ALTER {ADD|DROP|MODIFY...} COLUMN
ALTER_ORDER_BY,
ADD_INDEX,
DROP_INDEX,
MATERIALIZE_INDEX,
CLEAR_INDEX,
INDEX, /// allows to execute ALTER ORDER BY or ALTER {ADD|DROP...} INDEX
ADD_CONSTRAINT,
DROP_CONSTRAINT,
ALTER_CONSTRAINT, /// allows to execute ALTER {ADD|DROP} CONSTRAINT
MODIFY_TTL, /// allows to execute ALTER MODIFY TTL
MODIFY_SETTING, /// allows to execute ALTER MODIFY SETTING
MOVE_PARTITION,
FETCH_PARTITION,
FREEZE_PARTITION,
ALTER_TABLE, /// allows to execute ALTER TABLE ...
REFRESH_VIEW, /// allows to execute ALTER LIVE VIEW REFRESH
MODIFY_VIEW_QUERY, /// allows to execute ALTER TABLE MODIFY QUERY
ALTER_VIEW, /// allows to execute ALTER LIVE VIEW REFRESH, ALTER TABLE MODIFY QUERY
ALTER, /// allows to execute ALTER {TABLE|LIVE VIEW} ...
CREATE_DATABASE, /// allows to execute {CREATE|ATTACH} DATABASE
CREATE_TABLE, /// allows to execute {CREATE|ATTACH} TABLE
CREATE_VIEW, /// allows to execute {CREATE|ATTACH} VIEW
CREATE_DICTIONARY, /// allows to execute {CREATE|ATTACH} DICTIONARY
CREATE_TEMPORARY_TABLE, /// allows to create and manipulate temporary tables and views.
CREATE, /// allows to execute {CREATE|ATTACH} [TEMPORARY] {DATABASE|TABLE|VIEW|DICTIONARY}
ATTACH_DATABASE, /// allows to execute {CREATE|ATTACH} DATABASE
ATTACH_TABLE, /// allows to execute {CREATE|ATTACH} TABLE
ATTACH_VIEW, /// allows to execute {CREATE|ATTACH} VIEW
ATTACH_DICTIONARY, /// allows to execute {CREATE|ATTACH} DICTIONARY
ATTACH, /// allows to execute {CREATE|ATTACH} {DATABASE|TABLE|VIEW|DICTIONARY}
DROP_DATABASE,
DROP_TABLE,
DROP_VIEW,
DROP_DICTIONARY,
DROP, /// allows to execute DROP {DATABASE|TABLE|VIEW|DICTIONARY}
DETACH_DATABASE,
DETACH_TABLE,
DETACH_VIEW,
DETACH_DICTIONARY,
DETACH, /// allows to execute DETACH {DATABASE|TABLE|VIEW|DICTIONARY}
TRUNCATE_TABLE,
TRUNCATE_VIEW,
TRUNCATE, /// allows to execute TRUNCATE {TABLE|VIEW}
OPTIMIZE, /// allows to execute OPTIMIZE TABLE
KILL_QUERY, /// allows to kill a query started by another user (anyone can kill his own queries)
KILL_MUTATION, /// allows to kill a mutation
KILL, /// allows to execute KILL {MUTATION|QUERY}
CREATE_USER, /// allows to create, alter and drop users, roles, quotas, row policies.
ALTER_USER,
DROP_USER,
CREATE_ROLE,
DROP_ROLE,
CREATE_POLICY,
ALTER_POLICY,
DROP_POLICY,
CREATE_QUOTA,
ALTER_QUOTA,
DROP_QUOTA,
SHUTDOWN,
DROP_CACHE,
RELOAD_CONFIG,
RELOAD_DICTIONARY,
STOP_MERGES,
STOP_TTL_MERGES,
STOP_FETCHES,
STOP_MOVES,
STOP_DISTRIBUTED_SENDS,
STOP_REPLICATED_SENDS,
STOP_REPLICATION_QUEUES,
SYNC_REPLICA,
RESTART_REPLICA,
FLUSH_DISTRIBUTED,
FLUSH_LOGS,
SYSTEM, /// allows to execute SYSTEM {SHUTDOWN|RELOAD CONFIG|...}
dictGet, /// allows to execute functions dictGet, dictHas, dictGetHierarchy, dictIsIn
dictHas, /// allows to execute functions dictGet, dictHas, dictGetHierarchy, dictIsIn
dictGetHierarchy, /// allows to execute functions dictGet, dictHas, dictGetHierarchy, dictIsIn
dictIsIn, /// allows to execute functions dictGet, dictHas, dictGetHierarchy, dictIsIn
addressToLine, /// allows to execute function addressToLine
addressToSymbol, /// allows to execute function addressToSymbol
demangle, /// allows to execute function demangle
INTROSPECTION, /// allows to execute functions addressToLine, addressToSymbol, demangle
file,
url,
input,
values,
numbers,
merge,
remote,
mysql,
odbc,
jdbc,
hdfs,
s3,
TABLE_FUNCTIONS, /// allows to execute any table function
};
constexpr size_t MAX_ACCESS_TYPE = static_cast<size_t>(AccessType::TABLE_FUNCTIONS) + 1;
std::string_view toString(AccessType type);
namespace impl
{
template <typename = void>
class AccessTypeToKeywordConverter
{
public:
static const AccessTypeToKeywordConverter & instance()
{
static const AccessTypeToKeywordConverter res;
return res;
}
std::string_view convert(AccessType type) const
{
return access_type_to_keyword_mapping[static_cast<size_t>(type)];
}
private:
void addToMapping(AccessType type, const std::string_view & str)
{
String str2{str};
boost::replace_all(str2, "_", " ");
if (islower(str2[0]))
str2 += "()";
access_type_to_keyword_mapping[static_cast<size_t>(type)] = str2;
}
AccessTypeToKeywordConverter()
{
#define ACCESS_TYPE_TO_KEYWORD_CASE(type) \
addToMapping(AccessType::type, #type)
ACCESS_TYPE_TO_KEYWORD_CASE(NONE);
ACCESS_TYPE_TO_KEYWORD_CASE(ALL);
ACCESS_TYPE_TO_KEYWORD_CASE(SHOW);
ACCESS_TYPE_TO_KEYWORD_CASE(EXISTS);
ACCESS_TYPE_TO_KEYWORD_CASE(SELECT);
ACCESS_TYPE_TO_KEYWORD_CASE(INSERT);
ACCESS_TYPE_TO_KEYWORD_CASE(UPDATE);
ACCESS_TYPE_TO_KEYWORD_CASE(DELETE);
ACCESS_TYPE_TO_KEYWORD_CASE(ADD_COLUMN);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_COLUMN);
ACCESS_TYPE_TO_KEYWORD_CASE(MODIFY_COLUMN);
ACCESS_TYPE_TO_KEYWORD_CASE(COMMENT_COLUMN);
ACCESS_TYPE_TO_KEYWORD_CASE(CLEAR_COLUMN);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_COLUMN);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_ORDER_BY);
ACCESS_TYPE_TO_KEYWORD_CASE(ADD_INDEX);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_INDEX);
ACCESS_TYPE_TO_KEYWORD_CASE(MATERIALIZE_INDEX);
ACCESS_TYPE_TO_KEYWORD_CASE(CLEAR_INDEX);
ACCESS_TYPE_TO_KEYWORD_CASE(INDEX);
ACCESS_TYPE_TO_KEYWORD_CASE(ADD_CONSTRAINT);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_CONSTRAINT);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_CONSTRAINT);
ACCESS_TYPE_TO_KEYWORD_CASE(MODIFY_TTL);
ACCESS_TYPE_TO_KEYWORD_CASE(MODIFY_SETTING);
ACCESS_TYPE_TO_KEYWORD_CASE(MOVE_PARTITION);
ACCESS_TYPE_TO_KEYWORD_CASE(FETCH_PARTITION);
ACCESS_TYPE_TO_KEYWORD_CASE(FREEZE_PARTITION);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_TABLE);
ACCESS_TYPE_TO_KEYWORD_CASE(REFRESH_VIEW);
ACCESS_TYPE_TO_KEYWORD_CASE(MODIFY_VIEW_QUERY);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_VIEW);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_DATABASE);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_TABLE);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_VIEW);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_DICTIONARY);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_TEMPORARY_TABLE);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE);
ACCESS_TYPE_TO_KEYWORD_CASE(ATTACH_DATABASE);
ACCESS_TYPE_TO_KEYWORD_CASE(ATTACH_TABLE);
ACCESS_TYPE_TO_KEYWORD_CASE(ATTACH_VIEW);
ACCESS_TYPE_TO_KEYWORD_CASE(ATTACH_DICTIONARY);
ACCESS_TYPE_TO_KEYWORD_CASE(ATTACH);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_DATABASE);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_TABLE);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_VIEW);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_DICTIONARY);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP);
ACCESS_TYPE_TO_KEYWORD_CASE(DETACH_DATABASE);
ACCESS_TYPE_TO_KEYWORD_CASE(DETACH_TABLE);
ACCESS_TYPE_TO_KEYWORD_CASE(DETACH_VIEW);
ACCESS_TYPE_TO_KEYWORD_CASE(DETACH_DICTIONARY);
ACCESS_TYPE_TO_KEYWORD_CASE(DETACH);
ACCESS_TYPE_TO_KEYWORD_CASE(TRUNCATE_TABLE);
ACCESS_TYPE_TO_KEYWORD_CASE(TRUNCATE_VIEW);
ACCESS_TYPE_TO_KEYWORD_CASE(TRUNCATE);
ACCESS_TYPE_TO_KEYWORD_CASE(OPTIMIZE);
ACCESS_TYPE_TO_KEYWORD_CASE(KILL_QUERY);
ACCESS_TYPE_TO_KEYWORD_CASE(KILL_MUTATION);
ACCESS_TYPE_TO_KEYWORD_CASE(KILL);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_USER);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_USER);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_USER);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_ROLE);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_ROLE);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_POLICY);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_POLICY);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_POLICY);
ACCESS_TYPE_TO_KEYWORD_CASE(CREATE_QUOTA);
ACCESS_TYPE_TO_KEYWORD_CASE(ALTER_QUOTA);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_QUOTA);
ACCESS_TYPE_TO_KEYWORD_CASE(SHUTDOWN);
ACCESS_TYPE_TO_KEYWORD_CASE(DROP_CACHE);
ACCESS_TYPE_TO_KEYWORD_CASE(RELOAD_CONFIG);
ACCESS_TYPE_TO_KEYWORD_CASE(RELOAD_DICTIONARY);
ACCESS_TYPE_TO_KEYWORD_CASE(STOP_MERGES);
ACCESS_TYPE_TO_KEYWORD_CASE(STOP_TTL_MERGES);
ACCESS_TYPE_TO_KEYWORD_CASE(STOP_FETCHES);
ACCESS_TYPE_TO_KEYWORD_CASE(STOP_MOVES);
ACCESS_TYPE_TO_KEYWORD_CASE(STOP_DISTRIBUTED_SENDS);
ACCESS_TYPE_TO_KEYWORD_CASE(STOP_REPLICATED_SENDS);
ACCESS_TYPE_TO_KEYWORD_CASE(STOP_REPLICATION_QUEUES);
ACCESS_TYPE_TO_KEYWORD_CASE(SYNC_REPLICA);
ACCESS_TYPE_TO_KEYWORD_CASE(RESTART_REPLICA);
ACCESS_TYPE_TO_KEYWORD_CASE(FLUSH_DISTRIBUTED);
ACCESS_TYPE_TO_KEYWORD_CASE(FLUSH_LOGS);
ACCESS_TYPE_TO_KEYWORD_CASE(SYSTEM);
ACCESS_TYPE_TO_KEYWORD_CASE(dictGet);
ACCESS_TYPE_TO_KEYWORD_CASE(dictHas);
ACCESS_TYPE_TO_KEYWORD_CASE(dictGetHierarchy);
ACCESS_TYPE_TO_KEYWORD_CASE(dictIsIn);
ACCESS_TYPE_TO_KEYWORD_CASE(addressToLine);
ACCESS_TYPE_TO_KEYWORD_CASE(addressToSymbol);
ACCESS_TYPE_TO_KEYWORD_CASE(demangle);
ACCESS_TYPE_TO_KEYWORD_CASE(INTROSPECTION);
ACCESS_TYPE_TO_KEYWORD_CASE(file);
ACCESS_TYPE_TO_KEYWORD_CASE(url);
ACCESS_TYPE_TO_KEYWORD_CASE(input);
ACCESS_TYPE_TO_KEYWORD_CASE(values);
ACCESS_TYPE_TO_KEYWORD_CASE(numbers);
ACCESS_TYPE_TO_KEYWORD_CASE(merge);
ACCESS_TYPE_TO_KEYWORD_CASE(remote);
ACCESS_TYPE_TO_KEYWORD_CASE(mysql);
ACCESS_TYPE_TO_KEYWORD_CASE(odbc);
ACCESS_TYPE_TO_KEYWORD_CASE(jdbc);
ACCESS_TYPE_TO_KEYWORD_CASE(hdfs);
ACCESS_TYPE_TO_KEYWORD_CASE(s3);
ACCESS_TYPE_TO_KEYWORD_CASE(TABLE_FUNCTIONS);
#undef ACCESS_TYPE_TO_KEYWORD_CASE
}
std::array<String, MAX_ACCESS_TYPE> access_type_to_keyword_mapping;
};
}
inline std::string_view toKeyword(AccessType type) { return impl::AccessTypeToKeywordConverter<>::instance().convert(type); }
}

3
dbms/src/Access/IAccessEntity.cpp
View File

@ -1,6 +1,7 @@
#include <Access/IAccessEntity.h>
#include <Access/Quota.h>
#include <Access/RowPolicy.h>
#include <Access/User.h>
#include <common/demangle.h>
@ -8,6 +9,8 @@ namespace DB
{
String IAccessEntity::getTypeName(std::type_index type)
{
if (type == typeid(User))
return "User";
if (type == typeid(Quota))
return "Quota";
if (type == typeid(RowPolicy))

32
dbms/src/Access/IAccessStorage.cpp
View File

@ -299,44 +299,24 @@ std::vector<UUID> IAccessStorage::tryUpdate(const std::vector<UUID> & ids, const
}
IAccessStorage::SubscriptionPtr IAccessStorage::subscribeForChanges(std::type_index type, const OnChangedHandler & handler) const
ext::scope_guard IAccessStorage::subscribeForChanges(std::type_index type, const OnChangedHandler & handler) const
{
return subscribeForChangesImpl(type, handler);
}
IAccessStorage::SubscriptionPtr IAccessStorage::subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const
ext::scope_guard IAccessStorage::subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const
{
return subscribeForChangesImpl(id, handler);
}
IAccessStorage::SubscriptionPtr IAccessStorage::subscribeForChanges(const std::vector<UUID> & ids, const OnChangedHandler & handler) const
ext::scope_guard IAccessStorage::subscribeForChanges(const std::vector<UUID> & ids, const OnChangedHandler & handler) const
{
if (ids.empty())
return nullptr;
if (ids.size() == 1)
return subscribeForChangesImpl(ids[0], handler);
std::vector<SubscriptionPtr> subscriptions;
subscriptions.reserve(ids.size());
ext::scope_guard subscriptions;
for (const auto & id : ids)
{
auto subscription = subscribeForChangesImpl(id, handler);
if (subscription)
subscriptions.push_back(std::move(subscription));
}
class SubscriptionImpl : public Subscription
{
public:
SubscriptionImpl(std::vector<SubscriptionPtr> subscriptions_)
: subscriptions(std::move(subscriptions_)) {}
private:
std::vector<SubscriptionPtr> subscriptions;
};
return std::make_unique<SubscriptionImpl>(std::move(subscriptions));
subscriptions.join(subscribeForChangesImpl(id, handler));
return subscriptions;
}

20
dbms/src/Access/IAccessStorage.h
View File

@ -3,6 +3,7 @@
#include <Access/IAccessEntity.h>
#include <Core/Types.h>
#include <Core/UUID.h>
#include <ext/scope_guard.h>
#include <functional>
#include <optional>
#include <vector>
@ -109,26 +110,19 @@ public:
/// Updates multiple entities in the storage. Returns the list of successfully updated.
std::vector<UUID> tryUpdate(const std::vector<UUID> & ids, const UpdateFunc & update_func);
class Subscription
{
public:
virtual ~Subscription() {}
};
using SubscriptionPtr = std::unique_ptr<Subscription>;
using OnChangedHandler = std::function<void(const UUID & /* id */, const AccessEntityPtr & /* new or changed entity, null if removed */)>;
/// Subscribes for all changes.
/// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only).
SubscriptionPtr subscribeForChanges(std::type_index type, const OnChangedHandler & handler) const;
ext::scope_guard subscribeForChanges(std::type_index type, const OnChangedHandler & handler) const;
template <typename EntityType>
SubscriptionPtr subscribeForChanges(OnChangedHandler handler) const { return subscribeForChanges(typeid(EntityType), handler); }
ext::scope_guard subscribeForChanges(OnChangedHandler handler) const { return subscribeForChanges(typeid(EntityType), handler); }
/// Subscribes for changes of a specific entry.
/// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only).
SubscriptionPtr subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const;
SubscriptionPtr subscribeForChanges(const std::vector<UUID> & ids, const OnChangedHandler & handler) const;
ext::scope_guard subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const;
ext::scope_guard subscribeForChanges(const std::vector<UUID> & ids, const OnChangedHandler & handler) const;
bool hasSubscription(std::type_index type) const;
bool hasSubscription(const UUID & id) const;
@ -142,8 +136,8 @@ protected:
virtual UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) = 0;
virtual void removeImpl(const UUID & id) = 0;
virtual void updateImpl(const UUID & id, const UpdateFunc & update_func) = 0;
virtual SubscriptionPtr subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const = 0;
virtual SubscriptionPtr subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const = 0;
virtual ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const = 0;
virtual ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const = 0;
virtual bool hasSubscriptionImpl(const UUID & id) const = 0;
virtual bool hasSubscriptionImpl(std::type_index type) const = 0;

91
dbms/src/Access/MemoryAccessStorage.cpp
View File

@ -6,7 +6,7 @@
namespace DB
{
MemoryAccessStorage::MemoryAccessStorage(const String & storage_name_)
: IAccessStorage(storage_name_), shared_ptr_to_this{std::make_shared<const MemoryAccessStorage *>(this)}
: IAccessStorage(storage_name_)
{
}
@ -256,85 +256,38 @@ void MemoryAccessStorage::prepareNotifications(const Entry & entry, bool remove,
}
IAccessStorage::SubscriptionPtr MemoryAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const
ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const
{
class SubscriptionImpl : public Subscription
std::lock_guard lock{mutex};
auto handler_it = handlers_by_type.emplace(type, handler);
return [this, handler_it]
{
public:
SubscriptionImpl(
const MemoryAccessStorage & storage_,
std::type_index type_,
const OnChangedHandler & handler_)
: storage_weak(storage_.shared_ptr_to_this)
{
std::lock_guard lock{storage_.mutex};
handler_it = storage_.handlers_by_type.emplace(type_, handler_);
}
~SubscriptionImpl() override
{
auto storage = storage_weak.lock();
if (storage)
{
std::lock_guard lock{(*storage)->mutex};
(*storage)->handlers_by_type.erase(handler_it);
}
}
private:
std::weak_ptr<const MemoryAccessStorage *> storage_weak;
std::unordered_multimap<std::type_index, OnChangedHandler>::iterator handler_it;
std::lock_guard lock2{mutex};
handlers_by_type.erase(handler_it);
};
return std::make_unique<SubscriptionImpl>(*this, type, handler);
}
IAccessStorage::SubscriptionPtr MemoryAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const
ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const
{
class SubscriptionImpl : public Subscription
std::lock_guard lock{mutex};
auto it = entries.find(id);
if (it == entries.end())
return {};
const Entry & entry = it->second;
auto handler_it = entry.handlers_by_id.insert(entry.handlers_by_id.end(), handler);
return [this, id, handler_it]
{
public:
SubscriptionImpl(
const MemoryAccessStorage & storage_,
const UUID & id_,
const OnChangedHandler & handler_)
: storage_weak(storage_.shared_ptr_to_this),
id(id_)
std::lock_guard lock2{mutex};
auto it2 = entries.find(id);
if (it2 != entries.end())
{
std::lock_guard lock{storage_.mutex};
auto it = storage_.entries.find(id);
if (it == storage_.entries.end())
{
storage_weak.reset();
return;
}
const Entry & entry = it->second;
handler_it = entry.handlers_by_id.insert(entry.handlers_by_id.end(), handler_);
const Entry & entry2 = it2->second;
entry2.handlers_by_id.erase(handler_it);
}
~SubscriptionImpl() override
{
auto storage = storage_weak.lock();
if (storage)
{
std::lock_guard lock{(*storage)->mutex};
auto it = (*storage)->entries.find(id);
if (it != (*storage)->entries.end())
{
const Entry & entry = it->second;
entry.handlers_by_id.erase(handler_it);
}
}
}
private:
std::weak_ptr<const MemoryAccessStorage *> storage_weak;
UUID id;
std::list<OnChangedHandler>::iterator handler_it;
};
return std::make_unique<SubscriptionImpl>(*this, id, handler);
}

5
dbms/src/Access/MemoryAccessStorage.h
View File

@ -29,8 +29,8 @@ private:
UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override;
void removeImpl(const UUID & id) override;
void updateImpl(const UUID & id, const UpdateFunc & update_func) override;
SubscriptionPtr subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
SubscriptionPtr subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override;
ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override;
bool hasSubscriptionImpl(const UUID & id) const override;
bool hasSubscriptionImpl(std::type_index type) const override;
@ -60,6 +60,5 @@ private:
std::unordered_map<UUID, Entry> entries; /// We want to search entries both by ID and by the pair of name and type.
std::unordered_map<NameTypePair, Entry *, Hash> names; /// and by the pair of name and type.
mutable std::unordered_multimap<std::type_index, OnChangedHandler> handlers_by_type;
std::shared_ptr<const MemoryAccessStorage *> shared_ptr_to_this; /// We need weak pointers to `this` to implement subscriptions.
};
}

32
dbms/src/Access/MultipleAccessStorage.cpp
View File

@ -185,41 +185,21 @@ void MultipleAccessStorage::updateImpl(const UUID & id, const UpdateFunc & updat
}
IAccessStorage::SubscriptionPtr MultipleAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const
ext::scope_guard MultipleAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const
{
auto storage = findStorage(id);
if (!storage)
return nullptr;
return {};
return storage->subscribeForChanges(id, handler);
}
IAccessStorage::SubscriptionPtr MultipleAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const
ext::scope_guard MultipleAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const
{
std::vector<SubscriptionPtr> subscriptions;
ext::scope_guard subscriptions;
for (const auto & nested_storage : nested_storages)
{
auto subscription = nested_storage->subscribeForChanges(type, handler);
if (subscription)
subscriptions.emplace_back(std::move(subscription));
}
if (subscriptions.empty())
return nullptr;
if (subscriptions.size() == 1)
return std::move(subscriptions[0]);
class SubscriptionImpl : public Subscription
{
public:
SubscriptionImpl(std::vector<SubscriptionPtr> subscriptions_)
: subscriptions(std::move(subscriptions_)) {}
private:
std::vector<SubscriptionPtr> subscriptions;
};
return std::make_unique<SubscriptionImpl>(std::move(subscriptions));
subscriptions.join(nested_storage->subscribeForChanges(type, handler));
return subscriptions;
}

4
dbms/src/Access/MultipleAccessStorage.h
View File

@ -38,8 +38,8 @@ protected:
UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override;
void removeImpl(const UUID & id) override;
void updateImpl(const UUID & id, const UpdateFunc & update_func) override;
SubscriptionPtr subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
SubscriptionPtr subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override;
ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override;
bool hasSubscriptionImpl(const UUID & id) const override;
bool hasSubscriptionImpl(std::type_index type) const override;

4
dbms/src/Access/QuotaContextFactory.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <Access/QuotaContext.h>
#include <Access/IAccessStorage.h>
#include <ext/scope_guard.h>
#include <memory>
#include <mutex>
#include <unordered_map>
@ -56,7 +56,7 @@ private:
mutable std::mutex mutex;
std::unordered_map<UUID /* quota id */, QuotaInfo> all_quotas;
bool all_quotas_read = false;
IAccessStorage::SubscriptionPtr subscription;
ext::scope_guard subscription;
std::vector<std::weak_ptr<QuotaContext>> contexts;
};
}

4
dbms/src/Access/RowPolicyContextFactory.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <Access/RowPolicyContext.h>
#include <Access/IAccessStorage.h>
#include <ext/scope_guard.h>
#include <mutex>
#include <unordered_map>
#include <unordered_set>
@ -46,7 +46,7 @@ private:
const AccessControlManager & access_control_manager;
std::unordered_map<UUID, PolicyInfo> all_policies;
bool all_policies_read = false;
IAccessStorage::SubscriptionPtr subscription;
ext::scope_guard subscription;
std::vector<std::weak_ptr<RowPolicyContext>> contexts;
std::mutex mutex;
};

16
dbms/src/Access/User.cpp Normal file
View File

@ -0,0 +1,16 @@
#include <Access/User.h>
namespace DB
{
bool User::equal(const IAccessEntity & other) const
{
if (!IAccessEntity::equal(other))
return false;
const auto & other_user = typeid_cast<const User &>(other);
return (authentication == other_user.authentication) && (allowed_client_hosts == other_user.allowed_client_hosts)
&& (access == other_user.access) && (profile == other_user.profile);
}
}

26
dbms/src/Access/User.h Normal file
View File

@ -0,0 +1,26 @@
#pragma once
#include <Access/IAccessEntity.h>
#include <Access/Authentication.h>
#include <Access/AllowedClientHosts.h>
#include <Access/AccessRights.h>
#include <Core/Types.h>
namespace DB
{
/** User and ACL.
*/
struct User : public IAccessEntity
{
Authentication authentication;
AllowedClientHosts allowed_client_hosts;
AccessRights access;
String profile;
bool equal(const IAccessEntity & other) const override;
std::shared_ptr<IAccessEntity> clone() const override { return cloneImpl<User>(); }
};
using UserPtr = std::shared_ptr<const User>;
}

150
dbms/src/Access/UsersConfigAccessStorage.cpp
View File

@ -1,6 +1,8 @@
#include <Access/UsersConfigAccessStorage.h>
#include <Access/Quota.h>
#include <Access/RowPolicy.h>
#include <Access/User.h>
#include <Dictionaries/IDictionary.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/quoteString.h>
#include <Poco/Util/AbstractConfiguration.h>
@ -10,10 +12,19 @@
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int UNKNOWN_ADDRESS_PATTERN_TYPE;
}
namespace
{
char getTypeChar(std::type_index type)
{
if (type == typeid(User))
return 'U';
if (type == typeid(Quota))
return 'Q';
if (type == typeid(RowPolicy))
@ -37,6 +48,139 @@ namespace
UUID generateID(const IAccessEntity & entity) { return generateID(entity.getType(), entity.getFullName()); }
UserPtr parseUser(const Poco::Util::AbstractConfiguration & config, const String & user_name)
{
auto user = std::make_shared<User>();
user->setName(user_name);
String user_config = "users." + user_name;
bool has_password = config.has(user_config + ".password");
bool has_password_sha256_hex = config.has(user_config + ".password_sha256_hex");
bool has_password_double_sha1_hex = config.has(user_config + ".password_double_sha1_hex");
if (has_password + has_password_sha256_hex + has_password_double_sha1_hex > 1)
throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex' is used to specify password for user " + user_name + ". Must be only one of them.",
ErrorCodes::BAD_ARGUMENTS);
if (!has_password && !has_password_sha256_hex && !has_password_double_sha1_hex)
throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS);
if (has_password)
{
user->authentication = Authentication{Authentication::PLAINTEXT_PASSWORD};
user->authentication.setPassword(config.getString(user_config + ".password"));
}
else if (has_password_sha256_hex)
{
user->authentication = Authentication{Authentication::SHA256_PASSWORD};
user->authentication.setPasswordHashHex(config.getString(user_config + ".password_sha256_hex"));
}
else if (has_password_double_sha1_hex)
{
user->authentication = Authentication{Authentication::DOUBLE_SHA1_PASSWORD};
user->authentication.setPasswordHashHex(config.getString(user_config + ".password_double_sha1_hex"));
}
user->profile = config.getString(user_config + ".profile");
/// Fill list of allowed hosts.
const auto networks_config = user_config + ".networks";
if (config.has(networks_config))
{
Poco::Util::AbstractConfiguration::Keys keys;
config.keys(networks_config, keys);
for (const String & key : keys)
{
String value = config.getString(networks_config + "." + key);
if (key.starts_with("ip"))
user->allowed_client_hosts.addSubnet(value);
else if (key.starts_with("host_regexp"))
user->allowed_client_hosts.addHostRegexp(value);
else if (key.starts_with("host"))
user->allowed_client_hosts.addHostName(value);
else
throw Exception("Unknown address pattern type: " + key, ErrorCodes::UNKNOWN_ADDRESS_PATTERN_TYPE);
}
}
/// Fill list of allowed databases.
const auto databases_config = user_config + ".allow_databases";
std::optional<Strings> databases;
if (config.has(databases_config))
{
Poco::Util::AbstractConfiguration::Keys keys;
config.keys(databases_config, keys);
databases.emplace();
databases->reserve(keys.size());
for (const auto & key : keys)
{
const auto database_name = config.getString(databases_config + "." + key);
databases->push_back(database_name);
}
}
/// Fill list of allowed dictionaries.
const auto dictionaries_config = user_config + ".allow_dictionaries";
std::optional<Strings> dictionaries;
if (config.has(dictionaries_config))
{
Poco::Util::AbstractConfiguration::Keys keys;
config.keys(dictionaries_config, keys);
dictionaries.emplace();
dictionaries->reserve(keys.size());
for (const auto & key : keys)
{
const auto dictionary_name = config.getString(dictionaries_config + "." + key);
dictionaries->push_back(dictionary_name);
}
}
user->access.grant(AccessType::ALL); /// By default all databases are accessible.
if (databases)
{
user->access.fullRevoke(AccessFlags::databaseLevel());
for (const String & database : *databases)
user->access.grant(AccessFlags::databaseLevel(), database);
user->access.grant(AccessFlags::databaseLevel(), "system"); /// Anyone has access to the "system" database.
}
if (dictionaries)
{
user->access.fullRevoke(AccessType::dictGet, IDictionary::NO_DATABASE_TAG);
for (const String & dictionary : *dictionaries)
user->access.grant(AccessType::dictGet, IDictionary::NO_DATABASE_TAG, dictionary);
}
else if (databases)
user->access.grant(AccessType::dictGet, IDictionary::NO_DATABASE_TAG);
return user;
}
std::vector<AccessEntityPtr> parseUsers(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log)
{
Poco::Util::AbstractConfiguration::Keys user_names;
config.keys("users", user_names);
std::vector<AccessEntityPtr> users;
users.reserve(user_names.size());
for (const auto & user_name : user_names)
{
try
{
users.push_back(parseUser(config, user_name));
}
catch (...)
{
tryLogCurrentException(log, "Could not parse user " + backQuote(user_name));
}
}
return users;
}
QuotaPtr parseQuota(const Poco::Util::AbstractConfiguration & config, const String & quota_name, const Strings & user_names)
{
auto quota = std::make_shared<Quota>();
@ -192,6 +336,8 @@ UsersConfigAccessStorage::~UsersConfigAccessStorage() {}
void UsersConfigAccessStorage::loadFromConfig(const Poco::Util::AbstractConfiguration & config)
{
std::vector<std::pair<UUID, AccessEntityPtr>> all_entities;
for (const auto & entity : parseUsers(config, getLogger()))
all_entities.emplace_back(generateID(*entity), entity);
for (const auto & entity : parseQuotas(config, getLogger()))
all_entities.emplace_back(generateID(*entity), entity);
for (const auto & entity : parseRowPolicies(config, getLogger()))
@ -250,13 +396,13 @@ void UsersConfigAccessStorage::updateImpl(const UUID & id, const UpdateFunc &)
}
IAccessStorage::SubscriptionPtr UsersConfigAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const
ext::scope_guard UsersConfigAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const
{
return memory_storage.subscribeForChanges(id, handler);
}
IAccessStorage::SubscriptionPtr UsersConfigAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const
ext::scope_guard UsersConfigAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const
{
return memory_storage.subscribeForChanges(type, handler);
}

4
dbms/src/Access/UsersConfigAccessStorage.h
View File

@ -32,8 +32,8 @@ private:
UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override;
void removeImpl(const UUID & id) override;
void updateImpl(const UUID & id, const UpdateFunc & update_func) override;
SubscriptionPtr subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
SubscriptionPtr subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override;
ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override;
bool hasSubscriptionImpl(const UUID & id) const override;
bool hasSubscriptionImpl(std::type_index type) const override;

4
dbms/src/AggregateFunctions/AggregateFunctionSumMap.cpp
View File

@ -83,6 +83,8 @@ AggregateFunctionPtr createAggregateFunctionSumMap(const std::string & name, con
AggregateFunctionPtr res(createWithNumericBasedType<Function>(*keys_type, keys_type, values_types, arguments));
if (!res)
res.reset(createWithDecimalType<Function>(*keys_type, keys_type, values_types, arguments));
if (!res)
res.reset(createWithStringType<Function>(*keys_type, keys_type, values_types, arguments));
if (!res)
throw Exception("Illegal type of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
@ -106,6 +108,8 @@ AggregateFunctionPtr createAggregateFunctionSumMapFiltered(const std::string & n
AggregateFunctionPtr res(createWithNumericBasedType<Function>(*keys_type, keys_type, values_types, keys_to_keep, arguments, params));
if (!res)
res.reset(createWithDecimalType<Function>(*keys_type, keys_type, values_types, keys_to_keep, arguments, params));
if (!res)
res.reset(createWithStringType<Function>(*keys_type, keys_type, values_types, keys_to_keep, arguments, params));
if (!res)
throw Exception("Illegal type of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);

13
dbms/src/AggregateFunctions/AggregateFunctionSumMap.h
View File

@ -10,6 +10,7 @@
#include <Columns/ColumnTuple.h>
#include <Columns/ColumnVector.h>
#include <Columns/ColumnDecimal.h>
#include <Columns/ColumnString.h>
#include <Common/FieldVisitors.h>
#include <Common/assert_cast.h>
@ -56,8 +57,6 @@ class AggregateFunctionSumMapBase : public IAggregateFunctionDataHelper<
AggregateFunctionSumMapData<NearestFieldType<T>>, Derived>
{
private:
using ColVecType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;
DataTypePtr keys_type;
DataTypes values_types;
@ -84,9 +83,10 @@ public:
void add(AggregateDataPtr place, const IColumn ** columns, const size_t row_num, Arena *) const override
{
// Column 0 contains array of keys of known type
Field key_field;
const ColumnArray & array_column0 = assert_cast<const ColumnArray &>(*columns[0]);
const IColumn::Offsets & offsets0 = array_column0.getOffsets();
const auto & keys_vec = static_cast<const ColVecType &>(array_column0.getData());
const IColumn & key_column = array_column0.getData();
const size_t keys_vec_offset = offsets0[row_num - 1];
const size_t keys_vec_size = (offsets0[row_num] - keys_vec_offset);
@ -111,7 +111,8 @@ public:
using IteratorType = typename MapType::iterator;
array_column.getData().get(values_vec_offset + i, value);
const auto & key = keys_vec.getElement(keys_vec_offset + i);
key_column.get(keys_vec_offset + i, key_field);
auto && key = key_field.get<T>();
if (!keepKey(key))
{
@ -121,7 +122,7 @@ public:
IteratorType it;
if constexpr (IsDecimalNumber<T>)
{
UInt32 scale = keys_vec.getData().getScale();
UInt32 scale = static_cast<const ColumnDecimal<T> &>(key_column).getData().getScale();
it = merged_maps.find(DecimalField<T>(key, scale));
}
else
@ -139,7 +140,7 @@ public:
if constexpr (IsDecimalNumber<T>)
{
UInt32 scale = keys_vec.getData().getScale();
UInt32 scale = static_cast<const ColumnDecimal<T> &>(key_column).getData().getScale();
merged_maps.emplace(DecimalField<T>(key, scale), std::move(new_values));
}
else

9
dbms/src/AggregateFunctions/Helpers.h
View File

@ -149,4 +149,13 @@ static IAggregateFunction * createWithTwoNumericTypes(const IDataType & first_ty
return nullptr;
}
template <template <typename> class AggregateFunctionTemplate, typename... TArgs>
static IAggregateFunction * createWithStringType(const IDataType & argument_type, TArgs && ... args)
{
WhichDataType which(argument_type);
if (which.idx == TypeIndex::String) return new AggregateFunctionTemplate<String>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::FixedString) return new AggregateFunctionTemplate<String>(std::forward<TArgs>(args)...);
return nullptr;
}
}

4
dbms/src/Common/CurrentThread.cpp
View File

@ -7,7 +7,7 @@
#include <Common/TaskStatsInfoGetter.h>
#include <Interpreters/ProcessList.h>
#include <Interpreters/Context.h>
#include <common/getThreadNumber.h>
#include <common/getThreadId.h>
#include <Poco/Logger.h>
@ -34,7 +34,7 @@ bool CurrentThread::isInitialized()
ThreadStatus & CurrentThread::get()
{
if (unlikely(!current_thread))
throw Exception("Thread #" + std::to_string(getThreadNumber()) + " status was not initialized", ErrorCodes::LOGICAL_ERROR);
throw Exception("Thread #" + std::to_string(getThreadId()) + " status was not initialized", ErrorCodes::LOGICAL_ERROR);
return *current_thread;
}

4
dbms/src/Common/ErrorCodes.cpp
View File

@ -469,7 +469,7 @@ namespace ErrorCodes
extern const int ACCESS_ENTITY_FOUND_DUPLICATES = 494;
extern const int ACCESS_ENTITY_STORAGE_READONLY = 495;
extern const int QUOTA_REQUIRES_CLIENT_KEY = 496;
extern const int NOT_ENOUGH_PRIVILEGES = 497;
extern const int ACCESS_DENIED = 497;
extern const int LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED = 498;
extern const int S3_ERROR = 499;
extern const int CANNOT_CREATE_DATABASE = 501;
@ -479,6 +479,8 @@ namespace ErrorCodes
extern const int CANNOT_DELETE_DIRECTORY = 505;
extern const int UNEXPECTED_ERROR_CODE = 506;
extern const int UNABLE_TO_SKIP_UNUSED_SHARDS = 507;
extern const int UNKNOWN_ACCESS_TYPE = 508;
extern const int INVALID_GRANT = 509;
extern const int KEEPER_EXCEPTION = 999;
extern const int POCO_EXCEPTION = 1000;

12
dbms/src/Common/QueryProfiler.cpp
View File

@ -63,14 +63,14 @@ namespace
sizeof(UInt8) + // number of stack frames
sizeof(StackTrace::Frames) + // collected stack trace, maximum capacity
sizeof(TimerType) + // timer type
sizeof(UInt32); // thread_number
sizeof(UInt64); // thread_id
char buffer[buf_size];
WriteBufferFromFileDescriptorDiscardOnFailure out(trace_pipe.fds_rw[1], buf_size, buffer);
StringRef query_id = CurrentThread::getQueryId();
query_id.size = std::min(query_id.size, QUERY_ID_MAX_LEN);
UInt32 thread_number = CurrentThread::get().thread_number;
UInt64 thread_id = CurrentThread::get().thread_id;
const auto signal_context = *reinterpret_cast<ucontext_t *>(context);
const StackTrace stack_trace(signal_context);
@ -85,7 +85,7 @@ namespace
writePODBinary(stack_trace.getFrames()[i], out);
writePODBinary(timer_type, out);
writePODBinary(thread_number, out);
writePODBinary(thread_id, out);
out.next();
}
@ -103,7 +103,7 @@ namespace ErrorCodes
}
template <typename ProfilerImpl>
QueryProfilerBase<ProfilerImpl>::QueryProfilerBase(const Int32 thread_id, const int clock_type, UInt32 period, const int pause_signal_)
QueryProfilerBase<ProfilerImpl>::QueryProfilerBase(const UInt64 thread_id, const int clock_type, UInt32 period, const int pause_signal_)
: log(&Logger::get("QueryProfiler"))
, pause_signal(pause_signal_)
{
@ -200,7 +200,7 @@ void QueryProfilerBase<ProfilerImpl>::tryCleanup()
template class QueryProfilerBase<QueryProfilerReal>;
template class QueryProfilerBase<QueryProfilerCpu>;
QueryProfilerReal::QueryProfilerReal(const Int32 thread_id, const UInt32 period)
QueryProfilerReal::QueryProfilerReal(const UInt64 thread_id, const UInt32 period)
: QueryProfilerBase(thread_id, CLOCK_REALTIME, period, SIGUSR1)
{}
@ -209,7 +209,7 @@ void QueryProfilerReal::signalHandler(int sig, siginfo_t * info, void * context)
writeTraceInfo(TimerType::Real, sig, info, context);
}
QueryProfilerCpu::QueryProfilerCpu(const Int32 thread_id, const UInt32 period)
QueryProfilerCpu::QueryProfilerCpu(const UInt64 thread_id, const UInt32 period)
: QueryProfilerBase(thread_id, CLOCK_THREAD_CPUTIME_ID, period, SIGUSR2)
{}

6
dbms/src/Common/QueryProfiler.h
View File

@ -36,7 +36,7 @@ template <typename ProfilerImpl>
class QueryProfilerBase
{
public:
QueryProfilerBase(const Int32 thread_id, const int clock_type, UInt32 period, const int pause_signal_);
QueryProfilerBase(const UInt64 thread_id, const int clock_type, UInt32 period, const int pause_signal_);
~QueryProfilerBase();
private:
@ -60,7 +60,7 @@ private:
class QueryProfilerReal : public QueryProfilerBase<QueryProfilerReal>
{
public:
QueryProfilerReal(const Int32 thread_id, const UInt32 period);
QueryProfilerReal(const UInt64 thread_id, const UInt32 period);
static void signalHandler(int sig, siginfo_t * info, void * context);
};
@ -69,7 +69,7 @@ public:
class QueryProfilerCpu : public QueryProfilerBase<QueryProfilerCpu>
{
public:
QueryProfilerCpu(const Int32 thread_id, const UInt32 period);
QueryProfilerCpu(const UInt64 thread_id, const UInt32 period);
static void signalHandler(int sig, siginfo_t * info, void * context);
};

16
dbms/src/Common/TaskStatsInfoGetter.cpp
View File

@ -14,7 +14,6 @@
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <syscall.h>
#include <linux/genetlink.h>
#include <linux/netlink.h>
#include <linux/taskstats.h>
@ -287,16 +286,6 @@ void TaskStatsInfoGetter::getStat(::taskstats & out_stats, pid_t tid)
}
static thread_local pid_t current_tid = 0;
pid_t TaskStatsInfoGetter::getCurrentTID()
{
if (!current_tid)
current_tid = syscall(SYS_gettid); /// This call is always successful. - man gettid
return current_tid;
}
TaskStatsInfoGetter::~TaskStatsInfoGetter()
{
if (netlink_socket_fd >= 0)
@ -331,11 +320,6 @@ void TaskStatsInfoGetter::getStat(::taskstats &, pid_t)
{
}
pid_t TaskStatsInfoGetter::getCurrentTID()
{
return 0;
}
TaskStatsInfoGetter::~TaskStatsInfoGetter()
{
}

3
dbms/src/Common/TaskStatsInfoGetter.h
View File

@ -18,9 +18,6 @@ public:
void getStat(::taskstats & stat, pid_t tid);
/// Make a syscall and returns Linux thread id
static pid_t getCurrentTID();
/// Whether the current process has permissions (sudo or cap_net_admin capabilties) to get taskstats info
static bool checkPermissions();

7
dbms/src/Common/ThreadStatus.cpp
View File

@ -8,7 +8,7 @@
#include <Common/ThreadStatus.h>
#include <Poco/Logger.h>
#include <common/getThreadNumber.h>
#include <common/getThreadId.h>
namespace DB
@ -28,14 +28,13 @@ thread_local ThreadStatus * current_thread = nullptr;
TasksStatsCounters TasksStatsCounters::current()
{
TasksStatsCounters res;
CurrentThread::get().taskstats_getter->getStat(res.stat, CurrentThread::get().os_thread_id);
CurrentThread::get().taskstats_getter->getStat(res.stat, CurrentThread::get().thread_id);
return res;
}
ThreadStatus::ThreadStatus()
{
thread_number = getThreadNumber();
os_thread_id = TaskStatsInfoGetter::getCurrentTID();
thread_id = getThreadId();
last_rusage = std::make_unique<RUsageCounters>();
last_taskstats = std::make_unique<TasksStatsCounters>();

10
dbms/src/Common/ThreadStatus.h
View File

@ -60,12 +60,10 @@ public:
InternalTextLogsQueueWeakPtr logs_queue_ptr;
std::vector<UInt32> thread_numbers;
std::vector<UInt32> os_thread_ids;
std::vector<UInt64> thread_ids;
/// The first thread created this thread group
UInt32 master_thread_number = 0;
Int32 master_thread_os_id = -1;
UInt64 master_thread_id = 0;
LogsLevel client_logs_level = LogsLevel::none;
@ -89,10 +87,8 @@ public:
ThreadStatus();
~ThreadStatus();
/// Poco's thread number (the same number is used in logs)
UInt32 thread_number = 0;
/// Linux's PID (or TGID) (the same id is shown by ps util)
Int32 os_thread_id = -1;
UInt64 thread_id = 0;
/// Also called "nice" value. If it was changed to non-zero (when attaching query) - will be reset to zero when query is detached.
Int32 os_thread_priority = 0;

6
dbms/src/Common/TraceCollector.cpp
View File

@ -105,10 +105,10 @@ void TraceCollector::run()
TimerType timer_type;
readPODBinary(timer_type, in);
UInt32 thread_number;
readPODBinary(thread_number, in);
UInt64 thread_id;
readPODBinary(thread_id, in);
TraceLogElement element{std::time(nullptr), timer_type, thread_number, query_id, trace};
TraceLogElement element{std::time(nullptr), timer_type, thread_id, query_id, trace};
trace_log->add(element);
}
}

5
dbms/src/Common/tests/gtest_makeRegexpPatternFromGlobs.cpp
View File

@ -8,6 +8,11 @@ using namespace DB;
TEST(Common, makeRegexpPatternFromGlobs)
{
EXPECT_EQ(makeRegexpPatternFromGlobs("?"), "[^/]");
EXPECT_EQ(makeRegexpPatternFromGlobs("*"), "[^/]*");
EXPECT_EQ(makeRegexpPatternFromGlobs("/?"), "/[^/]");
EXPECT_EQ(makeRegexpPatternFromGlobs("/*"), "/[^/]*");
EXPECT_EQ(makeRegexpPatternFromGlobs("*_{{a,b,c,d}}/?.csv"), "[^/]*_\\{(a|b|c|d)\\}/[^/]\\.csv");
EXPECT_EQ(makeRegexpPatternFromGlobs("f{01..09}"), "f(1|2|3|4|5|6|7|8|9)");
EXPECT_EQ(makeRegexpPatternFromGlobs("f{01..9}"), "f(1|2|3|4|5|6|7|8|9)");
EXPECT_EQ(makeRegexpPatternFromGlobs("f{0001..0000009}"), "f(1|2|3|4|5|6|7|8|9)");

2
dbms/src/Compression/CachedCompressedReadBuffer.cpp
View File

@ -39,7 +39,7 @@ bool CachedCompressedReadBuffer::nextImpl()
{
/// If not, read it from the file.
initInput();
file_in->seek(file_pos);
file_in->seek(file_pos, SEEK_SET);
owned_cell = std::make_shared<UncompressedCacheCell>();

2
dbms/src/Compression/CompressedReadBufferFromFile.cpp
View File

@ -55,7 +55,7 @@ void CompressedReadBufferFromFile::seek(size_t offset_in_compressed_file, size_t
}
else
{
file_in.seek(offset_in_compressed_file);
file_in.seek(offset_in_compressed_file, SEEK_SET);
bytes += offset();
nextImpl();

1
dbms/src/Core/MySQLProtocol.h
View File

@ -7,6 +7,7 @@
#include <Common/PODArray.h>
#include <Core/Types.h>
#include <Interpreters/Context.h>
#include <Access/User.h>
#include <IO/copyData.h>
#include <IO/LimitReadBuffer.h>
#include <IO/ReadBuffer.h>

6
dbms/src/Core/Settings.h
View File

@ -189,7 +189,7 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, input_format_values_interpret_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.", 0) \
M(SettingBool, input_format_values_deduce_templates_of_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \
M(SettingBool, input_format_values_accurate_types_of_literals, true, "For Values format: when parsing and interpreting expressions using template, check actual type of literal to avoid possible overflow and precision issues.", 0) \
M(SettingString, input_format_avro_schema_registry_url, "", "For AvroConfluent format: Confluent Schema Registry URL.", 0) \
M(SettingURI, format_avro_schema_registry_url, {}, "For AvroConfluent format: Confluent Schema Registry URL.", 0) \
\
M(SettingBool, output_format_json_quote_64bit_integers, true, "Controls quoting of 64-bit integers in JSON output format.", 0) \
\
@ -314,7 +314,7 @@ struct Settings : public SettingsCollection<Settings>
M(SettingUInt64, max_bytes_in_join, 0, "Maximum size of the hash table for JOIN (in number of bytes in memory).", 0) \
M(SettingOverflowMode, join_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \
M(SettingBool, join_any_take_last_row, false, "When disabled (default) ANY JOIN will take the first found row for a key. When enabled, it will take the last row seen if there are multiple rows for the same key.", IMPORTANT) \
M(SettingBool, partial_merge_join, false, "Use partial merge join instead of hash join for LEFT and INNER JOINs.", 0) \
M(SettingBool, partial_merge_join, false, "Use partial merge join instead of hash join for joins (ANY|ALL|SEMI LEFT and ALL INNER are supported for now).", 0) \
M(SettingBool, partial_merge_join_optimizations, false, "Enable optimizations in partial merge join", 0) \
M(SettingUInt64, default_max_bytes_in_join, 100000000, "Maximum size of right-side table if limit is required but max_bytes_in_join is not set.", 0) \
M(SettingUInt64, partial_merge_join_rows_in_right_blocks, 10000, "Split right-hand joining data in blocks of specified size. It's a portion of data indexed by min-max values and possibly unloaded on disk.", 0) \
@ -338,6 +338,7 @@ struct Settings : public SettingsCollection<Settings>
M(SettingChar, format_csv_delimiter, ',', "The character to be considered as a delimiter in CSV data. If setting with a string, a string has to have a length of 1.", 0) \
M(SettingBool, format_csv_allow_single_quotes, 1, "If it is set to true, allow strings in single quotes.", 0) \
M(SettingBool, format_csv_allow_double_quotes, 1, "If it is set to true, allow strings in double quotes.", 0) \
M(SettingBool, output_format_csv_crlf_end_of_line, false, "If it is set true, end of line will be \\r\\n instead of \\n.", 0) \
M(SettingBool, input_format_csv_unquoted_null_literal_as_null, false, "Consider unquoted NULL literal as \\N", 0) \
\
M(SettingDateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic' and 'best_effort'.", 0) \
@ -388,6 +389,7 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, optimize_trivial_count_query, true, "Process trivial 'SELECT count() FROM table' query from metadata.", 0) \
M(SettingUInt64, mutations_sync, 0, "Wait for synchronous execution of ALTER TABLE UPDATE/DELETE queries (mutations). 0 - execute asynchronously. 1 - wait current server. 2 - wait all replicas if they exist.", 0) \
M(SettingBool, optimize_if_chain_to_miltiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \
M(SettingBool, allow_experimental_alter_materialized_view_structure, false, "Allow atomic alter on Materialized views. Work in progress.", 0) \
\
/** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
\

48
dbms/src/Core/SettingsCollection.cpp
View File

@ -396,6 +396,54 @@ void SettingEnum<EnumType, Tag>::set(const Field & x)
}
String SettingURI::toString() const
{
return value.toString();
}
Field SettingURI::toField() const
{
return value.toString();
}
void SettingURI::set(const Poco::URI & x)
{
value = x;
changed = true;
}
void SettingURI::set(const Field & x)
{
const String & s = safeGet<const String &>(x);
set(s);
}
void SettingURI::set(const String & x)
{
try {
Poco::URI uri(x);
set(uri);
}
catch (const Poco::Exception& e)
{
throw Exception{Exception::CreateFromPoco, e};
}
}
void SettingURI::serialize(WriteBuffer & buf, SettingsBinaryFormat) const
{
writeStringBinary(toString(), buf);
}
void SettingURI::deserialize(ReadBuffer & buf, SettingsBinaryFormat)
{
String s;
readStringBinary(s, buf);
set(s);
}
#define IMPLEMENT_SETTING_ENUM(ENUM_NAME, LIST_OF_NAMES_MACRO, ERROR_CODE_FOR_UNEXPECTED_NAME) \
IMPLEMENT_SETTING_ENUM_WITH_TAG(ENUM_NAME, void, LIST_OF_NAMES_MACRO, ERROR_CODE_FOR_UNEXPECTED_NAME)

21
dbms/src/Core/SettingsCollection.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <Poco/Timespan.h>
#include <Poco/URI.h>
#include <DataStreams/SizeLimits.h>
#include <Formats/FormatSettings.h>
#include <common/StringRef.h>
@ -196,6 +197,26 @@ struct SettingEnum
void deserialize(ReadBuffer & buf, SettingsBinaryFormat format);
};
struct SettingURI
{
Poco::URI value;
bool changed = false;
SettingURI(const Poco::URI & x = Poco::URI{}) : value(x) {}
operator Poco::URI() const { return value; }
SettingURI & operator= (const Poco::URI & x) { set(x); return *this; }
String toString() const;
Field toField() const;
void set(const Poco::URI & x);
void set(const Field & x);
void set(const String & x);
void serialize(WriteBuffer & buf, SettingsBinaryFormat format) const;
void deserialize(ReadBuffer & buf, SettingsBinaryFormat format);
};
enum class LoadBalancing
{

28
dbms/src/DataStreams/AddingMissedBlockInputStream.cpp
View File

@ -1,28 +0,0 @@
#include <DataStreams/AddingMissedBlockInputStream.h>
#include <Interpreters/addMissingDefaults.h>
namespace DB
{
AddingMissedBlockInputStream::AddingMissedBlockInputStream(
const BlockInputStreamPtr & input_,
const Block & header_,
const ColumnDefaults & column_defaults_,
const Context & context_)
: input(input_), header(header_),
column_defaults(column_defaults_), context(context_)
{
children.emplace_back(input);
}
Block AddingMissedBlockInputStream::readImpl()
{
Block src = children.back()->read();
if (!src)
return src;
return addMissingDefaults(src, header.getNamesAndTypesList(), column_defaults, context);
}
}

6
dbms/src/DataStreams/InternalTextLogsRowOutputStream.cpp
View File

@ -24,7 +24,7 @@ void InternalTextLogsRowOutputStream::write(const Block & block)
auto & column_host_name = typeid_cast<const ColumnString &>(*block.getByName("host_name").column);
auto & column_query_id = typeid_cast<const ColumnString &>(*block.getByName("query_id").column);
auto & array_thread_number = typeid_cast<const ColumnUInt32 &>(*block.getByName("thread_number").column).getData();
auto & array_thread_id = typeid_cast<const ColumnUInt64 &>(*block.getByName("thread_id").column).getData();
auto & array_priority = typeid_cast<const ColumnInt8 &>(*block.getByName("priority").column).getData();
auto & column_source = typeid_cast<const ColumnString &>(*block.getByName("source").column);
auto & column_text = typeid_cast<const ColumnString &>(*block.getByName("text").column);
@ -59,9 +59,9 @@ void InternalTextLogsRowOutputStream::write(const Block & block)
writeCString("}", wb);
}
UInt32 thread_number = array_thread_number[row_num];
UInt64 thread_id = array_thread_id[row_num];
writeCString(" [ ", wb);
writeIntText(thread_number, wb);
writeIntText(thread_id, wb);
writeCString(" ] <", wb);
Int8 priority = array_priority[row_num];

2
dbms/src/DataStreams/SummingSortedBlockInputStream.cpp
View File

@ -156,7 +156,7 @@ SummingSortedBlockInputStream::SummingSortedBlockInputStream(
|| endsWith(name, "Key")
|| endsWith(name, "Type"))
{
if (!nested_type.isValueRepresentedByInteger())
if (!nested_type.isValueRepresentedByInteger() && !isStringOrFixedString(nested_type))
break;
map_desc.key_col_nums.push_back(*column_num_it);

23
dbms/src/Databases/DatabaseOnDisk.cpp
View File

@ -245,21 +245,20 @@ void DatabaseOnDisk::renameTable(
ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const
{
ASTPtr ast;
bool has_table = tryGetTable(context, table_name) != nullptr;
auto table_metadata_path = getObjectMetadataPath(table_name);
ast = getCreateQueryFromMetadata(context, table_metadata_path, throw_on_error);
if (!ast && throw_on_error)
try
{
/// Handle system.* tables for which there are no table.sql files.
bool has_table = tryGetTable(context, table_name) != nullptr;
auto msg = has_table
? "There is no CREATE TABLE query for table "
: "There is no metadata file for table ";
throw Exception(msg + backQuote(table_name), ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
ast = getCreateQueryFromMetadata(context, table_metadata_path, throw_on_error);
}
catch (const Exception & e)
{
if (!has_table && e.code() == ErrorCodes::FILE_DOESNT_EXIST && throw_on_error)
throw Exception{"Table " + backQuote(table_name) + " doesn't exist",
ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY};
else if (throw_on_error)
throw;
}
return ast;
}

7
dbms/src/Databases/DatabaseOrdinary.cpp
View File

@ -237,7 +237,7 @@ void DatabaseOrdinary::alterTable(
ParserCreateQuery parser;
ASTPtr ast = parseQuery(parser, statement.data(), statement.data() + statement.size(), "in file " + table_metadata_path, 0);
const auto & ast_create_query = ast->as<ASTCreateQuery &>();
auto & ast_create_query = ast->as<ASTCreateQuery &>();
ASTPtr new_columns = InterpreterCreateQuery::formatColumns(metadata.columns);
ASTPtr new_indices = InterpreterCreateQuery::formatIndices(metadata.indices);
@ -247,6 +247,11 @@ void DatabaseOrdinary::alterTable(
ast_create_query.columns_list->setOrReplace(ast_create_query.columns_list->indices, new_indices);
ast_create_query.columns_list->setOrReplace(ast_create_query.columns_list->constraints, new_constraints);
if (metadata.select)
{
ast->replace(ast_create_query.select, metadata.select);
}
ASTStorage & storage_ast = *ast_create_query.storage;
/// ORDER BY may change, but cannot appear, it's required construction
if (metadata.order_by_ast && storage_ast.order_by)

1
dbms/src/Databases/IDatabase.h
View File

@ -28,7 +28,6 @@ namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
extern const int CANNOT_GET_CREATE_DICTIONARY_QUERY;
}

14
dbms/src/Dictionaries/IDictionary.h
View File

@ -28,8 +28,22 @@ struct IDictionaryBase : public IExternalLoadable
virtual const std::string & getDatabase() const = 0;
virtual const std::string & getName() const = 0;
virtual const std::string & getFullName() const = 0;
const std::string & getLoadableName() const override { return getFullName(); }
/// Specifies that no database is used.
/// Sometimes we cannot simply use an empty string for that because an empty string is
/// usually replaced with the current database.
static constexpr char NO_DATABASE_TAG[] = "<no_database>";
std::string_view getDatabaseOrNoDatabaseTag() const
{
const std::string & database = getDatabase();
if (!database.empty())
return database;
return NO_DATABASE_TAG;
}
virtual std::string getTypeName() const = 0;
virtual size_t getBytesAllocated() const = 0;

2
dbms/src/Disks/DiskLocal.cpp
View File

@ -200,7 +200,7 @@ void DiskLocal::copyFile(const String & from_path, const String & to_path)
Poco::File(disk_path + from_path).copyTo(disk_path + to_path);
}
std::unique_ptr<ReadBuffer> DiskLocal::readFile(const String & path, size_t buf_size) const
std::unique_ptr<SeekableReadBuffer> DiskLocal::readFile(const String & path, size_t buf_size) const
{
return std::make_unique<ReadBufferFromFile>(disk_path + path, buf_size);
}

2
dbms/src/Disks/DiskLocal.h
View File

@ -66,7 +66,7 @@ public:
void copyFile(const String & from_path, const String & to_path) override;
std::unique_ptr<ReadBuffer> readFile(const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE) const override;
std::unique_ptr<SeekableReadBuffer> readFile(const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE) const override;
std::unique_ptr<WriteBuffer> writeFile(const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, WriteMode mode = WriteMode::Rewrite) override;

42
dbms/src/Disks/DiskMemory.cpp
View File

@ -2,6 +2,7 @@
#include "DiskFactory.h"
#include <IO/ReadBufferFromString.h>
#include <IO/SeekableReadBuffer.h>
#include <IO/WriteBufferFromString.h>
#include <Interpreters/Context.h>
@ -36,6 +37,36 @@ private:
std::vector<String>::iterator iter;
};
void WriteIndirectBuffer::finalize()
{
next();
WriteBufferFromVector::finalize();
auto iter = disk->files.find(path);
if (iter == disk->files.end())
throw Exception("File '" + path + "' does not exist", ErrorCodes::FILE_DOESNT_EXIST);
// Resize to the actual number of bytes written to string.
value.resize(count());
if (mode == WriteMode::Rewrite)
disk->files.insert_or_assign(path, DiskMemory::FileData{iter->second.type, value});
else if (mode == WriteMode::Append)
disk->files.insert_or_assign(path, DiskMemory::FileData{iter->second.type, iter->second.data + value});
}
WriteIndirectBuffer::~WriteIndirectBuffer()
{
try
{
finalize();
}
catch (...)
{
tryLogCurrentException(__PRETTY_FUNCTION__);
}
}
ReservationPtr DiskMemory::reserve(UInt64 /*bytes*/)
{
@ -94,7 +125,7 @@ size_t DiskMemory::getFileSize(const String & path) const
if (iter == files.end())
throw Exception("File '" + path + "' does not exist", ErrorCodes::FILE_DOESNT_EXIST);
return iter->second.data.size();
return iter->second.data.length();
}
void DiskMemory::createDirectory(const String & path)
@ -218,7 +249,7 @@ void DiskMemory::copyFile(const String & /*from_path*/, const String & /*to_path
throw Exception("Method copyFile is not implemented for memory disks", ErrorCodes::NOT_IMPLEMENTED);
}
std::unique_ptr<ReadBuffer> DiskMemory::readFile(const String & path, size_t /*buf_size*/) const
std::unique_ptr<SeekableReadBuffer> DiskMemory::readFile(const String & path, size_t /*buf_size*/) const
{
std::lock_guard lock(mutex);
@ -241,13 +272,10 @@ std::unique_ptr<WriteBuffer> DiskMemory::writeFile(const String & path, size_t /
throw Exception(
"Failed to create file '" + path + "'. Directory " + parent_path + " does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST);
iter = files.emplace(path, FileData{FileType::File}).first;
files.emplace(path, FileData{FileType::File});
}
if (mode == WriteMode::Append)
return std::make_unique<WriteBufferFromString>(iter->second.data, WriteBufferFromString::AppendModeTag{});
else
return std::make_unique<WriteBufferFromString>(iter->second.data);
return std::make_unique<WriteIndirectBuffer>(this, path, mode);
}
void DiskMemory::remove(const String & path)

39
dbms/src/Disks/DiskMemory.h
View File

@ -1,17 +1,33 @@
#pragma once
#include <Disks/IDisk.h>
#include <mutex>
#include <memory>
#include <mutex>
#include <unordered_map>
#include <utility>
#include <Disks/IDisk.h>
#include <IO/WriteBufferFromString.h>
namespace DB
{
class DiskMemory;
class ReadBuffer;
class WriteBuffer;
// This class is responsible to update files metadata after buffer is finalized.
class WriteIndirectBuffer : public WriteBufferFromOwnString
{
public:
WriteIndirectBuffer(DiskMemory * disk_, String path_, WriteMode mode_) : disk(disk_), path(std::move(path_)), mode(mode_) {}
~WriteIndirectBuffer() override;
void finalize() override;
private:
DiskMemory * disk;
String path;
WriteMode mode;
};
/** Implementation of Disk intended only for testing purposes.
* All filesystem objects are stored in memory and lost on server restart.
@ -22,7 +38,7 @@ class WriteBuffer;
class DiskMemory : public IDisk
{
public:
DiskMemory(const String & name_) : name(name_), disk_path("memory://" + name_ + '/') { }
DiskMemory(const String & name_) : name(name_), disk_path("memory://" + name_ + '/') {}
const String & getName() const override { return name; }
@ -60,12 +76,10 @@ public:
void copyFile(const String & from_path, const String & to_path) override;
std::unique_ptr<ReadBuffer> readFile(const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE) const override;
std::unique_ptr<SeekableReadBuffer> readFile(const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE) const override;
std::unique_ptr<WriteBuffer> writeFile(
const String & path,
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
WriteMode mode = WriteMode::Rewrite) override;
std::unique_ptr<WriteBuffer>
writeFile(const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, WriteMode mode = WriteMode::Rewrite) override;
void remove(const String & path) override;
@ -76,6 +90,8 @@ private:
void replaceFileImpl(const String & from_path, const String & to_path);
private:
friend class WriteIndirectBuffer;
enum class FileType
{
File,
@ -87,7 +103,8 @@ private:
FileType type;
String data;
explicit FileData(FileType type_) : type(type_) { }
FileData(FileType type_, String data_) : type(type_), data(std::move(data_)) {}
explicit FileData(FileType type_) : type(type_), data("") {}
};
using Files = std::unordered_map<String, FileData>; /// file path -> file data

391
dbms/src/Disks/DiskS3.cpp
View File

@ -4,12 +4,13 @@
# include "DiskFactory.h"
# include <random>
# include <IO/S3Common.h>
# include <IO/ReadBufferFromS3.h>
# include <IO/WriteBufferFromS3.h>
# include <utility>
# include <IO/ReadBufferFromFile.h>
# include <IO/WriteBufferFromFile.h>
# include <IO/ReadBufferFromS3.h>
# include <IO/ReadHelpers.h>
# include <IO/S3Common.h>
# include <IO/WriteBufferFromFile.h>
# include <IO/WriteBufferFromS3.h>
# include <IO/WriteHelpers.h>
# include <Poco/File.h>
# include <Common/checkStackSize.h>
@ -27,6 +28,9 @@ namespace ErrorCodes
extern const int FILE_ALREADY_EXISTS;
extern const int FILE_DOESNT_EXIST;
extern const int PATH_ACCESS_DENIED;
extern const int SEEK_POSITION_OUT_OF_BOUND;
extern const int CANNOT_SEEK_THROUGH_FILE;
extern const int UNKNOWN_FORMAT;
}
namespace
@ -41,33 +45,209 @@ namespace
}
}
String readKeyFromFile(const String & path)
/**
* S3 metadata file layout:
* Number of S3 objects, Total size of all S3 objects.
* Each S3 object represents path where object located in S3 and size of object.
*/
struct Metadata
{
String key;
ReadBufferFromFile buf(path, 1024); /* reasonable buffer size for small file */
readStringUntilEOF(key, buf);
return key;
}
// Metadata file version.
const UInt32 VERSION = 1;
void writeKeyToFile(const String & key, const String & path)
using PathAndSize = std::pair<String, size_t>;
// Path to metadata file on local FS.
String metadata_file_path;
// S3 object references count.
UInt32 s3_objects_count;
// Total size of all S3 objects.
size_t total_size;
// S3 objects paths and their sizes.
std::vector<PathAndSize> s3_objects;
explicit Metadata(const Poco::File & file) : Metadata(file.path(), false) {}
// Load metadata by path or create empty if `create` flag is set.
explicit Metadata(const String & file_path, bool create = false)
: metadata_file_path(file_path), s3_objects_count(0), total_size(0), s3_objects(0)
{
if (create)
return;
ReadBufferFromFile buf(file_path, 1024); /* reasonable buffer size for small file */
UInt32 version;
readIntText(version, buf);
if (version != VERSION)
throw Exception(
"Unknown metadata file version. Path: " + file_path + ", Version: " + std::to_string(version)
+ ", Expected version: " + std::to_string(VERSION),
ErrorCodes::UNKNOWN_FORMAT);
assertChar('\n', buf);
readIntText(s3_objects_count, buf);
assertChar('\t', buf);
readIntText(total_size, buf);
assertChar('\n', buf);
s3_objects.resize(s3_objects_count);
for (UInt32 i = 0; i < s3_objects_count; ++i)
{
String path;
size_t size;
readIntText(size, buf);
assertChar('\t', buf);
readEscapedString(path, buf);
assertChar('\n', buf);
s3_objects[i] = std::make_pair(path, size);
}
}
void addObject(const String & path, size_t size)
{
++s3_objects_count;
total_size += size;
s3_objects.emplace_back(path, size);
}
void save()
{
WriteBufferFromFile buf(metadata_file_path, 1024);
writeIntText(VERSION, buf);
writeChar('\n', buf);
writeIntText(s3_objects_count, buf);
writeChar('\t', buf);
writeIntText(total_size, buf);
writeChar('\n', buf);
for (UInt32 i = 0; i < s3_objects_count; ++i)
{
auto path_and_size = s3_objects[i];
writeIntText(path_and_size.second, buf);
writeChar('\t', buf);
writeEscapedString(path_and_size.first, buf);
writeChar('\n', buf);
}
buf.finalize();
}
};
// Reads data from S3.
// It supports reading from multiple S3 paths that resides in Metadata.
class ReadIndirectBufferFromS3 : public BufferWithOwnMemory<SeekableReadBuffer>
{
WriteBufferFromFile buf(path, 1024);
writeString(key, buf);
buf.next();
}
public:
ReadIndirectBufferFromS3(
std::shared_ptr<Aws::S3::S3Client> client_ptr_, const String & bucket_, Metadata metadata_, size_t buf_size_)
: BufferWithOwnMemory(buf_size_)
, client_ptr(std::move(client_ptr_))
, bucket(bucket_)
, metadata(std::move(metadata_))
, buf_size(buf_size_)
, offset(0)
, initialized(false)
, current_buf_idx(0)
, current_buf(nullptr)
{
}
/// Stores data in S3 and the object key in file in local filesystem.
off_t seek(off_t offset_, int whence) override
{
if (initialized)
throw Exception("Seek is allowed only before first read attempt from the buffer.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
if (whence != SEEK_SET)
throw Exception("Only SEEK_SET mode is allowed.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
if (offset_ < 0 || metadata.total_size <= static_cast<UInt64>(offset_))
throw Exception(
"Seek position is out of bounds. "
"Offset: "
+ std::to_string(offset_) + ", Max: " + std::to_string(metadata.total_size),
ErrorCodes::SEEK_POSITION_OUT_OF_BOUND);
offset = offset_;
return offset;
}
private:
std::unique_ptr<ReadBufferFromS3> initialize()
{
for (UInt32 i = 0; i < metadata.s3_objects_count; ++i)
{
current_buf_idx = i;
auto path = metadata.s3_objects[i].first;
auto size = metadata.s3_objects[i].second;
if (size > offset)
{
auto buf = std::make_unique<ReadBufferFromS3>(client_ptr, bucket, path, buf_size);
buf->seek(offset, SEEK_SET);
return buf;
}
offset -= size;
}
return nullptr;
}
bool nextImpl() override
{
// Find first available buffer that fits to given offset.
if (!initialized)
{
current_buf = initialize();
initialized = true;
}
// If current buffer has remaining data - use it.
if (current_buf && current_buf->next())
{
working_buffer = current_buf->buffer();
return true;
}
// If there is no available buffers - nothing to read.
if (current_buf_idx + 1 >= metadata.s3_objects_count)
return false;
++current_buf_idx;
auto path = metadata.s3_objects[current_buf_idx].first;
current_buf = std::make_unique<ReadBufferFromS3>(client_ptr, bucket, path, buf_size);
current_buf->next();
working_buffer = current_buf->buffer();
return true;
}
private:
std::shared_ptr<Aws::S3::S3Client> client_ptr;
const String & bucket;
Metadata metadata;
size_t buf_size;
size_t offset;
bool initialized;
UInt32 current_buf_idx;
std::unique_ptr<ReadBufferFromS3> current_buf;
};
/// Stores data in S3 and adds the object key (S3 path) and object size to metadata file on local FS.
class WriteIndirectBufferFromS3 : public WriteBufferFromS3
{
public:
WriteIndirectBufferFromS3(
std::shared_ptr<Aws::S3::S3Client> & client_ptr_,
const String & bucket_,
const String & metadata_path_,
Metadata metadata_,
const String & s3_path_,
size_t min_upload_part_size,
size_t buf_size_)
: WriteBufferFromS3(client_ptr_, bucket_, s3_path_, DEFAULT_BLOCK_SIZE, buf_size_)
, metadata_path(metadata_path_)
: WriteBufferFromS3(client_ptr_, bucket_, s3_path_, min_upload_part_size, buf_size_)
, metadata(std::move(metadata_))
, s3_path(s3_path_)
{
}
@ -75,7 +255,8 @@ namespace
void finalize() override
{
WriteBufferFromS3::finalize();
writeKeyToFile(s3_path, metadata_path);
metadata.addObject(s3_path, total_size);
metadata.save();
finalized = true;
}
@ -96,8 +277,8 @@ namespace
private:
bool finalized = false;
const String metadata_path;
const String s3_path;
Metadata metadata;
String s3_path;
};
}
@ -156,12 +337,14 @@ private:
};
DiskS3::DiskS3(String name_, std::shared_ptr<Aws::S3::S3Client> client_, String bucket_, String s3_root_path_, String metadata_path_)
DiskS3::DiskS3(String name_, std::shared_ptr<Aws::S3::S3Client> client_, String bucket_, String s3_root_path_,
String metadata_path_, size_t min_upload_part_size_)
: name(std::move(name_))
, client(std::move(client_))
, bucket(std::move(bucket_))
, s3_root_path(std::move(s3_root_path_))
, metadata_path(std::move(metadata_path_))
, min_upload_part_size(min_upload_part_size_)
{
}
@ -189,19 +372,8 @@ bool DiskS3::isDirectory(const String & path) const
size_t DiskS3::getFileSize(const String & path) const
{
Aws::S3::Model::GetObjectRequest request;
request.SetBucket(bucket);
request.SetKey(getS3Path(path));
auto outcome = client->GetObject(request);
if (!outcome.IsSuccess())
{
auto & err = outcome.GetError();
throw Exception(err.GetMessage(), static_cast<int>(err.GetErrorType()));
}
else
{
return outcome.GetResult().GetContentLength();
}
Metadata metadata(metadata_path + path);
return metadata.total_size;
}
void DiskS3::createDirectory(const String & path)
@ -229,7 +401,7 @@ void DiskS3::clearDirectory(const String & path)
void DiskS3::moveFile(const String & from_path, const String & to_path)
{
if (exists(to_path))
throw Exception("File already exists " + to_path, ErrorCodes::FILE_ALREADY_EXISTS);
throw Exception("File already exists: " + to_path, ErrorCodes::FILE_ALREADY_EXISTS);
Poco::File(metadata_path + from_path).renameTo(metadata_path + to_path);
}
@ -253,86 +425,112 @@ void DiskS3::copyFile(const String & from_path, const String & to_path)
if (exists(to_path))
remove(to_path);
String s3_from_path = readKeyFromFile(metadata_path + from_path);
String s3_to_path = s3_root_path + getRandomName();
Metadata from(metadata_path + from_path);
Metadata to(metadata_path + to_path, true);
Aws::S3::Model::CopyObjectRequest req;
req.SetBucket(bucket);
req.SetCopySource(s3_from_path);
req.SetKey(s3_to_path);
throwIfError(client->CopyObject(req));
writeKeyToFile(s3_to_path, metadata_path + to_path);
for (UInt32 i = 0; i < from.s3_objects_count; ++i)
{
auto path = from.s3_objects[i].first;
auto size = from.s3_objects[i].second;
auto new_path = s3_root_path + getRandomName();
Aws::S3::Model::CopyObjectRequest req;
req.SetBucket(bucket);
req.SetCopySource(path);
req.SetKey(new_path);
throwIfError(client->CopyObject(req));
to.addObject(new_path, size);
}
to.save();
}
std::unique_ptr<ReadBuffer> DiskS3::readFile(const String & path, size_t buf_size) const
std::unique_ptr<SeekableReadBuffer> DiskS3::readFile(const String & path, size_t buf_size) const
{
return std::make_unique<ReadBufferFromS3>(client, bucket, getS3Path(path), buf_size);
Metadata metadata(metadata_path + path);
LOG_DEBUG(
&Logger::get("DiskS3"),
"Read from file by path: " << backQuote(metadata_path + path) << " Existing S3 objects: " << metadata.s3_objects_count);
return std::make_unique<ReadIndirectBufferFromS3>(client, bucket, metadata, buf_size);
}
std::unique_ptr<WriteBuffer> DiskS3::writeFile(const String & path, size_t buf_size, WriteMode mode)
{
if (!exists(path) || mode == WriteMode::Rewrite)
bool exist = exists(path);
// Reference to store new S3 object.
auto s3_path = s3_root_path + getRandomName();
if (!exist || mode == WriteMode::Rewrite)
{
String new_s3_path = s3_root_path + getRandomName();
return std::make_unique<WriteIndirectBufferFromS3>(client, bucket, metadata_path + path, new_s3_path, buf_size);
// If metadata file exists - remove and create new.
if (exist)
remove(path);
Metadata metadata(metadata_path + path, true);
// Save empty metadata to disk to have ability to get file size while buffer is not finalized.
metadata.save();
LOG_DEBUG(&Logger::get("DiskS3"), "Write to file by path: " << backQuote(metadata_path + path) << " New S3 path: " << s3_path);
return std::make_unique<WriteIndirectBufferFromS3>(client, bucket, metadata, s3_path, min_upload_part_size, buf_size);
}
else
{
auto old_s3_path = getS3Path(path);
ReadBufferFromS3 read_buffer(client, bucket, old_s3_path, buf_size);
auto writeBuffer = std::make_unique<WriteIndirectBufferFromS3>(client, bucket, metadata_path + path, old_s3_path, buf_size);
std::vector<char> buffer(buf_size);
while (!read_buffer.eof())
writeBuffer->write(buffer.data(), read_buffer.read(buffer.data(), buf_size));
return writeBuffer;
Metadata metadata(metadata_path + path);
LOG_DEBUG(
&Logger::get("DiskS3"),
"Append to file by path: " << backQuote(metadata_path + path) << " New S3 path: " << s3_path
<< " Existing S3 objects: " << metadata.s3_objects_count);
return std::make_unique<WriteIndirectBufferFromS3>(client, bucket, metadata, s3_path, min_upload_part_size, buf_size);
}
}
void DiskS3::remove(const String & path)
{
LOG_DEBUG(&Logger::get("DiskS3"), "Remove file by path: " << backQuote(metadata_path + path));
Poco::File file(metadata_path + path);
if (file.isFile())
{
Aws::S3::Model::DeleteObjectRequest request;
request.SetBucket(bucket);
request.SetKey(getS3Path(path));
throwIfError(client->DeleteObject(request));
Metadata metadata(file);
for (UInt32 i = 0; i < metadata.s3_objects_count; ++i)
{
auto s3_path = metadata.s3_objects[i].first;
// TODO: Make operation idempotent. Do not throw exception if key is already deleted.
Aws::S3::Model::DeleteObjectRequest request;
request.SetBucket(bucket);
request.SetKey(s3_path);
throwIfError(client->DeleteObject(request));
}
}
file.remove();
}
void DiskS3::removeRecursive(const String & path)
{
checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks.
checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks.
Poco::File file(metadata_path + path);
if (file.isFile())
{
Aws::S3::Model::DeleteObjectRequest request;
request.SetBucket(bucket);
request.SetKey(getS3Path(path));
throwIfError(client->DeleteObject(request));
remove(metadata_path + path);
}
else
{
for (auto it{iterateDirectory(path)}; it->isValid(); it->next())
removeRecursive(it->path());
file.remove();
}
file.remove();
}
String DiskS3::getS3Path(const String & path) const
{
if (!exists(path))
throw Exception("File not found: " + path, ErrorCodes::FILE_DOESNT_EXIST);
return readKeyFromFile(metadata_path + path);
}
String DiskS3::getRandomName() const
{
std::uniform_int_distribution<int> distribution('a', 'z');
String res(32, ' '); /// The number of bits of entropy should be not less than 128.
String res(32, ' '); /// The number of bits of entropy should be not less than 128.
for (auto & c : res)
c = distribution(thread_local_rng);
return res;
@ -390,6 +588,26 @@ DiskS3Reservation::~DiskS3Reservation()
}
}
inline void checkWriteAccess(std::shared_ptr<DiskS3> & disk)
{
auto file = disk->writeFile("test_acl", DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite);
file->write("test", 4);
}
inline void checkReadAccess(const String & disk_name, std::shared_ptr<DiskS3> & disk)
{
auto file = disk->readFile("test_acl", DBMS_DEFAULT_BUFFER_SIZE);
String buf(4, '0');
file->readStrict(buf.data(), 4);
if (buf != "test")
throw Exception("No read access to S3 bucket in disk " + disk_name, ErrorCodes::PATH_ACCESS_DENIED);
}
inline void checkRemoveAccess(std::shared_ptr<DiskS3> & disk)
{
disk->remove("test_acl");
}
void registerDiskS3(DiskFactory & factory)
{
auto creator = [](const String & name,
@ -410,24 +628,13 @@ void registerDiskS3(DiskFactory & factory)
String metadata_path = context.getPath() + "disks/" + name + "/";
auto s3disk = std::make_shared<DiskS3>(name, client, uri.bucket, uri.key, metadata_path);
auto s3disk = std::make_shared<DiskS3>(name, client, uri.bucket, uri.key, metadata_path,
context.getSettingsRef().s3_min_upload_part_size);
/// This code is used only to check access to the corresponding disk.
{
auto file = s3disk->writeFile("test_acl", DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite);
file->write("test", 4);
}
{
auto file = s3disk->readFile("test_acl", DBMS_DEFAULT_BUFFER_SIZE);
String buf(4, '0');
file->readStrict(buf.data(), 4);
if (buf != "test")
throw Exception("No read accecss to S3 bucket in disk " + name, ErrorCodes::PATH_ACCESS_DENIED);
}
{
s3disk->remove("test_acl");
}
checkWriteAccess(s3disk);
checkReadAccess(name, s3disk);
checkRemoveAccess(s3disk);
return s3disk;
};

8
dbms/src/Disks/DiskS3.h
View File

@ -21,7 +21,8 @@ class DiskS3 : public IDisk
public:
friend class DiskS3Reservation;
DiskS3(String name_, std::shared_ptr<Aws::S3::S3Client> client_, String bucket_, String s3_root_path_, String metadata_path_);
DiskS3(String name_, std::shared_ptr<Aws::S3::S3Client> client_, String bucket_, String s3_root_path_,
String metadata_path_, size_t min_upload_part_size_);
const String & getName() const override { return name; }
@ -61,7 +62,7 @@ public:
void copyFile(const String & from_path, const String & to_path) override;
std::unique_ptr<ReadBuffer> readFile(const String & path, size_t buf_size) const override;
std::unique_ptr<SeekableReadBuffer> readFile(const String & path, size_t buf_size) const override;
std::unique_ptr<WriteBuffer> writeFile(const String & path, size_t buf_size, WriteMode mode) override;
@ -70,8 +71,6 @@ public:
void removeRecursive(const String & path) override;
private:
String getS3Path(const String & path) const;
String getRandomName() const;
bool tryReserve(UInt64 bytes);
@ -82,6 +81,7 @@ private:
const String bucket;
const String s3_root_path;
const String metadata_path;
size_t min_upload_part_size;
UInt64 reserved_bytes = 0;
UInt64 reservation_count = 0;

6
dbms/src/Disks/IDisk.h
View File

@ -25,7 +25,7 @@ using DiskDirectoryIteratorPtr = std::unique_ptr<IDiskDirectoryIterator>;
class IReservation;
using ReservationPtr = std::unique_ptr<IReservation>;
class ReadBuffer;
class SeekableReadBuffer;
class WriteBuffer;
/**
@ -121,8 +121,8 @@ public:
/// Copy the file from `from_path` to `to_path`.
virtual void copyFile(const String & from_path, const String & to_path) = 0;
/// Open the file for read and return ReadBuffer object.
virtual std::unique_ptr<ReadBuffer> readFile(const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE) const = 0;
/// Open the file for read and return SeekableReadBuffer object.
virtual std::unique_ptr<SeekableReadBuffer> readFile(const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE) const = 0;
/// Open the file for write and return WriteBuffer object.
virtual std::unique_ptr<WriteBuffer> writeFile(const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, WriteMode mode = WriteMode::Rewrite) = 0;

48
dbms/src/Disks/tests/gtest_disk.cpp
View File

@ -1,13 +1,12 @@
#include <gtest/gtest.h>
#include <Disks/DiskLocal.h>
#include <Disks/DiskMemory.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include "gtest_disk.h"
#if !__clang__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsuggest-override"
# pragma GCC diagnostic push
# pragma GCC diagnostic ignored "-Wsuggest-override"
#endif
template <typename T>
@ -33,6 +32,12 @@ void destroyDisk(DB::DiskPtr & disk)
disk.reset();
}
template <>
void destroyDisk<DB::DiskMemory>(DB::DiskPtr & disk)
{
disk.reset();
}
template <>
void destroyDisk<DB::DiskLocal>(DB::DiskPtr & disk)
{
@ -92,6 +97,41 @@ TYPED_TEST(DiskTest, writeFile)
}
TYPED_TEST(DiskTest, readFile)
{
const auto & disk = this->getDisk();
{
std::unique_ptr<DB::WriteBuffer> out = disk->writeFile("test_file");
writeString("test data", *out);
}
// Test SEEK_SET
{
DB::String data;
std::unique_ptr<DB::SeekableReadBuffer> in = disk->readFile("test_file");
in->seek(5, SEEK_SET);
readString(data, *in);
EXPECT_EQ("data", data);
}
// Test SEEK_CUR
{
std::unique_ptr<DB::SeekableReadBuffer> in = disk->readFile("test_file");
String buf(4, '0');
in->readStrict(buf.data(), 4);
EXPECT_EQ("test", buf);
// Skip whitespace
in->seek(1, SEEK_CUR);
in->readStrict(buf.data(), 4);
EXPECT_EQ("data", buf);
}
}
TYPED_TEST(DiskTest, iterateDirectory)
{
const auto & disk = this->getDisk();

21
dbms/src/Disks/tests/gtest_disk.h Normal file
View File

@ -0,0 +1,21 @@
#include <Disks/DiskLocal.h>
#include <Disks/DiskMemory.h>
#include <Disks/IDisk.h>
template <typename T>
DB::DiskPtr createDisk();
template <>
DB::DiskPtr createDisk<DB::DiskMemory>();
template <>
DB::DiskPtr createDisk<DB::DiskLocal>();
template <typename T>
void destroyDisk(DB::DiskPtr & disk);
template <>
void destroyDisk<DB::DiskLocal>(DB::DiskPtr & disk);
template <>
void destroyDisk<DB::DiskMemory>(DB::DiskPtr & disk);

13
dbms/src/Formats/FormatFactory.cpp
View File

@ -14,7 +14,7 @@
#include <DataStreams/NativeBlockInputStream.h>
#include <Processors/Formats/Impl/ValuesBlockInputFormat.h>
#include <Processors/Formats/Impl/MySQLOutputFormat.h>
#include <Poco/URI.h>
namespace DB
{
@ -68,7 +68,15 @@ static FormatSettings getInputFormatSetting(const Settings & settings, const Con
format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter;
format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter;
format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter;
format_settings.avro.schema_registry_url = settings.input_format_avro_schema_registry_url;
/// Validate avro_schema_registry_url with RemoteHostFilter when non-empty and in Server context
if (context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER))
{
const Poco::URI & avro_schema_registry_url = settings.format_avro_schema_registry_url;
if (!avro_schema_registry_url.empty())
context.getRemoteHostFilter().checkURL(avro_schema_registry_url);
}
format_settings.avro.schema_registry_url = settings.format_avro_schema_registry_url.toString();
return format_settings;
}
@ -82,6 +90,7 @@ static FormatSettings getOutputFormatSetting(const Settings & settings, const Co
format_settings.csv.delimiter = settings.format_csv_delimiter;
format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes;
format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes;
format_settings.csv.crlf_end_of_line = settings.output_format_csv_crlf_end_of_line;
format_settings.pretty.max_rows = settings.output_format_pretty_max_rows;
format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width;
format_settings.pretty.color = settings.output_format_pretty_color;

1
dbms/src/Formats/FormatSettings.h
View File

@ -29,6 +29,7 @@ struct FormatSettings
bool allow_double_quotes = true;
bool unquoted_null_literal_as_null = false;
bool empty_as_default = false;
bool crlf_end_of_line = false;
};
CSV csv;

52
dbms/src/Functions/FunctionsExternalDictionaries.h
View File

@ -18,6 +18,8 @@
#include <Columns/ColumnString.h>
#include <Columns/ColumnTuple.h>
#include <Access/AccessFlags.h>
#include <Interpreters/Context.h>
#include <Interpreters/ExternalDictionariesLoader.h>
@ -48,7 +50,6 @@ namespace ErrorCodes
extern const int TYPE_MISMATCH;
extern const int ILLEGAL_COLUMN;
extern const int BAD_ARGUMENTS;
extern const int DICTIONARY_ACCESS_DENIED;
}
/** Functions that use plug-ins (external) dictionaries_loader.
@ -126,12 +127,7 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getFullName()))
{
throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getFullName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
}
context.checkAccess(AccessType::dictHas, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName());
if (!executeDispatchSimple<FlatDictionary>(block, arguments, result, dict_ptr) &&
!executeDispatchSimple<HashedDictionary>(block, arguments, result, dict_ptr) &&
@ -301,12 +297,7 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getFullName()))
{
throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getFullName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
}
context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName());
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&
!executeDispatch<HashedDictionary>(block, arguments, result, dict_ptr) &&
@ -487,12 +478,7 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getFullName()))
{
throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getFullName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
}
context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName());
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&
!executeDispatch<HashedDictionary>(block, arguments, result, dict_ptr) &&
@ -829,12 +815,7 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getName()))
{
throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
}
context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName());
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&
!executeDispatch<HashedDictionary>(block, arguments, result, dict_ptr) &&
@ -1093,12 +1074,7 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getName()))
{
throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
}
context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName());
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&
!executeDispatch<HashedDictionary>(block, arguments, result, dict_ptr) &&
@ -1670,12 +1646,7 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getName()))
{
throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
}
context.checkAccess(AccessType::dictGetHierarchy, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName());
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr) &&
!executeDispatch<HashedDictionary>(block, arguments, result, dict_ptr) &&
@ -1839,12 +1810,7 @@ private:
auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue<String>());
const auto dict_ptr = dict.get();
if (!context.hasDictionaryAccessRights(dict_ptr->getName()))
{
throw Exception{"For function " + getName() + ", cannot access dictionary "
+ dict->getName() + " on database " + context.getCurrentDatabase(), ErrorCodes::DICTIONARY_ACCESS_DENIED};
}
context.checkAccess(AccessType::dictIsIn, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName());
if (!executeDispatch<FlatDictionary>(block, arguments, result, dict_ptr)
&& !executeDispatch<HashedDictionary>(block, arguments, result, dict_ptr)

6
dbms/src/Functions/addressToLine.cpp
View File

@ -13,6 +13,7 @@
#include <Functions/FunctionFactory.h>
#include <IO/WriteBufferFromArena.h>
#include <IO/WriteHelpers.h>
#include <Access/AccessFlags.h>
#include <Interpreters/Context.h>
#include <mutex>
@ -28,7 +29,6 @@ namespace ErrorCodes
extern const int ILLEGAL_COLUMN;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int FUNCTION_NOT_ALLOWED;
}
class FunctionAddressToLine : public IFunction
@ -37,9 +37,7 @@ public:
static constexpr auto name = "addressToLine";
static FunctionPtr create(const Context & context)
{
if (!context.getSettingsRef().allow_introspection_functions)
throw Exception("Introspection functions are disabled, because setting 'allow_introspection_functions' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED);
context.checkAccess(AccessType::addressToLine);
return std::make_shared<FunctionAddressToLine>();
}

5
dbms/src/Functions/addressToSymbol.cpp
View File

@ -7,6 +7,7 @@
#include <Functions/IFunctionImpl.h>
#include <Functions/FunctionHelpers.h>
#include <Functions/FunctionFactory.h>
#include <Access/AccessFlags.h>
#include <Interpreters/Context.h>
#include <IO/WriteHelpers.h>
@ -19,7 +20,6 @@ namespace ErrorCodes
extern const int ILLEGAL_COLUMN;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int FUNCTION_NOT_ALLOWED;
}
class FunctionAddressToSymbol : public IFunction
@ -28,8 +28,7 @@ public:
static constexpr auto name = "addressToSymbol";
static FunctionPtr create(const Context & context)
{
if (!context.getSettingsRef().allow_introspection_functions)
throw Exception("Introspection functions are disabled, because setting 'allow_introspection_functions' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED);
context.checkAccess(AccessType::addressToSymbol);
return std::make_shared<FunctionAddressToSymbol>();
}

6
dbms/src/Functions/demange.cpp
View File

@ -6,6 +6,7 @@
#include <Functions/FunctionHelpers.h>
#include <Functions/FunctionFactory.h>
#include <IO/WriteHelpers.h>
#include <Access/AccessFlags.h>
#include <Interpreters/Context.h>
@ -17,7 +18,6 @@ namespace ErrorCodes
extern const int ILLEGAL_COLUMN;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int FUNCTION_NOT_ALLOWED;
}
class FunctionDemangle : public IFunction
@ -26,9 +26,7 @@ public:
static constexpr auto name = "demangle";
static FunctionPtr create(const Context & context)
{
if (!context.getSettingsRef().allow_introspection_functions)
throw Exception("Introspection functions are disabled, because setting 'allow_introspection_functions' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED);
context.checkAccess(AccessType::demangle);
return std::make_shared<FunctionDemangle>();
}

13
dbms/src/Functions/h3EdgeLengthM.cpp
View File

@ -4,14 +4,22 @@
# include <DataTypes/DataTypesNumber.h>
# include <Functions/FunctionFactory.h>
# include <Functions/IFunction.h>
# include <IO/WriteHelpers.h>
# include <Common/typeid_cast.h>
# include <ext/range.h>
# include <h3api.h>
# include <constants.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ARGUMENT_OUT_OF_BOUND;
}
// Average metric edge length of H3 hexagon. The edge length `e` for given resolution `res` can
// be used for converting metric search radius `radius` to hexagon search ring size `k` that is
// used by `H3kRing` function. For small enough search area simple flat approximation can be used,
@ -50,7 +58,10 @@ public:
for (const auto row : ext::range(0, input_rows_count))
{
const int resolution = col_hindex->getUInt(row);
const UInt64 resolution = col_hindex->getUInt(row);
if (resolution > MAX_H3_RES)
throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
Float64 res = edgeLengthM(resolution);

66
dbms/src/Functions/h3GetBaseCell.cpp Normal file
View File

@ -0,0 +1,66 @@
#include "config_functions.h"
#if USE_H3
# include <Columns/ColumnsNumber.h>
# include <DataTypes/DataTypesNumber.h>
# include <Functions/FunctionFactory.h>
# include <Functions/IFunction.h>
# include <Common/typeid_cast.h>
# include <ext/range.h>
# include <h3api.h>
namespace DB
{
class FunctionH3GetBaseCell : public IFunction
{
public:
static constexpr auto name = "h3GetBaseCell";
static FunctionPtr create(const Context &) { return std::make_shared<FunctionH3GetBaseCell>(); }
std::string getName() const override { return name; }
size_t getNumberOfArguments() const override { return 1; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
auto arg = arguments[0].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return std::make_shared<DataTypeUInt8>();
}
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
{
const auto col_hindex = block.getByPosition(arguments[0]).column.get();
auto dst = ColumnVector<UInt8>::create();
auto & dst_data = dst->getData();
dst_data.resize(input_rows_count);
for (const auto row : ext::range(0, input_rows_count))
{
const UInt64 hindex = col_hindex->getUInt(row);
UInt8 res = h3GetBaseCell(hindex);
dst_data[row] = res;
}
block.getByPosition(result).column = std::move(dst);
}
};
void registerFunctionH3GetBaseCell(FunctionFactory & factory)
{
factory.registerFunction<FunctionH3GetBaseCell>();
}
}
#endif

83
dbms/src/Functions/h3HexAreaM2.cpp Normal file
View File

@ -0,0 +1,83 @@
#include "config_functions.h"
#if USE_H3
# include <Columns/ColumnsNumber.h>
# include <DataTypes/DataTypesNumber.h>
# include <Functions/FunctionFactory.h>
# include <Functions/IFunction.h>
# include <IO/WriteHelpers.h>
# include <Common/typeid_cast.h>
# include <ext/range.h>
# if __has_include(<h3/h3api.h>)
# include <h3/h3api.h>
# include <h3/constants.h>
# else
# include <h3api.h>
# include <constants.h>
# endif
namespace DB
{
namespace ErrorCodes
{
extern const int ARGUMENT_OUT_OF_BOUND;
}
class FunctionH3HexAreaM2 : public IFunction
{
public:
static constexpr auto name = "h3HexAreaM2";
static FunctionPtr create(const Context &) { return std::make_shared<FunctionH3HexAreaM2>(); }
std::string getName() const override { return name; }
size_t getNumberOfArguments() const override { return 1; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
auto arg = arguments[0].get();
if (!WhichDataType(arg).isUInt8())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt8",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return std::make_shared<DataTypeFloat64>();
}
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
{
const auto col_hindex = block.getByPosition(arguments[0]).column.get();
auto dst = ColumnVector<Float64>::create();
auto & dst_data = dst->getData();
dst_data.resize(input_rows_count);
for (const auto row : ext::range(0, input_rows_count))
{
const UInt64 resolution = col_hindex->getUInt(row);
if (resolution > MAX_H3_RES)
throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
Float64 res = hexAreaM2(resolution);
dst_data[row] = res;
}
block.getByPosition(result).column = std::move(dst);
}
};
void registerFunctionH3HexAreaM2(FunctionFactory & factory)
{
factory.registerFunction<FunctionH3HexAreaM2>();
}
}
#endif

74
dbms/src/Functions/h3IndexesAreNeighbors.cpp Normal file
View File

@ -0,0 +1,74 @@
#include "config_functions.h"
#if USE_H3
# include <Columns/ColumnsNumber.h>
# include <DataTypes/DataTypesNumber.h>
# include <Functions/FunctionFactory.h>
# include <Functions/IFunction.h>
# include <Common/typeid_cast.h>
# include <ext/range.h>
# include <h3api.h>
namespace DB
{
class FunctionH3IndexesAreNeighbors : public IFunction
{
public:
static constexpr auto name = "h3IndexesAreNeighbors";
static FunctionPtr create(const Context &) { return std::make_shared<FunctionH3IndexesAreNeighbors>(); }
std::string getName() const override { return name; }
size_t getNumberOfArguments() const override { return 2; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
auto arg = arguments[0].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
arg = arguments[1].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be UInt64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return std::make_shared<DataTypeUInt8>();
}
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
{
const auto col_hindex_origin = block.getByPosition(arguments[0]).column.get();
const auto col_hindex_dest = block.getByPosition(arguments[1]).column.get();
auto dst = ColumnVector<UInt8>::create();
auto & dst_data = dst->getData();
dst_data.resize(input_rows_count);
for (const auto row : ext::range(0, input_rows_count))
{
const UInt64 hindex_origin = col_hindex_origin->getUInt(row);
const UInt64 hindex_dest = col_hindex_dest->getUInt(row);
UInt8 res = h3IndexesAreNeighbors(hindex_origin, hindex_dest);
dst_data[row] = res;
}
block.getByPosition(result).column = std::move(dst);
}
};
void registerFunctionH3IndexesAreNeighbors(FunctionFactory & factory)
{
factory.registerFunction<FunctionH3IndexesAreNeighbors>();
}
}
#endif

91
dbms/src/Functions/h3ToChildren.cpp Normal file
View File

@ -0,0 +1,91 @@
#include "config_functions.h"
#if USE_H3
# include <Columns/ColumnArray.h>
# include <Columns/ColumnsNumber.h>
# include <DataTypes/DataTypeArray.h>
# include <DataTypes/DataTypesNumber.h>
# include <Functions/FunctionFactory.h>
# include <Functions/IFunction.h>
# include <Common/typeid_cast.h>
# include <ext/range.h>
# include <h3api.h>
namespace DB
{
class FunctionH3ToChildren : public IFunction
{
public:
static constexpr auto name = "h3ToChildren";
static FunctionPtr create(const Context &) { return std::make_shared<FunctionH3ToChildren>(); }
std::string getName() const override { return name; }
size_t getNumberOfArguments() const override { return 2; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
auto arg = arguments[0].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
arg = arguments[1].get();
if (!WhichDataType(arg).isUInt8())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be UInt8",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>());
}
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
{
const auto col_hindex = block.getByPosition(arguments[0]).column.get();
const auto col_resolution = block.getByPosition(arguments[1]).column.get();
auto dst = ColumnArray::create(ColumnUInt64::create());
auto & dst_data = dst->getData();
auto & dst_offsets = dst->getOffsets();
dst_offsets.resize(input_rows_count);
auto current_offset = 0;
std::vector<H3Index> hindex_vec;
for (const auto row : ext::range(0, input_rows_count))
{
const UInt64 parent_hindex = col_hindex->getUInt(row);
const UInt8 child_resolution = col_resolution->getUInt(row);
const auto vec_size = maxH3ToChildrenSize(parent_hindex, child_resolution);
hindex_vec.resize(vec_size);
h3ToChildren(parent_hindex, child_resolution, hindex_vec.data());
dst_data.reserve(dst_data.size() + vec_size);
for (auto hindex : hindex_vec)
{
if (hindex != 0)
{
++current_offset;
dst_data.insert(hindex);
}
}
dst_offsets[row] = current_offset;
}
block.getByPosition(result).column = std::move(dst);
}
};
void registerFunctionH3ToChildren(FunctionFactory & factory)
{
factory.registerFunction<FunctionH3ToChildren>();
}
}
#endif

74
dbms/src/Functions/h3ToParent.cpp Normal file
View File

@ -0,0 +1,74 @@
#include "config_functions.h"
#if USE_H3
# include <Columns/ColumnsNumber.h>
# include <DataTypes/DataTypesNumber.h>
# include <Functions/FunctionFactory.h>
# include <Functions/IFunction.h>
# include <Common/typeid_cast.h>
# include <ext/range.h>
# include <h3api.h>
namespace DB
{
class FunctionH3ToParent : public IFunction
{
public:
static constexpr auto name = "h3ToParent";
static FunctionPtr create(const Context &) { return std::make_shared<FunctionH3ToParent>(); }
std::string getName() const override { return name; }
size_t getNumberOfArguments() const override { return 2; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
auto arg = arguments[0].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
arg = arguments[1].get();
if (!WhichDataType(arg).isUInt8())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be UInt8",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return std::make_shared<DataTypeUInt64>();
}
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
{
const auto col_hindex = block.getByPosition(arguments[0]).column.get();
const auto col_resolution = block.getByPosition(arguments[1]).column.get();
auto dst = ColumnVector<UInt64>::create();
auto & dst_data = dst->getData();
dst_data.resize(input_rows_count);
for (const auto row : ext::range(0, input_rows_count))
{
const UInt64 hindex = col_hindex->getUInt(row);
const UInt8 resolution = col_resolution->getUInt(row);
UInt64 res = h3ToParent(hindex, resolution);
dst_data[row] = res;
}
block.getByPosition(result).column = std::move(dst);
}
};
void registerFunctionH3ToParent(FunctionFactory & factory)
{
factory.registerFunction<FunctionH3ToParent>();
}
}
#endif

82
dbms/src/Functions/h3ToString.cpp Normal file
View File

@ -0,0 +1,82 @@
#include "config_functions.h"
#if USE_H3
# include <Columns/ColumnString.h>
# include <DataTypes/DataTypeString.h>
# include <Functions/FunctionFactory.h>
# include <Functions/IFunction.h>
# include <Common/typeid_cast.h>
# include <h3api.h>
# define H3_INDEX_STRING_LENGTH 17 // includes \0 terminator
namespace DB
{
class FunctionH3ToString : public IFunction
{
public:
static constexpr auto name = "h3ToString";
static FunctionPtr create(const Context &) { return std::make_shared<FunctionH3ToString>(); }
std::string getName() const override { return name; }
size_t getNumberOfArguments() const override { return 1; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
auto arg = arguments[0].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return std::make_shared<DataTypeString>();
}
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
{
const auto col_hindex = block.getByPosition(arguments[0]).column.get();
auto col_res = ColumnString::create();
auto & vec_res = col_res->getChars();
auto & vec_offsets = col_res->getOffsets();
vec_offsets.resize(input_rows_count);
vec_res.resize_fill(input_rows_count * H3_INDEX_STRING_LENGTH, '\0');
char * begin = reinterpret_cast<char *>(vec_res.data());
char * pos = begin;
for (size_t i = 0; i < input_rows_count; ++i)
{
const UInt64 hindex = col_hindex->getUInt(i);
if (!h3IsValid(hindex))
{
throw Exception("Invalid H3 index: " + std::to_string(hindex), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
h3ToString(hindex, pos, H3_INDEX_STRING_LENGTH);
// move to end of the index
while (*pos != '\0')
{
pos++;
}
vec_offsets[i] = ++pos - begin;
}
vec_res.resize(pos - begin);
block.getByPosition(result).column = std::move(col_res);
}
};
void registerFunctionH3ToString(FunctionFactory & factory)
{
factory.registerFunction<FunctionH3ToString>();
}
}
#endif

14
dbms/src/Functions/registerFunctionsGeo.cpp
View File

@ -19,6 +19,13 @@ void registerFunctionH3EdgeLengthM(FunctionFactory &);
void registerFunctionH3GetResolution(FunctionFactory &);
void registerFunctionH3IsValid(FunctionFactory &);
void registerFunctionH3KRing(FunctionFactory &);
void registerFunctionH3GetBaseCell(FunctionFactory &);
void registerFunctionH3ToParent(FunctionFactory &);
void registerFunctionH3ToChildren(FunctionFactory &);
void registerFunctionH3IndexesAreNeighbors(FunctionFactory &);
void registerFunctionStringToH3(FunctionFactory &);
void registerFunctionH3ToString(FunctionFactory &);
void registerFunctionH3HexAreaM2(FunctionFactory &);
#endif
@ -38,6 +45,13 @@ void registerFunctionsGeo(FunctionFactory & factory)
registerFunctionH3GetResolution(factory);
registerFunctionH3IsValid(factory);
registerFunctionH3KRing(factory);
registerFunctionH3GetBaseCell(factory);
registerFunctionH3ToParent(factory);
registerFunctionH3ToChildren(factory);
registerFunctionH3IndexesAreNeighbors(factory);
registerFunctionStringToH3(factory);
registerFunctionH3ToString(factory);
registerFunctionH3HexAreaM2(factory);
#endif
}

97
dbms/src/Functions/stringToH3.cpp Normal file
View File

@ -0,0 +1,97 @@
#include "config_functions.h"
#if USE_H3
# include <Functions/GatherUtils/GatherUtils.h>
# include <Functions/GatherUtils/Sources.h>
# include <DataTypes/DataTypeString.h>
# include <DataTypes/DataTypesNumber.h>
# include <Columns/ColumnString.h>
# include <Functions/FunctionFactory.h>
# include <Functions/IFunction.h>
# include <Common/typeid_cast.h>
# include <h3api.h>
namespace DB
{
using namespace GatherUtils;
class FunctionStringToH3 : public IFunction
{
public:
static constexpr auto name = "stringToH3";
static FunctionPtr create(const Context &) { return std::make_shared<FunctionStringToH3>(); }
std::string getName() const override { return name; }
size_t getNumberOfArguments() const override { return 1; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
auto arg = arguments[0].get();
if (!WhichDataType(arg).isStringOrFixedString())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be String or FixedString",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return std::make_shared<DataTypeUInt64>();
}
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
{
const auto col_hindex = block.getByPosition(arguments[0]).column.get();
auto dst = ColumnVector<UInt64>::create();
auto & dst_data = dst->getData();
dst_data.resize(input_rows_count);
if (auto * h3index = checkAndGetColumn<ColumnString>(col_hindex))
execute<StringSource>(StringSource(*h3index), dst_data);
else if (auto * h3index_fixed = checkAndGetColumn<ColumnFixedString>(col_hindex))
execute<FixedStringSource>(FixedStringSource(*h3index_fixed), dst_data);
else if (const ColumnConst * h3index_const = checkAndGetColumnConst<ColumnString>(col_hindex))
execute<ConstSource<StringSource>>(ConstSource<StringSource>(*h3index_const), dst_data);
else if (const ColumnConst * h3index_const_fixed = checkAndGetColumnConst<ColumnFixedString>(col_hindex))
execute<ConstSource<FixedStringSource>>(ConstSource<FixedStringSource>(*h3index_const_fixed), dst_data);
else
throw Exception("Illegal column as argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN);
block.getByPosition(result).column = std::move(dst);
}
private:
template <typename H3IndexSource>
static void execute(H3IndexSource h3index_source, PaddedPODArray<UInt64> & res_data)
{
size_t row_num = 0;
while (!h3index_source.isEnd())
{
auto h3index = h3index_source.getWhole();
// covert to std::string and get the c_str to have the delimiting \0 at the end.
auto h3index_str = StringRef(h3index.data, h3index.size).toString();
res_data[row_num] = stringToH3(h3index_str.c_str());
if (res_data[row_num] == 0)
{
throw Exception("Invalid H3 index: " + h3index_str, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
h3index_source.next();
++row_num;
}
}
};
void registerFunctionStringToH3(FunctionFactory & factory)
{
factory.registerFunction<FunctionStringToH3>();
}
}
#endif

2
dbms/src/IO/MMapReadBufferFromFileDescriptor.cpp
View File

@ -105,7 +105,7 @@ off_t MMapReadBufferFromFileDescriptor::getPositionInFile()
return count();
}
off_t MMapReadBufferFromFileDescriptor::doSeek(off_t offset, int whence)
off_t MMapReadBufferFromFileDescriptor::seek(off_t offset, int whence)
{
off_t new_pos;
if (whence == SEEK_SET)

6
dbms/src/IO/MMapReadBufferFromFileDescriptor.h
View File

@ -13,14 +13,14 @@ namespace DB
*/
class MMapReadBufferFromFileDescriptor : public ReadBufferFromFileBase
{
public:
off_t seek(off_t off, int whence) override;
protected:
MMapReadBufferFromFileDescriptor() {}
void init(int fd_, size_t offset, size_t length_);
void init(int fd_, size_t offset);
off_t doSeek(off_t off, int whence) override;
public:
MMapReadBufferFromFileDescriptor(int fd_, size_t offset_, size_t length_);

2
dbms/src/IO/ReadBufferAIO.cpp
View File

@ -149,7 +149,7 @@ bool ReadBufferAIO::nextImpl()
return true;
}
off_t ReadBufferAIO::doSeek(off_t off, int whence)
off_t ReadBufferAIO::seek(off_t off, int whence)
{
off_t new_pos_in_file;

4
dbms/src/IO/ReadBufferAIO.h
View File

@ -40,11 +40,11 @@ public:
std::string getFileName() const override { return filename; }
int getFD() const override { return fd; }
off_t seek(off_t off, int whence) override;
private:
///
bool nextImpl() override;
///
off_t doSeek(off_t off, int whence) override;
/// Synchronously read the data.
void synchronousRead();
/// Get data from an asynchronous request.

10
dbms/src/IO/ReadBufferFromFileBase.cpp
View File

@ -3,13 +3,12 @@
namespace DB
{
ReadBufferFromFileBase::ReadBufferFromFileBase()
: BufferWithOwnMemory<ReadBuffer>(0)
ReadBufferFromFileBase::ReadBufferFromFileBase() : BufferWithOwnMemory<SeekableReadBuffer>(0)
{
}
ReadBufferFromFileBase::ReadBufferFromFileBase(size_t buf_size, char * existing_memory, size_t alignment)
: BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment)
: BufferWithOwnMemory<SeekableReadBuffer>(buf_size, existing_memory, alignment)
{
}
@ -17,9 +16,4 @@ ReadBufferFromFileBase::~ReadBufferFromFileBase()
{
}
off_t ReadBufferFromFileBase::seek(off_t off, int whence)
{
return doSeek(off, whence);
}
}

6
dbms/src/IO/ReadBufferFromFileBase.h
View File

@ -7,18 +7,18 @@
#include <IO/ReadBuffer.h>
#include <IO/BufferWithOwnMemory.h>
#include <port/clock.h>
#include "SeekableReadBuffer.h"
namespace DB
{
class ReadBufferFromFileBase : public BufferWithOwnMemory<ReadBuffer>
class ReadBufferFromFileBase : public BufferWithOwnMemory<SeekableReadBuffer>
{
public:
ReadBufferFromFileBase();
ReadBufferFromFileBase(size_t buf_size, char * existing_memory, size_t alignment);
ReadBufferFromFileBase(ReadBufferFromFileBase &&) = default;
~ReadBufferFromFileBase() override;
off_t seek(off_t off, int whence = SEEK_SET);
virtual off_t getPositionInFile() = 0;
virtual std::string getFileName() const = 0;
virtual int getFD() const = 0;
@ -44,8 +44,6 @@ protected:
ProfileCallback profile_callback;
clockid_t clock_type{};
/// Children implementation should be able to seek backwards
virtual off_t doSeek(off_t off, int whence) = 0;
};
}

2
dbms/src/IO/ReadBufferFromFileDescriptor.cpp
View File

@ -99,7 +99,7 @@ bool ReadBufferFromFileDescriptor::nextImpl()
/// If 'offset' is small enough to stay in buffer after seek, then true seek in file does not happen.
off_t ReadBufferFromFileDescriptor::doSeek(off_t offset, int whence)
off_t ReadBufferFromFileDescriptor::seek(off_t offset, int whence)
{
off_t new_pos;
if (whence == SEEK_SET)

4
dbms/src/IO/ReadBufferFromFileDescriptor.h
View File

@ -37,10 +37,10 @@ public:
return pos_in_file - (working_buffer.end() - pos);
}
private:
/// If 'offset' is small enough to stay in buffer after seek, then true seek in file does not happen.
off_t doSeek(off_t offset, int whence) override;
off_t seek(off_t off, int whence) override;
private:
/// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout.
bool poll(size_t timeout_microseconds);
};

46
dbms/src/IO/ReadBufferFromMemory.cpp Normal file
View File

@ -0,0 +1,46 @@
#include "ReadBufferFromMemory.h"
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_SEEK_THROUGH_FILE;
extern const int SEEK_POSITION_OUT_OF_BOUND;
}
off_t ReadBufferFromMemory::seek(off_t offset, int whence)
{
if (whence == SEEK_SET)
{
if (offset >= 0 && working_buffer.begin() + offset < working_buffer.end())
{
pos = working_buffer.begin() + offset;
return size_t(pos - working_buffer.begin());
}
else
throw Exception(
"Seek position is out of bounds. "
"Offset: "
+ std::to_string(offset) + ", Max: " + std::to_string(size_t(working_buffer.end() - working_buffer.begin())),
ErrorCodes::SEEK_POSITION_OUT_OF_BOUND);
}
else if (whence == SEEK_CUR)
{
Position new_pos = pos + offset;
if (new_pos >= working_buffer.begin() && new_pos < working_buffer.end())
{
pos = new_pos;
return size_t(pos - working_buffer.begin());
}
else
throw Exception(
"Seek position is out of bounds. "
"Offset: "
+ std::to_string(offset) + ", Max: " + std::to_string(size_t(working_buffer.end() - working_buffer.begin())),
ErrorCodes::SEEK_POSITION_OUT_OF_BOUND);
}
else
throw Exception("Only SEEK_SET and SEEK_CUR seek modes allowed.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
}
}

18
dbms/src/IO/ReadBufferFromMemory.h
View File

@ -1,26 +1,30 @@
#pragma once
#include <IO/ReadBuffer.h>
#include "SeekableReadBuffer.h"
namespace DB
{
/** Allows to read from memory range.
* In comparison with just ReadBuffer, it only adds convenient constructors, that do const_cast.
* In fact, ReadBuffer will not modify data in buffer, but it requires non-const pointer.
*/
class ReadBufferFromMemory : public ReadBuffer
class ReadBufferFromMemory : public SeekableReadBuffer
{
public:
ReadBufferFromMemory(const char * buf, size_t size)
: ReadBuffer(const_cast<char *>(buf), size, 0) {}
ReadBufferFromMemory(const char * buf, size_t size) : SeekableReadBuffer(const_cast<char *>(buf), size, 0) {}
ReadBufferFromMemory(const unsigned char * buf, size_t size)
: ReadBuffer(const_cast<char *>(reinterpret_cast<const char *>(buf)), size, 0) {}
: SeekableReadBuffer(const_cast<char *>(reinterpret_cast<const char *>(buf)), size, 0)
{
}
ReadBufferFromMemory(const signed char * buf, size_t size)
: ReadBuffer(const_cast<char *>(reinterpret_cast<const char *>(buf)), size, 0) {}
: SeekableReadBuffer(const_cast<char *>(reinterpret_cast<const char *>(buf)), size, 0)
{
}
off_t seek(off_t off, int whence) override;
};
}

78
dbms/src/IO/ReadBufferFromS3.cpp
View File

@ -2,44 +2,39 @@
#if USE_AWS_S3
#include <IO/ReadBufferFromS3.h>
#include <IO/ReadBufferFromIStream.h>
# include <IO/ReadBufferFromIStream.h>
# include <IO/ReadBufferFromS3.h>
#include <common/logger_useful.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/S3Client.h>
# include <aws/s3/S3Client.h>
# include <aws/s3/model/GetObjectRequest.h>
# include <common/logger_useful.h>
# include <utility>
namespace DB
{
namespace ErrorCodes
{
extern const int S3_ERROR;
extern const int CANNOT_SEEK_THROUGH_FILE;
extern const int SEEK_POSITION_OUT_OF_BOUND;
}
ReadBufferFromS3::ReadBufferFromS3(const std::shared_ptr<Aws::S3::S3Client> & client_ptr,
const String & bucket,
const String & key,
size_t buffer_size_): ReadBuffer(nullptr, 0)
ReadBufferFromS3::ReadBufferFromS3(
std::shared_ptr<Aws::S3::S3Client> client_ptr_, const String & bucket_, const String & key_, size_t buffer_size_)
: SeekableReadBuffer(nullptr, 0), client_ptr(std::move(client_ptr_)), bucket(bucket_), key(key_), buffer_size(buffer_size_)
{
Aws::S3::Model::GetObjectRequest req;
req.SetBucket(bucket);
req.SetKey(key);
Aws::S3::Model::GetObjectOutcome outcome = client_ptr->GetObject(req);
if (outcome.IsSuccess())
{
read_result = outcome.GetResultWithOwnership();
impl = std::make_unique<ReadBufferFromIStream>(read_result.GetBody(), buffer_size_);
}
else
throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
}
bool ReadBufferFromS3::nextImpl()
{
if (!initialized)
{
impl = initialize();
initialized = true;
}
if (!impl->next())
return false;
internal_buffer = impl->buffer();
@ -47,6 +42,43 @@ bool ReadBufferFromS3::nextImpl()
return true;
}
off_t ReadBufferFromS3::seek(off_t offset_, int whence)
{
if (initialized)
throw Exception("Seek is allowed only before first read attempt from the buffer.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
if (whence != SEEK_SET)
throw Exception("Only SEEK_SET mode is allowed.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
if (offset_ < 0)
throw Exception("Seek position is out of bounds. Offset: " + std::to_string(offset_), ErrorCodes::SEEK_POSITION_OUT_OF_BOUND);
offset = offset_;
return offset;
}
std::unique_ptr<ReadBuffer> ReadBufferFromS3::initialize()
{
LOG_TRACE(log, "Read S3 object. Bucket: " + bucket + ", Key: " + key + ", Offset: " + std::to_string(offset));
Aws::S3::Model::GetObjectRequest req;
req.SetBucket(bucket);
req.SetKey(key);
if (offset != 0)
req.SetRange("bytes=" + std::to_string(offset) + "-");
Aws::S3::Model::GetObjectOutcome outcome = client_ptr->GetObject(req);
if (outcome.IsSuccess())
{
read_result = outcome.GetResultWithOwnership();
return std::make_unique<ReadBufferFromIStream>(read_result.GetBody(), buffer_size);
}
else
throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
}
}
#endif

41
dbms/src/IO/ReadBufferFromS3.h
View File

@ -4,37 +4,50 @@
#if USE_AWS_S3
#include <memory>
# include <memory>
#include <IO/HTTPCommon.h>
#include <IO/ReadBuffer.h>
#include <aws/s3/model/GetObjectResult.h>
# include <IO/HTTPCommon.h>
# include <IO/ReadBuffer.h>
# include <aws/s3/model/GetObjectResult.h>
# include "SeekableReadBuffer.h"
namespace Aws::S3
{
class S3Client;
class S3Client;
}
namespace DB
{
/** Perform S3 HTTP GET request and provide response to read.
*/
class ReadBufferFromS3 : public ReadBuffer
/**
* Perform S3 HTTP GET request and provide response to read.
*/
class ReadBufferFromS3 : public SeekableReadBuffer
{
private:
Logger * log = &Logger::get("ReadBufferFromS3");
std::shared_ptr<Aws::S3::S3Client> client_ptr;
String bucket;
String key;
size_t buffer_size;
bool initialized = false;
off_t offset = 0;
Aws::S3::Model::GetObjectResult read_result;
protected:
std::unique_ptr<ReadBuffer> impl;
Logger * log = &Logger::get("ReadBufferFromS3");
public:
explicit ReadBufferFromS3(const std::shared_ptr<Aws::S3::S3Client> & client_ptr,
const String & bucket,
const String & key,
explicit ReadBufferFromS3(
std::shared_ptr<Aws::S3::S3Client> client_ptr_,
const String & bucket_,
const String & key_,
size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE);
bool nextImpl() override;
off_t seek(off_t off, int whence) override;
private:
std::unique_ptr<ReadBuffer> initialize();
};
}

25
dbms/src/IO/SeekableReadBuffer.h Normal file
View File

@ -0,0 +1,25 @@
#pragma once
#include <IO/ReadBuffer.h>
namespace DB
{
class SeekableReadBuffer : public ReadBuffer
{
public:
SeekableReadBuffer(Position ptr, size_t size)
: ReadBuffer(ptr, size) {}
SeekableReadBuffer(Position ptr, size_t size, size_t offset)
: ReadBuffer(ptr, size, offset) {}
/**
* Shifts buffer current position to given offset.
* @param off Offset.
* @param whence Seek mode (@see SEEK_SET, @see SEEK_CUR).
* @return New position from the begging of underlying buffer / file.
*/
virtual off_t seek(off_t off, int whence) = 0;
};
}

69
dbms/src/IO/WriteBufferFromS3.cpp
View File

@ -2,21 +2,20 @@
#if USE_AWS_S3
#include <IO/WriteBufferFromS3.h>
#include <IO/WriteHelpers.h>
# include <IO/WriteBufferFromS3.h>
# include <IO/WriteHelpers.h>
#include <common/logger_useful.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/CreateMultipartUploadRequest.h>
#include <aws/s3/model/UploadPartRequest.h>
#include <aws/s3/model/CompleteMultipartUploadRequest.h>
# include <aws/s3/S3Client.h>
# include <aws/s3/model/CompleteMultipartUploadRequest.h>
# include <aws/s3/model/CreateMultipartUploadRequest.h>
# include <aws/s3/model/UploadPartRequest.h>
# include <common/logger_useful.h>
#include <utility>
# include <utility>
namespace DB
{
// S3 protocol does not allow to have multipart upload with more than 10000 parts.
// In case server does not return an error on exceeding that number, we print a warning
// because custom S3 implementation may allow relaxed requirements on that.
@ -34,15 +33,14 @@ WriteBufferFromS3::WriteBufferFromS3(
const String & bucket_,
const String & key_,
size_t minimum_upload_part_size_,
size_t buffer_size_
)
size_t buffer_size_)
: BufferWithOwnMemory<WriteBuffer>(buffer_size_, nullptr, 0)
, bucket(bucket_)
, key(key_)
, client_ptr(std::move(client_ptr_))
, minimum_upload_part_size {minimum_upload_part_size_}
, temporary_buffer {std::make_unique<WriteBufferFromString>(buffer_string)}
, last_part_size {0}
, minimum_upload_part_size{minimum_upload_part_size_}
, temporary_buffer{std::make_unique<WriteBufferFromOwnString>()}
, last_part_size{0}
{
initiate();
}
@ -60,9 +58,9 @@ void WriteBufferFromS3::nextImpl()
if (last_part_size > minimum_upload_part_size)
{
temporary_buffer->finalize();
writePart(buffer_string);
writePart(temporary_buffer->str());
last_part_size = 0;
temporary_buffer = std::make_unique<WriteBufferFromString>(buffer_string);
temporary_buffer = std::make_unique<WriteBufferFromOwnString>();
}
}
@ -70,11 +68,9 @@ void WriteBufferFromS3::nextImpl()
void WriteBufferFromS3::finalize()
{
next();
temporary_buffer->finalize();
if (!buffer_string.empty())
{
writePart(buffer_string);
}
writePart(temporary_buffer->str());
complete();
}
@ -104,7 +100,7 @@ void WriteBufferFromS3::initiate()
if (outcome.IsSuccess())
{
upload_id = outcome.GetResult().GetUploadId();
LOG_DEBUG(log, "Multipart upload initiated. Upload id = " + upload_id);
LOG_DEBUG(log, "Multipart upload initiated. Upload id: " << upload_id);
}
else
throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
@ -113,6 +109,9 @@ void WriteBufferFromS3::initiate()
void WriteBufferFromS3::writePart(const String & data)
{
if (data.empty())
return;
if (part_tags.size() == S3_WARN_MAX_PARTS)
{
// Don't throw exception here by ourselves but leave the decision to take by S3 server.
@ -130,11 +129,18 @@ void WriteBufferFromS3::writePart(const String & data)
auto outcome = client_ptr->UploadPart(req);
LOG_TRACE(
log, "Writing part. Bucket: " << bucket << ", Key: " << key << ", Upload_id: " << upload_id << ", Data size: " << data.size());
if (outcome.IsSuccess())
{
auto etag = outcome.GetResult().GetETag();
part_tags.push_back(etag);
LOG_DEBUG(log, "Write part " + std::to_string(part_tags.size()) + " finished. Upload id = " + upload_id + ". Etag = " + etag);
total_size += data.size();
LOG_DEBUG(
log,
"Writing part finished. "
<< "Total parts: " << part_tags.size() << ", Upload_id: " << upload_id << ", Etag: " << etag);
}
else
throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
@ -143,24 +149,29 @@ void WriteBufferFromS3::writePart(const String & data)
void WriteBufferFromS3::complete()
{
LOG_DEBUG(log, "Completing multipart upload. Bucket: " + bucket + ", Key: " + key + ", Upload_id: " + upload_id);
Aws::S3::Model::CompleteMultipartUploadRequest req;
req.SetBucket(bucket);
req.SetKey(key);
req.SetUploadId(upload_id);
Aws::S3::Model::CompletedMultipartUpload multipart_upload;
for (size_t i = 0; i < part_tags.size(); ++i)
if (!part_tags.empty())
{
Aws::S3::Model::CompletedPart part;
multipart_upload.AddParts(part.WithETag(part_tags[i]).WithPartNumber(i + 1));
}
Aws::S3::Model::CompletedMultipartUpload multipart_upload;
for (size_t i = 0; i < part_tags.size(); ++i)
{
Aws::S3::Model::CompletedPart part;
multipart_upload.AddParts(part.WithETag(part_tags[i]).WithPartNumber(i + 1));
}
req.SetMultipartUpload(multipart_upload);
req.SetMultipartUpload(multipart_upload);
}
auto outcome = client_ptr->CompleteMultipartUpload(req);
if (outcome.IsSuccess())
LOG_DEBUG(log, "Multipart upload completed. Upload_id = " + upload_id);
LOG_DEBUG(log, "Multipart upload completed. Upload_id: " << upload_id);
else
throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR);
}

Some files were not shown because too many files have changed in this diff Show More