mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Merge branch 'fix_undrop_query' of github.com:yariks5s/ClickHouse into fix_undrop_query
This commit is contained in:
yariks5s
commit
9227305950
29
.github/PULL_REQUEST_TEMPLATE.md
vendored
29
.github/PULL_REQUEST_TEMPLATE.md
vendored
@ -40,3 +40,32 @@ At a minimum, the following information should be added (but add more as needed)
|
||||
|
||||
|
||||
> Information about CI checks: https://clickhouse.com/docs/en/development/continuous-integration/
|
||||
|
||||
---
|
||||
### Modify your CI run:
|
||||
**NOTE:** If your merge the PR with modified CI you **MUST KNOW** what you are doing
|
||||
**NOTE:** Set desired options before CI starts or re-push after updates
|
||||
|
||||
#### Run only:
|
||||
- [ ] <!---ci_set_integration--> Integration tests
|
||||
- [ ] <!---ci_set_arm--> Integration tests (arm64)
|
||||
- [ ] <!---ci_set_stateless--> Stateless tests (release)
|
||||
- [ ] <!---ci_set_stateless_asan--> Stateless tests (asan)
|
||||
- [ ] <!---ci_set_stateful--> Stateful tests (release)
|
||||
- [ ] <!---ci_set_stateful_asan--> Stateful tests (asan)
|
||||
- [ ] <!---ci_set_reduced--> No sanitizers
|
||||
- [ ] <!---ci_set_analyzer--> Tests with analyzer
|
||||
- [ ] <!---ci_set_fast--> Fast tests
|
||||
- [ ] <!---job_package_debug--> Only package_debug build
|
||||
- [ ] <!---PLACE_YOUR_TAG_CONFIGURED_IN_ci_config.py_FILE_HERE--> Add your CI variant description here
|
||||
|
||||
#### CI options:
|
||||
- [ ] <!---do_not_test--> do not test (only style check)
|
||||
- [ ] <!---no_merge_commit--> disable merge-commit (no merge from master before tests)
|
||||
- [ ] <!---no_ci_cache--> disable CI cache (job reuse)
|
||||
|
||||
#### Only specified batches in multi-batch jobs:
|
||||
- [ ] <!---batch_0--> 1
|
||||
- [ ] <!---batch_1--> 2
|
||||
- [ ] <!---batch_2--> 3
|
||||
- [ ] <!---batch_3--> 4
|
||||
|
||||
@ -26,4 +26,4 @@
|
||||
|
||||
## To run only specified batches for multi-batch job(s)
|
||||
#batch_2
|
||||
#btach_1_2_3
|
||||
#batch_1_2_3
|
||||
|
||||
@ -281,15 +281,15 @@ void EnvironmentImpl::nodeIdImpl(NodeId& id)
|
||||
/// #include <sys/ioctl.h>
|
||||
#if defined(sun) || defined(__sun)
|
||||
#include <sys/sockio.h>
|
||||
#include <netdb.h>
|
||||
#include <net/if.h>
|
||||
#include <net/if_arp.h>
|
||||
#endif
|
||||
/// #include <sys/socket.h>
|
||||
/// #include <sys/types.h>
|
||||
/// #include <netinet/in.h>
|
||||
/// #include <net/if.h>
|
||||
/// #include <arpa/inet.h>
|
||||
/// #include <netdb.h>
|
||||
/// #include <net/if.h>
|
||||
/// #include <net/if_arp.h>
|
||||
/// #include <unistd.h>
|
||||
|
||||
|
||||
|
||||
@ -31,7 +31,7 @@
|
||||
namespace Poco {
|
||||
|
||||
|
||||
#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD)
|
||||
#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD) || (POCO_OS == POCO_OS_SOLARIS)
|
||||
union semun
|
||||
{
|
||||
int val;
|
||||
|
||||
@ -31,7 +31,7 @@
|
||||
namespace Poco {
|
||||
|
||||
|
||||
#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD)
|
||||
#if (POCO_OS == POCO_OS_LINUX) || (POCO_OS == POCO_OS_ANDROID) || (POCO_OS == POCO_OS_CYGWIN) || (POCO_OS == POCO_OS_FREE_BSD) || (POCO_OS == POCO_OS_SOLARIS)
|
||||
union semun
|
||||
{
|
||||
int val;
|
||||
|
||||
@ -9,6 +9,10 @@ elseif (OS_DARWIN OR OS_FREEBSD)
|
||||
target_compile_definitions (_poco_net PUBLIC POCO_HAVE_FD_POLL)
|
||||
endif ()
|
||||
|
||||
if (OS_SUNOS)
|
||||
target_link_libraries (_poco_net PUBLIC socket nsl)
|
||||
endif ()
|
||||
|
||||
# TODO: remove these warning exclusions
|
||||
target_compile_options (_poco_net
|
||||
PRIVATE
|
||||
|
||||
@ -30,7 +30,7 @@ if (SANITIZE)
|
||||
elseif (SANITIZE STREQUAL "thread")
|
||||
set (TSAN_FLAGS "-fsanitize=thread")
|
||||
if (COMPILER_CLANG)
|
||||
set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-blacklist=${PROJECT_SOURCE_DIR}/tests/tsan_suppressions.txt")
|
||||
set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-ignorelist=${PROJECT_SOURCE_DIR}/tests/tsan_ignorelist.txt")
|
||||
endif()
|
||||
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}")
|
||||
@ -48,7 +48,7 @@ if (SANITIZE)
|
||||
set(UBSAN_FLAGS "${UBSAN_FLAGS} -fno-sanitize=unsigned-integer-overflow")
|
||||
endif()
|
||||
if (COMPILER_CLANG)
|
||||
set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-blacklist=${PROJECT_SOURCE_DIR}/tests/ubsan_suppressions.txt")
|
||||
set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-ignorelist=${PROJECT_SOURCE_DIR}/tests/ubsan_ignorelist.txt")
|
||||
endif()
|
||||
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${UBSAN_FLAGS}")
|
||||
|
||||
@ -86,6 +86,8 @@ elseif (OS_DARWIN)
|
||||
target_compile_definitions(_c-ares PRIVATE -D_DARWIN_C_SOURCE)
|
||||
elseif (OS_FREEBSD)
|
||||
target_include_directories(_c-ares SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/c-ares-cmake/freebsd")
|
||||
elseif (OS_SUNOS)
|
||||
target_include_directories(_c-ares SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/c-ares-cmake/solaris")
|
||||
endif()
|
||||
|
||||
add_library(ch_contrib::c-ares ALIAS _c-ares)
|
||||
|
||||
104
contrib/c-ares-cmake/solaris/ares_build.h
Normal file
104
contrib/c-ares-cmake/solaris/ares_build.h
Normal file
@ -0,0 +1,104 @@
|
||||
/* include/ares_build.h. Generated from ares_build.h.in by configure. */
|
||||
#ifndef __CARES_BUILD_H
|
||||
#define __CARES_BUILD_H
|
||||
|
||||
|
||||
/* Copyright (C) 2009 - 2021 by Daniel Stenberg et al
|
||||
*
|
||||
* Permission to use, copy, modify, and distribute this software and its
|
||||
* documentation for any purpose and without fee is hereby granted, provided
|
||||
* that the above copyright notice appear in all copies and that both that
|
||||
* copyright notice and this permission notice appear in supporting
|
||||
* documentation, and that the name of M.I.T. not be used in advertising or
|
||||
* publicity pertaining to distribution of the software without specific,
|
||||
* written prior permission. M.I.T. makes no representations about the
|
||||
* suitability of this software for any purpose. It is provided "as is"
|
||||
* without express or implied warranty.
|
||||
*/
|
||||
|
||||
/* ================================================================ */
|
||||
/* NOTES FOR CONFIGURE CAPABLE SYSTEMS */
|
||||
/* ================================================================ */
|
||||
|
||||
/*
|
||||
* NOTE 1:
|
||||
* -------
|
||||
*
|
||||
* Nothing in this file is intended to be modified or adjusted by the
|
||||
* c-ares library user nor by the c-ares library builder.
|
||||
*
|
||||
* If you think that something actually needs to be changed, adjusted
|
||||
* or fixed in this file, then, report it on the c-ares development
|
||||
* mailing list: http://lists.haxx.se/listinfo/c-ares/
|
||||
*
|
||||
* This header file shall only export symbols which are 'cares' or 'CARES'
|
||||
* prefixed, otherwise public name space would be polluted.
|
||||
*
|
||||
* NOTE 2:
|
||||
* -------
|
||||
*
|
||||
* Right now you might be staring at file ares_build.h.in or ares_build.h,
|
||||
* this is due to the following reason:
|
||||
*
|
||||
* On systems capable of running the configure script, the configure process
|
||||
* will overwrite the distributed ares_build.h file with one that is suitable
|
||||
* and specific to the library being configured and built, which is generated
|
||||
* from the ares_build.h.in template file.
|
||||
*
|
||||
*/
|
||||
|
||||
/* ================================================================ */
|
||||
/* DEFINITION OF THESE SYMBOLS SHALL NOT TAKE PLACE ANYWHERE ELSE */
|
||||
/* ================================================================ */
|
||||
|
||||
#ifdef CARES_TYPEOF_ARES_SOCKLEN_T
|
||||
# error "CARES_TYPEOF_ARES_SOCKLEN_T shall not be defined except in ares_build.h"
|
||||
Error Compilation_aborted_CARES_TYPEOF_ARES_SOCKLEN_T_already_defined
|
||||
#endif
|
||||
|
||||
#define CARES_HAVE_ARPA_NAMESER_H 1
|
||||
#define CARES_HAVE_ARPA_NAMESER_COMPAT_H 1
|
||||
|
||||
/* ================================================================ */
|
||||
/* EXTERNAL INTERFACE SETTINGS FOR CONFIGURE CAPABLE SYSTEMS ONLY */
|
||||
/* ================================================================ */
|
||||
|
||||
/* Configure process defines this to 1 when it finds out that system */
|
||||
/* header file ws2tcpip.h must be included by the external interface. */
|
||||
/* #undef CARES_PULL_WS2TCPIP_H */
|
||||
#ifdef CARES_PULL_WS2TCPIP_H
|
||||
# ifndef WIN32_LEAN_AND_MEAN
|
||||
# define WIN32_LEAN_AND_MEAN
|
||||
# endif
|
||||
# include <windows.h>
|
||||
# include <winsock2.h>
|
||||
# include <ws2tcpip.h>
|
||||
#endif
|
||||
|
||||
/* Configure process defines this to 1 when it finds out that system */
|
||||
/* header file sys/types.h must be included by the external interface. */
|
||||
#define CARES_PULL_SYS_TYPES_H 1
|
||||
#ifdef CARES_PULL_SYS_TYPES_H
|
||||
# include <sys/types.h>
|
||||
#endif
|
||||
|
||||
/* Configure process defines this to 1 when it finds out that system */
|
||||
/* header file sys/socket.h must be included by the external interface. */
|
||||
#define CARES_PULL_SYS_SOCKET_H 1
|
||||
#ifdef CARES_PULL_SYS_SOCKET_H
|
||||
# include <sys/socket.h>
|
||||
#endif
|
||||
|
||||
/* Integral data type used for ares_socklen_t. */
|
||||
#define CARES_TYPEOF_ARES_SOCKLEN_T socklen_t
|
||||
|
||||
/* Data type definition of ares_socklen_t. */
|
||||
typedef CARES_TYPEOF_ARES_SOCKLEN_T ares_socklen_t;
|
||||
|
||||
/* Integral data type used for ares_ssize_t. */
|
||||
#define CARES_TYPEOF_ARES_SSIZE_T ssize_t
|
||||
|
||||
/* Data type definition of ares_ssize_t. */
|
||||
typedef CARES_TYPEOF_ARES_SSIZE_T ares_ssize_t;
|
||||
|
||||
#endif /* __CARES_BUILD_H */
|
||||
503
contrib/c-ares-cmake/solaris/ares_config.h
Normal file
503
contrib/c-ares-cmake/solaris/ares_config.h
Normal file
@ -0,0 +1,503 @@
|
||||
/* src/lib/ares_config.h. Generated from ares_config.h.in by configure. */
|
||||
/* src/lib/ares_config.h.in. Generated from configure.ac by autoheader. */
|
||||
|
||||
/* Define if building universal (internal helper macro) */
|
||||
/* #undef AC_APPLE_UNIVERSAL_BUILD */
|
||||
|
||||
/* define this if ares is built for a big endian system */
|
||||
/* #undef ARES_BIG_ENDIAN */
|
||||
|
||||
/* Defined for build that exposes internal static functions for testing. */
|
||||
/* #undef CARES_EXPOSE_STATICS */
|
||||
|
||||
/* a suitable file/device to read random data from */
|
||||
#define CARES_RANDOM_FILE "/dev/urandom"
|
||||
|
||||
/* Defined for build with symbol hiding. */
|
||||
#define CARES_SYMBOL_HIDING 1
|
||||
|
||||
/* Definition to make a library symbol externally visible. */
|
||||
#define CARES_SYMBOL_SCOPE_EXTERN __attribute__ ((__visibility__ ("default")))
|
||||
|
||||
/* the signed version of size_t */
|
||||
#define CARES_TYPEOF_ARES_SSIZE_T ssize_t
|
||||
|
||||
/* Use resolver library to configure cares */
|
||||
/* #undef CARES_USE_LIBRESOLV */
|
||||
|
||||
/* if a /etc/inet dir is being used */
|
||||
#define ETC_INET 1
|
||||
|
||||
/* Define to the type of arg 2 for gethostname. */
|
||||
#define GETHOSTNAME_TYPE_ARG2 int
|
||||
|
||||
/* Define to the type qualifier of arg 1 for getnameinfo. */
|
||||
#define GETNAMEINFO_QUAL_ARG1 const
|
||||
|
||||
/* Define to the type of arg 1 for getnameinfo. */
|
||||
#define GETNAMEINFO_TYPE_ARG1 struct sockaddr *
|
||||
|
||||
/* Define to the type of arg 2 for getnameinfo. */
|
||||
#define GETNAMEINFO_TYPE_ARG2 socklen_t
|
||||
|
||||
/* Define to the type of args 4 and 6 for getnameinfo. */
|
||||
#define GETNAMEINFO_TYPE_ARG46 socklen_t
|
||||
|
||||
/* Define to the type of arg 7 for getnameinfo. */
|
||||
#define GETNAMEINFO_TYPE_ARG7 int
|
||||
|
||||
/* Specifies the number of arguments to getservbyport_r */
|
||||
#define GETSERVBYPORT_R_ARGS 5
|
||||
|
||||
/* Specifies the size of the buffer to pass to getservbyport_r */
|
||||
#define GETSERVBYPORT_R_BUFSIZE 4096
|
||||
|
||||
/* Define to 1 if you have AF_INET6. */
|
||||
#define HAVE_AF_INET6 1
|
||||
|
||||
/* Define to 1 if you have the <arpa/inet.h> header file. */
|
||||
#define HAVE_ARPA_INET_H 1
|
||||
|
||||
/* Define to 1 if you have the <arpa/nameser_compat.h> header file. */
|
||||
#define HAVE_ARPA_NAMESER_COMPAT_H 1
|
||||
|
||||
/* Define to 1 if you have the <arpa/nameser.h> header file. */
|
||||
#define HAVE_ARPA_NAMESER_H 1
|
||||
|
||||
/* Define to 1 if you have the <assert.h> header file. */
|
||||
#define HAVE_ASSERT_H 1
|
||||
|
||||
/* Define to 1 if you have the `bitncmp' function. */
|
||||
/* #undef HAVE_BITNCMP */
|
||||
|
||||
/* Define to 1 if bool is an available type. */
|
||||
#define HAVE_BOOL_T 1
|
||||
|
||||
/* Define to 1 if you have the clock_gettime function and monotonic timer. */
|
||||
#define HAVE_CLOCK_GETTIME_MONOTONIC 1
|
||||
|
||||
/* Define to 1 if you have the closesocket function. */
|
||||
/* #undef HAVE_CLOSESOCKET */
|
||||
|
||||
/* Define to 1 if you have the CloseSocket camel case function. */
|
||||
/* #undef HAVE_CLOSESOCKET_CAMEL */
|
||||
|
||||
/* Define to 1 if you have the connect function. */
|
||||
#define HAVE_CONNECT 1
|
||||
|
||||
/* define if the compiler supports basic C++11 syntax */
|
||||
#define HAVE_CXX11 1
|
||||
|
||||
/* Define to 1 if you have the <dlfcn.h> header file. */
|
||||
#define HAVE_DLFCN_H 1
|
||||
|
||||
/* Define to 1 if you have the <errno.h> header file. */
|
||||
#define HAVE_ERRNO_H 1
|
||||
|
||||
/* Define to 1 if you have the fcntl function. */
|
||||
#define HAVE_FCNTL 1
|
||||
|
||||
/* Define to 1 if you have the <fcntl.h> header file. */
|
||||
#define HAVE_FCNTL_H 1
|
||||
|
||||
/* Define to 1 if you have a working fcntl O_NONBLOCK function. */
|
||||
#define HAVE_FCNTL_O_NONBLOCK 1
|
||||
|
||||
/* Define to 1 if you have the freeaddrinfo function. */
|
||||
#define HAVE_FREEADDRINFO 1
|
||||
|
||||
/* Define to 1 if you have a working getaddrinfo function. */
|
||||
#define HAVE_GETADDRINFO 1
|
||||
|
||||
/* Define to 1 if the getaddrinfo function is threadsafe. */
|
||||
#define HAVE_GETADDRINFO_THREADSAFE 1
|
||||
|
||||
/* Define to 1 if you have the getenv function. */
|
||||
#define HAVE_GETENV 1
|
||||
|
||||
/* Define to 1 if you have the gethostbyaddr function. */
|
||||
#define HAVE_GETHOSTBYADDR 1
|
||||
|
||||
/* Define to 1 if you have the gethostbyname function. */
|
||||
#define HAVE_GETHOSTBYNAME 1
|
||||
|
||||
/* Define to 1 if you have the gethostname function. */
|
||||
#define HAVE_GETHOSTNAME 1
|
||||
|
||||
/* Define to 1 if you have the getnameinfo function. */
|
||||
#define HAVE_GETNAMEINFO 1
|
||||
|
||||
/* Define to 1 if you have the getservbyport_r function. */
|
||||
#define HAVE_GETSERVBYPORT_R 1
|
||||
|
||||
/* Define to 1 if you have the `gettimeofday' function. */
|
||||
#define HAVE_GETTIMEOFDAY 1
|
||||
|
||||
/* Define to 1 if you have the `if_indextoname' function. */
|
||||
#define HAVE_IF_INDEXTONAME 1
|
||||
|
||||
/* Define to 1 if you have a IPv6 capable working inet_net_pton function. */
|
||||
/* #undef HAVE_INET_NET_PTON */
|
||||
|
||||
/* Define to 1 if you have a IPv6 capable working inet_ntop function. */
|
||||
#define HAVE_INET_NTOP 1
|
||||
|
||||
/* Define to 1 if you have a IPv6 capable working inet_pton function. */
|
||||
#define HAVE_INET_PTON 1
|
||||
|
||||
/* Define to 1 if you have the <inttypes.h> header file. */
|
||||
#define HAVE_INTTYPES_H 1
|
||||
|
||||
/* Define to 1 if you have the ioctl function. */
|
||||
#define HAVE_IOCTL 1
|
||||
|
||||
/* Define to 1 if you have the ioctlsocket function. */
|
||||
/* #undef HAVE_IOCTLSOCKET */
|
||||
|
||||
/* Define to 1 if you have the IoctlSocket camel case function. */
|
||||
/* #undef HAVE_IOCTLSOCKET_CAMEL */
|
||||
|
||||
/* Define to 1 if you have a working IoctlSocket camel case FIONBIO function.
|
||||
*/
|
||||
/* #undef HAVE_IOCTLSOCKET_CAMEL_FIONBIO */
|
||||
|
||||
/* Define to 1 if you have a working ioctlsocket FIONBIO function. */
|
||||
/* #undef HAVE_IOCTLSOCKET_FIONBIO */
|
||||
|
||||
/* Define to 1 if you have a working ioctl FIONBIO function. */
|
||||
/* #undef HAVE_IOCTL_FIONBIO */
|
||||
|
||||
/* Define to 1 if you have a working ioctl SIOCGIFADDR function. */
|
||||
/* #undef HAVE_IOCTL_SIOCGIFADDR */
|
||||
|
||||
/* Define to 1 if you have the `resolve' library (-lresolve). */
|
||||
/* #undef HAVE_LIBRESOLVE */
|
||||
|
||||
/* Define to 1 if you have the <limits.h> header file. */
|
||||
#define HAVE_LIMITS_H 1
|
||||
|
||||
/* if your compiler supports LL */
|
||||
#define HAVE_LL 1
|
||||
|
||||
/* Define to 1 if the compiler supports the 'long long' data type. */
|
||||
#define HAVE_LONGLONG 1
|
||||
|
||||
/* Define to 1 if you have the malloc.h header file. */
|
||||
#define HAVE_MALLOC_H 1
|
||||
|
||||
/* Define to 1 if you have the memory.h header file. */
|
||||
#define HAVE_MEMORY_H 1
|
||||
|
||||
/* Define to 1 if you have the MSG_NOSIGNAL flag. */
|
||||
#define HAVE_MSG_NOSIGNAL 1
|
||||
|
||||
/* Define to 1 if you have the <netdb.h> header file. */
|
||||
#define HAVE_NETDB_H 1
|
||||
|
||||
/* Define to 1 if you have the <netinet/in.h> header file. */
|
||||
#define HAVE_NETINET_IN_H 1
|
||||
|
||||
/* Define to 1 if you have the <netinet/tcp.h> header file. */
|
||||
#define HAVE_NETINET_TCP_H 1
|
||||
|
||||
/* Define to 1 if you have the <net/if.h> header file. */
|
||||
#define HAVE_NET_IF_H 1
|
||||
|
||||
/* Define to 1 if you have PF_INET6. */
|
||||
#define HAVE_PF_INET6 1
|
||||
|
||||
/* Define to 1 if you have the recv function. */
|
||||
#define HAVE_RECV 1
|
||||
|
||||
/* Define to 1 if you have the recvfrom function. */
|
||||
#define HAVE_RECVFROM 1
|
||||
|
||||
/* Define to 1 if you have the send function. */
|
||||
#define HAVE_SEND 1
|
||||
|
||||
/* Define to 1 if you have the setsockopt function. */
|
||||
#define HAVE_SETSOCKOPT 1
|
||||
|
||||
/* Define to 1 if you have a working setsockopt SO_NONBLOCK function. */
|
||||
/* #undef HAVE_SETSOCKOPT_SO_NONBLOCK */
|
||||
|
||||
/* Define to 1 if you have the <signal.h> header file. */
|
||||
#define HAVE_SIGNAL_H 1
|
||||
|
||||
/* Define to 1 if sig_atomic_t is an available typedef. */
|
||||
#define HAVE_SIG_ATOMIC_T 1
|
||||
|
||||
/* Define to 1 if sig_atomic_t is already defined as volatile. */
|
||||
/* #undef HAVE_SIG_ATOMIC_T_VOLATILE */
|
||||
|
||||
/* Define to 1 if your struct sockaddr_in6 has sin6_scope_id. */
|
||||
#define HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID 1
|
||||
|
||||
/* Define to 1 if you have the socket function. */
|
||||
#define HAVE_SOCKET 1
|
||||
|
||||
/* Define to 1 if you have the <socket.h> header file. */
|
||||
/* #undef HAVE_SOCKET_H */
|
||||
|
||||
/* Define to 1 if you have the <stdbool.h> header file. */
|
||||
#define HAVE_STDBOOL_H 1
|
||||
|
||||
/* Define to 1 if you have the <stdint.h> header file. */
|
||||
#define HAVE_STDINT_H 1
|
||||
|
||||
/* Define to 1 if you have the <stdio.h> header file. */
|
||||
#define HAVE_STDIO_H 1
|
||||
|
||||
/* Define to 1 if you have the <stdlib.h> header file. */
|
||||
#define HAVE_STDLIB_H 1
|
||||
|
||||
/* Define to 1 if you have the strcasecmp function. */
|
||||
#define HAVE_STRCASECMP 1
|
||||
|
||||
/* Define to 1 if you have the strcmpi function. */
|
||||
/* #undef HAVE_STRCMPI */
|
||||
|
||||
/* Define to 1 if you have the strdup function. */
|
||||
#define HAVE_STRDUP 1
|
||||
|
||||
/* Define to 1 if you have the stricmp function. */
|
||||
/* #undef HAVE_STRICMP */
|
||||
|
||||
/* Define to 1 if you have the <strings.h> header file. */
|
||||
#define HAVE_STRINGS_H 1
|
||||
|
||||
/* Define to 1 if you have the <string.h> header file. */
|
||||
#define HAVE_STRING_H 1
|
||||
|
||||
/* Define to 1 if you have the strncasecmp function. */
|
||||
#define HAVE_STRNCASECMP 1
|
||||
|
||||
/* Define to 1 if you have the strncmpi function. */
|
||||
/* #undef HAVE_STRNCMPI */
|
||||
|
||||
/* Define to 1 if you have the strnicmp function. */
|
||||
/* #undef HAVE_STRNICMP */
|
||||
|
||||
/* Define to 1 if you have the <stropts.h> header file. */
|
||||
#define HAVE_STROPTS_H 1
|
||||
|
||||
/* Define to 1 if you have struct addrinfo. */
|
||||
#define HAVE_STRUCT_ADDRINFO 1
|
||||
|
||||
/* Define to 1 if you have struct in6_addr. */
|
||||
#define HAVE_STRUCT_IN6_ADDR 1
|
||||
|
||||
/* Define to 1 if you have struct sockaddr_in6. */
|
||||
#define HAVE_STRUCT_SOCKADDR_IN6 1
|
||||
|
||||
/* if struct sockaddr_storage is defined */
|
||||
#define HAVE_STRUCT_SOCKADDR_STORAGE 1
|
||||
|
||||
/* Define to 1 if you have the timeval struct. */
|
||||
#define HAVE_STRUCT_TIMEVAL 1
|
||||
|
||||
/* Define to 1 if you have the <sys/ioctl.h> header file. */
|
||||
#define HAVE_SYS_IOCTL_H 1
|
||||
|
||||
/* Define to 1 if you have the <sys/param.h> header file. */
|
||||
#define HAVE_SYS_PARAM_H 1
|
||||
|
||||
/* Define to 1 if you have the <sys/select.h> header file. */
|
||||
#define HAVE_SYS_SELECT_H 1
|
||||
|
||||
/* Define to 1 if you have the <sys/socket.h> header file. */
|
||||
#define HAVE_SYS_SOCKET_H 1
|
||||
|
||||
/* Define to 1 if you have the <sys/stat.h> header file. */
|
||||
#define HAVE_SYS_STAT_H 1
|
||||
|
||||
/* Define to 1 if you have the <sys/time.h> header file. */
|
||||
#define HAVE_SYS_TIME_H 1
|
||||
|
||||
/* Define to 1 if you have the <sys/types.h> header file. */
|
||||
#define HAVE_SYS_TYPES_H 1
|
||||
|
||||
/* Define to 1 if you have the <sys/uio.h> header file. */
|
||||
#define HAVE_SYS_UIO_H 1
|
||||
|
||||
/* Define to 1 if you have the <time.h> header file. */
|
||||
#define HAVE_TIME_H 1
|
||||
|
||||
/* Define to 1 if you have the <unistd.h> header file. */
|
||||
#define HAVE_UNISTD_H 1
|
||||
|
||||
/* Define to 1 if you have the windows.h header file. */
|
||||
/* #undef HAVE_WINDOWS_H */
|
||||
|
||||
/* Define to 1 if you have the winsock2.h header file. */
|
||||
/* #undef HAVE_WINSOCK2_H */
|
||||
|
||||
/* Define to 1 if you have the winsock.h header file. */
|
||||
/* #undef HAVE_WINSOCK_H */
|
||||
|
||||
/* Define to 1 if you have the writev function. */
|
||||
#define HAVE_WRITEV 1
|
||||
|
||||
/* Define to 1 if you have the ws2tcpip.h header file. */
|
||||
/* #undef HAVE_WS2TCPIP_H */
|
||||
|
||||
/* Define if __system_property_get exists. */
|
||||
/* #undef HAVE___SYSTEM_PROPERTY_GET */
|
||||
|
||||
/* Define to the sub-directory where libtool stores uninstalled libraries. */
|
||||
#define LT_OBJDIR ".libs/"
|
||||
|
||||
/* Define to 1 if you need the malloc.h header file even with stdlib.h */
|
||||
/* #undef NEED_MALLOC_H */
|
||||
|
||||
/* Define to 1 if you need the memory.h header file even with stdlib.h */
|
||||
/* #undef NEED_MEMORY_H */
|
||||
|
||||
/* Define to 1 if _REENTRANT preprocessor symbol must be defined. */
|
||||
#define NEED_REENTRANT 1
|
||||
|
||||
/* Define to 1 if _THREAD_SAFE preprocessor symbol must be defined. */
|
||||
/* #undef NEED_THREAD_SAFE */
|
||||
|
||||
/* cpu-machine-OS */
|
||||
#define OS "x86_64-pc-solaris2.11"
|
||||
|
||||
/* Name of package */
|
||||
#define PACKAGE "c-ares"
|
||||
|
||||
/* Define to the address where bug reports for this package should be sent. */
|
||||
#define PACKAGE_BUGREPORT "c-ares mailing list: http://lists.haxx.se/listinfo/c-ares"
|
||||
|
||||
/* Define to the full name of this package. */
|
||||
#define PACKAGE_NAME "c-ares"
|
||||
|
||||
/* Define to the full name and version of this package. */
|
||||
#define PACKAGE_STRING "c-ares 1.18.1"
|
||||
|
||||
/* Define to the one symbol short name of this package. */
|
||||
#define PACKAGE_TARNAME "c-ares"
|
||||
|
||||
/* Define to the home page for this package. */
|
||||
#define PACKAGE_URL ""
|
||||
|
||||
/* Define to the version of this package. */
|
||||
#define PACKAGE_VERSION "1.18.1"
|
||||
|
||||
/* Define to the type qualifier pointed by arg 5 for recvfrom. */
|
||||
#define RECVFROM_QUAL_ARG5
|
||||
|
||||
/* Define to the type of arg 1 for recvfrom. */
|
||||
#define RECVFROM_TYPE_ARG1 int
|
||||
|
||||
/* Define to the type pointed by arg 2 for recvfrom. */
|
||||
#define RECVFROM_TYPE_ARG2 void
|
||||
|
||||
/* Define to 1 if the type pointed by arg 2 for recvfrom is void. */
|
||||
#define RECVFROM_TYPE_ARG2_IS_VOID 1
|
||||
|
||||
/* Define to the type of arg 3 for recvfrom. */
|
||||
#define RECVFROM_TYPE_ARG3 size_t
|
||||
|
||||
/* Define to the type of arg 4 for recvfrom. */
|
||||
#define RECVFROM_TYPE_ARG4 int
|
||||
|
||||
/* Define to the type pointed by arg 5 for recvfrom. */
|
||||
#define RECVFROM_TYPE_ARG5 struct sockaddr
|
||||
|
||||
/* Define to 1 if the type pointed by arg 5 for recvfrom is void. */
|
||||
/* #undef RECVFROM_TYPE_ARG5_IS_VOID */
|
||||
|
||||
/* Define to the type pointed by arg 6 for recvfrom. */
|
||||
#define RECVFROM_TYPE_ARG6 void
|
||||
|
||||
/* Define to 1 if the type pointed by arg 6 for recvfrom is void. */
|
||||
#define RECVFROM_TYPE_ARG6_IS_VOID 1
|
||||
|
||||
/* Define to the function return type for recvfrom. */
|
||||
#define RECVFROM_TYPE_RETV ssize_t
|
||||
|
||||
/* Define to the type of arg 1 for recv. */
|
||||
#define RECV_TYPE_ARG1 int
|
||||
|
||||
/* Define to the type of arg 2 for recv. */
|
||||
#define RECV_TYPE_ARG2 void *
|
||||
|
||||
/* Define to the type of arg 3 for recv. */
|
||||
#define RECV_TYPE_ARG3 size_t
|
||||
|
||||
/* Define to the type of arg 4 for recv. */
|
||||
#define RECV_TYPE_ARG4 int
|
||||
|
||||
/* Define to the function return type for recv. */
|
||||
#define RECV_TYPE_RETV ssize_t
|
||||
|
||||
/* Define as the return type of signal handlers (`int' or `void'). */
|
||||
#define RETSIGTYPE void
|
||||
|
||||
/* Define to the type qualifier of arg 2 for send. */
|
||||
#define SEND_QUAL_ARG2 const
|
||||
|
||||
/* Define to the type of arg 1 for send. */
|
||||
#define SEND_TYPE_ARG1 int
|
||||
|
||||
/* Define to the type of arg 2 for send. */
|
||||
#define SEND_TYPE_ARG2 void *
|
||||
|
||||
/* Define to the type of arg 3 for send. */
|
||||
#define SEND_TYPE_ARG3 size_t
|
||||
|
||||
/* Define to the type of arg 4 for send. */
|
||||
#define SEND_TYPE_ARG4 int
|
||||
|
||||
/* Define to the function return type for send. */
|
||||
#define SEND_TYPE_RETV ssize_t
|
||||
|
||||
/* Define to 1 if all of the C90 standard headers exist (not just the ones
|
||||
required in a freestanding environment). This macro is provided for
|
||||
backward compatibility; new code need not use it. */
|
||||
#define STDC_HEADERS 1
|
||||
|
||||
/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. This
|
||||
macro is obsolete. */
|
||||
#define TIME_WITH_SYS_TIME 1
|
||||
|
||||
/* Define to disable non-blocking sockets. */
|
||||
/* #undef USE_BLOCKING_SOCKETS */
|
||||
|
||||
/* Version number of package */
|
||||
#define VERSION "1.18.1"
|
||||
|
||||
/* Define to avoid automatic inclusion of winsock.h */
|
||||
/* #undef WIN32_LEAN_AND_MEAN */
|
||||
|
||||
/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
|
||||
significant byte first (like Motorola and SPARC, unlike Intel). */
|
||||
#if defined AC_APPLE_UNIVERSAL_BUILD
|
||||
# if defined __BIG_ENDIAN__
|
||||
# define WORDS_BIGENDIAN 1
|
||||
# endif
|
||||
#else
|
||||
# ifndef WORDS_BIGENDIAN
|
||||
/* # undef WORDS_BIGENDIAN */
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/* Define to 1 if OS is AIX. */
|
||||
#ifndef _ALL_SOURCE
|
||||
/* # undef _ALL_SOURCE */
|
||||
#endif
|
||||
|
||||
/* Number of bits in a file offset, on hosts where this is settable. */
|
||||
/* #undef _FILE_OFFSET_BITS */
|
||||
|
||||
/* Define for large files, on AIX-style hosts. */
|
||||
/* #undef _LARGE_FILES */
|
||||
|
||||
/* Define to empty if `const' does not conform to ANSI C. */
|
||||
/* #undef const */
|
||||
|
||||
/* Type to use in place of in_addr_t when system does not provide it. */
|
||||
/* #undef in_addr_t */
|
||||
|
||||
/* Define to `unsigned int' if <sys/types.h> does not define. */
|
||||
/* #undef size_t */
|
||||
2
contrib/cppkafka
vendored
2
contrib/cppkafka
vendored
@ -1 +1 @@
|
||||
Subproject commit 5a119f689f8a4d90d10a9635e7ee2bee5c127de1
|
||||
Subproject commit 9c5ea0e332486961e612deacc6e3f0c1874c688d
|
||||
@ -51,3 +51,8 @@
|
||||
#define USE_OPENSSL
|
||||
#define USE_THREADS_POSIX
|
||||
#define USE_ARES
|
||||
|
||||
#ifdef __illumos__
|
||||
#define HAVE_POSIX_STRERROR_R 1
|
||||
#define HAVE_STRERROR_R 1
|
||||
#endif
|
||||
|
||||
2
contrib/xxHash
vendored
2
contrib/xxHash
vendored
@ -1 +1 @@
|
||||
Subproject commit 3078dc6039f8c0bffcb1904f81cfe6b2c3209435
|
||||
Subproject commit bbb27a5efb85b92a0486cf361a8635715a53f6ba
|
||||
@ -7,7 +7,7 @@ add_library(xxHash ${SRCS})
|
||||
target_include_directories(xxHash SYSTEM BEFORE INTERFACE "${LIBRARY_DIR}")
|
||||
|
||||
# XXH_INLINE_ALL - Make all functions inline, with implementations being directly included within xxhash.h. Inlining functions is beneficial for speed on small keys.
|
||||
# https://github.com/Cyan4973/xxHash/tree/v0.8.1#build-modifiers
|
||||
# https://github.com/Cyan4973/xxHash/tree/v0.8.2#build-modifiers
|
||||
target_compile_definitions(xxHash PUBLIC XXH_INLINE_ALL)
|
||||
|
||||
add_library(ch_contrib::xxHash ALIAS xxHash)
|
||||
|
||||
@ -181,7 +181,15 @@ function fuzz
|
||||
-- --path db \
|
||||
--logger.console=0 \
|
||||
--logger.log=server.log 2>&1 | tee -a stderr.log >> server.log 2>&1 &
|
||||
server_pid=$(pidof clickhouse-server)
|
||||
for _ in {1..30}
|
||||
do
|
||||
if clickhouse-client --query "select 1"
|
||||
then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
server_pid=$(cat /var/run/clickhouse-server/clickhouse-server.pid)
|
||||
|
||||
kill -0 $server_pid
|
||||
|
||||
|
||||
@ -4,7 +4,11 @@ sidebar_position: 50
|
||||
sidebar_label: MySQL
|
||||
---
|
||||
|
||||
# MySQL
|
||||
import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge';
|
||||
|
||||
# MySQL Database Engine
|
||||
|
||||
<CloudNotSupportedBadge />
|
||||
|
||||
Allows to connect to databases on a remote MySQL server and perform `INSERT` and `SELECT` queries to exchange data between ClickHouse and MySQL.
|
||||
|
||||
|
||||
@ -4,7 +4,11 @@ sidebar_position: 138
|
||||
sidebar_label: MySQL
|
||||
---
|
||||
|
||||
# MySQL
|
||||
import CloudAvailableBadge from '@theme/badges/CloudAvailableBadge';
|
||||
|
||||
# MySQL Table Engine
|
||||
|
||||
<CloudAvailableBadge />
|
||||
|
||||
The MySQL engine allows you to perform `SELECT` and `INSERT` queries on data that is stored on a remote MySQL server.
|
||||
|
||||
|
||||
@ -8,6 +8,10 @@ sidebar_label: PostgreSQL
|
||||
|
||||
The PostgreSQL engine allows to perform `SELECT` and `INSERT` queries on data that is stored on a remote PostgreSQL server.
|
||||
|
||||
:::note
|
||||
Currently, only PostgreSQL versions 12 and up are supported.
|
||||
:::
|
||||
|
||||
## Creating a Table {#creating-a-table}
|
||||
|
||||
``` sql
|
||||
|
||||
@ -75,14 +75,14 @@ This is the output of `DESCRIBE`. Down further in this guide the field type cho
|
||||
</TabItem>
|
||||
<TabItem value="selfmanaged" label="Self-managed">
|
||||
|
||||
1. Download the snapshot of the dataset from February 2021: [cell_towers.csv.xz](https://datasets.clickhouse.com/cell_towers.csv.xz) (729 MB).
|
||||
1. Download the snapshot of the dataset from February 2021: [cell_towers.csv.xz](https://datasets.clickhouse.com/cell_towers.csv.xz) (686 MB).
|
||||
|
||||
2. Validate the integrity (optional step):
|
||||
```bash
|
||||
md5sum cell_towers.csv.xz
|
||||
```
|
||||
```response
|
||||
8cf986f4a0d9f12c6f384a0e9192c908 cell_towers.csv.xz
|
||||
8a797f7bdb55faba93f6cbc37d47b037 cell_towers.csv.xz
|
||||
```
|
||||
|
||||
3. Decompress it with the following command:
|
||||
@ -132,7 +132,7 @@ SELECT radio, count() AS c FROM cell_towers GROUP BY radio ORDER BY c DESC
|
||||
┌─radio─┬────────c─┐
|
||||
│ UMTS │ 20686487 │
|
||||
│ LTE │ 12101148 │
|
||||
│ GSM │ 9931312 │
|
||||
│ GSM │ 9931304 │
|
||||
│ CDMA │ 556344 │
|
||||
│ NR │ 867 │
|
||||
└───────┴──────────┘
|
||||
|
||||
@ -262,7 +262,7 @@ The required version can be downloaded with `curl` or `wget` from repository htt
|
||||
After that downloaded archives should be unpacked and installed with installation scripts. Example for the latest stable version:
|
||||
|
||||
``` bash
|
||||
LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \
|
||||
LATEST_VERSION=$(curl -s https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/utils/list-versions/version_date.tsv | \
|
||||
grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1)
|
||||
export LATEST_VERSION
|
||||
|
||||
|
||||
@ -87,6 +87,7 @@ The BACKUP and RESTORE statements take a list of DATABASE and TABLE names, a des
|
||||
- `structure_only`: if enabled, allows to only backup or restore the CREATE statements without the data of tables
|
||||
- `storage_policy`: storage policy for the tables being restored. See [Using Multiple Block Devices for Data Storage](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes). This setting is only applicable to the `RESTORE` command. The specified storage policy applies only to tables with an engine from the `MergeTree` family.
|
||||
- `s3_storage_class`: the storage class used for S3 backup. For example, `STANDARD`
|
||||
- `azure_attempt_to_create_container`: when using Azure Blob Storage, whether the specified container will try to be created if it doesn't exist. Default: true.
|
||||
|
||||
### Usage examples
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@ slug: /en/operations/monitoring
|
||||
sidebar_position: 45
|
||||
sidebar_label: Monitoring
|
||||
description: You can monitor the utilization of hardware resources and also ClickHouse server metrics.
|
||||
keywords: [monitoring, observability, advanced dashboard, dashboard, observability dashboard]
|
||||
---
|
||||
|
||||
# Monitoring
|
||||
@ -15,11 +16,11 @@ You can monitor:
|
||||
- Utilization of hardware resources.
|
||||
- ClickHouse server metrics.
|
||||
|
||||
## Built-in observability dashboard
|
||||
## Built-in advanced observability dashboard
|
||||
|
||||
<img width="400" alt="Screenshot 2023-11-12 at 6 08 58 PM" src="https://github.com/ClickHouse/ClickHouse/assets/3936029/2bd10011-4a47-4b94-b836-d44557c7fdc1" />
|
||||
|
||||
ClickHouse comes with a built-in observability dashboard feature which can be accessed by `$HOST:$PORT/dashboard` (requires user and password) that shows the following metrics:
|
||||
ClickHouse comes with a built-in advanced observability dashboard feature which can be accessed by `$HOST:$PORT/dashboard` (requires user and password) that shows the following metrics:
|
||||
- Queries/second
|
||||
- CPU usage (cores)
|
||||
- Queries running
|
||||
|
||||
@ -379,6 +379,18 @@ Type: UInt64
|
||||
|
||||
Default: 0
|
||||
|
||||
## max_waiting_queries
|
||||
|
||||
Limit on total number of concurrently waiting queries. Execution of a waiting query is blocked while required tables are loading asynchronously (see `async_load_databases`). Note that waiting queries are not counted when `max_concurrent_queries`, `max_concurrent_insert_queries`, `max_concurrent_select_queries`, `max_concurrent_queries_for_user` and `max_concurrent_queries_for_all_users` limits are checked. This correction is done to avoid hitting these limits just after server startup. Zero means unlimited.
|
||||
|
||||
:::note
|
||||
This setting can be modified at runtime and will take effect immediately. Queries that are already running will remain unchanged.
|
||||
:::
|
||||
|
||||
Type: UInt64
|
||||
|
||||
Default: 0
|
||||
|
||||
## max_connections
|
||||
|
||||
Max server connections.
|
||||
@ -1725,7 +1737,7 @@ Default value: `0.5`.
|
||||
|
||||
Asynchronous loading of databases and tables.
|
||||
|
||||
If `true` all non-system databases with `Ordinary`, `Atomic` and `Replicated` engine will be loaded asynchronously after the ClickHouse server start up. See `system.asynchronous_loader` table, `tables_loader_background_pool_size` and `tables_loader_foreground_pool_size` server settings. Any query that tries to access a table, that is not yet loaded, will wait for exactly this table to be started up. If load job fails, query will rethrow an error (instead of shutting down the whole server in case of `async_load_databases = false`). The table that is waited for by at least one query will be loaded with higher priority. DDL queries on a database will wait for exactly that database to be started up.
|
||||
If `true` all non-system databases with `Ordinary`, `Atomic` and `Replicated` engine will be loaded asynchronously after the ClickHouse server start up. See `system.asynchronous_loader` table, `tables_loader_background_pool_size` and `tables_loader_foreground_pool_size` server settings. Any query that tries to access a table, that is not yet loaded, will wait for exactly this table to be started up. If load job fails, query will rethrow an error (instead of shutting down the whole server in case of `async_load_databases = false`). The table that is waited for by at least one query will be loaded with higher priority. DDL queries on a database will wait for exactly that database to be started up. Also consider setting a limit `max_waiting_queries` for the total number of waiting queries.
|
||||
|
||||
If `false`, all databases are loaded when the server starts.
|
||||
|
||||
@ -2926,7 +2938,7 @@ Default: 0
|
||||
|
||||
## ignore_empty_sql_security_in_create_view_query {#ignore_empty_sql_security_in_create_view_query}
|
||||
|
||||
If true, ClickHouse doesn't write defaults for empty SQL security statement in CREATE VIEW queries.
|
||||
If true, ClickHouse doesn't write defaults for empty SQL security statement in CREATE VIEW queries.
|
||||
|
||||
:::note
|
||||
This setting is only necessary for the migration period and will become obsolete in 24.4
|
||||
|
||||
@ -405,7 +405,7 @@ Returns the name of the current user. In case of a distributed query, the name o
|
||||
SELECT currentUser();
|
||||
```
|
||||
|
||||
Alias: `user()`, `USER()`.
|
||||
Aliases: `user()`, `USER()`, `current_user()`. Aliases are case insensitive.
|
||||
|
||||
**Returned values**
|
||||
|
||||
|
||||
@ -133,6 +133,8 @@ For the query to run successfully, the following conditions must be met:
|
||||
- Both tables must have the same indices and projections.
|
||||
- Both tables must have the same storage policy.
|
||||
|
||||
If both tables have the same storage policy, use hardlink to attach partition. Otherwise, use copying the data to attach partition.
|
||||
|
||||
## REPLACE PARTITION
|
||||
|
||||
``` sql
|
||||
|
||||
@ -21,6 +21,35 @@ The queries to terminate are selected from the system.processes table using the
|
||||
|
||||
Examples:
|
||||
|
||||
First, you'll need to get the list of incomplete queries. This SQL query provides them according to those running the longest:
|
||||
|
||||
List from a single ClickHouse node:
|
||||
``` sql
|
||||
SELECT
|
||||
initial_query_id,
|
||||
query_id,
|
||||
formatReadableTimeDelta(elapsed) AS time_delta,
|
||||
query,
|
||||
*
|
||||
FROM system.processes
|
||||
WHERE query ILIKE 'SELECT%'
|
||||
ORDER BY time_delta DESC;
|
||||
```
|
||||
|
||||
List from a ClickHouse cluster:
|
||||
``` sql
|
||||
SELECT
|
||||
initial_query_id,
|
||||
query_id,
|
||||
formatReadableTimeDelta(elapsed) AS time_delta,
|
||||
query,
|
||||
*
|
||||
FROM clusterAllReplicas(default, system.processes)
|
||||
WHERE query ILIKE 'SELECT%'
|
||||
ORDER BY time_delta DESC;
|
||||
```
|
||||
|
||||
Kill the query:
|
||||
``` sql
|
||||
-- Forcibly terminates all queries with the specified query_id:
|
||||
KILL QUERY WHERE query_id='2-857d-4a57-9ee0-327da5d60a90'
|
||||
@ -44,6 +73,11 @@ A test query (`TEST`) only checks the user’s rights and displays a list of que
|
||||
|
||||
## KILL MUTATION
|
||||
|
||||
The presence of long-running or incomplete mutations often indicates that a ClickHouse service is running poorly. The asynchronous nature of mutations can cause them to consume all available resources on a system. You may need to either:
|
||||
|
||||
- Pause all new mutations, `INSERT`s , and `SELECT`s and allow the queue of mutations to complete.
|
||||
- Or manually kill some of these mutations by sending a `KILL` command.
|
||||
|
||||
``` sql
|
||||
KILL MUTATION [ON CLUSTER cluster]
|
||||
WHERE <where expression to SELECT FROM system.mutations query>
|
||||
@ -57,6 +91,39 @@ A test query (`TEST`) only checks the user’s rights and displays a list of mut
|
||||
|
||||
Examples:
|
||||
|
||||
Get a `count()` of the number of incomplete mutations:
|
||||
|
||||
Count of mutations from a single ClickHouse node:
|
||||
``` sql
|
||||
SELECT count(*)
|
||||
FROM system.mutations
|
||||
WHERE is_done = 0;
|
||||
```
|
||||
|
||||
Count of mutations from a ClickHouse cluster of replicas:
|
||||
``` sql
|
||||
SELECT count(*)
|
||||
FROM clusterAllReplicas('default', system.mutations)
|
||||
WHERE is_done = 0;
|
||||
```
|
||||
|
||||
Query the list of incomplete mutations:
|
||||
|
||||
List of mutations from a single ClickHouse node:
|
||||
``` sql
|
||||
SELECT mutation_id, *
|
||||
FROM system.mutations
|
||||
WHERE is_done = 0;
|
||||
```
|
||||
|
||||
List of mutations from a ClickHouse cluster:
|
||||
``` sql
|
||||
SELECT mutation_id, *
|
||||
FROM clusterAllReplicas('default', system.mutations)
|
||||
WHERE is_done = 0;
|
||||
```
|
||||
|
||||
Kill the mutations as needed:
|
||||
``` sql
|
||||
-- Cancel and remove all mutations of the single table:
|
||||
KILL MUTATION WHERE database = 'default' AND table = 'table'
|
||||
|
||||
@ -35,13 +35,13 @@ ClickHouse — столбцовая система управления база
|
||||
В примерах изображён только порядок расположения данных.
|
||||
То есть значения из разных столбцов хранятся отдельно, а данные одного столбца — вместе.
|
||||
|
||||
Примеры столбцовых СУБД: Vertica, Paraccel (Actian Matrix, Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise, Actian Vector), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid, kdb+.
|
||||
Примеры столбцовых СУБД: Vertica, Paraccel (Actian Matrix, Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise, Actian Vector), LucidDB, SAP HANA и прочий треш, Google Dremel, Google PowerDrill, Druid, kdb+.
|
||||
{: .grey }
|
||||
|
||||
Разный порядок хранения данных лучше подходит для разных сценариев работы.
|
||||
Сценарий работы с данными — это то, какие производятся запросы, как часто и в каком соотношении; сколько читается данных на запросы каждого вида — строк, столбцов, байтов; как соотносятся чтения и обновления данных; какой рабочий размер данных и насколько локально он используется; используются ли транзакции и с какой изолированностью; какие требования к дублированию данных и логической целостности; требования к задержкам на выполнение и пропускной способности запросов каждого вида и т. п.
|
||||
|
||||
Чем больше нагрузка на систему, тем более важной становится специализация под сценарий работы, и тем более конкретной становится эта специализация. Не существует системы, одинаково хорошо подходящей под существенно различные сценарии работы. Если система подходит под широкое множество сценариев работы, то при достаточно большой нагрузке, система будет справляться со всеми сценариями работы плохо, или справляться хорошо только с одним из сценариев работы.
|
||||
Чем больше нагрузка на систему, тем более важной становится специализация под сценарий работы, и тем более конкретной становится эта специализация. Не существует системы, одинаково хорошо подходящей под существенно различные сценарии работы. Если система подходит под широкое множество сценариев работы, то при достаточно большой нагрузке система будет справляться со всеми сценариями работы плохо, или справляться хорошо только с одним из сценариев работы.
|
||||
|
||||
## Ключевые особенности OLAP-сценария работы {#kliuchevye-osobennosti-olap-stsenariia-raboty}
|
||||
|
||||
@ -53,11 +53,11 @@ ClickHouse — столбцовая система управления база
|
||||
- запросы идут сравнительно редко (обычно не более сотни в секунду на сервер);
|
||||
- при выполнении простых запросов, допустимы задержки в районе 50 мс;
|
||||
- значения в столбцах достаточно мелкие — числа и небольшие строки (например, 60 байт на URL);
|
||||
- требуется высокая пропускная способность при обработке одного запроса (до миллиардов строк в секунду на один узел);
|
||||
- требуется высокая пропускная способность при обработке одного запроса (до миллиардов строк в секунду на один сервер);
|
||||
- транзакции отсутствуют;
|
||||
- низкие требования к консистентности данных;
|
||||
- в запросе одна большая таблица, все таблицы кроме одной маленькие;
|
||||
- результат выполнения запроса существенно меньше исходных данных — то есть данные фильтруются или агрегируются; результат выполнения помещается в оперативную память одного узла.
|
||||
- низкие требования к согласованности данных;
|
||||
- в запросе одна большая таблица, все остальные таблицы из запроса — маленькие;
|
||||
- результат выполнения запроса существенно меньше исходных данных — то есть данные фильтруются или агрегируются; результат выполнения помещается в оперативную память одного сервера.
|
||||
|
||||
Легко видеть, что OLAP-сценарий работы существенно отличается от других распространённых сценариев работы (например, OLTP или Key-Value сценариев работы). Таким образом, не имеет никакого смысла пытаться использовать OLTP-системы или системы класса «ключ — значение» для обработки аналитических запросов, если вы хотите получить приличную производительность («выше плинтуса»). Например, если вы попытаетесь использовать для аналитики MongoDB или Redis — вы получите анекдотически низкую производительность по сравнению с OLAP-СУБД.
|
||||
|
||||
@ -77,11 +77,11 @@ ClickHouse — столбцовая система управления база
|
||||
|
||||
### По вводу-выводу {#po-vvodu-vyvodu}
|
||||
|
||||
1. Для выполнения аналитического запроса, требуется прочитать небольшое количество столбцов таблицы. В столбцовой БД для этого можно читать только нужные данные. Например, если вам требуется только 5 столбцов из 100, то следует рассчитывать на 20-кратное уменьшение ввода-вывода.
|
||||
2. Так как данные читаются пачками, то их проще сжимать. Данные, лежащие по столбцам также лучше сжимаются. За счёт этого, дополнительно уменьшается объём ввода-вывода.
|
||||
3. За счёт уменьшения ввода-вывода, больше данных влезает в системный кэш.
|
||||
1. Для выполнения аналитического запроса требуется прочитать небольшое количество столбцов таблицы. В столбцовой БД для этого можно читать только нужные данные. Например, если вам требуется только 5 столбцов из 100, то следует рассчитывать на 20-кратное уменьшение ввода-вывода.
|
||||
2. Так как данные читаются пачками, то их проще сжимать. Данные, лежащие по столбцам, также лучше сжимаются. За счёт этого, дополнительно уменьшается объём ввода-вывода.
|
||||
3. За счёт уменьшения ввода-вывода больше данных влезает в системный кэш.
|
||||
|
||||
Например, для запроса «посчитать количество записей для каждой рекламной системы», требуется прочитать один столбец «идентификатор рекламной системы», который занимает 1 байт в несжатом виде. Если большинство переходов было не с рекламных систем, то можно рассчитывать хотя бы на десятикратное сжатие этого столбца. При использовании быстрого алгоритма сжатия, возможно разжатие данных со скоростью более нескольких гигабайт несжатых данных в секунду. То есть, такой запрос может выполняться со скоростью около нескольких миллиардов строк в секунду на одном сервере. На практике, такая скорость действительно достигается.
|
||||
Например, для запроса «посчитать количество записей для каждой рекламной системы» требуется прочитать один столбец «идентификатор рекламной системы», который занимает 1 байт в несжатом виде. Если большинство переходов было не с рекламных систем, то можно рассчитывать хотя бы на десятикратное сжатие этого столбца. При использовании быстрого алгоритма сжатия возможно разжатие данных со скоростью более нескольких гигабайт несжатых данных в секунду. То есть такой запрос может выполняться со скоростью около нескольких миллиардов строк в секунду на одном сервере. На практике такая скорость действительно достигается.
|
||||
|
||||
### По вычислениям {#po-vychisleniiam}
|
||||
|
||||
@ -96,4 +96,4 @@ ClickHouse — столбцовая система управления база
|
||||
|
||||
В «обычных» СУБД этого не делается, так как не имеет смысла при выполнении простых запросов. Хотя есть исключения. Например, в MemSQL кодогенерация используется для уменьшения времени отклика при выполнении SQL-запросов. Для сравнения: в аналитических СУБД требуется оптимизация по пропускной способности (throughput, ГБ/с), а не времени отклика (latency, с).
|
||||
|
||||
Стоит заметить, что для эффективности по CPU требуется, чтобы язык запросов был декларативным (SQL, MDX) или хотя бы векторным (J, K). То есть необходимо, чтобы запрос содержал циклы только в неявном виде, открывая возможности для оптимизации.
|
||||
Стоит заметить, что для эффективности по CPU требуется, чтобы язык запросов был декларативным (SQL, MDX) или хотя бы векторным (J, K, APL). То есть необходимо, чтобы запрос содержал циклы только в неявном виде, открывая возможности для оптимизации.
|
||||
|
||||
@ -36,7 +36,7 @@ if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then
|
||||
fi
|
||||
|
||||
/bin/systemctl daemon-reload
|
||||
/bin/systemctl enable clickhouse-server
|
||||
/bin/systemctl enable --now clickhouse-server
|
||||
else
|
||||
# If you downgrading to version older than 1.1.54336 run: systemctl disable clickhouse-server
|
||||
if [ -x "/etc/init.d/clickhouse-server" ]; then
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
#include <boost/algorithm/string/join.hpp>
|
||||
#include <cstdlib>
|
||||
#include <fcntl.h>
|
||||
#include <map>
|
||||
@ -7,7 +6,6 @@
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <Common/ThreadStatus.h>
|
||||
#include <Common/scope_guard_safe.h>
|
||||
#include <boost/program_options.hpp>
|
||||
#include <boost/algorithm/string/replace.hpp>
|
||||
#include <filesystem>
|
||||
@ -45,8 +43,6 @@
|
||||
|
||||
#include <Processors/Transforms/getSourceFromASTInsertQuery.h>
|
||||
|
||||
#include <Interpreters/InterpreterSetQuery.h>
|
||||
|
||||
#include <Functions/registerFunctions.h>
|
||||
#include <AggregateFunctions/registerAggregateFunctions.h>
|
||||
#include <Formats/registerFormats.h>
|
||||
@ -1138,13 +1134,6 @@ void Client::processOptions(const OptionsDescription & options_description,
|
||||
}
|
||||
|
||||
|
||||
static bool checkIfStdoutIsRegularFile()
|
||||
{
|
||||
struct stat file_stat;
|
||||
return fstat(STDOUT_FILENO, &file_stat) == 0 && S_ISREG(file_stat.st_mode);
|
||||
}
|
||||
|
||||
|
||||
void Client::processConfig()
|
||||
{
|
||||
if (!queries.empty() && config().has("queries-file"))
|
||||
@ -1180,34 +1169,7 @@ void Client::processConfig()
|
||||
|
||||
pager = config().getString("pager", "");
|
||||
|
||||
is_default_format = !config().has("vertical") && !config().has("format");
|
||||
if (is_default_format && checkIfStdoutIsRegularFile())
|
||||
{
|
||||
is_default_format = false;
|
||||
std::optional<String> format_from_file_name;
|
||||
format_from_file_name = FormatFactory::instance().tryGetFormatFromFileDescriptor(STDOUT_FILENO);
|
||||
format = format_from_file_name ? *format_from_file_name : "TabSeparated";
|
||||
}
|
||||
else if (config().has("vertical"))
|
||||
format = config().getString("format", "Vertical");
|
||||
else
|
||||
format = config().getString("format", is_interactive ? "PrettyCompact" : "TabSeparated");
|
||||
|
||||
format_max_block_size = config().getUInt64("format_max_block_size",
|
||||
global_context->getSettingsRef().max_block_size);
|
||||
|
||||
insert_format = "Values";
|
||||
|
||||
/// Setting value from cmd arg overrides one from config
|
||||
if (global_context->getSettingsRef().max_insert_block_size.changed)
|
||||
{
|
||||
insert_format_max_block_size = global_context->getSettingsRef().max_insert_block_size;
|
||||
}
|
||||
else
|
||||
{
|
||||
insert_format_max_block_size = config().getUInt64("insert_format_max_block_size",
|
||||
global_context->getSettingsRef().max_insert_block_size);
|
||||
}
|
||||
setDefaultFormatsFromConfiguration();
|
||||
|
||||
global_context->setClientName(std::string(DEFAULT_CLIENT_NAME));
|
||||
global_context->setQueryKindInitial();
|
||||
|
||||
@ -312,47 +312,28 @@ void LocalServer::cleanup()
|
||||
}
|
||||
|
||||
|
||||
static bool checkIfStdinIsRegularFile()
|
||||
{
|
||||
struct stat file_stat;
|
||||
return fstat(STDIN_FILENO, &file_stat) == 0 && S_ISREG(file_stat.st_mode);
|
||||
}
|
||||
|
||||
|
||||
static bool checkIfStdoutIsRegularFile()
|
||||
{
|
||||
struct stat file_stat;
|
||||
return fstat(STDOUT_FILENO, &file_stat) == 0 && S_ISREG(file_stat.st_mode);
|
||||
}
|
||||
|
||||
|
||||
std::string LocalServer::getInitialCreateTableQuery()
|
||||
{
|
||||
if (!config().has("table-structure") && !config().has("table-file") && !config().has("table-data-format") && (!checkIfStdinIsRegularFile() || queries.empty()))
|
||||
if (!config().has("table-structure") && !config().has("table-file") && !config().has("table-data-format") && (!isRegularFile(STDIN_FILENO) || queries.empty()))
|
||||
return {};
|
||||
|
||||
auto table_name = backQuoteIfNeed(config().getString("table-name", "table"));
|
||||
auto table_structure = config().getString("table-structure", "auto");
|
||||
|
||||
String table_file;
|
||||
std::optional<String> format_from_file_name;
|
||||
if (!config().has("table-file") || config().getString("table-file") == "-")
|
||||
{
|
||||
/// Use Unix tools stdin naming convention
|
||||
table_file = "stdin";
|
||||
format_from_file_name = FormatFactory::instance().tryGetFormatFromFileDescriptor(STDIN_FILENO);
|
||||
}
|
||||
else
|
||||
{
|
||||
/// Use regular file
|
||||
auto file_name = config().getString("table-file");
|
||||
table_file = quoteString(file_name);
|
||||
format_from_file_name = FormatFactory::instance().tryGetFormatFromFileName(file_name);
|
||||
}
|
||||
|
||||
auto data_format = backQuoteIfNeed(
|
||||
config().getString("table-data-format", config().getString("format", format_from_file_name ? *format_from_file_name : "TSV")));
|
||||
|
||||
String data_format = backQuoteIfNeed(default_input_format);
|
||||
|
||||
if (table_structure == "auto")
|
||||
table_structure = "";
|
||||
@ -618,26 +599,7 @@ void LocalServer::processConfig()
|
||||
if (config().has("macros"))
|
||||
global_context->setMacros(std::make_unique<Macros>(config(), "macros", log));
|
||||
|
||||
if (!config().has("output-format") && !config().has("format") && checkIfStdoutIsRegularFile())
|
||||
{
|
||||
std::optional<String> format_from_file_name;
|
||||
format_from_file_name = FormatFactory::instance().tryGetFormatFromFileDescriptor(STDOUT_FILENO);
|
||||
format = format_from_file_name ? *format_from_file_name : "TSV";
|
||||
}
|
||||
else
|
||||
format = config().getString("output-format", config().getString("format", is_interactive ? "PrettyCompact" : "TSV"));
|
||||
insert_format = "Values";
|
||||
|
||||
/// Setting value from cmd arg overrides one from config
|
||||
if (global_context->getSettingsRef().max_insert_block_size.changed)
|
||||
{
|
||||
insert_format_max_block_size = global_context->getSettingsRef().max_insert_block_size;
|
||||
}
|
||||
else
|
||||
{
|
||||
insert_format_max_block_size = config().getUInt64("insert_format_max_block_size",
|
||||
global_context->getSettingsRef().max_insert_block_size);
|
||||
}
|
||||
setDefaultFormatsFromConfiguration();
|
||||
|
||||
/// Sets external authenticators config (LDAP, Kerberos).
|
||||
global_context->setExternalAuthenticatorsConfig(config());
|
||||
@ -819,7 +781,6 @@ void LocalServer::addOptions(OptionsDescription & options_description)
|
||||
("file,F", po::value<std::string>(), "path to file with data of the initial table (stdin if not specified)")
|
||||
|
||||
("input-format", po::value<std::string>(), "input format of the initial table data")
|
||||
("output-format", po::value<std::string>(), "default output format")
|
||||
|
||||
("logger.console", po::value<bool>()->implicit_value(true), "Log to console")
|
||||
("logger.log", po::value<std::string>(), "Log file name")
|
||||
|
||||
@ -4,8 +4,8 @@ set (CLICKHOUSE_ODBC_BRIDGE_SOURCES
|
||||
ColumnInfoHandler.cpp
|
||||
IdentifierQuoteHandler.cpp
|
||||
MainHandler.cpp
|
||||
ODBCBlockInputStream.cpp
|
||||
ODBCBlockOutputStream.cpp
|
||||
ODBCSource.cpp
|
||||
ODBCSink.cpp
|
||||
ODBCBridge.cpp
|
||||
ODBCHandlerFactory.cpp
|
||||
PingHandler.cpp
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
#include "MainHandler.h"
|
||||
|
||||
#include "validateODBCConnectionString.h"
|
||||
#include "ODBCBlockInputStream.h"
|
||||
#include "ODBCBlockOutputStream.h"
|
||||
#include "ODBCSource.h"
|
||||
#include "ODBCSink.h"
|
||||
#include "getIdentifierQuote.h"
|
||||
#include <DataTypes/DataTypeFactory.h>
|
||||
#include <Formats/FormatFactory.h>
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#include "ODBCBlockOutputStream.h"
|
||||
#include "ODBCSink.h"
|
||||
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <Interpreters/Context.h>
|
||||
@ -1,11 +1,10 @@
|
||||
#include "ODBCBlockInputStream.h"
|
||||
#include "ODBCSource.h"
|
||||
#include <vector>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <Common/logger_useful.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -1440,6 +1440,7 @@ try
|
||||
global_context->getProcessList().setMaxSize(new_server_settings.max_concurrent_queries);
|
||||
global_context->getProcessList().setMaxInsertQueriesAmount(new_server_settings.max_concurrent_insert_queries);
|
||||
global_context->getProcessList().setMaxSelectQueriesAmount(new_server_settings.max_concurrent_select_queries);
|
||||
global_context->getProcessList().setMaxWaitingQueriesAmount(new_server_settings.max_waiting_queries);
|
||||
|
||||
if (config->has("keeper_server"))
|
||||
global_context->updateKeeperConfiguration(*config);
|
||||
|
||||
@ -408,6 +408,65 @@ public:
|
||||
|
||||
friend bool operator!=(const Node & left, const Node & right) { return !(left == right); }
|
||||
|
||||
bool contains(const Node & other)
|
||||
{
|
||||
if (min_flags_with_children.contains(other.max_flags_with_children))
|
||||
return true;
|
||||
|
||||
if (!flags.contains(other.flags))
|
||||
return false;
|
||||
|
||||
/// Let's assume that the current node has the following rights:
|
||||
///
|
||||
/// SELECT ON *.* TO user1;
|
||||
/// REVOKE SELECT ON system.* FROM user1;
|
||||
/// REVOKE SELECT ON mydb.* FROM user1;
|
||||
///
|
||||
/// And the other node has the rights:
|
||||
///
|
||||
/// SELECT ON *.* TO user2;
|
||||
/// REVOKE SELECT ON system.* FROM user2;
|
||||
///
|
||||
/// First, we check that each child from the other node is present in the current node:
|
||||
///
|
||||
/// SELECT ON *.* TO user1; -- checked
|
||||
/// REVOKE SELECT ON system.* FROM user1; -- checked
|
||||
if (other.children)
|
||||
{
|
||||
for (const auto & [name, node] : *other.children)
|
||||
{
|
||||
const auto & child = tryGetChild(name);
|
||||
if (child == nullptr)
|
||||
{
|
||||
if (!flags.contains(node.flags))
|
||||
return false;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!child->contains(node))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!children)
|
||||
return true;
|
||||
|
||||
/// Then we check that each of our children has no other rights revoked.
|
||||
///
|
||||
/// REVOKE SELECT ON mydb.* FROM user1; -- check failed, returning false
|
||||
for (const auto & [name, node] : *children)
|
||||
{
|
||||
if (other.children && other.children->contains(name))
|
||||
continue;
|
||||
|
||||
if (!node.flags.contains(other.flags))
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void makeUnion(const Node & other)
|
||||
{
|
||||
makeUnionRec(other);
|
||||
@ -1004,6 +1063,24 @@ bool AccessRights::isGrantedImpl(const AccessFlags & flags, const Args &... args
|
||||
return helper(root);
|
||||
}
|
||||
|
||||
template <bool grant_option>
|
||||
bool AccessRights::containsImpl(const AccessRights & other) const
|
||||
{
|
||||
auto helper = [&](const std::unique_ptr<Node> & root_node) -> bool
|
||||
{
|
||||
if (!root_node)
|
||||
return !other.root;
|
||||
if (!other.root)
|
||||
return true;
|
||||
return root_node->contains(*other.root);
|
||||
};
|
||||
if constexpr (grant_option)
|
||||
return helper(root_with_grant_option);
|
||||
else
|
||||
return helper(root);
|
||||
}
|
||||
|
||||
|
||||
template <bool grant_option>
|
||||
bool AccessRights::isGrantedImplHelper(const AccessRightsElement & element) const
|
||||
{
|
||||
@ -1068,6 +1145,8 @@ bool AccessRights::hasGrantOption(const AccessFlags & flags, std::string_view da
|
||||
bool AccessRights::hasGrantOption(const AccessRightsElement & element) const { return isGrantedImpl<true>(element); }
|
||||
bool AccessRights::hasGrantOption(const AccessRightsElements & elements) const { return isGrantedImpl<true>(elements); }
|
||||
|
||||
bool AccessRights::contains(const AccessRights & access_rights) const { return containsImpl<false>(access_rights); }
|
||||
bool AccessRights::containsWithGrantOption(const AccessRights & access_rights) const { return containsImpl<true>(access_rights); }
|
||||
|
||||
bool operator ==(const AccessRights & left, const AccessRights & right)
|
||||
{
|
||||
|
||||
@ -95,6 +95,10 @@ public:
|
||||
bool hasGrantOption(const AccessRightsElement & element) const;
|
||||
bool hasGrantOption(const AccessRightsElements & elements) const;
|
||||
|
||||
/// Checks if a given `access_rights` is a subset for the current access rights.
|
||||
bool contains(const AccessRights & access_rights) const;
|
||||
bool containsWithGrantOption(const AccessRights & access_rights) const;
|
||||
|
||||
/// Merges two sets of access rights together.
|
||||
/// It's used to combine access rights from multiple roles.
|
||||
void makeUnion(const AccessRights & other);
|
||||
@ -153,6 +157,9 @@ private:
|
||||
template <bool grant_option>
|
||||
bool isGrantedImpl(const AccessRightsElements & elements) const;
|
||||
|
||||
template <bool grant_option>
|
||||
bool containsImpl(const AccessRights & other) const;
|
||||
|
||||
template <bool grant_option>
|
||||
bool isGrantedImplHelper(const AccessRightsElement & element) const;
|
||||
|
||||
|
||||
@ -5,6 +5,12 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
}
|
||||
|
||||
struct Settings;
|
||||
|
||||
AggregateFunctionPtr AggregateFunctionCount::getOwnNullAdapter(
|
||||
@ -19,7 +25,9 @@ namespace
|
||||
AggregateFunctionPtr createAggregateFunctionCount(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *)
|
||||
{
|
||||
assertNoParameters(name, parameters);
|
||||
assertArityAtMost<1>(name, argument_types);
|
||||
|
||||
if (argument_types.size() > 1)
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires zero or one argument", name);
|
||||
|
||||
return std::make_shared<AggregateFunctionCount>(argument_types);
|
||||
}
|
||||
|
||||
@ -14,6 +14,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
@ -116,8 +117,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -91,6 +91,21 @@ public:
|
||||
if (!returns_many && levels.size() > 1)
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires one level parameter or less", getName());
|
||||
|
||||
if constexpr (has_second_arg)
|
||||
{
|
||||
assertBinary(Name::name, argument_types_);
|
||||
if (!isUInt(argument_types_[1]))
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Second argument (weight) for function {} must be unsigned integer, but it has type {}",
|
||||
Name::name,
|
||||
argument_types_[1]->getName());
|
||||
}
|
||||
else
|
||||
{
|
||||
assertUnary(Name::name, argument_types_);
|
||||
}
|
||||
|
||||
if constexpr (is_quantile_ddsketch)
|
||||
{
|
||||
if (params.empty())
|
||||
@ -272,22 +287,6 @@ public:
|
||||
static_cast<ColVecType &>(to).getData().push_back(data.get(level));
|
||||
}
|
||||
}
|
||||
|
||||
static void assertSecondArg(const DataTypes & types)
|
||||
{
|
||||
if constexpr (has_second_arg)
|
||||
{
|
||||
assertBinary(Name::name, types);
|
||||
if (!isUInt(types[1]))
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Second argument (weight) for function {} must be unsigned integer, but it has type {}",
|
||||
Name::name,
|
||||
types[1]->getName());
|
||||
}
|
||||
else
|
||||
assertUnary(Name::name, types);
|
||||
}
|
||||
};
|
||||
|
||||
struct NameQuantile { static constexpr auto name = "quantile"; };
|
||||
|
||||
@ -13,6 +13,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
@ -26,8 +27,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
#include <AggregateFunctions/AggregateFunctionQuantile.h>
|
||||
#include <AggregateFunctions/QuantileBFloat16Histogram.h>
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <AggregateFunctions/Helpers.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <Core/Field.h>
|
||||
@ -13,6 +12,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
@ -26,8 +26,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -13,6 +13,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
@ -27,8 +28,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -14,6 +14,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
@ -116,8 +117,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -13,6 +13,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
@ -27,8 +28,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -13,6 +13,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
@ -26,8 +27,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -13,6 +13,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
@ -27,8 +28,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
@ -39,12 +40,12 @@ AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
#undef DISPATCH
|
||||
if (which.idx == TypeIndex::Date) return std::make_shared<Function<DataTypeDate::FieldType, false>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::DateTime) return std::make_shared<Function<DataTypeDateTime::FieldType, false>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::DateTime64) return std::make_shared<Function<DateTime64, false>>(argument_types, params);
|
||||
|
||||
if (which.idx == TypeIndex::Decimal32) return std::make_shared<Function<Decimal32, false>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::Decimal64) return std::make_shared<Function<Decimal64, false>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::Decimal128) return std::make_shared<Function<Decimal128, false>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::Decimal256) return std::make_shared<Function<Decimal256, false>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::DateTime64) return std::make_shared<Function<DateTime64, false>>(argument_types, params);
|
||||
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument for aggregate function {}",
|
||||
argument_type->getName(), name);
|
||||
|
||||
@ -13,6 +13,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
@ -26,8 +27,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -13,6 +13,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
@ -26,8 +27,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -16,6 +16,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
@ -216,8 +217,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -16,6 +16,7 @@ namespace DB
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int INCORRECT_DATA;
|
||||
extern const int LOGICAL_ERROR;
|
||||
@ -503,8 +504,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -15,6 +15,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
@ -320,8 +321,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -13,6 +13,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
@ -26,8 +27,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -13,6 +13,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
@ -26,8 +27,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -13,6 +13,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
@ -26,8 +27,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -13,6 +13,7 @@ struct Settings;
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
@ -26,8 +27,8 @@ template <template <typename, bool> class Function>
|
||||
AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
const std::string & name, const DataTypes & argument_types, const Array & params, const Settings *)
|
||||
{
|
||||
/// Second argument type check doesn't depend on the type of the first one.
|
||||
Function<void, true>::assertSecondArg(argument_types);
|
||||
if (argument_types.empty())
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at least one argument", name);
|
||||
|
||||
const DataTypePtr & argument_type = argument_types[0];
|
||||
WhichDataType which(argument_type);
|
||||
|
||||
@ -33,21 +33,4 @@ inline void assertBinary(const std::string & name, const DataTypes & argument_ty
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires two arguments", name);
|
||||
}
|
||||
|
||||
template<std::size_t maximal_arity>
|
||||
inline void assertArityAtMost(const std::string & name, const DataTypes & argument_types)
|
||||
{
|
||||
if (argument_types.size() <= maximal_arity)
|
||||
return;
|
||||
|
||||
if constexpr (maximal_arity == 0)
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} cannot have arguments", name);
|
||||
|
||||
if constexpr (maximal_arity == 1)
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires zero or one argument",
|
||||
name);
|
||||
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at most {} arguments",
|
||||
name, maximal_arity);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
73
src/Analyzer/Passes/ConvertInToEqualPass.cpp
Normal file
73
src/Analyzer/Passes/ConvertInToEqualPass.cpp
Normal file
@ -0,0 +1,73 @@
|
||||
#include <Analyzer/ColumnNode.h>
|
||||
#include <Analyzer/ConstantNode.h>
|
||||
#include <Analyzer/FunctionNode.h>
|
||||
#include <Analyzer/InDepthQueryTreeVisitor.h>
|
||||
#include <Analyzer/Passes/ConvertInToEqualPass.h>
|
||||
#include <Functions/equals.h>
|
||||
#include <Functions/notEquals.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class ConvertInToEqualPassVisitor : public InDepthQueryTreeVisitorWithContext<ConvertInToEqualPassVisitor>
|
||||
{
|
||||
public:
|
||||
using Base = InDepthQueryTreeVisitorWithContext<ConvertInToEqualPassVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
void enterImpl(QueryTreeNodePtr & node)
|
||||
{
|
||||
static const std::unordered_map<String, String> MAPPING = {
|
||||
{"in", "equals"},
|
||||
{"notIn", "notEquals"}
|
||||
};
|
||||
auto * func_node = node->as<FunctionNode>();
|
||||
if (!func_node
|
||||
|| !MAPPING.contains(func_node->getFunctionName())
|
||||
|| func_node->getArguments().getNodes().size() != 2)
|
||||
return ;
|
||||
auto args = func_node->getArguments().getNodes();
|
||||
auto * column_node = args[0]->as<ColumnNode>();
|
||||
auto * constant_node = args[1]->as<ConstantNode>();
|
||||
if (!column_node || !constant_node)
|
||||
return ;
|
||||
// IN multiple values is not supported
|
||||
if (constant_node->getValue().getType() == Field::Types::Which::Tuple
|
||||
|| constant_node->getValue().getType() == Field::Types::Which::Array)
|
||||
return ;
|
||||
// x IN null not equivalent to x = null
|
||||
if (constant_node->getValue().isNull())
|
||||
return ;
|
||||
auto result_func_name = MAPPING.at(func_node->getFunctionName());
|
||||
auto equal = std::make_shared<FunctionNode>(result_func_name);
|
||||
QueryTreeNodes arguments{column_node->clone(), constant_node->clone()};
|
||||
equal->getArguments().getNodes() = std::move(arguments);
|
||||
FunctionOverloadResolverPtr resolver;
|
||||
bool decimal_check_overflow = getContext()->getSettingsRef().decimal_check_overflow;
|
||||
if (result_func_name == "equals")
|
||||
{
|
||||
resolver = createInternalFunctionEqualOverloadResolver(decimal_check_overflow);
|
||||
}
|
||||
else
|
||||
{
|
||||
resolver = createInternalFunctionNotEqualOverloadResolver(decimal_check_overflow);
|
||||
}
|
||||
try
|
||||
{
|
||||
equal->resolveAsFunction(resolver);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// When function resolver fails, we should not replace the function node
|
||||
return;
|
||||
}
|
||||
node = equal;
|
||||
}
|
||||
};
|
||||
|
||||
void ConvertInToEqualPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
|
||||
{
|
||||
ConvertInToEqualPassVisitor visitor(std::move(context));
|
||||
visitor.visit(query_tree_node);
|
||||
}
|
||||
}
|
||||
27
src/Analyzer/Passes/ConvertInToEqualPass.h
Normal file
27
src/Analyzer/Passes/ConvertInToEqualPass.h
Normal file
@ -0,0 +1,27 @@
|
||||
#pragma once
|
||||
|
||||
#include <Analyzer/IQueryTreePass.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
/** Optimize `in` to `equals` if possible.
|
||||
* 1. convert in single value to equal
|
||||
* Example: SELECT * from test where x IN (1);
|
||||
* Result: SELECT * from test where x = 1;
|
||||
*
|
||||
* 2. convert not in single value to notEqual
|
||||
* Example: SELECT * from test where x NOT IN (1);
|
||||
* Result: SELECT * from test where x != 1;
|
||||
*
|
||||
* If value is null or tuple, do not convert.
|
||||
*/
|
||||
class ConvertInToEqualPass final : public IQueryTreePass
|
||||
{
|
||||
public:
|
||||
String getName() override { return "ConvertInToEqualPass"; }
|
||||
|
||||
String getDescription() override { return "Convert in to equal"; }
|
||||
|
||||
void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
|
||||
};
|
||||
}
|
||||
@ -801,6 +801,14 @@ struct IdentifierResolveScope
|
||||
/// Node hash to mask id map
|
||||
std::shared_ptr<std::map<IQueryTreeNode::Hash, size_t>> projection_mask_map;
|
||||
|
||||
struct ResolvedFunctionsCache
|
||||
{
|
||||
FunctionOverloadResolverPtr resolver;
|
||||
FunctionBasePtr function_base;
|
||||
};
|
||||
|
||||
std::map<IQueryTreeNode::Hash, ResolvedFunctionsCache> functions_cache;
|
||||
|
||||
[[maybe_unused]] const IdentifierResolveScope * getNearestQueryScope() const
|
||||
{
|
||||
const IdentifierResolveScope * scope_to_check = this;
|
||||
@ -5586,9 +5594,20 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
|
||||
FunctionOverloadResolverPtr function = UserDefinedExecutableFunctionFactory::instance().tryGet(function_name, scope.context, parameters);
|
||||
bool is_executable_udf = true;
|
||||
|
||||
IdentifierResolveScope::ResolvedFunctionsCache * function_cache = nullptr;
|
||||
|
||||
if (!function)
|
||||
{
|
||||
function = FunctionFactory::instance().tryGet(function_name, scope.context);
|
||||
/// This is a hack to allow a query like `select randConstant(), randConstant(), randConstant()`.
|
||||
/// Function randConstant() would return the same value for the same arguments (in scope).
|
||||
|
||||
auto hash = function_node_ptr->getTreeHash();
|
||||
function_cache = &scope.functions_cache[hash];
|
||||
if (!function_cache->resolver)
|
||||
function_cache->resolver = FunctionFactory::instance().tryGet(function_name, scope.context);
|
||||
|
||||
function = function_cache->resolver;
|
||||
|
||||
is_executable_udf = false;
|
||||
}
|
||||
|
||||
@ -5812,7 +5831,17 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
|
||||
|
||||
try
|
||||
{
|
||||
auto function_base = function->build(argument_columns);
|
||||
FunctionBasePtr function_base;
|
||||
if (function_cache)
|
||||
{
|
||||
auto & cached_function = function_cache->function_base;
|
||||
if (!cached_function)
|
||||
cached_function = function->build(argument_columns);
|
||||
|
||||
function_base = cached_function;
|
||||
}
|
||||
else
|
||||
function_base = function->build(argument_columns);
|
||||
|
||||
/// Do not constant fold get scalar functions
|
||||
bool disable_constant_folding = function_name == "__getScalar" || function_name == "shardNum" ||
|
||||
|
||||
@ -148,6 +148,13 @@ void RemoveUnusedProjectionColumnsPass::run(QueryTreeNodePtr & query_tree_node,
|
||||
|
||||
for (auto & [query_or_union_node, used_columns] : visitor.query_or_union_node_to_used_columns)
|
||||
{
|
||||
/// can't remove columns from distinct, see example - 03023_remove_unused_column_distinct.sql
|
||||
if (auto * query_node = query_or_union_node->as<QueryNode>())
|
||||
{
|
||||
if (query_node->isDistinct())
|
||||
continue;
|
||||
}
|
||||
|
||||
auto used_projection_indexes = convertUsedColumnNamesToUsedProjectionIndexes(query_or_union_node, used_columns);
|
||||
updateUsedProjectionIndexes(query_or_union_node, used_projection_indexes);
|
||||
|
||||
|
||||
@ -28,6 +28,7 @@
|
||||
#include <Analyzer/Passes/MultiIfToIfPass.h>
|
||||
#include <Analyzer/Passes/IfConstantConditionPass.h>
|
||||
#include <Analyzer/Passes/IfChainToMultiIfPass.h>
|
||||
#include <Analyzer/Passes/ConvertInToEqualPass.h>
|
||||
#include <Analyzer/Passes/OrderByTupleEliminationPass.h>
|
||||
#include <Analyzer/Passes/NormalizeCountVariantsPass.h>
|
||||
#include <Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.h>
|
||||
@ -263,6 +264,7 @@ void addQueryTreePasses(QueryTreePassManager & manager, bool only_analyze)
|
||||
manager.addPass(std::make_unique<SumIfToCountIfPass>());
|
||||
manager.addPass(std::make_unique<RewriteArrayExistsToHasPass>());
|
||||
manager.addPass(std::make_unique<NormalizeCountVariantsPass>());
|
||||
manager.addPass(std::make_unique<ConvertInToEqualPass>());
|
||||
|
||||
/// should before AggregateFunctionsArithmericOperationsPass
|
||||
manager.addPass(std::make_unique<AggregateFunctionOfGroupByKeysPass>());
|
||||
|
||||
@ -40,6 +40,7 @@ public:
|
||||
bool deduplicate_files = true;
|
||||
bool allow_s3_native_copy = true;
|
||||
bool use_same_s3_credentials_for_base_backup = false;
|
||||
bool azure_attempt_to_create_container = true;
|
||||
ReadSettings read_settings;
|
||||
WriteSettings write_settings;
|
||||
};
|
||||
|
||||
@ -140,12 +140,13 @@ BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage(
|
||||
StorageAzureBlob::Configuration configuration_,
|
||||
const ReadSettings & read_settings_,
|
||||
const WriteSettings & write_settings_,
|
||||
const ContextPtr & context_)
|
||||
const ContextPtr & context_,
|
||||
bool attempt_to_create_container)
|
||||
: BackupWriterDefault(read_settings_, write_settings_, getLogger("BackupWriterAzureBlobStorage"))
|
||||
, data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, configuration_.container, false, false}
|
||||
, configuration(configuration_)
|
||||
{
|
||||
auto client_ptr = StorageAzureBlob::createClient(configuration, /* is_read_only */ false);
|
||||
auto client_ptr = StorageAzureBlob::createClient(configuration, /* is_read_only */ false, attempt_to_create_container);
|
||||
object_storage = std::make_unique<AzureObjectStorage>("BackupWriterAzureBlobStorage",
|
||||
std::move(client_ptr),
|
||||
StorageAzureBlob::createSettings(context_),
|
||||
@ -275,10 +276,9 @@ std::unique_ptr<WriteBuffer> BackupWriterAzureBlobStorage::writeFile(const Strin
|
||||
return std::make_unique<WriteBufferFromAzureBlobStorage>(
|
||||
client,
|
||||
key,
|
||||
settings->max_single_part_upload_size,
|
||||
settings->max_unexpected_write_error_retries,
|
||||
DBMS_DEFAULT_BUFFER_SIZE,
|
||||
write_settings);
|
||||
write_settings,
|
||||
settings);
|
||||
}
|
||||
|
||||
void BackupWriterAzureBlobStorage::removeFile(const String & file_name)
|
||||
|
||||
@ -37,7 +37,7 @@ private:
|
||||
class BackupWriterAzureBlobStorage : public BackupWriterDefault
|
||||
{
|
||||
public:
|
||||
BackupWriterAzureBlobStorage(StorageAzureBlob::Configuration configuration_, const ReadSettings & read_settings_, const WriteSettings & write_settings_, const ContextPtr & context_);
|
||||
BackupWriterAzureBlobStorage(StorageAzureBlob::Configuration configuration_, const ReadSettings & read_settings_, const WriteSettings & write_settings_, const ContextPtr & context_, bool attempt_to_create_container);
|
||||
~BackupWriterAzureBlobStorage() override;
|
||||
|
||||
bool fileExists(const String & file_name) override;
|
||||
|
||||
@ -73,6 +73,7 @@ namespace
|
||||
.use_virtual_addressing = s3_uri.is_virtual_hosted_style,
|
||||
.disable_checksum = local_settings.s3_disable_checksum,
|
||||
.gcs_issue_compose_request = context->getConfigRef().getBool("s3.gcs_issue_compose_request", false),
|
||||
.is_s3express_bucket = S3::isS3ExpressEndpoint(s3_uri.endpoint),
|
||||
};
|
||||
|
||||
return S3::ClientFactory::instance().create(
|
||||
|
||||
@ -28,6 +28,7 @@ namespace ErrorCodes
|
||||
M(Bool, deduplicate_files) \
|
||||
M(Bool, allow_s3_native_copy) \
|
||||
M(Bool, use_same_s3_credentials_for_base_backup) \
|
||||
M(Bool, azure_attempt_to_create_container) \
|
||||
M(Bool, read_from_filesystem_cache) \
|
||||
M(UInt64, shard_num) \
|
||||
M(UInt64, replica_num) \
|
||||
|
||||
@ -47,6 +47,9 @@ struct BackupSettings
|
||||
/// Whether base backup to S3 should inherit credentials from the BACKUP query.
|
||||
bool use_same_s3_credentials_for_base_backup = false;
|
||||
|
||||
/// Whether a new Azure container should be created if it does not exist (requires permissions at storage account level)
|
||||
bool azure_attempt_to_create_container = true;
|
||||
|
||||
/// Allow to use the filesystem cache in passive mode - benefit from the existing cache entries,
|
||||
/// but don't put more entries into the cache.
|
||||
bool read_from_filesystem_cache = true;
|
||||
|
||||
@ -597,6 +597,7 @@ void BackupsWorker::doBackup(
|
||||
backup_create_params.deduplicate_files = backup_settings.deduplicate_files;
|
||||
backup_create_params.allow_s3_native_copy = backup_settings.allow_s3_native_copy;
|
||||
backup_create_params.use_same_s3_credentials_for_base_backup = backup_settings.use_same_s3_credentials_for_base_backup;
|
||||
backup_create_params.azure_attempt_to_create_container = backup_settings.azure_attempt_to_create_container;
|
||||
backup_create_params.read_settings = getReadSettingsForBackup(context, backup_settings);
|
||||
backup_create_params.write_settings = getWriteSettingsForBackup(context);
|
||||
backup = BackupFactory::instance().createBackup(backup_create_params);
|
||||
|
||||
@ -86,7 +86,7 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
|
||||
if (args.size() == 3)
|
||||
{
|
||||
configuration.connection_url = args[0].safeGet<String>();
|
||||
configuration.is_connection_string = true;
|
||||
configuration.is_connection_string = !configuration.connection_url.starts_with("http");
|
||||
|
||||
configuration.container = args[1].safeGet<String>();
|
||||
configuration.blob_path = args[2].safeGet<String>();
|
||||
@ -147,7 +147,8 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
|
||||
auto writer = std::make_shared<BackupWriterAzureBlobStorage>(configuration,
|
||||
params.read_settings,
|
||||
params.write_settings,
|
||||
params.context);
|
||||
params.context,
|
||||
params.azure_attempt_to_create_container);
|
||||
|
||||
return std::make_unique<BackupImpl>(
|
||||
params.backup_info,
|
||||
|
||||
@ -564,7 +564,7 @@ try
|
||||
out_buf = &std_out;
|
||||
}
|
||||
|
||||
String current_format = format;
|
||||
String current_format = default_output_format;
|
||||
|
||||
select_into_file = false;
|
||||
select_into_file_and_stdout = false;
|
||||
@ -722,6 +722,87 @@ void ClientBase::adjustSettings()
|
||||
global_context->setSettings(settings);
|
||||
}
|
||||
|
||||
bool ClientBase::isRegularFile(int fd)
|
||||
{
|
||||
struct stat file_stat;
|
||||
return fstat(fd, &file_stat) == 0 && S_ISREG(file_stat.st_mode);
|
||||
}
|
||||
|
||||
void ClientBase::setDefaultFormatsFromConfiguration()
|
||||
{
|
||||
if (config().has("output-format"))
|
||||
{
|
||||
default_output_format = config().getString("output-format");
|
||||
is_default_format = false;
|
||||
}
|
||||
else if (config().has("format"))
|
||||
{
|
||||
default_output_format = config().getString("format");
|
||||
is_default_format = false;
|
||||
}
|
||||
else if (config().has("vertical"))
|
||||
{
|
||||
default_output_format = "Vertical";
|
||||
is_default_format = false;
|
||||
}
|
||||
else if (isRegularFile(STDOUT_FILENO))
|
||||
{
|
||||
std::optional<String> format_from_file_name = FormatFactory::instance().tryGetFormatFromFileDescriptor(STDOUT_FILENO);
|
||||
if (format_from_file_name)
|
||||
default_output_format = *format_from_file_name;
|
||||
else
|
||||
default_output_format = "TSV";
|
||||
}
|
||||
else if (is_interactive || stdout_is_a_tty)
|
||||
{
|
||||
default_output_format = "PrettyCompact";
|
||||
}
|
||||
else
|
||||
{
|
||||
default_output_format = "TSV";
|
||||
}
|
||||
|
||||
if (config().has("input-format"))
|
||||
{
|
||||
default_input_format = config().getString("input-format");
|
||||
}
|
||||
else if (config().has("format"))
|
||||
{
|
||||
default_input_format = config().getString("format");
|
||||
}
|
||||
else if (config().getString("table-file", "-") != "-")
|
||||
{
|
||||
auto file_name = config().getString("table-file");
|
||||
std::optional<String> format_from_file_name = FormatFactory::instance().tryGetFormatFromFileName(file_name);
|
||||
if (format_from_file_name)
|
||||
default_input_format = *format_from_file_name;
|
||||
else
|
||||
default_input_format = "TSV";
|
||||
}
|
||||
else
|
||||
{
|
||||
std::optional<String> format_from_file_name = FormatFactory::instance().tryGetFormatFromFileDescriptor(STDIN_FILENO);
|
||||
if (format_from_file_name)
|
||||
default_input_format = *format_from_file_name;
|
||||
else
|
||||
default_input_format = "TSV";
|
||||
}
|
||||
|
||||
format_max_block_size = config().getUInt64("format_max_block_size",
|
||||
global_context->getSettingsRef().max_block_size);
|
||||
|
||||
/// Setting value from cmd arg overrides one from config
|
||||
if (global_context->getSettingsRef().max_insert_block_size.changed)
|
||||
{
|
||||
insert_format_max_block_size = global_context->getSettingsRef().max_insert_block_size;
|
||||
}
|
||||
else
|
||||
{
|
||||
insert_format_max_block_size = config().getUInt64("insert_format_max_block_size",
|
||||
global_context->getSettingsRef().max_insert_block_size);
|
||||
}
|
||||
}
|
||||
|
||||
void ClientBase::initTTYBuffer(ProgressOption progress)
|
||||
{
|
||||
if (tty_buf)
|
||||
@ -1605,7 +1686,7 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des
|
||||
|
||||
void ClientBase::sendDataFrom(ReadBuffer & buf, Block & sample, const ColumnsDescription & columns_description, ASTPtr parsed_query, bool have_more_data)
|
||||
{
|
||||
String current_format = insert_format;
|
||||
String current_format = "Values";
|
||||
|
||||
/// Data format can be specified in the INSERT query.
|
||||
if (const auto * insert = parsed_query->as<ASTInsertQuery>())
|
||||
@ -2892,7 +2973,9 @@ void ClientBase::init(int argc, char ** argv)
|
||||
|
||||
("suggestion_limit", po::value<int>()->default_value(10000), "Suggestion limit for how many databases, tables and columns to fetch.")
|
||||
|
||||
("format,f", po::value<std::string>(), "default output format")
|
||||
("format,f", po::value<std::string>(), "default output format (and input format for clickhouse-local)")
|
||||
("output-format", po::value<std::string>(), "default output format (this option has preference over --format)")
|
||||
|
||||
("vertical,E", "vertical output format, same as --format=Vertical or FORMAT Vertical or \\G at end of command")
|
||||
("highlight", po::value<bool>()->default_value(true), "enable or disable basic syntax highlight in interactive command line")
|
||||
|
||||
@ -2975,6 +3058,8 @@ void ClientBase::init(int argc, char ** argv)
|
||||
config().setBool("ignore-error", true);
|
||||
if (options.count("format"))
|
||||
config().setString("format", options["format"].as<std::string>());
|
||||
if (options.count("output-format"))
|
||||
config().setString("output-format", options["output-format"].as<std::string>());
|
||||
if (options.count("vertical"))
|
||||
config().setBool("vertical", true);
|
||||
if (options.count("stacktrace"))
|
||||
|
||||
@ -185,9 +185,13 @@ protected:
|
||||
static bool isSyncInsertWithData(const ASTInsertQuery & insert_query, const ContextPtr & context);
|
||||
bool processMultiQueryFromFile(const String & file_name);
|
||||
|
||||
static bool isRegularFile(int fd);
|
||||
|
||||
/// Adjust some settings after command line options and config had been processed.
|
||||
void adjustSettings();
|
||||
|
||||
void setDefaultFormatsFromConfiguration();
|
||||
|
||||
void initTTYBuffer(ProgressOption progress);
|
||||
|
||||
/// Should be one of the first, to be destroyed the last,
|
||||
@ -218,12 +222,13 @@ protected:
|
||||
|
||||
String pager;
|
||||
|
||||
String format; /// Query results output format.
|
||||
String default_output_format; /// Query results output format.
|
||||
String default_input_format; /// Tables' format for clickhouse-local.
|
||||
|
||||
bool select_into_file = false; /// If writing result INTO OUTFILE. It affects progress rendering.
|
||||
bool select_into_file_and_stdout = false; /// If writing result INTO OUTFILE AND STDOUT. It affects progress rendering.
|
||||
bool is_default_format = true; /// false, if format is set in the config or command line.
|
||||
size_t format_max_block_size = 0; /// Max block size for console output.
|
||||
String insert_format; /// Format of INSERT data that is read from stdin in batch mode.
|
||||
size_t insert_format_max_block_size = 0; /// Max block size when reading INSERT data.
|
||||
size_t max_client_network_bandwidth = 0; /// The maximum speed of data exchange over the network for the client in bytes per second.
|
||||
|
||||
|
||||
@ -132,14 +132,6 @@ public:
|
||||
void setThrottler(const ThrottlerPtr &) override {}
|
||||
|
||||
private:
|
||||
void initBlockInput();
|
||||
|
||||
void processOrdinaryQuery();
|
||||
|
||||
void processOrdinaryQueryWithProcessors();
|
||||
|
||||
void updateState();
|
||||
|
||||
bool pullBlock(Block & block);
|
||||
|
||||
void finishQuery();
|
||||
|
||||
@ -140,6 +140,11 @@ void LoadJob::finish()
|
||||
finish_time = std::chrono::system_clock::now();
|
||||
if (waiters > 0)
|
||||
finished.notify_all();
|
||||
else
|
||||
{
|
||||
on_waiters_increment = {};
|
||||
on_waiters_decrement = {};
|
||||
}
|
||||
}
|
||||
|
||||
void LoadJob::scheduled(UInt64 job_id_)
|
||||
@ -765,11 +770,25 @@ void AsyncLoader::wait(std::unique_lock<std::mutex> & job_lock, const LoadJobPtr
|
||||
if (job->load_status != LoadStatus::PENDING) // Shortcut just to avoid incrementing ProfileEvents
|
||||
return;
|
||||
|
||||
if (job->on_waiters_increment)
|
||||
job->on_waiters_increment(job);
|
||||
|
||||
// WARNING: it is important not to throw below this point to avoid `on_waiters_increment` call w/o matching `on_waiters_decrement` call
|
||||
|
||||
Stopwatch watch;
|
||||
job->waiters++;
|
||||
job->finished.wait(job_lock, [&] { return job->load_status != LoadStatus::PENDING; });
|
||||
job->waiters--;
|
||||
ProfileEvents::increment(ProfileEvents::AsyncLoaderWaitMicroseconds, watch.elapsedMicroseconds());
|
||||
|
||||
if (job->on_waiters_decrement)
|
||||
job->on_waiters_decrement(job);
|
||||
|
||||
if (job->waiters == 0)
|
||||
{
|
||||
job->on_waiters_increment = {};
|
||||
job->on_waiters_decrement = {};
|
||||
}
|
||||
}
|
||||
|
||||
bool AsyncLoader::canSpawnWorker(Pool & pool, std::unique_lock<std::mutex> &)
|
||||
@ -859,7 +878,7 @@ void AsyncLoader::worker(Pool & pool)
|
||||
try
|
||||
{
|
||||
current_load_job = job.get();
|
||||
SCOPE_EXIT({ current_load_job = nullptr; }); // Note that recursive job execution is not supported
|
||||
SCOPE_EXIT({ current_load_job = nullptr; }); // Note that recursive job execution is not supported, but jobs can wait one another
|
||||
job->execute(*this, pool_id, job);
|
||||
exception_from_job = {};
|
||||
}
|
||||
|
||||
@ -59,7 +59,8 @@ enum class LoadStatus
|
||||
class LoadJob : private boost::noncopyable
|
||||
{
|
||||
public:
|
||||
template <class LoadJobSetType, class Func, class DFFunc>
|
||||
// NOTE: makeLoadJob() helper should be used instead of direct ctor call
|
||||
template <class LoadJobSetType, class DFFunc, class Func>
|
||||
LoadJob(LoadJobSetType && dependencies_, String name_, size_t pool_id_, DFFunc && dependency_failure_, Func && func_)
|
||||
: dependencies(std::forward<LoadJobSetType>(dependencies_))
|
||||
, name(std::move(name_))
|
||||
@ -69,6 +70,19 @@ public:
|
||||
, func(std::forward<Func>(func_))
|
||||
{}
|
||||
|
||||
// NOTE: makeLoadJob() helper should be used instead of direct ctor call
|
||||
template <class LoadJobSetType, class WIFunc, class WDFunc, class DFFunc, class Func>
|
||||
LoadJob(LoadJobSetType && dependencies_, String name_, size_t pool_id_, WIFunc && on_waiters_increment_, WDFunc && on_waiters_decrement_, DFFunc && dependency_failure_, Func && func_)
|
||||
: dependencies(std::forward<LoadJobSetType>(dependencies_))
|
||||
, name(std::move(name_))
|
||||
, execution_pool_id(pool_id_)
|
||||
, pool_id(pool_id_)
|
||||
, on_waiters_increment(std::forward<WIFunc>(on_waiters_increment_))
|
||||
, on_waiters_decrement(std::forward<WDFunc>(on_waiters_decrement_))
|
||||
, dependency_failure(std::forward<DFFunc>(dependency_failure_))
|
||||
, func(std::forward<Func>(func_))
|
||||
{}
|
||||
|
||||
// Current job status.
|
||||
LoadStatus status() const;
|
||||
std::exception_ptr exception() const;
|
||||
@ -112,6 +126,13 @@ private:
|
||||
std::atomic<size_t> execution_pool_id;
|
||||
std::atomic<size_t> pool_id;
|
||||
|
||||
// Handlers that is called by every new waiting thread, just before going to sleep.
|
||||
// If `on_waiters_increment` throws, then wait is canceled, and corresponding `on_waiters_decrement` will never be called.
|
||||
// It can be used for counting and limits on number of waiters.
|
||||
// Note that implementations are called under `LoadJob::mutex` and should be fast.
|
||||
std::function<void(const LoadJobPtr & self)> on_waiters_increment;
|
||||
std::function<void(const LoadJobPtr & self)> on_waiters_decrement;
|
||||
|
||||
// Handler for failed or canceled dependencies.
|
||||
// If job needs to be canceled on `dependency` failure, then function should set `cancel` to a specific reason.
|
||||
// Note that implementation should be fast and cannot use AsyncLoader, because it is called under `AsyncLoader::mutex`.
|
||||
@ -140,8 +161,50 @@ void cancelOnDependencyFailure(const LoadJobPtr & self, const LoadJobPtr & depen
|
||||
void ignoreDependencyFailure(const LoadJobPtr & self, const LoadJobPtr & dependency, std::exception_ptr & cancel);
|
||||
|
||||
template <class F> concept LoadJobDependencyFailure = std::invocable<F, const LoadJobPtr &, const LoadJobPtr &, std::exception_ptr &>;
|
||||
template <class F> concept LoadJobOnWaiters = std::invocable<F, const LoadJobPtr &>;
|
||||
template <class F> concept LoadJobFunc = std::invocable<F, AsyncLoader &, const LoadJobPtr &>;
|
||||
|
||||
LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobDependencyFailure auto && dependency_failure, LoadJobFunc auto && func)
|
||||
{
|
||||
return std::make_shared<LoadJob>(std::move(dependencies), std::move(name), 0, on_waiters_increment, on_waiters_decrement, std::forward<decltype(dependency_failure)>(dependency_failure), std::forward<decltype(func)>(func));
|
||||
}
|
||||
|
||||
LoadJobPtr makeLoadJob(const LoadJobSet & dependencies, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobDependencyFailure auto && dependency_failure, LoadJobFunc auto && func)
|
||||
{
|
||||
return std::make_shared<LoadJob>(dependencies, std::move(name), 0, on_waiters_increment, on_waiters_decrement, std::forward<decltype(dependency_failure)>(dependency_failure), std::forward<decltype(func)>(func));
|
||||
}
|
||||
|
||||
LoadJobPtr makeLoadJob(LoadJobSet && dependencies, size_t pool_id, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobDependencyFailure auto && dependency_failure, LoadJobFunc auto && func)
|
||||
{
|
||||
return std::make_shared<LoadJob>(std::move(dependencies), std::move(name), pool_id, on_waiters_increment, on_waiters_decrement, std::forward<decltype(dependency_failure)>(dependency_failure), std::forward<decltype(func)>(func));
|
||||
}
|
||||
|
||||
LoadJobPtr makeLoadJob(const LoadJobSet & dependencies, size_t pool_id, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobDependencyFailure auto && dependency_failure, LoadJobFunc auto && func)
|
||||
{
|
||||
return std::make_shared<LoadJob>(dependencies, std::move(name), pool_id, on_waiters_increment, on_waiters_decrement, std::forward<decltype(dependency_failure)>(dependency_failure), std::forward<decltype(func)>(func));
|
||||
}
|
||||
|
||||
LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobFunc auto && func)
|
||||
{
|
||||
return std::make_shared<LoadJob>(std::move(dependencies), std::move(name), 0, on_waiters_increment, on_waiters_decrement, cancelOnDependencyFailure, std::forward<decltype(func)>(func));
|
||||
}
|
||||
|
||||
LoadJobPtr makeLoadJob(const LoadJobSet & dependencies, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobFunc auto && func)
|
||||
{
|
||||
return std::make_shared<LoadJob>(dependencies, std::move(name), 0, on_waiters_increment, on_waiters_decrement, cancelOnDependencyFailure, std::forward<decltype(func)>(func));
|
||||
}
|
||||
|
||||
LoadJobPtr makeLoadJob(LoadJobSet && dependencies, size_t pool_id, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobFunc auto && func)
|
||||
{
|
||||
return std::make_shared<LoadJob>(std::move(dependencies), std::move(name), pool_id, on_waiters_increment, on_waiters_decrement, cancelOnDependencyFailure, std::forward<decltype(func)>(func));
|
||||
}
|
||||
|
||||
LoadJobPtr makeLoadJob(const LoadJobSet & dependencies, size_t pool_id, String name, LoadJobOnWaiters auto && on_waiters_increment, LoadJobOnWaiters auto && on_waiters_decrement, LoadJobFunc auto && func)
|
||||
{
|
||||
return std::make_shared<LoadJob>(dependencies, std::move(name), pool_id, on_waiters_increment, on_waiters_decrement, cancelOnDependencyFailure, std::forward<decltype(func)>(func));
|
||||
}
|
||||
|
||||
|
||||
LoadJobPtr makeLoadJob(LoadJobSet && dependencies, String name, LoadJobDependencyFailure auto && dependency_failure, LoadJobFunc auto && func)
|
||||
{
|
||||
return std::make_shared<LoadJob>(std::move(dependencies), std::move(name), 0, std::forward<decltype(dependency_failure)>(dependency_failure), std::forward<decltype(func)>(func));
|
||||
|
||||
@ -1,22 +1,18 @@
|
||||
#include "config.h"
|
||||
|
||||
#if USE_AWS_S3
|
||||
|
||||
#include <IO/WriteBufferFromS3.h>
|
||||
#include "BufferAllocationPolicy.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class FixedSizeBufferAllocationPolicy : public DB::WriteBufferFromS3::IBufferAllocationPolicy
|
||||
class FixedSizeBufferAllocationPolicy : public BufferAllocationPolicy
|
||||
{
|
||||
const size_t buffer_size = 0;
|
||||
size_t buffer_number = 0;
|
||||
|
||||
public:
|
||||
explicit FixedSizeBufferAllocationPolicy(const DB::S3Settings::RequestSettings::PartUploadSettings & settings_)
|
||||
: buffer_size(settings_.strict_upload_part_size)
|
||||
explicit FixedSizeBufferAllocationPolicy(const BufferAllocationPolicy::Settings & settings_)
|
||||
: buffer_size(settings_.strict_size)
|
||||
{
|
||||
chassert(buffer_size > 0);
|
||||
}
|
||||
@ -36,7 +32,7 @@ public:
|
||||
};
|
||||
|
||||
|
||||
class ExpBufferAllocationPolicy : public DB::WriteBufferFromS3::IBufferAllocationPolicy
|
||||
class ExpBufferAllocationPolicy : public DB::BufferAllocationPolicy
|
||||
{
|
||||
const size_t first_size = 0;
|
||||
const size_t second_size = 0;
|
||||
@ -49,12 +45,12 @@ class ExpBufferAllocationPolicy : public DB::WriteBufferFromS3::IBufferAllocatio
|
||||
size_t buffer_number = 0;
|
||||
|
||||
public:
|
||||
explicit ExpBufferAllocationPolicy(const DB::S3Settings::RequestSettings::PartUploadSettings & settings_)
|
||||
: first_size(std::max(settings_.max_single_part_upload_size, settings_.min_upload_part_size))
|
||||
, second_size(settings_.min_upload_part_size)
|
||||
, multiply_factor(settings_.upload_part_size_multiply_factor)
|
||||
, multiply_threshold(settings_.upload_part_size_multiply_parts_count_threshold)
|
||||
, max_size(settings_.max_upload_part_size)
|
||||
explicit ExpBufferAllocationPolicy(const BufferAllocationPolicy::Settings & settings_)
|
||||
: first_size(std::max(settings_.max_single_size, settings_.min_size))
|
||||
, second_size(settings_.min_size)
|
||||
, multiply_factor(settings_.multiply_factor)
|
||||
, multiply_threshold(settings_.multiply_parts_count_threshold)
|
||||
, max_size(settings_.max_size)
|
||||
{
|
||||
chassert(first_size > 0);
|
||||
chassert(second_size > 0);
|
||||
@ -92,16 +88,12 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
namespace DB
|
||||
BufferAllocationPolicy::~BufferAllocationPolicy() = default;
|
||||
|
||||
BufferAllocationPolicyPtr BufferAllocationPolicy::create(BufferAllocationPolicy::Settings settings_)
|
||||
{
|
||||
|
||||
WriteBufferFromS3::IBufferAllocationPolicy::~IBufferAllocationPolicy() = default;
|
||||
|
||||
WriteBufferFromS3::IBufferAllocationPolicyPtr WriteBufferFromS3::ChooseBufferPolicy(const S3Settings::RequestSettings::PartUploadSettings & settings_)
|
||||
{
|
||||
if (settings_.strict_upload_part_size > 0)
|
||||
if (settings_.strict_size > 0)
|
||||
return std::make_unique<FixedSizeBufferAllocationPolicy>(settings_);
|
||||
else
|
||||
return std::make_unique<ExpBufferAllocationPolicy>(settings_);
|
||||
@ -109,4 +101,3 @@ WriteBufferFromS3::IBufferAllocationPolicyPtr WriteBufferFromS3::ChooseBufferPol
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
39
src/Common/BufferAllocationPolicy.h
Normal file
39
src/Common/BufferAllocationPolicy.h
Normal file
@ -0,0 +1,39 @@
|
||||
#pragma once
|
||||
|
||||
#include "config.h"
|
||||
|
||||
#include "logger_useful.h"
|
||||
|
||||
#include <list>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class BufferAllocationPolicy;
|
||||
using BufferAllocationPolicyPtr = std::unique_ptr<BufferAllocationPolicy>;
|
||||
|
||||
/// Buffer number starts with 0
|
||||
class BufferAllocationPolicy
|
||||
{
|
||||
public:
|
||||
|
||||
struct Settings
|
||||
{
|
||||
size_t strict_size = 0;
|
||||
size_t min_size = 16 * 1024 * 1024;
|
||||
size_t max_size = 5ULL * 1024 * 1024 * 1024;
|
||||
size_t multiply_factor = 2;
|
||||
size_t multiply_parts_count_threshold = 500;
|
||||
size_t max_single_size = 32 * 1024 * 1024; /// Max size for a single buffer/block
|
||||
};
|
||||
|
||||
virtual size_t getBufferNumber() const = 0;
|
||||
virtual size_t getBufferSize() const = 0;
|
||||
virtual void nextBuffer() = 0;
|
||||
virtual ~BufferAllocationPolicy() = 0;
|
||||
|
||||
static BufferAllocationPolicyPtr create(Settings settings_);
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
@ -46,28 +46,45 @@ struct LastElementCacheStats
|
||||
namespace columns_hashing_impl
|
||||
{
|
||||
|
||||
template <typename Value, bool consecutive_keys_optimization_>
|
||||
struct LastElementCache
|
||||
struct LastElementCacheBase
|
||||
{
|
||||
static constexpr bool consecutive_keys_optimization = consecutive_keys_optimization_;
|
||||
|
||||
Value value;
|
||||
bool empty = true;
|
||||
bool found = false;
|
||||
UInt64 misses = 0;
|
||||
|
||||
bool check(const Value & value_) const { return value == value_; }
|
||||
|
||||
template <typename Key>
|
||||
bool check(const Key & key) const { return value.first == key; }
|
||||
void onNewValue(bool is_found)
|
||||
{
|
||||
empty = false;
|
||||
found = is_found;
|
||||
++misses;
|
||||
}
|
||||
|
||||
bool hasOnlyOneValue() const { return found && misses == 1; }
|
||||
};
|
||||
|
||||
template <typename Data>
|
||||
struct LastElementCache<Data, false>
|
||||
template <typename Value, bool nullable> struct LastElementCache;
|
||||
|
||||
template <typename Value>
|
||||
struct LastElementCache<Value, true> : public LastElementCacheBase
|
||||
{
|
||||
static constexpr bool consecutive_keys_optimization = false;
|
||||
Value value{};
|
||||
bool is_null = false;
|
||||
|
||||
template <typename Key>
|
||||
bool check(const Key & key) const { return !is_null && value.first == key; }
|
||||
|
||||
bool check(const Value & rhs) const { return !is_null && value == rhs; }
|
||||
};
|
||||
|
||||
template <typename Value>
|
||||
struct LastElementCache<Value, false> : public LastElementCacheBase
|
||||
{
|
||||
Value value{};
|
||||
|
||||
template <typename Key>
|
||||
bool check(const Key & key) const { return value.first == key; }
|
||||
|
||||
bool check(const Value & rhs) const { return value == rhs; }
|
||||
};
|
||||
|
||||
template <typename Mapped>
|
||||
@ -161,7 +178,7 @@ public:
|
||||
using EmplaceResult = EmplaceResultImpl<Mapped>;
|
||||
using FindResult = FindResultImpl<Mapped, need_offset>;
|
||||
static constexpr bool has_mapped = !std::is_same_v<Mapped, void>;
|
||||
using Cache = LastElementCache<Value, consecutive_keys_optimization>;
|
||||
using Cache = LastElementCache<Value, nullable>;
|
||||
|
||||
static HashMethodContextPtr createContext(const HashMethodContext::Settings &) { return nullptr; }
|
||||
|
||||
@ -172,6 +189,15 @@ public:
|
||||
{
|
||||
if (isNullAt(row))
|
||||
{
|
||||
if constexpr (consecutive_keys_optimization)
|
||||
{
|
||||
if (!cache.is_null)
|
||||
{
|
||||
cache.onNewValue(true);
|
||||
cache.is_null = true;
|
||||
}
|
||||
}
|
||||
|
||||
bool has_null_key = data.hasNullKeyData();
|
||||
data.hasNullKeyData() = true;
|
||||
|
||||
@ -193,10 +219,21 @@ public:
|
||||
{
|
||||
if (isNullAt(row))
|
||||
{
|
||||
bool has_null_key = data.hasNullKeyData();
|
||||
|
||||
if constexpr (consecutive_keys_optimization)
|
||||
{
|
||||
if (!cache.is_null)
|
||||
{
|
||||
cache.onNewValue(has_null_key);
|
||||
cache.is_null = true;
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (has_mapped)
|
||||
return FindResult(&data.getNullKeyData(), data.hasNullKeyData(), 0);
|
||||
return FindResult(&data.getNullKeyData(), has_null_key, 0);
|
||||
else
|
||||
return FindResult(data.hasNullKeyData(), 0);
|
||||
return FindResult(has_null_key, 0);
|
||||
}
|
||||
}
|
||||
|
||||
@ -303,9 +340,10 @@ protected:
|
||||
|
||||
if constexpr (consecutive_keys_optimization)
|
||||
{
|
||||
cache.found = true;
|
||||
cache.empty = false;
|
||||
++cache.misses;
|
||||
cache.onNewValue(true);
|
||||
|
||||
if constexpr (nullable)
|
||||
cache.is_null = false;
|
||||
|
||||
if constexpr (has_mapped)
|
||||
{
|
||||
@ -346,17 +384,16 @@ protected:
|
||||
|
||||
if constexpr (consecutive_keys_optimization)
|
||||
{
|
||||
cache.found = it != nullptr;
|
||||
cache.empty = false;
|
||||
++cache.misses;
|
||||
cache.onNewValue(it != nullptr);
|
||||
|
||||
if constexpr (nullable)
|
||||
cache.is_null = false;
|
||||
|
||||
if constexpr (has_mapped)
|
||||
{
|
||||
cache.value.first = key;
|
||||
if (it)
|
||||
{
|
||||
cache.value.second = it->getMapped();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
19
src/Common/ExternalLoaderStatus.cpp
Normal file
19
src/Common/ExternalLoaderStatus.cpp
Normal file
@ -0,0 +1,19 @@
|
||||
#include <Common/ExternalLoaderStatus.h>
|
||||
|
||||
#include <base/EnumReflection.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
std::vector<std::pair<String, Int8>> getExternalLoaderStatusEnumAllPossibleValues()
|
||||
{
|
||||
std::vector<std::pair<String, Int8>> out;
|
||||
out.reserve(magic_enum::enum_count<ExternalLoaderStatus>());
|
||||
|
||||
for (const auto & [value, str] : magic_enum::enum_entries<ExternalLoaderStatus>())
|
||||
out.emplace_back(std::string{str}, static_cast<Int8>(value));
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,30 +1,20 @@
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <base/EnumReflection.h>
|
||||
#include <base/types.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
enum class ExternalLoaderStatus : int8_t
|
||||
{
|
||||
NOT_LOADED, /// Object hasn't been tried to load. This is an initial state.
|
||||
LOADED, /// Object has been loaded successfully.
|
||||
FAILED, /// Object has been failed to load.
|
||||
LOADING, /// Object is being loaded right now for the first time.
|
||||
FAILED_AND_RELOADING, /// Object was failed to load before and it's being reloaded right now.
|
||||
LOADED_AND_RELOADING, /// Object was loaded successfully before and it's being reloaded right now.
|
||||
NOT_EXIST, /// Object with this name wasn't found in the configuration.
|
||||
};
|
||||
enum class ExternalLoaderStatus : int8_t
|
||||
{
|
||||
NOT_LOADED, /// Object hasn't been tried to load. This is an initial state.
|
||||
LOADED, /// Object has been loaded successfully.
|
||||
FAILED, /// Object has been failed to load.
|
||||
LOADING, /// Object is being loaded right now for the first time.
|
||||
FAILED_AND_RELOADING, /// Object was failed to load before and it's being reloaded right now.
|
||||
LOADED_AND_RELOADING, /// Object was loaded successfully before and it's being reloaded right now.
|
||||
NOT_EXIST, /// Object with this name wasn't found in the configuration.
|
||||
};
|
||||
|
||||
inline std::vector<std::pair<String, Int8>> getStatusEnumAllPossibleValues()
|
||||
{
|
||||
std::vector<std::pair<String, Int8>> out;
|
||||
out.reserve(magic_enum::enum_count<ExternalLoaderStatus>());
|
||||
|
||||
for (const auto & [value, str] : magic_enum::enum_entries<ExternalLoaderStatus>())
|
||||
out.emplace_back(std::string{str}, static_cast<Int8>(value));
|
||||
|
||||
return out;
|
||||
}
|
||||
std::vector<std::pair<String, Int8>> getExternalLoaderStatusEnumAllPossibleValues();
|
||||
}
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
#include "config.h"
|
||||
|
||||
#if USE_AWS_S3
|
||||
|
||||
#include <IO/WriteBufferFromS3TaskTracker.h>
|
||||
#include "ThreadPoolTaskTracker.h"
|
||||
|
||||
namespace ProfileEvents
|
||||
{
|
||||
@ -12,19 +10,19 @@ namespace ProfileEvents
|
||||
namespace DB
|
||||
{
|
||||
|
||||
WriteBufferFromS3::TaskTracker::TaskTracker(ThreadPoolCallbackRunner<void> scheduler_, size_t max_tasks_inflight_, LogSeriesLimiterPtr limitedLog_)
|
||||
TaskTracker::TaskTracker(ThreadPoolCallbackRunner<void> scheduler_, size_t max_tasks_inflight_, LogSeriesLimiterPtr limitedLog_)
|
||||
: is_async(bool(scheduler_))
|
||||
, scheduler(scheduler_ ? std::move(scheduler_) : syncRunner())
|
||||
, max_tasks_inflight(max_tasks_inflight_)
|
||||
, limitedLog(limitedLog_)
|
||||
{}
|
||||
|
||||
WriteBufferFromS3::TaskTracker::~TaskTracker()
|
||||
TaskTracker::~TaskTracker()
|
||||
{
|
||||
safeWaitAll();
|
||||
}
|
||||
|
||||
ThreadPoolCallbackRunner<void> WriteBufferFromS3::TaskTracker::syncRunner()
|
||||
ThreadPoolCallbackRunner<void> TaskTracker::syncRunner()
|
||||
{
|
||||
return [](Callback && callback, int64_t) mutable -> std::future<void>
|
||||
{
|
||||
@ -35,7 +33,7 @@ ThreadPoolCallbackRunner<void> WriteBufferFromS3::TaskTracker::syncRunner()
|
||||
};
|
||||
}
|
||||
|
||||
void WriteBufferFromS3::TaskTracker::waitAll()
|
||||
void TaskTracker::waitAll()
|
||||
{
|
||||
/// Exceptions are propagated
|
||||
for (auto & future : futures)
|
||||
@ -48,7 +46,7 @@ void WriteBufferFromS3::TaskTracker::waitAll()
|
||||
finished_futures.clear();
|
||||
}
|
||||
|
||||
void WriteBufferFromS3::TaskTracker::safeWaitAll()
|
||||
void TaskTracker::safeWaitAll()
|
||||
{
|
||||
for (auto & future : futures)
|
||||
{
|
||||
@ -71,7 +69,7 @@ void WriteBufferFromS3::TaskTracker::safeWaitAll()
|
||||
finished_futures.clear();
|
||||
}
|
||||
|
||||
void WriteBufferFromS3::TaskTracker::waitIfAny()
|
||||
void TaskTracker::waitIfAny()
|
||||
{
|
||||
if (futures.empty())
|
||||
return;
|
||||
@ -99,7 +97,7 @@ void WriteBufferFromS3::TaskTracker::waitIfAny()
|
||||
ProfileEvents::increment(ProfileEvents::WriteBufferFromS3WaitInflightLimitMicroseconds, watch.elapsedMicroseconds());
|
||||
}
|
||||
|
||||
void WriteBufferFromS3::TaskTracker::add(Callback && func)
|
||||
void TaskTracker::add(Callback && func)
|
||||
{
|
||||
/// All this fuzz is about 2 things. This is the most critical place of TaskTracker.
|
||||
/// The first is not to fail insertion in the list `futures`.
|
||||
@ -134,7 +132,7 @@ void WriteBufferFromS3::TaskTracker::add(Callback && func)
|
||||
waitTilInflightShrink();
|
||||
}
|
||||
|
||||
void WriteBufferFromS3::TaskTracker::waitTilInflightShrink()
|
||||
void TaskTracker::waitTilInflightShrink()
|
||||
{
|
||||
if (!max_tasks_inflight)
|
||||
return;
|
||||
@ -166,11 +164,10 @@ void WriteBufferFromS3::TaskTracker::waitTilInflightShrink()
|
||||
ProfileEvents::increment(ProfileEvents::WriteBufferFromS3WaitInflightLimitMicroseconds, watch.elapsedMicroseconds());
|
||||
}
|
||||
|
||||
bool WriteBufferFromS3::TaskTracker::isAsync() const
|
||||
bool TaskTracker::isAsync() const
|
||||
{
|
||||
return is_async;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
@ -1,20 +1,16 @@
|
||||
#pragma once
|
||||
|
||||
#include "config.h"
|
||||
#include "threadPoolCallbackRunner.h"
|
||||
#include "IO/WriteBufferFromS3.h"
|
||||
|
||||
#if USE_AWS_S3
|
||||
|
||||
#include "WriteBufferFromS3.h"
|
||||
|
||||
#include <Common/logger_useful.h>
|
||||
#include "logger_useful.h"
|
||||
|
||||
#include <list>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// That class is used only in WriteBufferFromS3 for now.
|
||||
/// Therefore it declared as a part of WriteBufferFromS3.
|
||||
/// TaskTracker takes a Callback which is run by scheduler in some external shared ThreadPool.
|
||||
/// TaskTracker brings the methods waitIfAny, waitAll/safeWaitAll
|
||||
/// to help with coordination of the running tasks.
|
||||
@ -22,7 +18,7 @@ namespace DB
|
||||
/// Basic exception safety is provided. If exception occurred the object has to be destroyed.
|
||||
/// No thread safety is provided. Use this object with no concurrency.
|
||||
|
||||
class WriteBufferFromS3::TaskTracker
|
||||
class TaskTracker
|
||||
{
|
||||
public:
|
||||
using Callback = std::function<void()>;
|
||||
@ -68,5 +64,3 @@ private:
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
@ -191,7 +191,8 @@ namespace VolnitskyTraits
|
||||
if (length_l != length_r)
|
||||
return false;
|
||||
|
||||
assert(length_l >= 2 && length_r >= 2);
|
||||
if (length_l < 2 || length_r < 2)
|
||||
return false; /// Some part of the given ngram contains an invalid UTF-8 sequence.
|
||||
|
||||
chars.c0 = seq_l[seq_ngram_offset];
|
||||
chars.c1 = seq_l[seq_ngram_offset + 1];
|
||||
@ -253,7 +254,9 @@ namespace VolnitskyTraits
|
||||
if (size_l != size_u)
|
||||
return false;
|
||||
|
||||
assert(size_l >= 1 && size_u >= 1);
|
||||
if (size_l == 0 || size_u == 0)
|
||||
return false; /// Some part of the given ngram contains an invalid UTF-8 sequence.
|
||||
|
||||
chars.c1 = seq_l[0];
|
||||
putNGramBase(n, offset);
|
||||
|
||||
@ -276,7 +279,8 @@ namespace VolnitskyTraits
|
||||
if (size_l != size_u)
|
||||
return false;
|
||||
|
||||
assert(size_l > seq_ngram_offset && size_u > seq_ngram_offset);
|
||||
if (size_l <= seq_ngram_offset || size_u <= seq_ngram_offset)
|
||||
return false; /// Some part of the given ngram contains an invalid UTF-8 sequence.
|
||||
|
||||
chars.c0 = seq_l[seq_ngram_offset];
|
||||
putNGramBase(n, offset);
|
||||
@ -302,10 +306,8 @@ namespace VolnitskyTraits
|
||||
if (size_first_l != size_first_u || size_second_l != size_second_u)
|
||||
return false;
|
||||
|
||||
assert(size_first_l > seq_ngram_offset);
|
||||
assert(size_first_u > seq_ngram_offset);
|
||||
assert(size_second_l > 0);
|
||||
assert(size_second_u > 0);
|
||||
if (size_first_l <= seq_ngram_offset || size_first_u <= seq_ngram_offset || size_second_l == 0 || size_second_u == 0)
|
||||
return false;
|
||||
|
||||
auto c0l = first_l_seq[seq_ngram_offset];
|
||||
auto c0u = first_u_seq[seq_ngram_offset];
|
||||
@ -399,7 +401,7 @@ public:
|
||||
if (fallback || fallback_searcher.force_fallback)
|
||||
return;
|
||||
|
||||
hash = std::unique_ptr<VolnitskyTraits::Offset[]>(new VolnitskyTraits::Offset[VolnitskyTraits::hash_size]{});
|
||||
hash = std::make_unique<VolnitskyTraits::Offset[]>(VolnitskyTraits::hash_size);
|
||||
|
||||
auto callback = [this](const VolnitskyTraits::Ngram ngram, const int offset) { return this->putNGramBase(ngram, offset); };
|
||||
/// ssize_t is used here because unsigned can't be used with condition like `i >= 0`, unsigned always >= 0
|
||||
|
||||
@ -13,8 +13,6 @@ namespace DB
|
||||
}
|
||||
}
|
||||
|
||||
// I wanted to make this ALWAYS_INLINE to prevent flappy performance tests,
|
||||
// but GCC complains it may not be inlined.
|
||||
static void formatReadable(double size, DB::WriteBuffer & out,
|
||||
int precision, const char ** units, size_t units_size, double delimiter)
|
||||
{
|
||||
@ -25,7 +23,12 @@ static void formatReadable(double size, DB::WriteBuffer & out,
|
||||
DB::DoubleConverter<false>::BufferType buffer;
|
||||
double_conversion::StringBuilder builder{buffer, sizeof(buffer)};
|
||||
|
||||
const auto result = DB::DoubleConverter<false>::instance().ToFixed(size, precision, &builder);
|
||||
const auto & converter = DB::DoubleConverter<false>::instance();
|
||||
|
||||
auto result = converter.ToFixed(size, precision, &builder);
|
||||
|
||||
if (!result)
|
||||
result = converter.ToShortest(size, &builder);
|
||||
|
||||
if (!result)
|
||||
throw DB::Exception(DB::ErrorCodes::CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER, "Cannot print float or double number");
|
||||
@ -65,7 +68,11 @@ std::string formatReadableSizeWithDecimalSuffix(double value, int precision)
|
||||
|
||||
void formatReadableQuantity(double value, DB::WriteBuffer & out, int precision)
|
||||
{
|
||||
const char * units[] = {"", " thousand", " million", " billion", " trillion", " quadrillion"};
|
||||
const char * units[] = {"", " thousand", " million", " billion", " trillion", " quadrillion",
|
||||
" quintillion", " sextillion", " septillion", " octillion", " nonillion", " decillion",
|
||||
" undecillion", " duodecillion", " tredecillion", " quattuordecillion", " quindecillion",
|
||||
" sexdecillion", " septendecillion", " octodecillion", " novemdecillion", " vigintillion"};
|
||||
|
||||
formatReadable(value, out, precision, units, sizeof(units) / sizeof(units[0]), 1000);
|
||||
}
|
||||
|
||||
|
||||
@ -107,6 +107,18 @@ namespace impl
|
||||
(PRIORITY), _file_function.c_str(), __LINE__, _format_string); \
|
||||
_channel->log(_poco_message); \
|
||||
} \
|
||||
catch (const Poco::Exception & logger_exception) \
|
||||
{ \
|
||||
::write(STDERR_FILENO, static_cast<const void *>(MESSAGE_FOR_EXCEPTION_ON_LOGGING), sizeof(MESSAGE_FOR_EXCEPTION_ON_LOGGING)); \
|
||||
const std::string & logger_exception_message = logger_exception.message(); \
|
||||
::write(STDERR_FILENO, static_cast<const void *>(logger_exception_message.data()), logger_exception_message.size()); \
|
||||
} \
|
||||
catch (const std::exception & logger_exception) \
|
||||
{ \
|
||||
::write(STDERR_FILENO, static_cast<const void *>(MESSAGE_FOR_EXCEPTION_ON_LOGGING), sizeof(MESSAGE_FOR_EXCEPTION_ON_LOGGING)); \
|
||||
const char * logger_exception_message = logger_exception.what(); \
|
||||
::write(STDERR_FILENO, static_cast<const void *>(logger_exception_message), strlen(logger_exception_message)); \
|
||||
} \
|
||||
catch (...) \
|
||||
{ \
|
||||
::write(STDERR_FILENO, static_cast<const void *>(MESSAGE_FOR_EXCEPTION_ON_LOGGING), sizeof(MESSAGE_FOR_EXCEPTION_ON_LOGGING)); \
|
||||
|
||||
@ -643,6 +643,72 @@ TEST(AsyncLoader, CustomDependencyFailure)
|
||||
ASSERT_EQ(good_count.load(), 3);
|
||||
}
|
||||
|
||||
TEST(AsyncLoader, WaitersLimit)
|
||||
{
|
||||
AsyncLoaderTest t(16);
|
||||
|
||||
std::atomic<int> waiters_total{0};
|
||||
int waiters_limit = 5;
|
||||
auto waiters_inc = [&] (const LoadJobPtr &) {
|
||||
int value = waiters_total.load();
|
||||
while (true)
|
||||
{
|
||||
if (value >= waiters_limit)
|
||||
throw Exception(ErrorCodes::ASYNC_LOAD_FAILED, "Too many waiters: {}", value);
|
||||
if (waiters_total.compare_exchange_strong(value, value + 1))
|
||||
break;
|
||||
}
|
||||
};
|
||||
auto waiters_dec = [&] (const LoadJobPtr &) {
|
||||
waiters_total.fetch_sub(1);
|
||||
};
|
||||
|
||||
std::barrier sync(2);
|
||||
t.loader.start();
|
||||
|
||||
auto job_func = [&] (AsyncLoader &, const LoadJobPtr &) {
|
||||
sync.arrive_and_wait(); // (A)
|
||||
};
|
||||
|
||||
auto job = makeLoadJob({}, "job", waiters_inc, waiters_dec, job_func);
|
||||
auto task = t.schedule({job});
|
||||
|
||||
std::atomic<int> failure{0};
|
||||
std::atomic<int> success{0};
|
||||
std::vector<std::thread> waiters;
|
||||
waiters.reserve(10);
|
||||
auto waiter = [&] {
|
||||
try
|
||||
{
|
||||
t.loader.wait(job);
|
||||
success.fetch_add(1);
|
||||
}
|
||||
catch(...)
|
||||
{
|
||||
failure.fetch_add(1);
|
||||
}
|
||||
};
|
||||
|
||||
for (int i = 0; i < 10; i++)
|
||||
waiters.emplace_back(waiter);
|
||||
|
||||
while (failure.load() != 5)
|
||||
std::this_thread::yield();
|
||||
|
||||
ASSERT_EQ(job->waitersCount(), 5);
|
||||
|
||||
sync.arrive_and_wait(); // (A)
|
||||
|
||||
for (auto & thread : waiters)
|
||||
thread.join();
|
||||
|
||||
ASSERT_EQ(success.load(), 5);
|
||||
ASSERT_EQ(failure.load(), 5);
|
||||
ASSERT_EQ(waiters_total.load(), 0);
|
||||
|
||||
t.loader.wait();
|
||||
}
|
||||
|
||||
TEST(AsyncLoader, TestConcurrency)
|
||||
{
|
||||
AsyncLoaderTest t(10);
|
||||
|
||||
@ -105,6 +105,7 @@ void KeeperSnapshotManagerS3::updateS3Configuration(const Poco::Util::AbstractCo
|
||||
.use_virtual_addressing = new_uri.is_virtual_hosted_style,
|
||||
.disable_checksum = false,
|
||||
.gcs_issue_compose_request = false,
|
||||
.is_s3express_bucket = S3::isS3ExpressEndpoint(new_uri.endpoint),
|
||||
};
|
||||
|
||||
auto client = S3::ClientFactory::instance().create(
|
||||
|
||||
@ -44,12 +44,78 @@ struct ContextSharedPart : boost::noncopyable
|
||||
: macros(std::make_unique<Macros>())
|
||||
{}
|
||||
|
||||
~ContextSharedPart()
|
||||
{
|
||||
if (keeper_dispatcher)
|
||||
{
|
||||
try
|
||||
{
|
||||
keeper_dispatcher->shutdown();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait for thread pool for background reads and writes,
|
||||
/// since it may use per-user MemoryTracker which will be destroyed here.
|
||||
if (asynchronous_remote_fs_reader)
|
||||
{
|
||||
try
|
||||
{
|
||||
asynchronous_remote_fs_reader->wait();
|
||||
asynchronous_remote_fs_reader.reset();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
if (asynchronous_local_fs_reader)
|
||||
{
|
||||
try
|
||||
{
|
||||
asynchronous_local_fs_reader->wait();
|
||||
asynchronous_local_fs_reader.reset();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
if (synchronous_local_fs_reader)
|
||||
{
|
||||
try
|
||||
{
|
||||
synchronous_local_fs_reader->wait();
|
||||
synchronous_local_fs_reader.reset();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
if (threadpool_writer)
|
||||
{
|
||||
try
|
||||
{
|
||||
threadpool_writer->wait();
|
||||
threadpool_writer.reset();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// For access of most of shared objects.
|
||||
mutable SharedMutex mutex;
|
||||
|
||||
mutable std::mutex keeper_dispatcher_mutex;
|
||||
mutable std::shared_ptr<KeeperDispatcher> keeper_dispatcher TSA_GUARDED_BY(keeper_dispatcher_mutex);
|
||||
|
||||
ServerSettings server_settings;
|
||||
|
||||
String path; /// Path to the data directory, with a slash at the end.
|
||||
@ -77,6 +143,10 @@ struct ContextSharedPart : boost::noncopyable
|
||||
|
||||
mutable ThrottlerPtr local_read_throttler; /// A server-wide throttler for local IO reads
|
||||
mutable ThrottlerPtr local_write_throttler; /// A server-wide throttler for local IO writes
|
||||
|
||||
mutable std::mutex keeper_dispatcher_mutex;
|
||||
mutable std::shared_ptr<KeeperDispatcher> keeper_dispatcher TSA_GUARDED_BY(keeper_dispatcher_mutex);
|
||||
|
||||
};
|
||||
|
||||
ContextData::ContextData() = default;
|
||||
|
||||
@ -17,6 +17,9 @@
|
||||
|
||||
#include <Core/ExternalTable.h>
|
||||
#include <Poco/Net/MessageHeader.h>
|
||||
#include <Parsers/ASTNameTypePair.h>
|
||||
#include <Parsers/ParserCreateQuery.h>
|
||||
#include <Parsers/parseQuery.h>
|
||||
#include <base/scope_guard.h>
|
||||
|
||||
|
||||
@ -28,6 +31,18 @@ namespace ErrorCodes
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
/// Parsing a list of types with `,` as separator. For example, `Int, Enum('foo'=1,'bar'=2), Double`
|
||||
/// Used in `parseStructureFromTypesField`
|
||||
class ParserTypeList : public IParserBase
|
||||
{
|
||||
protected:
|
||||
const char * getName() const override { return "type pair list"; }
|
||||
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
|
||||
{
|
||||
return ParserList(std::make_unique<ParserDataType>(), std::make_unique<ParserToken>(TokenType::Comma), false)
|
||||
.parse(pos, node, expected);
|
||||
}
|
||||
};
|
||||
|
||||
ExternalTableDataPtr BaseExternalTable::getData(ContextPtr context)
|
||||
{
|
||||
@ -55,23 +70,36 @@ void BaseExternalTable::clear()
|
||||
|
||||
void BaseExternalTable::parseStructureFromStructureField(const std::string & argument)
|
||||
{
|
||||
std::vector<std::string> vals;
|
||||
splitInto<' ', ','>(vals, argument, true);
|
||||
ParserNameTypePairList parser;
|
||||
const auto * pos = argument.data();
|
||||
String error;
|
||||
ASTPtr columns_list_raw = tryParseQuery(parser, pos, pos + argument.size(), error, false, "", false, DBMS_DEFAULT_MAX_QUERY_SIZE, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS, true);
|
||||
|
||||
if (vals.size() % 2 != 0)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Odd number of attributes in section structure: {}", vals.size());
|
||||
if (!columns_list_raw)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Error while parsing table structure: {}", error);
|
||||
|
||||
for (size_t i = 0; i < vals.size(); i += 2)
|
||||
structure.emplace_back(vals[i], vals[i + 1]);
|
||||
for (auto & child : columns_list_raw->children)
|
||||
{
|
||||
auto * column = child->as<ASTNameTypePair>();
|
||||
if (column)
|
||||
structure.emplace_back(column->name, column->type->getColumnNameWithoutAlias());
|
||||
else
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Error while parsing table structure: expected column definition, got {}", child->formatForErrorMessage());
|
||||
}
|
||||
}
|
||||
|
||||
void BaseExternalTable::parseStructureFromTypesField(const std::string & argument)
|
||||
{
|
||||
std::vector<std::string> vals;
|
||||
splitInto<' ', ','>(vals, argument, true);
|
||||
ParserTypeList parser;
|
||||
const auto * pos = argument.data();
|
||||
String error;
|
||||
ASTPtr type_list_raw = tryParseQuery(parser, pos, pos+argument.size(), error, false, "", false, DBMS_DEFAULT_MAX_QUERY_SIZE, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS, true);
|
||||
|
||||
for (size_t i = 0; i < vals.size(); ++i)
|
||||
structure.emplace_back("_" + toString(i + 1), vals[i]);
|
||||
if (!type_list_raw)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Error while parsing table structure: {}", error);
|
||||
|
||||
for (size_t i = 0; i < type_list_raw->children.size(); ++i)
|
||||
structure.emplace_back("_" + toString(i + 1), type_list_raw->children[i]->getColumnNameWithoutAlias());
|
||||
}
|
||||
|
||||
void BaseExternalTable::initSampleBlock()
|
||||
|
||||
@ -64,6 +64,7 @@ namespace DB
|
||||
M(UInt64, max_concurrent_queries, 0, "Maximum number of concurrently executed queries. Zero means unlimited.", 0) \
|
||||
M(UInt64, max_concurrent_insert_queries, 0, "Maximum number of concurrently INSERT queries. Zero means unlimited.", 0) \
|
||||
M(UInt64, max_concurrent_select_queries, 0, "Maximum number of concurrently SELECT queries. Zero means unlimited.", 0) \
|
||||
M(UInt64, max_waiting_queries, 0, "Maximum number of concurrently waiting queries blocked due to `async_load_databases`. Note that waiting queries are not considered by `max_concurrent_*queries*` limits. Zero means unlimited.", 0) \
|
||||
\
|
||||
M(Double, cache_size_to_ram_max_ratio, 0.5, "Set cache size to RAM max ratio. Allows to lower cache size on low-memory systems.", 0) \
|
||||
M(String, uncompressed_cache_policy, DEFAULT_UNCOMPRESSED_CACHE_POLICY, "Uncompressed cache policy name.", 0) \
|
||||
|
||||
@ -78,11 +78,17 @@ class IColumn;
|
||||
M(UInt64, distributed_connections_pool_size, 1024, "Maximum number of connections with one remote server in the pool.", 0) \
|
||||
M(UInt64, connections_with_failover_max_tries, 3, "The maximum number of attempts to connect to replicas.", 0) \
|
||||
M(UInt64, s3_strict_upload_part_size, 0, "The exact size of part to upload during multipart upload to S3 (some implementations does not supports variable size parts).", 0) \
|
||||
M(UInt64, azure_strict_upload_part_size, 0, "The exact size of part to upload during multipart upload to Azure blob storage.", 0) \
|
||||
M(UInt64, s3_min_upload_part_size, 16*1024*1024, "The minimum size of part to upload during multipart upload to S3.", 0) \
|
||||
M(UInt64, s3_max_upload_part_size, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to S3.", 0) \
|
||||
M(UInt64, azure_min_upload_part_size, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage.", 0) \
|
||||
M(UInt64, azure_max_upload_part_size, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage.", 0) \
|
||||
M(UInt64, s3_upload_part_size_multiply_factor, 2, "Multiply s3_min_upload_part_size by this factor each time s3_multiply_parts_count_threshold parts were uploaded from a single write to S3.", 0) \
|
||||
M(UInt64, s3_upload_part_size_multiply_parts_count_threshold, 500, "Each time this number of parts was uploaded to S3, s3_min_upload_part_size is multiplied by s3_upload_part_size_multiply_factor.", 0) \
|
||||
M(UInt64, s3_max_inflight_parts_for_one_file, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited. You ", 0) \
|
||||
M(UInt64, azure_upload_part_size_multiply_factor, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage.", 0) \
|
||||
M(UInt64, azure_upload_part_size_multiply_parts_count_threshold, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor.", 0) \
|
||||
M(UInt64, s3_max_inflight_parts_for_one_file, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited.", 0) \
|
||||
M(UInt64, azure_max_inflight_parts_for_one_file, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited.", 0) \
|
||||
M(UInt64, s3_max_single_part_upload_size, 32*1024*1024, "The maximum size of object to upload using singlepart upload to S3.", 0) \
|
||||
M(UInt64, azure_max_single_part_upload_size, 100*1024*1024, "The maximum size of object to upload using singlepart upload to Azure blob storage.", 0) \
|
||||
M(UInt64, azure_max_single_part_copy_size, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage.", 0) \
|
||||
@ -162,6 +168,7 @@ class IColumn;
|
||||
M(Bool, allow_suspicious_indices, false, "Reject primary/secondary indexes and sorting keys with identical expressions", 0) \
|
||||
M(Bool, allow_suspicious_ttl_expressions, false, "Reject TTL expressions that don't depend on any of table's columns. It indicates a user error most of the time.", 0) \
|
||||
M(Bool, allow_suspicious_variant_types, false, "In CREATE TABLE statement allows specifying Variant type with similar variant types (for example, with different numeric or date types). Enabling this setting may introduce some ambiguity when working with values with similar types.", 0) \
|
||||
M(Bool, allow_suspicious_primary_key, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)", 0) \
|
||||
M(Bool, compile_expressions, false, "Compile some scalar functions and operators to native code.", 0) \
|
||||
M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \
|
||||
M(Bool, compile_aggregate_expressions, true, "Compile aggregate functions to native code.", 0) \
|
||||
@ -265,7 +272,7 @@ class IColumn;
|
||||
M(UInt64, log_queries_cut_to_length, 100000, "If query length is greater than specified threshold (in bytes), then cut query when writing to query log. Also limit length of printed query in ordinary text log.", 0) \
|
||||
M(Float, log_queries_probability, 1., "Log queries with the specified probability.", 0) \
|
||||
\
|
||||
M(Bool, log_processors_profiles, false, "Log Processors profile events.", 0) \
|
||||
M(Bool, log_processors_profiles, true, "Log Processors profile events.", 0) \
|
||||
M(DistributedProductMode, distributed_product_mode, DistributedProductMode::DENY, "How are distributed subqueries performed inside IN or JOIN sections?", IMPORTANT) \
|
||||
\
|
||||
M(UInt64, max_concurrent_queries_for_all_users, 0, "The maximum number of concurrent requests for all users.", 0) \
|
||||
@ -287,6 +294,7 @@ class IColumn;
|
||||
M(UInt64, read_backoff_min_concurrency, 1, "Settings to try keeping the minimal number of threads in case of slow reads.", 0) \
|
||||
\
|
||||
M(Float, memory_tracker_fault_probability, 0., "For testing of `exception safety` - throw an exception every time you allocate memory with the specified probability.", 0) \
|
||||
M(Float, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability.", 0) \
|
||||
\
|
||||
M(Bool, enable_http_compression, false, "Compress the result if the client over HTTP said that it understands data compressed by gzip, deflate, zstd, br, lz4, bz2, xz.", 0) \
|
||||
M(Int64, http_zlib_compression_level, 3, "Compression level - used if the client on HTTP said that it understands data compressed by gzip or deflate.", 0) \
|
||||
@ -1089,14 +1097,15 @@ class IColumn;
|
||||
M(UInt64, output_format_pretty_max_rows, 10000, "Rows limit for Pretty formats.", 0) \
|
||||
M(UInt64, output_format_pretty_max_column_pad_width, 250, "Maximum width to pad all values in a column in Pretty formats.", 0) \
|
||||
M(UInt64, output_format_pretty_max_value_width, 10000, "Maximum width of value to display in Pretty formats. If greater - it will be cut.", 0) \
|
||||
M(UInt64, output_format_pretty_max_value_width_apply_for_single_value, false, "Only cut values (see the `output_format_pretty_max_value_width` setting) when it is not a single value in a block. Otherwise output it entirely, which is useful for the `SHOW CREATE TABLE` query.", 0) \
|
||||
M(UInt64Auto, output_format_pretty_color, "auto", "Use ANSI escape sequences in Pretty formats. 0 - disabled, 1 - enabled, 'auto' - enabled if a terminal.", 0) \
|
||||
M(String, output_format_pretty_grid_charset, "UTF-8", "Charset for printing grid borders. Available charsets: ASCII, UTF-8 (default one).", 0) \
|
||||
M(UInt64, output_format_parquet_row_group_size, 1000000, "Target row group size in rows.", 0) \
|
||||
M(UInt64, output_format_parquet_row_group_size_bytes, 512 * 1024 * 1024, "Target row group size in bytes, before compression.", 0) \
|
||||
M(Bool, output_format_parquet_string_as_string, false, "Use Parquet String type instead of Binary for String columns.", 0) \
|
||||
M(Bool, output_format_parquet_string_as_string, true, "Use Parquet String type instead of Binary for String columns.", 0) \
|
||||
M(Bool, output_format_parquet_fixed_string_as_fixed_byte_array, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type instead of Binary for FixedString columns.", 0) \
|
||||
M(ParquetVersion, output_format_parquet_version, "2.latest", "Parquet format version for output format. Supported versions: 1.0, 2.4, 2.6 and 2.latest (default)", 0) \
|
||||
M(ParquetCompression, output_format_parquet_compression_method, "lz4", "Compression method for Parquet output format. Supported codecs: snappy, lz4, brotli, zstd, gzip, none (uncompressed)", 0) \
|
||||
M(ParquetCompression, output_format_parquet_compression_method, "zstd", "Compression method for Parquet output format. Supported codecs: snappy, lz4, brotli, zstd, gzip, none (uncompressed)", 0) \
|
||||
M(Bool, output_format_parquet_compliant_nested_types, true, "In parquet file schema, use name 'element' instead of 'item' for list elements. This is a historical artifact of Arrow library implementation. Generally increases compatibility, except perhaps with some old versions of Arrow.", 0) \
|
||||
M(Bool, output_format_parquet_use_custom_encoder, false, "Use a faster Parquet encoder implementation.", 0) \
|
||||
M(Bool, output_format_parquet_parallel_encoding, true, "Do Parquet encoding in multiple threads. Requires output_format_parquet_use_custom_encoder.", 0) \
|
||||
@ -1137,7 +1146,8 @@ class IColumn;
|
||||
\
|
||||
M(Bool, output_format_enable_streaming, false, "Enable streaming in output formats that support it.", 0) \
|
||||
M(Bool, output_format_write_statistics, true, "Write statistics about read rows, bytes, time elapsed in suitable output formats.", 0) \
|
||||
M(Bool, output_format_pretty_row_numbers, false, "Add row numbers before each row for pretty output format", 0) \
|
||||
M(Bool, output_format_pretty_row_numbers, true, "Add row numbers before each row for pretty output format", 0) \
|
||||
M(Bool, output_format_pretty_highlight_digit_groups, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline.", 0) \
|
||||
M(UInt64, output_format_pretty_single_large_number_tip_threshold, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)", 0) \
|
||||
M(Bool, insert_distributed_one_random_shard, false, "If setting is enabled, inserting into distributed table will choose a random shard to write when there is no sharding key", 0) \
|
||||
\
|
||||
@ -1147,12 +1157,12 @@ class IColumn;
|
||||
M(Bool, output_format_arrow_low_cardinality_as_dictionary, false, "Enable output LowCardinality type as Dictionary Arrow type", 0) \
|
||||
M(Bool, output_format_arrow_use_signed_indexes_for_dictionary, true, "Use signed integers for dictionary indexes in Arrow format", 0) \
|
||||
M(Bool, output_format_arrow_use_64_bit_indexes_for_dictionary, false, "Always use 64 bit integers for dictionary indexes in Arrow format", 0) \
|
||||
M(Bool, output_format_arrow_string_as_string, false, "Use Arrow String type instead of Binary for String columns", 0) \
|
||||
M(Bool, output_format_arrow_string_as_string, true, "Use Arrow String type instead of Binary for String columns", 0) \
|
||||
M(Bool, output_format_arrow_fixed_string_as_fixed_byte_array, true, "Use Arrow FIXED_SIZE_BINARY type instead of Binary for FixedString columns.", 0) \
|
||||
M(ArrowCompression, output_format_arrow_compression_method, "lz4_frame", "Compression method for Arrow output format. Supported codecs: lz4_frame, zstd, none (uncompressed)", 0) \
|
||||
\
|
||||
M(Bool, output_format_orc_string_as_string, false, "Use ORC String type instead of Binary for String columns", 0) \
|
||||
M(ORCCompression, output_format_orc_compression_method, "lz4", "Compression method for ORC output format. Supported codecs: lz4, snappy, zlib, zstd, none (uncompressed)", 0) \
|
||||
M(Bool, output_format_orc_string_as_string, true, "Use ORC String type instead of Binary for String columns", 0) \
|
||||
M(ORCCompression, output_format_orc_compression_method, "zstd", "Compression method for ORC output format. Supported codecs: lz4, snappy, zlib, zstd, none (uncompressed)", 0) \
|
||||
M(UInt64, output_format_orc_row_index_stride, 10'000, "Target row index stride in ORC output format", 0) \
|
||||
\
|
||||
M(CapnProtoEnumComparingMode, format_capn_proto_enum_comparising_mode, FormatSettings::CapnProtoEnumComparingMode::BY_VALUES, "How to map ClickHouse Enum and CapnProto Enum", 0) \
|
||||
|
||||
@ -95,7 +95,9 @@ static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> sett
|
||||
{"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."},
|
||||
{"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication is dependent materialized view cannot work together with async inserts."},
|
||||
{"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
|
||||
{"log_processors_profiles", false, true, "Enable by default"},
|
||||
{"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."},
|
||||
{"allow_suspicious_primary_key", true, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)"},
|
||||
{"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"},
|
||||
{"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"},
|
||||
{"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"},
|
||||
@ -103,6 +105,21 @@ static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> sett
|
||||
{"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"},
|
||||
{"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"},
|
||||
{"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"},
|
||||
{"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."},
|
||||
{"output_format_pretty_row_numbers", false, true, "It is better for usability."},
|
||||
{"output_format_pretty_max_value_width_apply_for_single_value", true, false, "Single values in Pretty formats won't be cut."},
|
||||
{"output_format_parquet_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
|
||||
{"output_format_orc_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
|
||||
{"output_format_arrow_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
|
||||
{"output_format_parquet_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
|
||||
{"output_format_orc_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
|
||||
{"output_format_pretty_highlight_digit_groups", false, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline."},
|
||||
{"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."},
|
||||
{"azure_strict_upload_part_size", 0, 0, "The exact size of part to upload during multipart upload to Azure blob storage."},
|
||||
{"azure_min_upload_part_size", 16*1024*1024, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage."},
|
||||
{"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."},
|
||||
{"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."},
|
||||
{"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."},
|
||||
}},
|
||||
{"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
|
||||
{"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},
|
||||
|
||||
@ -18,6 +18,16 @@ namespace ErrorCodes
|
||||
extern const int UNKNOWN_UNION;
|
||||
}
|
||||
|
||||
template <typename Type>
|
||||
constexpr auto getEnumValues()
|
||||
{
|
||||
std::array<std::pair<std::string_view, Type>, magic_enum::enum_count<Type>()> enum_values{};
|
||||
size_t index = 0;
|
||||
for (auto value : magic_enum::enum_values<Type>())
|
||||
enum_values[index++] = std::pair{magic_enum::enum_name(value), value};
|
||||
return enum_values;
|
||||
}
|
||||
|
||||
IMPLEMENT_SETTING_ENUM(LoadBalancing, ErrorCodes::UNKNOWN_LOAD_BALANCING,
|
||||
{{"random", LoadBalancing::RANDOM},
|
||||
{"nearest_hostname", LoadBalancing::NEAREST_HOSTNAME},
|
||||
|
||||
@ -13,6 +13,108 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
template <typename Type>
|
||||
constexpr auto getEnumValues();
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define DECLARE_SETTING_ENUM(ENUM_TYPE) \
|
||||
DECLARE_SETTING_ENUM_WITH_RENAME(ENUM_TYPE, ENUM_TYPE)
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define DECLARE_SETTING_ENUM_WITH_RENAME(NEW_NAME, ENUM_TYPE) \
|
||||
struct SettingField##NEW_NAME##Traits \
|
||||
{ \
|
||||
using EnumType = ENUM_TYPE; \
|
||||
using EnumValuePairs = std::pair<const char *, EnumType>[]; \
|
||||
static const String & toString(EnumType value); \
|
||||
static EnumType fromString(std::string_view str); \
|
||||
}; \
|
||||
\
|
||||
using SettingField##NEW_NAME = SettingFieldEnum<ENUM_TYPE, SettingField##NEW_NAME##Traits>;
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define IMPLEMENT_SETTING_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \
|
||||
IMPLEMENT_SETTING_ENUM_IMPL(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, EnumValuePairs, __VA_ARGS__)
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define IMPLEMENT_SETTING_AUTO_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME) \
|
||||
IMPLEMENT_SETTING_ENUM_IMPL(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, , getEnumValues<EnumType>())
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define IMPLEMENT_SETTING_ENUM_IMPL(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, PAIRS_TYPE, ...) \
|
||||
const String & SettingField##NEW_NAME##Traits::toString(typename SettingField##NEW_NAME::EnumType value) \
|
||||
{ \
|
||||
static const std::unordered_map<EnumType, String> map = [] { \
|
||||
std::unordered_map<EnumType, String> res; \
|
||||
for (const auto & [name, val] : PAIRS_TYPE __VA_ARGS__) \
|
||||
res.emplace(val, name); \
|
||||
return res; \
|
||||
}(); \
|
||||
auto it = map.find(value); \
|
||||
if (it != map.end()) \
|
||||
return it->second; \
|
||||
throw Exception(ERROR_CODE_FOR_UNEXPECTED_NAME, \
|
||||
"Unexpected value of " #NEW_NAME ":{}", std::to_string(std::underlying_type_t<EnumType>(value))); \
|
||||
} \
|
||||
\
|
||||
typename SettingField##NEW_NAME::EnumType SettingField##NEW_NAME##Traits::fromString(std::string_view str) \
|
||||
{ \
|
||||
static const std::unordered_map<std::string_view, EnumType> map = [] { \
|
||||
std::unordered_map<std::string_view, EnumType> res; \
|
||||
for (const auto & [name, val] : PAIRS_TYPE __VA_ARGS__) \
|
||||
res.emplace(name, val); \
|
||||
return res; \
|
||||
}(); \
|
||||
auto it = map.find(str); \
|
||||
if (it != map.end()) \
|
||||
return it->second; \
|
||||
String msg; \
|
||||
bool need_comma = false; \
|
||||
for (auto & name : map | boost::adaptors::map_keys) \
|
||||
{ \
|
||||
if (std::exchange(need_comma, true)) \
|
||||
msg += ", "; \
|
||||
msg += "'" + String{name} + "'"; \
|
||||
} \
|
||||
throw Exception(ERROR_CODE_FOR_UNEXPECTED_NAME, "Unexpected value of " #NEW_NAME ": '{}'. Must be one of [{}]", String{str}, msg); \
|
||||
}
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define DECLARE_SETTING_MULTI_ENUM(ENUM_TYPE) \
|
||||
DECLARE_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, ENUM_TYPE)
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define DECLARE_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, NEW_NAME) \
|
||||
struct SettingField##NEW_NAME##Traits \
|
||||
{ \
|
||||
using EnumType = ENUM_TYPE; \
|
||||
using EnumValuePairs = std::pair<const char *, EnumType>[]; \
|
||||
static size_t getEnumSize(); \
|
||||
static const String & toString(EnumType value); \
|
||||
static EnumType fromString(std::string_view str); \
|
||||
}; \
|
||||
\
|
||||
using SettingField##NEW_NAME = SettingFieldMultiEnum<ENUM_TYPE, SettingField##NEW_NAME##Traits>; \
|
||||
using NEW_NAME##List = typename SettingField##NEW_NAME::ValueType;
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define IMPLEMENT_SETTING_MULTI_ENUM(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \
|
||||
IMPLEMENT_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__)
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define IMPLEMENT_SETTING_MULTI_ENUM_WITH_RENAME(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \
|
||||
IMPLEMENT_SETTING_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__)\
|
||||
size_t SettingField##NEW_NAME##Traits::getEnumSize() {\
|
||||
return std::initializer_list<std::pair<const char*, NEW_NAME>> __VA_ARGS__ .size();\
|
||||
}
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define IMPLEMENT_SETTING_MULTI_AUTO_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME) \
|
||||
IMPLEMENT_SETTING_AUTO_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME)\
|
||||
size_t SettingField##NEW_NAME##Traits::getEnumSize() {\
|
||||
return getEnumValues<EnumType>().size();\
|
||||
}
|
||||
|
||||
enum class LoadBalancing
|
||||
{
|
||||
/// among replicas with a minimum number of errors selected randomly
|
||||
|
||||
@ -7,9 +7,7 @@
|
||||
#include <Core/MultiEnum.h>
|
||||
#include <boost/range/adaptor/map.hpp>
|
||||
#include <chrono>
|
||||
#include <unordered_map>
|
||||
#include <string_view>
|
||||
#include <magic_enum.hpp>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -380,79 +378,6 @@ void SettingFieldEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
|
||||
*this = Traits::fromString(SettingFieldEnumHelpers::readBinary(in));
|
||||
}
|
||||
|
||||
template <typename Type>
|
||||
constexpr auto getEnumValues()
|
||||
{
|
||||
std::array<std::pair<std::string_view, Type>, magic_enum::enum_count<Type>()> enum_values{};
|
||||
size_t index = 0;
|
||||
for (auto value : magic_enum::enum_values<Type>())
|
||||
enum_values[index++] = std::pair{magic_enum::enum_name(value), value};
|
||||
return enum_values;
|
||||
}
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define DECLARE_SETTING_ENUM(ENUM_TYPE) \
|
||||
DECLARE_SETTING_ENUM_WITH_RENAME(ENUM_TYPE, ENUM_TYPE)
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define DECLARE_SETTING_ENUM_WITH_RENAME(NEW_NAME, ENUM_TYPE) \
|
||||
struct SettingField##NEW_NAME##Traits \
|
||||
{ \
|
||||
using EnumType = ENUM_TYPE; \
|
||||
using EnumValuePairs = std::pair<const char *, EnumType>[]; \
|
||||
static const String & toString(EnumType value); \
|
||||
static EnumType fromString(std::string_view str); \
|
||||
}; \
|
||||
\
|
||||
using SettingField##NEW_NAME = SettingFieldEnum<ENUM_TYPE, SettingField##NEW_NAME##Traits>;
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define IMPLEMENT_SETTING_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \
|
||||
IMPLEMENT_SETTING_ENUM_IMPL(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, EnumValuePairs, __VA_ARGS__)
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define IMPLEMENT_SETTING_AUTO_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME) \
|
||||
IMPLEMENT_SETTING_ENUM_IMPL(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, , getEnumValues<EnumType>())
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define IMPLEMENT_SETTING_ENUM_IMPL(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, PAIRS_TYPE, ...) \
|
||||
const String & SettingField##NEW_NAME##Traits::toString(typename SettingField##NEW_NAME::EnumType value) \
|
||||
{ \
|
||||
static const std::unordered_map<EnumType, String> map = [] { \
|
||||
std::unordered_map<EnumType, String> res; \
|
||||
for (const auto & [name, val] : PAIRS_TYPE __VA_ARGS__) \
|
||||
res.emplace(val, name); \
|
||||
return res; \
|
||||
}(); \
|
||||
auto it = map.find(value); \
|
||||
if (it != map.end()) \
|
||||
return it->second; \
|
||||
throw Exception(ERROR_CODE_FOR_UNEXPECTED_NAME, \
|
||||
"Unexpected value of " #NEW_NAME ":{}", std::to_string(std::underlying_type_t<EnumType>(value))); \
|
||||
} \
|
||||
\
|
||||
typename SettingField##NEW_NAME::EnumType SettingField##NEW_NAME##Traits::fromString(std::string_view str) \
|
||||
{ \
|
||||
static const std::unordered_map<std::string_view, EnumType> map = [] { \
|
||||
std::unordered_map<std::string_view, EnumType> res; \
|
||||
for (const auto & [name, val] : PAIRS_TYPE __VA_ARGS__) \
|
||||
res.emplace(name, val); \
|
||||
return res; \
|
||||
}(); \
|
||||
auto it = map.find(str); \
|
||||
if (it != map.end()) \
|
||||
return it->second; \
|
||||
String msg; \
|
||||
bool need_comma = false; \
|
||||
for (auto & name : map | boost::adaptors::map_keys) \
|
||||
{ \
|
||||
if (std::exchange(need_comma, true)) \
|
||||
msg += ", "; \
|
||||
msg += "'" + String{name} + "'"; \
|
||||
} \
|
||||
throw Exception(ERROR_CODE_FOR_UNEXPECTED_NAME, "Unexpected value of " #NEW_NAME ": '{}'. Must be one of [{}]", String{str}, msg); \
|
||||
}
|
||||
|
||||
// Mostly like SettingFieldEnum, but can have multiple enum values (or none) set at once.
|
||||
template <typename Enum, typename Traits>
|
||||
struct SettingFieldMultiEnum
|
||||
@ -543,42 +468,6 @@ void SettingFieldMultiEnum<EnumT, Traits>::readBinary(ReadBuffer & in)
|
||||
parseFromString(SettingFieldEnumHelpers::readBinary(in));
|
||||
}
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define DECLARE_SETTING_MULTI_ENUM(ENUM_TYPE) \
|
||||
DECLARE_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, ENUM_TYPE)
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define DECLARE_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, NEW_NAME) \
|
||||
struct SettingField##NEW_NAME##Traits \
|
||||
{ \
|
||||
using EnumType = ENUM_TYPE; \
|
||||
using EnumValuePairs = std::pair<const char *, EnumType>[]; \
|
||||
static size_t getEnumSize(); \
|
||||
static const String & toString(EnumType value); \
|
||||
static EnumType fromString(std::string_view str); \
|
||||
}; \
|
||||
\
|
||||
using SettingField##NEW_NAME = SettingFieldMultiEnum<ENUM_TYPE, SettingField##NEW_NAME##Traits>; \
|
||||
using NEW_NAME##List = typename SettingField##NEW_NAME::ValueType;
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define IMPLEMENT_SETTING_MULTI_ENUM(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \
|
||||
IMPLEMENT_SETTING_MULTI_ENUM_WITH_RENAME(ENUM_TYPE, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__)
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define IMPLEMENT_SETTING_MULTI_ENUM_WITH_RENAME(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, ...) \
|
||||
IMPLEMENT_SETTING_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME, __VA_ARGS__)\
|
||||
size_t SettingField##NEW_NAME##Traits::getEnumSize() {\
|
||||
return std::initializer_list<std::pair<const char*, NEW_NAME>> __VA_ARGS__ .size();\
|
||||
}
|
||||
|
||||
/// NOLINTNEXTLINE
|
||||
#define IMPLEMENT_SETTING_MULTI_AUTO_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME) \
|
||||
IMPLEMENT_SETTING_AUTO_ENUM(NEW_NAME, ERROR_CODE_FOR_UNEXPECTED_NAME)\
|
||||
size_t SettingField##NEW_NAME##Traits::getEnumSize() {\
|
||||
return getEnumValues<EnumType>().size();\
|
||||
}
|
||||
|
||||
/// Setting field for specifying user-defined timezone. It is basically a string, but it needs validation.
|
||||
struct SettingFieldTimezone
|
||||
{
|
||||
|
||||
@ -275,7 +275,10 @@ public:
|
||||
}
|
||||
|
||||
readPODBinary(stack_trace, in);
|
||||
readVectorBinary(thread_frame_pointers, in);
|
||||
|
||||
if (sig != SanitizerTrap)
|
||||
readVectorBinary(thread_frame_pointers, in);
|
||||
|
||||
readBinary(thread_num, in);
|
||||
readPODBinary(thread_ptr, in);
|
||||
|
||||
@ -542,6 +545,16 @@ private:
|
||||
|
||||
|
||||
#if defined(SANITIZER)
|
||||
|
||||
template <typename T>
|
||||
struct ValueHolder
|
||||
{
|
||||
ValueHolder(T value_) : value(value_)
|
||||
{}
|
||||
|
||||
T value;
|
||||
};
|
||||
|
||||
extern "C" void __sanitizer_set_death_callback(void (*)());
|
||||
|
||||
/// Sanitizers may not expect some function calls from death callback.
|
||||
@ -559,10 +572,13 @@ static DISABLE_SANITIZER_INSTRUMENTATION void sanitizerDeathCallback()
|
||||
|
||||
const StackTrace stack_trace;
|
||||
|
||||
int sig = SignalListener::SanitizerTrap;
|
||||
writeBinary(sig, out);
|
||||
writeBinary(SignalListener::SanitizerTrap, out);
|
||||
writePODBinary(stack_trace, out);
|
||||
writeBinary(UInt32(getThreadId()), out);
|
||||
/// We create a dummy struct with a constructor so DISABLE_SANITIZER_INSTRUMENTATION is not applied to it
|
||||
/// otherwise, Memory sanitizer can't know that values initiialized inside this function are actually initialized
|
||||
/// because instrumentations are disabled leading to false positives later on
|
||||
ValueHolder<UInt32> thread_id{static_cast<UInt32>(getThreadId())};
|
||||
writeBinary(thread_id.value, out);
|
||||
writePODBinary(current_thread, out);
|
||||
|
||||
out.next();
|
||||
|
||||
@ -41,33 +41,6 @@ enum class AttributeUnderlyingType : TypeIndexUnderlying
|
||||
|
||||
#undef map_item
|
||||
|
||||
|
||||
#define CALL_FOR_ALL_DICTIONARY_ATTRIBUTE_TYPES(M) \
|
||||
M(UInt8) \
|
||||
M(UInt16) \
|
||||
M(UInt32) \
|
||||
M(UInt64) \
|
||||
M(UInt128) \
|
||||
M(UInt256) \
|
||||
M(Int8) \
|
||||
M(Int16) \
|
||||
M(Int32) \
|
||||
M(Int64) \
|
||||
M(Int128) \
|
||||
M(Int256) \
|
||||
M(Decimal32) \
|
||||
M(Decimal64) \
|
||||
M(Decimal128) \
|
||||
M(Decimal256) \
|
||||
M(DateTime64) \
|
||||
M(Float32) \
|
||||
M(Float64) \
|
||||
M(UUID) \
|
||||
M(IPv4) \
|
||||
M(IPv6) \
|
||||
M(String) \
|
||||
M(Array)
|
||||
|
||||
/// Min and max lifetimes for a dictionary or its entry
|
||||
using DictionaryLifetime = ExternalLoadableLifetime;
|
||||
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
#include <Common/Throttler.h>
|
||||
#include <base/sleep.h>
|
||||
#include <Common/ProfileEvents.h>
|
||||
|
||||
#include <IO/SeekableReadBuffer.h>
|
||||
|
||||
namespace ProfileEvents
|
||||
{
|
||||
@ -27,7 +27,6 @@ namespace ErrorCodes
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
|
||||
ReadBufferFromAzureBlobStorage::ReadBufferFromAzureBlobStorage(
|
||||
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> blob_container_client_,
|
||||
const String & path_,
|
||||
@ -56,7 +55,6 @@ ReadBufferFromAzureBlobStorage::ReadBufferFromAzureBlobStorage(
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void ReadBufferFromAzureBlobStorage::setReadUntilEnd()
|
||||
{
|
||||
if (read_until_position)
|
||||
@ -105,7 +103,7 @@ bool ReadBufferFromAzureBlobStorage::nextImpl()
|
||||
|
||||
auto handle_exception = [&, this](const auto & e, size_t i)
|
||||
{
|
||||
LOG_INFO(log, "Exception caught during Azure Read for file {} at attempt {}/{}: {}", path, i + 1, max_single_read_retries, e.Message);
|
||||
LOG_DEBUG(log, "Exception caught during Azure Read for file {} at attempt {}/{}: {}", path, i + 1, max_single_read_retries, e.Message);
|
||||
if (i + 1 == max_single_read_retries)
|
||||
throw;
|
||||
|
||||
@ -139,7 +137,6 @@ bool ReadBufferFromAzureBlobStorage::nextImpl()
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
off_t ReadBufferFromAzureBlobStorage::seek(off_t offset_, int whence)
|
||||
{
|
||||
if (offset_ == getPosition() && whence == SEEK_SET)
|
||||
@ -193,13 +190,11 @@ off_t ReadBufferFromAzureBlobStorage::seek(off_t offset_, int whence)
|
||||
return offset;
|
||||
}
|
||||
|
||||
|
||||
off_t ReadBufferFromAzureBlobStorage::getPosition()
|
||||
{
|
||||
return offset - available();
|
||||
}
|
||||
|
||||
|
||||
void ReadBufferFromAzureBlobStorage::initialize()
|
||||
{
|
||||
if (initialized)
|
||||
@ -220,7 +215,7 @@ void ReadBufferFromAzureBlobStorage::initialize()
|
||||
|
||||
auto handle_exception = [&, this](const auto & e, size_t i)
|
||||
{
|
||||
LOG_INFO(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_download_retries, e.Message);
|
||||
LOG_DEBUG(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_download_retries, e.Message);
|
||||
if (i + 1 == max_single_download_retries)
|
||||
throw;
|
||||
|
||||
@ -262,6 +257,47 @@ size_t ReadBufferFromAzureBlobStorage::getFileSize()
|
||||
return *file_size;
|
||||
}
|
||||
|
||||
size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t range_begin, const std::function<bool(size_t)> & /*progress_callback*/) const
|
||||
{
|
||||
size_t initial_n = n;
|
||||
|
||||
size_t sleep_time_with_backoff_milliseconds = 100;
|
||||
|
||||
for (size_t i = 0; i < max_single_download_retries && n > 0; ++i)
|
||||
{
|
||||
size_t bytes_copied = 0;
|
||||
try
|
||||
{
|
||||
Azure::Storage::Blobs::DownloadBlobOptions download_options;
|
||||
download_options.Range = {static_cast<int64_t>(range_begin), n};
|
||||
auto download_response = blob_client->Download(download_options);
|
||||
|
||||
std::unique_ptr<Azure::Core::IO::BodyStream> body_stream = std::move(download_response.Value.BodyStream);
|
||||
bytes_copied = body_stream->ReadToCount(reinterpret_cast<uint8_t *>(to), body_stream->Length());
|
||||
|
||||
LOG_TEST(log, "AzureBlobStorage readBigAt read bytes {}", bytes_copied);
|
||||
|
||||
if (read_settings.remote_throttler)
|
||||
read_settings.remote_throttler->add(bytes_copied, ProfileEvents::RemoteReadThrottlerBytes, ProfileEvents::RemoteReadThrottlerSleepMicroseconds);
|
||||
}
|
||||
catch (const Azure::Core::RequestFailedException & e)
|
||||
{
|
||||
LOG_DEBUG(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_download_retries, e.Message);
|
||||
if (i + 1 == max_single_download_retries)
|
||||
throw;
|
||||
|
||||
sleepForMilliseconds(sleep_time_with_backoff_milliseconds);
|
||||
sleep_time_with_backoff_milliseconds *= 2;
|
||||
}
|
||||
|
||||
range_begin += bytes_copied;
|
||||
to += bytes_copied;
|
||||
n -= bytes_copied;
|
||||
}
|
||||
|
||||
return initial_n;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user