thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into dictionaries_ddl_interpreter

Browse Source
This commit is contained in:
alesapin 2019-10-14 11:49:18 +03:00
commit 155731a676
178 changed files with 2557 additions and 1045 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
CMakeLists.txt
View File

@ -118,16 +118,16 @@ endif ()
option (ENABLE_TESTS "Enables tests" ON)
if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64")
if (ARCH_AMD64)
option (USE_INTERNAL_MEMCPY "Use internal implementation of 'memcpy' function instead of provided by libc. Only for x86_64." ON)
endif ()
if (OS_LINUX AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0")
option (GLIBC_COMPATIBILITY "Set to TRUE to enable compatibility with older glibc libraries. Only for x86_64, Linux. Implies USE_INTERNAL_MEMCPY." ON)
endif ()
if (OS_LINUX AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0")
option (GLIBC_COMPATIBILITY "Set to TRUE to enable compatibility with older glibc libraries. Only for x86_64, Linux. Implies USE_INTERNAL_MEMCPY." ON)
endif ()
if (NOT CMAKE_VERSION VERSION_GREATER "3.9.0")
message (WARNING "CMake version must be greater than 3.9.0 for production builds.")
endif ()
if (NOT CMAKE_VERSION VERSION_GREATER "3.9.0")
message (WARNING "CMake version must be greater than 3.9.0 for production builds.")
endif ()
# Make sure the final executable has symbols exported

3
cmake/arch.cmake
View File

@ -1,3 +1,6 @@
if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64")
set (ARCH_AMD64 1)
endif ()
if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*)")
set (ARCH_AARCH64 1)
endif ()

12
contrib/jemalloc-cmake/CMakeLists.txt
View File

@ -45,9 +45,15 @@ endif ()
add_library(jemalloc STATIC ${SRCS})
target_include_directories(jemalloc PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}/include
${CMAKE_CURRENT_SOURCE_DIR}/include_linux_x86_64) # jemalloc.h
target_include_directories(jemalloc PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
if (ARCH_AMD64)
target_include_directories(jemalloc PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include_linux_x86_64)
elseif (ARCH_ARM)
target_include_directories(jemalloc PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include_linux_aarch64)
else ()
message (FATAL_ERROR "jemalloc can only be used on x86_64 or aarch64.")
endif ()
target_include_directories(jemalloc PRIVATE
${JEMALLOC_SOURCE_DIR}/include)

7
contrib/jemalloc-cmake/include_linux_aarch64/README Normal file
View File

@ -0,0 +1,7 @@
Here are pre-generated files from jemalloc on Linux aarch64.
You can obtain these files by running ./autogen.sh inside jemalloc source directory.
Added #define GNU_SOURCE
Added JEMALLOC_OVERRIDE___POSIX_MEMALIGN because why not.
Removed JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF because it's non standard.
Removed JEMALLOC_PURGE_MADVISE_FREE because it's available only from Linux 4.5.

382
contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h Normal file
View File

@ -0,0 +1,382 @@
/* include/jemalloc/internal/jemalloc_internal_defs.h. Generated from jemalloc_internal_defs.h.in by configure. */
#ifndef JEMALLOC_INTERNAL_DEFS_H_
#define JEMALLOC_INTERNAL_DEFS_H_
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
/*
* If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all
* public APIs to be prefixed. This makes it possible, with some care, to use
* multiple allocators simultaneously.
*/
/* #undef JEMALLOC_PREFIX */
/* #undef JEMALLOC_CPREFIX */
/*
* Define overrides for non-standard allocator-related functions if they are
* present on the system.
*/
#define JEMALLOC_OVERRIDE___LIBC_CALLOC
#define JEMALLOC_OVERRIDE___LIBC_FREE
#define JEMALLOC_OVERRIDE___LIBC_MALLOC
#define JEMALLOC_OVERRIDE___LIBC_MEMALIGN
#define JEMALLOC_OVERRIDE___LIBC_REALLOC
#define JEMALLOC_OVERRIDE___LIBC_VALLOC
#define JEMALLOC_OVERRIDE___POSIX_MEMALIGN
/*
* JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs.
* For shared libraries, symbol visibility mechanisms prevent these symbols
* from being exported, but for static libraries, naming collisions are a real
* possibility.
*/
#define JEMALLOC_PRIVATE_NAMESPACE je_
/*
* Hyper-threaded CPUs may need a special instruction inside spin loops in
* order to yield to another virtual CPU.
*/
#define CPU_SPINWAIT
/* 1 if CPU_SPINWAIT is defined, 0 otherwise. */
#define HAVE_CPU_SPINWAIT 0
/*
* Number of significant bits in virtual addresses. This may be less than the
* total number of bits in a pointer, e.g. on x64, for which the uppermost 16
* bits are the same as bit 47.
*/
#define LG_VADDR 48
/* Defined if C11 atomics are available. */
#define JEMALLOC_C11_ATOMICS 1
/* Defined if GCC __atomic atomics are available. */
#define JEMALLOC_GCC_ATOMIC_ATOMICS 1
/* Defined if GCC __sync atomics are available. */
#define JEMALLOC_GCC_SYNC_ATOMICS 1
/*
* Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and
* __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite
* __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the
* functions are defined in libgcc instead of being inlines).
*/
/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 */
/*
* Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and
* __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite
* __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the
* functions are defined in libgcc instead of being inlines).
*/
/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 */
/*
* Defined if __builtin_clz() and __builtin_clzl() are available.
*/
#define JEMALLOC_HAVE_BUILTIN_CLZ
/*
* Defined if os_unfair_lock_*() functions are available, as provided by Darwin.
*/
/* #undef JEMALLOC_OS_UNFAIR_LOCK */
/*
* Defined if OSSpin*() functions are available, as provided by Darwin, and
* documented in the spinlock(3) manual page.
*/
/* #undef JEMALLOC_OSSPIN */
/* Defined if syscall(2) is usable. */
#define JEMALLOC_USE_SYSCALL
/*
* Defined if secure_getenv(3) is available.
*/
#define JEMALLOC_HAVE_SECURE_GETENV
/*
* Defined if issetugid(2) is available.
*/
/* #undef JEMALLOC_HAVE_ISSETUGID */
/* Defined if pthread_atfork(3) is available. */
#define JEMALLOC_HAVE_PTHREAD_ATFORK
/* Defined if pthread_setname_np(3) is available. */
#define JEMALLOC_HAVE_PTHREAD_SETNAME_NP
/*
* Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
*/
#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE 1
/*
* Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.
*/
#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1
/*
* Defined if mach_absolute_time() is available.
*/
/* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */
/*
* Defined if _malloc_thread_cleanup() exists. At least in the case of
* FreeBSD, pthread_key_create() allocates, which if used during malloc
* bootstrapping will cause recursion into the pthreads library. Therefore, if
* _malloc_thread_cleanup() exists, use it as the basis for thread cleanup in
* malloc_tsd.
*/
/* #undef JEMALLOC_MALLOC_THREAD_CLEANUP */
/*
* Defined if threaded initialization is known to be safe on this platform.
* Among other things, it must be possible to initialize a mutex without
* triggering allocation in order for threaded allocation to be safe.
*/
#define JEMALLOC_THREADED_INIT
/*
* Defined if the pthreads implementation defines
* _pthread_mutex_init_calloc_cb(), in which case the function is used in order
* to avoid recursive allocation during mutex initialization.
*/
/* #undef JEMALLOC_MUTEX_INIT_CB */
/* Non-empty if the tls_model attribute is supported. */
#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))
/*
* JEMALLOC_DEBUG enables assertions and other sanity checks, and disables
* inline functions.
*/
/* #undef JEMALLOC_DEBUG */
/* JEMALLOC_STATS enables statistics calculation. */
#define JEMALLOC_STATS
/* JEMALLOC_EXPERIMENTAL_SMALLOCX_API enables experimental smallocx API. */
/* #undef JEMALLOC_EXPERIMENTAL_SMALLOCX_API */
/* JEMALLOC_PROF enables allocation profiling. */
/* #undef JEMALLOC_PROF */
/* Use libunwind for profile backtracing if defined. */
/* #undef JEMALLOC_PROF_LIBUNWIND */
/* Use libgcc for profile backtracing if defined. */
/* #undef JEMALLOC_PROF_LIBGCC */
/* Use gcc intrinsics for profile backtracing if defined. */
/* #undef JEMALLOC_PROF_GCC */
/*
* JEMALLOC_DSS enables use of sbrk(2) to allocate extents from the data storage
* segment (DSS).
*/
#define JEMALLOC_DSS
/* Support memory filling (junk/zero). */
#define JEMALLOC_FILL
/* Support utrace(2)-based tracing. */
/* #undef JEMALLOC_UTRACE */
/* Support optional abort() on OOM. */
/* #undef JEMALLOC_XMALLOC */
/* Support lazy locking (avoid locking unless a second thread is launched). */
/* #undef JEMALLOC_LAZY_LOCK */
/*
* Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size
* classes).
*/
/* #undef LG_QUANTUM */
/* One page is 2^LG_PAGE bytes. */
#define LG_PAGE 16
/*
* One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the
* system does not explicitly support huge pages; system calls that require
* explicit huge page support are separately configured.
*/
#define LG_HUGEPAGE 29
/*
* If defined, adjacent virtual memory mappings with identical attributes
* automatically coalesce, and they fragment when changes are made to subranges.
* This is the normal order of things for mmap()/munmap(), but on Windows
* VirtualAlloc()/VirtualFree() operations must be precisely matched, i.e.
* mappings do *not* coalesce/fragment.
*/
#define JEMALLOC_MAPS_COALESCE
/*
* If defined, retain memory for later reuse by default rather than using e.g.
* munmap() to unmap freed extents. This is enabled on 64-bit Linux because
* common sequences of mmap()/munmap() calls will cause virtual memory map
* holes.
*/
#define JEMALLOC_RETAIN
/* TLS is used to map arenas and magazine caches to threads. */
#define JEMALLOC_TLS
/*
* Used to mark unreachable code to quiet "end of non-void" compiler warnings.
* Don't use this directly; instead use unreachable() from util.h
*/
#define JEMALLOC_INTERNAL_UNREACHABLE __builtin_unreachable
/*
* ffs*() functions to use for bitmapping. Don't use these directly; instead,
* use ffs_*() from util.h.
*/
#define JEMALLOC_INTERNAL_FFSLL __builtin_ffsll
#define JEMALLOC_INTERNAL_FFSL __builtin_ffsl
#define JEMALLOC_INTERNAL_FFS __builtin_ffs
/*
* If defined, explicitly attempt to more uniformly distribute large allocation
* pointer alignments across all cache indices.
*/
#define JEMALLOC_CACHE_OBLIVIOUS
/*
* If defined, enable logging facilities. We make this a configure option to
* avoid taking extra branches everywhere.
*/
/* #undef JEMALLOC_LOG */
/*
* If defined, use readlinkat() (instead of readlink()) to follow
* /etc/malloc_conf.
*/
/* #undef JEMALLOC_READLINKAT */
/*
* Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings.
*/
/* #undef JEMALLOC_ZONE */
/*
* Methods for determining whether the OS overcommits.
* JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY: Linux's
* /proc/sys/vm.overcommit_memory file.
* JEMALLOC_SYSCTL_VM_OVERCOMMIT: FreeBSD's vm.overcommit sysctl.
*/
/* #undef JEMALLOC_SYSCTL_VM_OVERCOMMIT */
#define JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY
/* Defined if madvise(2) is available. */
#define JEMALLOC_HAVE_MADVISE
/*
* Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE
* arguments to madvise(2).
*/
#define JEMALLOC_HAVE_MADVISE_HUGE
/*
* Methods for purging unused pages differ between operating systems.
*
* madvise(..., MADV_FREE) : This marks pages as being unused, such that they
* will be discarded rather than swapped out.
* madvise(..., MADV_DONTNEED) : If JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS is
* defined, this immediately discards pages,
* such that new pages will be demand-zeroed if
* the address region is later touched;
* otherwise this behaves similarly to
* MADV_FREE, though typically with higher
* system overhead.
*/
#define JEMALLOC_PURGE_MADVISE_FREE
#define JEMALLOC_PURGE_MADVISE_DONTNEED
#define JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
/* Defined if madvise(2) is available but MADV_FREE is not (x86 Linux only). */
/* #undef JEMALLOC_DEFINE_MADVISE_FREE */
/*
* Defined if MADV_DO[NT]DUMP is supported as an argument to madvise.
*/
#define JEMALLOC_MADVISE_DONTDUMP
/*
* Defined if transparent huge pages (THPs) are supported via the
* MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled.
*/
/* #undef JEMALLOC_THP */
/* Define if operating system has alloca.h header. */
#define JEMALLOC_HAS_ALLOCA_H 1
/* C99 restrict keyword supported. */
#define JEMALLOC_HAS_RESTRICT 1
/* For use by hash code. */
/* #undef JEMALLOC_BIG_ENDIAN */
/* sizeof(int) == 2^LG_SIZEOF_INT. */
#define LG_SIZEOF_INT 2
/* sizeof(long) == 2^LG_SIZEOF_LONG. */
#define LG_SIZEOF_LONG 3
/* sizeof(long long) == 2^LG_SIZEOF_LONG_LONG. */
#define LG_SIZEOF_LONG_LONG 3
/* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */
#define LG_SIZEOF_INTMAX_T 3
/* glibc malloc hooks (__malloc_hook, __realloc_hook, __free_hook). */
#define JEMALLOC_GLIBC_MALLOC_HOOK
/* glibc memalign hook. */
#define JEMALLOC_GLIBC_MEMALIGN_HOOK
/* pthread support */
#define JEMALLOC_HAVE_PTHREAD
/* dlsym() support */
#define JEMALLOC_HAVE_DLSYM
/* Adaptive mutex support in pthreads. */
#define JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP
/* GNU specific sched_getcpu support */
#define JEMALLOC_HAVE_SCHED_GETCPU
/* GNU specific sched_setaffinity support */
#define JEMALLOC_HAVE_SCHED_SETAFFINITY
/*
* If defined, all the features necessary for background threads are present.
*/
#define JEMALLOC_BACKGROUND_THREAD 1
/*
* If defined, jemalloc symbols are not exported (doesn't work when
* JEMALLOC_PREFIX is not defined).
*/
/* #undef JEMALLOC_EXPORT */
/* config.malloc_conf options string. */
#define JEMALLOC_CONFIG_MALLOC_CONF ""
/* If defined, jemalloc takes the malloc/free/etc. symbol names. */
#define JEMALLOC_IS_MALLOC 1
/*
* Defined if strerror_r returns char * if _GNU_SOURCE is defined.
*/
#define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE
#endif /* JEMALLOC_INTERNAL_DEFS_H_ */

194
contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_preamble.h Normal file
View File

@ -0,0 +1,194 @@
#ifndef JEMALLOC_PREAMBLE_H
#define JEMALLOC_PREAMBLE_H
#include "jemalloc_internal_defs.h"
#include "jemalloc/internal/jemalloc_internal_decls.h"
#ifdef JEMALLOC_UTRACE
#include <sys/ktrace.h>
#endif
#define JEMALLOC_NO_DEMANGLE
#ifdef JEMALLOC_JET
# undef JEMALLOC_IS_MALLOC
# define JEMALLOC_N(n) jet_##n
# include "jemalloc/internal/public_namespace.h"
# define JEMALLOC_NO_RENAME
# include "jemalloc/jemalloc.h"
# undef JEMALLOC_NO_RENAME
#else
# define JEMALLOC_N(n) je_##n
# include "jemalloc/jemalloc.h"
#endif
#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN))
#include <libkern/OSAtomic.h>
#endif
#ifdef JEMALLOC_ZONE
#include <mach/mach_error.h>
#include <mach/mach_init.h>
#include <mach/vm_map.h>
#endif
#include "jemalloc/internal/jemalloc_internal_macros.h"
/*
* Note that the ordering matters here; the hook itself is name-mangled. We
* want the inclusion of hooks to happen early, so that we hook as much as
* possible.
*/
#ifndef JEMALLOC_NO_PRIVATE_NAMESPACE
# ifndef JEMALLOC_JET
# include "jemalloc/internal/private_namespace.h"
# else
# include "jemalloc/internal/private_namespace_jet.h"
# endif
#endif
#include "jemalloc/internal/test_hooks.h"
#ifdef JEMALLOC_DEFINE_MADVISE_FREE
# define JEMALLOC_MADV_FREE 8
#endif
static const bool config_debug =
#ifdef JEMALLOC_DEBUG
true
#else
false
#endif
;
static const bool have_dss =
#ifdef JEMALLOC_DSS
true
#else
false
#endif
;
static const bool have_madvise_huge =
#ifdef JEMALLOC_HAVE_MADVISE_HUGE
true
#else
false
#endif
;
static const bool config_fill =
#ifdef JEMALLOC_FILL
true
#else
false
#endif
;
static const bool config_lazy_lock =
#ifdef JEMALLOC_LAZY_LOCK
true
#else
false
#endif
;
static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF;
static const bool config_prof =
#ifdef JEMALLOC_PROF
true
#else
false
#endif
;
static const bool config_prof_libgcc =
#ifdef JEMALLOC_PROF_LIBGCC
true
#else
false
#endif
;
static const bool config_prof_libunwind =
#ifdef JEMALLOC_PROF_LIBUNWIND
true
#else
false
#endif
;
static const bool maps_coalesce =
#ifdef JEMALLOC_MAPS_COALESCE
true
#else
false
#endif
;
static const bool config_stats =
#ifdef JEMALLOC_STATS
true
#else
false
#endif
;
static const bool config_tls =
#ifdef JEMALLOC_TLS
true
#else
false
#endif
;
static const bool config_utrace =
#ifdef JEMALLOC_UTRACE
true
#else
false
#endif
;
static const bool config_xmalloc =
#ifdef JEMALLOC_XMALLOC
true
#else
false
#endif
;
static const bool config_cache_oblivious =
#ifdef JEMALLOC_CACHE_OBLIVIOUS
true
#else
false
#endif
;
/*
* Undocumented, for jemalloc development use only at the moment. See the note
* in jemalloc/internal/log.h.
*/
static const bool config_log =
#ifdef JEMALLOC_LOG
true
#else
false
#endif
;
#ifdef JEMALLOC_HAVE_SCHED_GETCPU
/* Currently percpu_arena depends on sched_getcpu. */
#define JEMALLOC_PERCPU_ARENA
#endif
static const bool have_percpu_arena =
#ifdef JEMALLOC_PERCPU_ARENA
true
#else
false
#endif
;
/*
* Undocumented, and not recommended; the application should take full
* responsibility for tracking provenance.
*/
static const bool force_ivsalloc =
#ifdef JEMALLOC_FORCE_IVSALLOC
true
#else
false
#endif
;
static const bool have_background_thread =
#ifdef JEMALLOC_BACKGROUND_THREAD
true
#else
false
#endif
;
#endif /* JEMALLOC_PREAMBLE_H */

43
contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_defs.h Normal file
View File

@ -0,0 +1,43 @@
/* include/jemalloc/jemalloc_defs.h. Generated from jemalloc_defs.h.in by configure. */
/* Defined if __attribute__((...)) syntax is supported. */
#define JEMALLOC_HAVE_ATTR
/* Defined if alloc_size attribute is supported. */
#define JEMALLOC_HAVE_ATTR_ALLOC_SIZE
/* Defined if format(printf, ...) attribute is supported. */
#define JEMALLOC_HAVE_ATTR_FORMAT_PRINTF
/*
* Define overrides for non-standard allocator-related functions if they are
* present on the system.
*/
#define JEMALLOC_OVERRIDE_MEMALIGN
#define JEMALLOC_OVERRIDE_VALLOC
/*
* At least Linux omits the "const" in:
*
* size_t malloc_usable_size(const void *ptr);
*
* Match the operating system's prototype.
*/
#define JEMALLOC_USABLE_SIZE_CONST
/*
* If defined, specify throw() for the public function prototypes when compiling
* with C++. The only justification for this is to match the prototypes that
* glibc defines.
*/
#define JEMALLOC_USE_CXX_THROW
#ifdef _MSC_VER
# ifdef _WIN64
# define LG_SIZEOF_PTR_WIN 3
# else
# define LG_SIZEOF_PTR_WIN 2
# endif
#endif
/* sizeof(void *) == 2^LG_SIZEOF_PTR. */
#define LG_SIZEOF_PTR 3

123
contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_macros.h Normal file
View File

@ -0,0 +1,123 @@
#include <stdlib.h>
#include <stdbool.h>
#include <stdint.h>
#include <limits.h>
#include <strings.h>
#define JEMALLOC_VERSION "5.1.0-97-gcd2931ad9bbd78208565716ab102e86d858c2fff"
#define JEMALLOC_VERSION_MAJOR 5
#define JEMALLOC_VERSION_MINOR 1
#define JEMALLOC_VERSION_BUGFIX 0
#define JEMALLOC_VERSION_NREV 97
#define JEMALLOC_VERSION_GID "cd2931ad9bbd78208565716ab102e86d858c2fff"
#define JEMALLOC_VERSION_GID_IDENT cd2931ad9bbd78208565716ab102e86d858c2fff
#define MALLOCX_LG_ALIGN(la) ((int)(la))
#if LG_SIZEOF_PTR == 2
# define MALLOCX_ALIGN(a) ((int)(ffs((int)(a))-1))
#else
# define MALLOCX_ALIGN(a) \
((int)(((size_t)(a) < (size_t)INT_MAX) ? ffs((int)(a))-1 : \
ffs((int)(((size_t)(a))>>32))+31))
#endif
#define MALLOCX_ZERO ((int)0x40)
/*
* Bias tcache index bits so that 0 encodes "automatic tcache management", and 1
* encodes MALLOCX_TCACHE_NONE.
*/
#define MALLOCX_TCACHE(tc) ((int)(((tc)+2) << 8))
#define MALLOCX_TCACHE_NONE MALLOCX_TCACHE(-1)
/*
* Bias arena index bits so that 0 encodes "use an automatically chosen arena".
*/
#define MALLOCX_ARENA(a) ((((int)(a))+1) << 20)
/*
* Use as arena index in "arena.<i>.{purge,decay,dss}" and
* "stats.arenas.<i>.*" mallctl interfaces to select all arenas. This
* definition is intentionally specified in raw decimal format to support
* cpp-based string concatenation, e.g.
*
* #define STRINGIFY_HELPER(x) #x
* #define STRINGIFY(x) STRINGIFY_HELPER(x)
*
* mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", NULL, NULL, NULL,
* 0);
*/
#define MALLCTL_ARENAS_ALL 4096
/*
* Use as arena index in "stats.arenas.<i>.*" mallctl interfaces to select
* destroyed arenas.
*/
#define MALLCTL_ARENAS_DESTROYED 4097
#if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW)
# define JEMALLOC_CXX_THROW throw()
#else
# define JEMALLOC_CXX_THROW
#endif
#if defined(_MSC_VER)
# define JEMALLOC_ATTR(s)
# define JEMALLOC_ALIGNED(s) __declspec(align(s))
# define JEMALLOC_ALLOC_SIZE(s)
# define JEMALLOC_ALLOC_SIZE2(s1, s2)
# ifndef JEMALLOC_EXPORT
# ifdef DLLEXPORT
# define JEMALLOC_EXPORT __declspec(dllexport)
# else
# define JEMALLOC_EXPORT __declspec(dllimport)
# endif
# endif
# define JEMALLOC_FORMAT_PRINTF(s, i)
# define JEMALLOC_NOINLINE __declspec(noinline)
# ifdef __cplusplus
# define JEMALLOC_NOTHROW __declspec(nothrow)
# else
# define JEMALLOC_NOTHROW
# endif
# define JEMALLOC_SECTION(s) __declspec(allocate(s))
# define JEMALLOC_RESTRICT_RETURN __declspec(restrict)
# if _MSC_VER >= 1900 && !defined(__EDG__)
# define JEMALLOC_ALLOCATOR __declspec(allocator)
# else
# define JEMALLOC_ALLOCATOR
# endif
#elif defined(JEMALLOC_HAVE_ATTR)
# define JEMALLOC_ATTR(s) __attribute__((s))
# define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s))
# ifdef JEMALLOC_HAVE_ATTR_ALLOC_SIZE
# define JEMALLOC_ALLOC_SIZE(s) JEMALLOC_ATTR(alloc_size(s))
# define JEMALLOC_ALLOC_SIZE2(s1, s2) JEMALLOC_ATTR(alloc_size(s1, s2))
# else
# define JEMALLOC_ALLOC_SIZE(s)
# define JEMALLOC_ALLOC_SIZE2(s1, s2)
# endif
# ifndef JEMALLOC_EXPORT
# define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default"))
# endif
# ifdef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF
# define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(gnu_printf, s, i))
# elif defined(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF)
# define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(printf, s, i))
# else
# define JEMALLOC_FORMAT_PRINTF(s, i)
# endif
# define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline)
# define JEMALLOC_NOTHROW JEMALLOC_ATTR(nothrow)
# define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s))
# define JEMALLOC_RESTRICT_RETURN
# define JEMALLOC_ALLOCATOR
#else
# define JEMALLOC_ATTR(s)
# define JEMALLOC_ALIGNED(s)
# define JEMALLOC_ALLOC_SIZE(s)
# define JEMALLOC_ALLOC_SIZE2(s1, s2)
# define JEMALLOC_EXPORT
# define JEMALLOC_FORMAT_PRINTF(s, i)
# define JEMALLOC_NOINLINE
# define JEMALLOC_NOTHROW
# define JEMALLOC_SECTION(s)
# define JEMALLOC_RESTRICT_RETURN
# define JEMALLOC_ALLOCATOR
#endif

66
contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_protos.h Normal file
View File

@ -0,0 +1,66 @@
/*
* The je_ prefix on the following public symbol declarations is an artifact
* of namespace management, and should be omitted in application code unless
* JEMALLOC_NO_DEMANGLE is defined (see jemalloc_mangle.h).
*/
extern JEMALLOC_EXPORT const char *je_malloc_conf;
extern JEMALLOC_EXPORT void (*je_malloc_message)(void *cbopaque,
const char *s);
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *je_malloc(size_t size)
JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1);
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *je_calloc(size_t num, size_t size)
JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2);
JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_posix_memalign(void **memptr,
size_t alignment, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(nonnull(1));
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *je_aligned_alloc(size_t alignment,
size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc)
JEMALLOC_ALLOC_SIZE(2);
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *je_realloc(void *ptr, size_t size)
JEMALLOC_CXX_THROW JEMALLOC_ALLOC_SIZE(2);
JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_free(void *ptr)
JEMALLOC_CXX_THROW;
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *je_mallocx(size_t size, int flags)
JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1);
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *je_rallocx(void *ptr, size_t size,
int flags) JEMALLOC_ALLOC_SIZE(2);
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_xallocx(void *ptr, size_t size,
size_t extra, int flags);
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_sallocx(const void *ptr,
int flags) JEMALLOC_ATTR(pure);
JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_dallocx(void *ptr, int flags);
JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_sdallocx(void *ptr, size_t size,
int flags);
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_nallocx(size_t size, int flags)
JEMALLOC_ATTR(pure);
JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctl(const char *name,
void *oldp, size_t *oldlenp, void *newp, size_t newlen);
JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctlnametomib(const char *name,
size_t *mibp, size_t *miblenp);
JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctlbymib(const size_t *mib,
size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen);
JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_malloc_stats_print(
void (*write_cb)(void *, const char *), void *je_cbopaque,
const char *opts);
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_malloc_usable_size(
JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW;
#ifdef JEMALLOC_OVERRIDE_MEMALIGN
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *je_memalign(size_t alignment, size_t size)
JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc);
#endif
#ifdef JEMALLOC_OVERRIDE_VALLOC
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *je_valloc(size_t size) JEMALLOC_CXX_THROW
JEMALLOC_ATTR(malloc);
#endif

77
contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_typedefs.h Normal file
View File

@ -0,0 +1,77 @@
typedef struct extent_hooks_s extent_hooks_t;
/*
* void *
* extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
* size_t alignment, bool *zero, bool *commit, unsigned arena_ind);
*/
typedef void *(extent_alloc_t)(extent_hooks_t *, void *, size_t, size_t, bool *,
bool *, unsigned);
/*
* bool
* extent_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size,
* bool committed, unsigned arena_ind);
*/
typedef bool (extent_dalloc_t)(extent_hooks_t *, void *, size_t, bool,
unsigned);
/*
* void
* extent_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size,
* bool committed, unsigned arena_ind);
*/
typedef void (extent_destroy_t)(extent_hooks_t *, void *, size_t, bool,
unsigned);
/*
* bool
* extent_commit(extent_hooks_t *extent_hooks, void *addr, size_t size,
* size_t offset, size_t length, unsigned arena_ind);
*/
typedef bool (extent_commit_t)(extent_hooks_t *, void *, size_t, size_t, size_t,
unsigned);
/*
* bool
* extent_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size,
* size_t offset, size_t length, unsigned arena_ind);
*/
typedef bool (extent_decommit_t)(extent_hooks_t *, void *, size_t, size_t,
size_t, unsigned);
/*
* bool
* extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size,
* size_t offset, size_t length, unsigned arena_ind);
*/
typedef bool (extent_purge_t)(extent_hooks_t *, void *, size_t, size_t, size_t,
unsigned);
/*
* bool
* extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size,
* size_t size_a, size_t size_b, bool committed, unsigned arena_ind);
*/
typedef bool (extent_split_t)(extent_hooks_t *, void *, size_t, size_t, size_t,
bool, unsigned);
/*
* bool
* extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
* void *addr_b, size_t size_b, bool committed, unsigned arena_ind);
*/
typedef bool (extent_merge_t)(extent_hooks_t *, void *, size_t, void *, size_t,
bool, unsigned);
struct extent_hooks_s {
extent_alloc_t *alloc;
extent_dalloc_t *dalloc;
extent_destroy_t *destroy;
extent_commit_t *commit;
extent_decommit_t *decommit;
extent_purge_t *purge_lazy;
extent_purge_t *purge_forced;
extent_split_t *split;
extent_merge_t *merge;
};

1
dbms/CMakeLists.txt
View File

@ -164,6 +164,7 @@ macro(add_object_library name common_path)
endif ()
endmacro()
add_object_library(clickhouse_access src/Access)
add_object_library(clickhouse_core src/Core)
add_object_library(clickhouse_compression src/Compression)
add_object_library(clickhouse_datastreams src/DataStreams)

12
dbms/programs/local/LocalServer.cpp
View File

@ -19,8 +19,8 @@
#include <Common/ClickHouseRevision.h>
#include <Common/ThreadStatus.h>
#include <Common/config_version.h>
#include <Common/quoteString.h>
#include <IO/ReadBufferFromString.h>
#include <IO/WriteBufferFromString.h>
#include <IO/WriteBufferFromFileDescriptor.h>
#include <IO/UseSSL.h>
#include <Parsers/parseQuery.h>
@ -221,14 +221,6 @@ catch (const Exception & e)
}
inline String getQuotedString(const String & s)
{
WriteBufferFromOwnString buf;
writeQuotedString(s, buf);
return buf.str();
}
std::string LocalServer::getInitialCreateTableQuery()
{
if (!config().has("table-structure"))
@ -241,7 +233,7 @@ std::string LocalServer::getInitialCreateTableQuery()
if (!config().has("table-file") || config().getString("table-file") == "-") /// Use Unix tools stdin naming convention
table_file = "stdin";
else /// Use regular file
table_file = getQuotedString(config().getString("table-file"));
table_file = quoteString(config().getString("table-file"));
return
"CREATE TABLE " + table_name +

6
dbms/programs/server/MySQLHandler.cpp
View File

@ -46,7 +46,7 @@ MySQLHandler::MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & so
, connection_id(connection_id_)
, public_key(public_key_)
, private_key(private_key_)
, auth_plugin(new Authentication::Native41())
, auth_plugin(new MySQLProtocol::Authentication::Native41())
{
server_capability_flags = CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION | CLIENT_PLUGIN_AUTH | CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA | CLIENT_CONNECT_WITH_DB | CLIENT_DEPRECATE_EOF;
if (ssl_enabled)
@ -231,8 +231,8 @@ void MySQLHandler::authenticate(const String & user_name, const String & auth_pl
{
// For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible (if password is specified using double SHA1). Otherwise SHA256 plugin is used.
auto user = connection_context.getUser(user_name);
if (user->password_double_sha1_hex.empty())
auth_plugin = std::make_unique<Authentication::Sha256Password>(public_key, private_key, log);
if (user->authentication.getType() != DB::Authentication::DOUBLE_SHA1_PASSWORD)
auth_plugin = std::make_unique<MySQLProtocol::Authentication::Sha256Password>(public_key, private_key, log);
try {
std::optional<String> auth_response = auth_plugin_name == auth_plugin->getName() ? std::make_optional<String>(initial_auth_response) : std::nullopt;

397
dbms/src/Access/AllowedClientHosts.cpp Normal file
View File

@ -0,0 +1,397 @@
#include <Access/AllowedClientHosts.h>
#include <Common/Exception.h>
#include <common/SimpleCache.h>
#include <Common/StringUtils/StringUtils.h>
#include <IO/ReadHelpers.h>
#include <Poco/Net/SocketAddress.h>
#include <Poco/RegularExpression.h>
#include <common/logger_useful.h>
#include <ext/scope_guard.h>
#include <boost/range/algorithm/find.hpp>
#include <boost/range/algorithm/find_first_of.hpp>
namespace DB
{
namespace ErrorCodes
{
extern const int DNS_ERROR;
extern const int IP_ADDRESS_NOT_ALLOWED;
}
namespace
{
using IPAddress = Poco::Net::IPAddress;
const AllowedClientHosts::IPSubnet ALL_ADDRESSES = AllowedClientHosts::IPSubnet{IPAddress{IPAddress::IPv6}, IPAddress{IPAddress::IPv6}};
IPAddress toIPv6(const IPAddress & addr)
{
if (addr.family() == IPAddress::IPv6)
return addr;
return IPAddress("::FFFF:" + addr.toString());
}
IPAddress maskToIPv6(const IPAddress & mask)
{
if (mask.family() == IPAddress::IPv6)
return mask;
return IPAddress(96, IPAddress::IPv6) | toIPv6(mask);
}
bool isAddressOfHostImpl(const IPAddress & address, const String & host)
{
IPAddress addr_v6 = toIPv6(address);
/// Resolve by hand, because Poco don't use AI_ALL flag but we need it.
addrinfo * ai = nullptr;
SCOPE_EXIT(
{
if (ai)
freeaddrinfo(ai);
});
addrinfo hints;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
hints.ai_flags |= AI_V4MAPPED | AI_ALL;
int ret = getaddrinfo(host.c_str(), nullptr, &hints, &ai);
if (0 != ret)
throw Exception("Cannot getaddrinfo: " + std::string(gai_strerror(ret)), ErrorCodes::DNS_ERROR);
for (; ai != nullptr; ai = ai->ai_next)
{
if (ai->ai_addrlen && ai->ai_addr)
{
if (ai->ai_family == AF_INET6)
{
if (addr_v6 == IPAddress(
&reinterpret_cast<sockaddr_in6*>(ai->ai_addr)->sin6_addr, sizeof(in6_addr),
reinterpret_cast<sockaddr_in6*>(ai->ai_addr)->sin6_scope_id))
{
return true;
}
}
else if (ai->ai_family == AF_INET)
{
if (addr_v6 == toIPv6(IPAddress(&reinterpret_cast<sockaddr_in *>(ai->ai_addr)->sin_addr, sizeof(in_addr))))
{
return true;
}
}
}
}
return false;
}
/// Cached version of isAddressOfHostImpl(). We need to cache DNS requests.
bool isAddressOfHost(const IPAddress & address, const String & host)
{
static SimpleCache<decltype(isAddressOfHostImpl), isAddressOfHostImpl> cache;
return cache(address, host);
}
String getHostByAddressImpl(const IPAddress & address)
{
Poco::Net::SocketAddress sock_addr(address, 0);
/// Resolve by hand, because Poco library doesn't have such functionality.
char host[1024];
int gai_errno = getnameinfo(sock_addr.addr(), sock_addr.length(), host, sizeof(host), nullptr, 0, NI_NAMEREQD);
if (0 != gai_errno)
throw Exception("Cannot getnameinfo: " + std::string(gai_strerror(gai_errno)), ErrorCodes::DNS_ERROR);
/// Check that PTR record is resolved back to client address
if (!isAddressOfHost(address, host))
throw Exception("Host " + String(host) + " isn't resolved back to " + address.toString(), ErrorCodes::DNS_ERROR);
return host;
}
/// Cached version of getHostByAddressImpl(). We need to cache DNS requests.
String getHostByAddress(const IPAddress & address)
{
static SimpleCache<decltype(getHostByAddressImpl), &getHostByAddressImpl> cache;
return cache(address);
}
}
String AllowedClientHosts::IPSubnet::toString() const
{
unsigned int prefix_length = mask.prefixLength();
if (IPAddress{prefix_length, mask.family()} == mask)
return prefix.toString() + "/" + std::to_string(prefix_length);
return prefix.toString() + "/" + mask.toString();
}
AllowedClientHosts::AllowedClientHosts()
{
}
AllowedClientHosts::AllowedClientHosts(AllAddressesTag)
{
addAllAddresses();
}
AllowedClientHosts::~AllowedClientHosts() = default;
AllowedClientHosts::AllowedClientHosts(const AllowedClientHosts & src)
{
*this = src;
}
AllowedClientHosts & AllowedClientHosts::operator =(const AllowedClientHosts & src)
{
addresses = src.addresses;
subnets = src.subnets;
host_names = src.host_names;
host_regexps = src.host_regexps;
compiled_host_regexps.clear();
return *this;
}
AllowedClientHosts::AllowedClientHosts(AllowedClientHosts && src)
{
*this = src;
}
AllowedClientHosts & AllowedClientHosts::operator =(AllowedClientHosts && src)
{
addresses = std::move(src.addresses);
subnets = std::move(src.subnets);
host_names = std::move(src.host_names);
host_regexps = std::move(src.host_regexps);
compiled_host_regexps = std::move(src.compiled_host_regexps);
return *this;
}
void AllowedClientHosts::clear()
{
addresses.clear();
subnets.clear();
host_names.clear();
host_regexps.clear();
compiled_host_regexps.clear();
}
bool AllowedClientHosts::empty() const
{
return addresses.empty() && subnets.empty() && host_names.empty() && host_regexps.empty();
}
void AllowedClientHosts::addAddress(const IPAddress & address)
{
IPAddress addr_v6 = toIPv6(address);
if (boost::range::find(addresses, addr_v6) == addresses.end())
addresses.push_back(addr_v6);
}
void AllowedClientHosts::addAddress(const String & address)
{
addAddress(IPAddress{address});
}
void AllowedClientHosts::addSubnet(const IPSubnet & subnet)
{
IPSubnet subnet_v6;
subnet_v6.prefix = toIPv6(subnet.prefix);
subnet_v6.mask = maskToIPv6(subnet.mask);
if (subnet_v6.mask == IPAddress(128, IPAddress::IPv6))
{
addAddress(subnet_v6.prefix);
return;
}
subnet_v6.prefix = subnet_v6.prefix & subnet_v6.mask;
if (boost::range::find(subnets, subnet_v6) == subnets.end())
subnets.push_back(subnet_v6);
}
void AllowedClientHosts::addSubnet(const IPAddress & prefix, const IPAddress & mask)
{
addSubnet(IPSubnet{prefix, mask});
}
void AllowedClientHosts::addSubnet(const IPAddress & prefix, size_t num_prefix_bits)
{
addSubnet(prefix, IPAddress(num_prefix_bits, prefix.family()));
}
void AllowedClientHosts::addSubnet(const String & subnet)
{
size_t slash = subnet.find('/');
if (slash == String::npos)
{
addAddress(subnet);
return;
}
IPAddress prefix{String{subnet, 0, slash}};
String mask(subnet, slash + 1, subnet.length() - slash - 1);
if (std::all_of(mask.begin(), mask.end(), isNumericASCII))
addSubnet(prefix, parseFromString<UInt8>(mask));
else
addSubnet(prefix, IPAddress{mask});
}
void AllowedClientHosts::addHostName(const String & host_name)
{
if (boost::range::find(host_names, host_name) == host_names.end())
host_names.push_back(host_name);
}
void AllowedClientHosts::addHostRegexp(const String & host_regexp)
{
if (boost::range::find(host_regexps, host_regexp) == host_regexps.end())
host_regexps.push_back(host_regexp);
}
void AllowedClientHosts::addAllAddresses()
{
clear();
addSubnet(ALL_ADDRESSES);
}
bool AllowedClientHosts::containsAllAddresses() const
{
return (boost::range::find(subnets, ALL_ADDRESSES) != subnets.end())
|| (boost::range::find(host_regexps, ".*") != host_regexps.end())
|| (boost::range::find(host_regexps, "$") != host_regexps.end());
}
bool AllowedClientHosts::contains(const IPAddress & address) const
{
return containsImpl(address, String(), nullptr);
}
void AllowedClientHosts::checkContains(const IPAddress & address, const String & user_name) const
{
String error;
if (!containsImpl(address, user_name, &error))
throw Exception(error, ErrorCodes::IP_ADDRESS_NOT_ALLOWED);
}
bool AllowedClientHosts::containsImpl(const IPAddress & address, const String & user_name, String * error) const
{
if (error)
error->clear();
/// Check `ip_addresses`.
IPAddress addr_v6 = toIPv6(address);
if (boost::range::find(addresses, addr_v6) != addresses.end())
return true;
/// Check `ip_subnets`.
for (const auto & subnet : subnets)
if ((addr_v6 & subnet.mask) == subnet.prefix)
return true;
/// Check `hosts`.
for (const String & host_name : host_names)
{
try
{
if (isAddressOfHost(address, host_name))
return true;
}
catch (Exception & e)
{
if (e.code() != ErrorCodes::DNS_ERROR)
e.rethrow();
/// Try to ignore DNS errors: if host cannot be resolved, skip it and try next.
LOG_WARNING(
&Logger::get("AddressPatterns"),
"Failed to check if the allowed client hosts contain address " << address.toString() << ". " << e.displayText()
<< ", code = " << e.code());
}
}
/// Check `host_regexps`.
if (!host_regexps.empty())
{
compileRegexps();
try
{
String resolved_host = getHostByAddress(address);
for (const auto & compiled_regexp : compiled_host_regexps)
{
if (compiled_regexp && compiled_regexp->match(resolved_host))
return true;
}
}
catch (Exception & e)
{
if (e.code() != ErrorCodes::DNS_ERROR)
e.rethrow();
/// Try to ignore DNS errors: if host cannot be resolved, skip it and try next.
LOG_WARNING(
&Logger::get("AddressPatterns"),
"Failed to check if the allowed client hosts contain address " << address.toString() << ". " << e.displayText()
<< ", code = " << e.code());
}
}
if (error)
{
if (user_name.empty())
*error = "It's not allowed to connect from address " + address.toString();
else
*error = "User " + user_name + " is not allowed to connect from address " + address.toString();
}
return false;
}
void AllowedClientHosts::compileRegexps() const
{
if (compiled_host_regexps.size() == host_regexps.size())
return;
size_t old_size = compiled_host_regexps.size();
compiled_host_regexps.reserve(host_regexps.size());
for (size_t i = old_size; i != host_regexps.size(); ++i)
compiled_host_regexps.emplace_back(std::make_unique<Poco::RegularExpression>(host_regexps[i]));
}
bool operator ==(const AllowedClientHosts & lhs, const AllowedClientHosts & rhs)
{
return (lhs.addresses == rhs.addresses) && (lhs.subnets == rhs.subnets) && (lhs.host_names == rhs.host_names)
&& (lhs.host_regexps == rhs.host_regexps);
}
}

103
dbms/src/Access/AllowedClientHosts.h Normal file
View File

@ -0,0 +1,103 @@
#pragma once
#include <Core/Types.h>
#include <Poco/Net/IPAddress.h>
#include <memory>
#include <vector>
namespace Poco
{
class RegularExpression;
}
namespace DB
{
/// Represents lists of hosts an user is allowed to connect to server from.
class AllowedClientHosts
{
public:
using IPAddress = Poco::Net::IPAddress;
struct IPSubnet
{
IPAddress prefix;
IPAddress mask;
String toString() const;
friend bool operator ==(const IPSubnet & lhs, const IPSubnet & rhs) { return (lhs.prefix == rhs.prefix) && (lhs.mask == rhs.mask); }
friend bool operator !=(const IPSubnet & lhs, const IPSubnet & rhs) { return !(lhs == rhs); }
};
struct AllAddressesTag {};
AllowedClientHosts();
explicit AllowedClientHosts(AllAddressesTag);
~AllowedClientHosts();
AllowedClientHosts(const AllowedClientHosts & src);
AllowedClientHosts & operator =(const AllowedClientHosts & src);
AllowedClientHosts(AllowedClientHosts && src);
AllowedClientHosts & operator =(AllowedClientHosts && src);
/// Removes all contained addresses. This will disallow all addresses.
void clear();
bool empty() const;
/// Allows exact IP address.
/// For example, 213.180.204.3 or 2a02:6b8::3
void addAddress(const IPAddress & address);
void addAddress(const String & address);
/// Allows an IP subnet.
void addSubnet(const IPSubnet & subnet);
void addSubnet(const String & subnet);
/// Allows an IP subnet.
/// For example, 312.234.1.1/255.255.255.0 or 2a02:6b8::3/FFFF:FFFF:FFFF:FFFF::
void addSubnet(const IPAddress & prefix, const IPAddress & mask);
/// Allows an IP subnet.
/// For example, 10.0.0.1/8 or 2a02:6b8::3/64
void addSubnet(const IPAddress & prefix, size_t num_prefix_bits);
/// Allows all addresses.
void addAllAddresses();
/// Allows an exact host. The `contains()` function will check that the provided address equals to one of that host's addresses.
void addHostName(const String & host_name);
/// Allows a regular expression for the host.
void addHostRegexp(const String & host_regexp);
const std::vector<IPAddress> & getAddresses() const { return addresses; }
const std::vector<IPSubnet> & getSubnets() const { return subnets; }
const std::vector<String> & getHostNames() const { return host_names; }
const std::vector<String> & getHostRegexps() const { return host_regexps; }
/// Checks if the provided address is in the list. Returns false if not.
bool contains(const IPAddress & address) const;
/// Checks if any address is allowed.
bool containsAllAddresses() const;
/// Checks if the provided address is in the list. Throws an exception if not.
/// `username` is only used for generating an error message if the address isn't in the list.
void checkContains(const IPAddress & address, const String & user_name = String()) const;
friend bool operator ==(const AllowedClientHosts & lhs, const AllowedClientHosts & rhs);
friend bool operator !=(const AllowedClientHosts & lhs, const AllowedClientHosts & rhs) { return !(lhs == rhs); }
private:
bool containsImpl(const IPAddress & address, const String & user_name, String * error) const;
void compileRegexps() const;
std::vector<IPAddress> addresses;
std::vector<IPSubnet> subnets;
std::vector<String> host_names;
std::vector<String> host_regexps;
mutable std::vector<std::unique_ptr<Poco::RegularExpression>> compiled_host_regexps;
};
}

207
dbms/src/Access/Authentication.cpp Normal file
View File

@ -0,0 +1,207 @@
#include <Access/Authentication.h>
#include <Common/Exception.h>
#include <common/StringRef.h>
#include <Core/Defines.h>
#include <Poco/SHA1Engine.h>
#include <boost/algorithm/hex.hpp>
#include "config_core.h"
#if USE_SSL
# include <openssl/sha.h>
#endif
namespace DB
{
namespace ErrorCodes
{
extern const int SUPPORT_IS_DISABLED;
extern const int REQUIRED_PASSWORD;
extern const int WRONG_PASSWORD;
extern const int BAD_ARGUMENTS;
extern const int LOGICAL_ERROR;
}
namespace
{
using Digest = Authentication::Digest;
Digest encodePlainText(const StringRef & text)
{
return Digest(text.data, text.data + text.size);
}
Digest encodeSHA256(const StringRef & text)
{
#if USE_SSL
Digest hash;
hash.resize(32);
SHA256_CTX ctx;
SHA256_Init(&ctx);
SHA256_Update(&ctx, reinterpret_cast<const UInt8 *>(text.data), text.size);
SHA256_Final(hash.data(), &ctx);
return hash;
#else
UNUSED(text);
throw DB::Exception("SHA256 passwords support is disabled, because ClickHouse was built without SSL library", DB::ErrorCodes::SUPPORT_IS_DISABLED);
#endif
}
Digest encodeSHA1(const StringRef & text)
{
Poco::SHA1Engine engine;
engine.update(text.data, text.size);
return engine.digest();
}
Digest encodeSHA1(const Digest & text)
{
return encodeSHA1(StringRef{reinterpret_cast<const char *>(text.data()), text.size()});
}
Digest encodeDoubleSHA1(const StringRef & text)
{
return encodeSHA1(encodeSHA1(text));
}
}
Authentication::Authentication(Authentication::Type type_)
: type(type_)
{
}
void Authentication::setPassword(const String & password_)
{
switch (type)
{
case NO_PASSWORD:
throw Exception("Cannot specify password for the 'NO_PASSWORD' authentication type", ErrorCodes::LOGICAL_ERROR);
case PLAINTEXT_PASSWORD:
setPasswordHashBinary(encodePlainText(password_));
return;
case SHA256_PASSWORD:
setPasswordHashBinary(encodeSHA256(password_));
return;
case DOUBLE_SHA1_PASSWORD:
setPasswordHashBinary(encodeDoubleSHA1(password_));
return;
}
throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
}
String Authentication::getPassword() const
{
if (type != PLAINTEXT_PASSWORD)
throw Exception("Cannot decode the password", ErrorCodes::LOGICAL_ERROR);
return String(password_hash.data(), password_hash.data() + password_hash.size());
}
void Authentication::setPasswordHashHex(const String & hash)
{
Digest digest;
digest.resize(hash.size() / 2);
boost::algorithm::unhex(hash.begin(), hash.end(), digest.data());
setPasswordHashBinary(digest);
}
String Authentication::getPasswordHashHex() const
{
String hex;
hex.resize(password_hash.size() * 2);
boost::algorithm::hex(password_hash.begin(), password_hash.end(), hex.data());
return hex;
}
void Authentication::setPasswordHashBinary(const Digest & hash)
{
switch (type)
{
case NO_PASSWORD:
throw Exception("Cannot specify password for the 'NO_PASSWORD' authentication type", ErrorCodes::LOGICAL_ERROR);
case PLAINTEXT_PASSWORD:
{
password_hash = hash;
return;
}
case SHA256_PASSWORD:
{
if (hash.size() != 32)
throw Exception(
"Password hash for the 'SHA256_PASSWORD' authentication type has length " + std::to_string(hash.size())
+ " but must be exactly 32 bytes.",
ErrorCodes::BAD_ARGUMENTS);
password_hash = hash;
return;
}
case DOUBLE_SHA1_PASSWORD:
{
if (hash.size() != 20)
throw Exception(
"Password hash for the 'DOUBLE_SHA1_PASSWORD' authentication type has length " + std::to_string(hash.size())
+ " but must be exactly 20 bytes.",
ErrorCodes::BAD_ARGUMENTS);
password_hash = hash;
return;
}
}
throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
}
bool Authentication::isCorrectPassword(const String & password_) const
{
switch (type)
{
case NO_PASSWORD:
return true;
case PLAINTEXT_PASSWORD:
return password_ == StringRef{reinterpret_cast<const char *>(password_hash.data()), password_hash.size()};
case SHA256_PASSWORD:
return encodeSHA256(password_) == password_hash;
case DOUBLE_SHA1_PASSWORD:
{
auto first_sha1 = encodeSHA1(password_);
/// If it was MySQL compatibility server, then first_sha1 already contains double SHA1.
if (first_sha1 == password_hash)
return true;
return encodeSHA1(first_sha1) == password_hash;
}
}
throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
}
void Authentication::checkPassword(const String & password_, const String & user_name) const
{
if (isCorrectPassword(password_))
return;
auto info_about_user_name = [&user_name]() { return user_name.empty() ? String() : " for user " + user_name; };
if (password_.empty() && (type != NO_PASSWORD))
throw Exception("Password required" + info_about_user_name(), ErrorCodes::REQUIRED_PASSWORD);
throw Exception("Wrong password" + info_about_user_name(), ErrorCodes::WRONG_PASSWORD);
}
bool operator ==(const Authentication & lhs, const Authentication & rhs)
{
return (lhs.type == rhs.type) && (lhs.password_hash == rhs.password_hash);
}
}

66
dbms/src/Access/Authentication.h Normal file
View File

@ -0,0 +1,66 @@
#pragma once
#include <Core/Types.h>
namespace DB
{
/// Authentication type and encrypted password for checking when an user logins.
class Authentication
{
public:
enum Type
{
/// User doesn't have to enter password.
NO_PASSWORD,
/// Password is stored as is.
PLAINTEXT_PASSWORD,
/// Password is encrypted in SHA256 hash.
SHA256_PASSWORD,
/// SHA1(SHA1(password)).
/// This kind of hash is used by the `mysql_native_password` authentication plugin.
DOUBLE_SHA1_PASSWORD,
};
using Digest = std::vector<UInt8>;
Authentication(Authentication::Type type = NO_PASSWORD);
Authentication(const Authentication & src) = default;
Authentication & operator =(const Authentication & src) = default;
Authentication(Authentication && src) = default;
Authentication & operator =(Authentication && src) = default;
Type getType() const { return type; }
/// Sets the password and encrypt it using the authentication type set in the constructor.
void setPassword(const String & password);
/// Returns the password. Allowed to use only for Type::PLAINTEXT_PASSWORD.
String getPassword() const;
/// Sets the password as a string of hexadecimal digits.
void setPasswordHashHex(const String & hash);
String getPasswordHashHex() const;
/// Sets the password in binary form.
void setPasswordHashBinary(const Digest & hash);
const Digest & getPasswordHashBinary() const { return password_hash; }
/// Checks if the provided password is correct. Returns false if not.
bool isCorrectPassword(const String & password) const;
/// Checks if the provided password is correct. Throws an exception if not.
/// `user_name` is only used for generating an error message if the password is incorrect.
void checkPassword(const String & password, const String & user_name = String()) const;
friend bool operator ==(const Authentication & lhs, const Authentication & rhs);
friend bool operator !=(const Authentication & lhs, const Authentication & rhs) { return !(lhs == rhs); }
private:
Type type = Type::NO_PASSWORD;
Digest password_hash;
};
}

0
dbms/src/Access/CMakeLists.txt Normal file
View File

2
dbms/src/AggregateFunctions/AggregateFunctionCount.cpp
View File

@ -12,8 +12,8 @@ namespace
AggregateFunctionPtr createAggregateFunctionCount(const std::string & name, const DataTypes & argument_types, const Array & parameters)
{
assertNoParameters(name, parameters);
assertArityAtMost<1>(name, argument_types);
/// 'count' accept any number of arguments and (in this case of non-Nullable types) simply ignore them.
return std::make_shared<AggregateFunctionCount>(argument_types);
}

65
dbms/src/AggregateFunctions/AggregateFunctionCount.h
View File

@ -113,69 +113,4 @@ public:
const char * getHeaderFilePath() const override { return __FILE__; }
};
/// Count number of calls where all arguments are not NULL.
class AggregateFunctionCountNotNullVariadic final : public IAggregateFunctionDataHelper<AggregateFunctionCountData, AggregateFunctionCountNotNullVariadic>
{
public:
AggregateFunctionCountNotNullVariadic(const DataTypes & arguments, const Array & params)
: IAggregateFunctionDataHelper<AggregateFunctionCountData, AggregateFunctionCountNotNullVariadic>(arguments, params)
{
number_of_arguments = arguments.size();
if (number_of_arguments == 1)
throw Exception("Logical error: single argument is passed to AggregateFunctionCountNotNullVariadic", ErrorCodes::LOGICAL_ERROR);
if (number_of_arguments > MAX_ARGS)
throw Exception("Maximum number of arguments for aggregate function with Nullable types is " + toString(size_t(MAX_ARGS)),
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
for (size_t i = 0; i < number_of_arguments; ++i)
is_nullable[i] = arguments[i]->isNullable();
}
String getName() const override { return "count"; }
DataTypePtr getReturnType() const override
{
return std::make_shared<DataTypeUInt64>();
}
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
{
for (size_t i = 0; i < number_of_arguments; ++i)
if (is_nullable[i] && assert_cast<const ColumnNullable &>(*columns[i]).isNullAt(row_num))
return;
++data(place).count;
}
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
{
data(place).count += data(rhs).count;
}
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
{
writeVarUInt(data(place).count, buf);
}
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
{
readVarUInt(data(place).count, buf);
}
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
{
assert_cast<ColumnUInt64 &>(to).getData().push_back(data(place).count);
}
const char * getHeaderFilePath() const override { return __FILE__; }
private:
enum { MAX_ARGS = 8 };
size_t number_of_arguments = 0;
std::array<char, MAX_ARGS> is_nullable; /// Plain array is better than std::vector due to one indirection less.
};
}

17
dbms/src/AggregateFunctions/AggregateFunctionGroupBitmap.cpp
View File

@ -1,9 +1,12 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/AggregateFunctionGroupBitmap.h>
#include <AggregateFunctions/Helpers.h>
#include <AggregateFunctions/FactoryHelpers.h>
#include <DataTypes/DataTypeAggregateFunction.h>
// TODO include this last because of a broken roaring header. See the comment
// inside.
#include <AggregateFunctions/AggregateFunctionGroupBitmap.h>
namespace DB
{
@ -36,15 +39,13 @@ AggregateFunctionPtr createAggregateFunctionBitmapL2(const std::string & name, c
assertUnary(name, argument_types);
DataTypePtr argument_type_ptr = argument_types[0];
WhichDataType which(*argument_type_ptr);
if (which.idx == TypeIndex::AggregateFunction)
{
const DataTypeAggregateFunction& datatype_aggfunc = dynamic_cast<const DataTypeAggregateFunction&>(*argument_type_ptr);
AggregateFunctionPtr aggfunc = datatype_aggfunc.getFunction();
argument_type_ptr = aggfunc->getArgumentTypes()[0];
}
if (which.idx != TypeIndex::AggregateFunction)
throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
const DataTypeAggregateFunction& datatype_aggfunc = dynamic_cast<const DataTypeAggregateFunction&>(*argument_type_ptr);
AggregateFunctionPtr aggfunc = datatype_aggfunc.getFunction();
argument_type_ptr = aggfunc->getArgumentTypes()[0];
AggregateFunctionPtr res(createWithUnsignedIntegerType<AggregateFunctionTemplate, AggregateFunctionGroupBitmapData>(*argument_type_ptr, argument_type_ptr));
if (!res)
throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);

9
dbms/src/AggregateFunctions/AggregateFunctionGroupBitmap.h
View File

@ -3,10 +3,13 @@
#include <Columns/ColumnVector.h>
#include <Common/assert_cast.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <AggregateFunctions/AggregateFunctionGroupBitmapData.h>
#include <DataTypes/DataTypesNumber.h>
#include <Columns/ColumnAggregateFunction.h>
// TODO include this last because of a broken roaring header. See the comment
// inside.
#include <AggregateFunctions/AggregateFunctionGroupBitmapData.h>
namespace DB
{
@ -71,7 +74,7 @@ public:
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
{
Data & data_lhs = this->data(place);
const Data & data_rhs = this->data(static_cast<const ColumnAggregateFunction &>(*columns[0]).getData()[row_num]);
const Data & data_rhs = this->data(assert_cast<const ColumnAggregateFunction &>(*columns[0]).getData()[row_num]);
if (!data_lhs.doneFirst)
{
data_lhs.doneFirst = true;
@ -110,7 +113,7 @@ public:
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
{
static_cast<ColumnVector<T> &>(to).getData().push_back(this->data(place).rbs.size());
assert_cast<ColumnVector<T> &>(to).getData().push_back(this->data(place).rbs.size());
}
const char * getHeaderFilePath() const override { return __FILE__; }

8
dbms/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h
View File

@ -1,14 +1,18 @@
#pragma once
#include <algorithm>
#include <roaring/roaring.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <boost/noncopyable.hpp>
#include <roaring/roaring.hh>
#include <Common/HashTable/SmallTable.h>
#include <Common/PODArray.h>
// Include this header last, because it is an auto-generated dump of questionable
// garbage that breaks the build (e.g. it changes _POSIX_C_SOURCE).
// TODO: find out what it is. On github, they have proper inteface headers like
// this one: https://github.com/RoaringBitmap/CRoaring/blob/master/include/roaring/roaring.h
#include <roaring/roaring.h>
namespace DB
{
/**

7
dbms/src/AggregateFunctions/AggregateFunctionNull.cpp
View File

@ -53,12 +53,7 @@ public:
/// Special case for 'count' function. It could be called with Nullable arguments
/// - that means - count number of calls, when all arguments are not NULL.
if (nested_function && nested_function->getName() == "count")
{
if (arguments.size() == 1)
return std::make_shared<AggregateFunctionCountNotNullUnary>(arguments[0], params);
else
return std::make_shared<AggregateFunctionCountNotNullVariadic>(arguments, params);
}
return std::make_shared<AggregateFunctionCountNotNullUnary>(arguments[0], params);
if (has_null_types)
return std::make_shared<AggregateFunctionNothing>(arguments, params);

19
dbms/src/AggregateFunctions/FactoryHelpers.h
View File

@ -2,6 +2,7 @@
#include <Core/Field.h>
#include <DataTypes/IDataType.h>
#include <IO/WriteHelpers.h>
namespace DB
@ -31,4 +32,22 @@ inline void assertBinary(const std::string & name, const DataTypes & argument_ty
throw Exception("Aggregate function " + name + " require two arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}
template<std::size_t maximal_arity>
inline void assertArityAtMost(const std::string & name, const DataTypes & argument_types)
{
if (argument_types.size() <= maximal_arity)
return;
if constexpr (maximal_arity == 0)
throw Exception("Aggregate function " + name + " cannot have arguments",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
if constexpr (maximal_arity == 1)
throw Exception("Aggregate function " + name + " requires zero or one argument",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
throw Exception("Aggregate function " + name + " requires at most " + toString(maximal_arity) + " arguments",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}
}

1
dbms/src/CMakeLists.txt
View File

@ -1,3 +1,4 @@
add_subdirectory (Access)
add_subdirectory (Columns)
add_subdirectory (Common)
add_subdirectory (Core)

1
dbms/src/Client/ConnectionPoolWithFailover.cpp
View File

@ -3,6 +3,7 @@
#include <Poco/Net/NetException.h>
#include <Poco/Net/DNS.h>
#include <Common/BitHelpers.h>
#include <Common/getFQDNOrHostName.h>
#include <Common/isLocalAddress.h>
#include <Common/ProfileEvents.h>

1
dbms/src/Columns/ColumnAggregateFunction.h
View File

@ -3,6 +3,7 @@
#include <AggregateFunctions/IAggregateFunction.h>
#include <Columns/IColumn.h>
#include <Common/PODArray.h>
#include <Core/Field.h>

1
dbms/src/Columns/ColumnArray.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <Columns/IColumn.h>
#include <Columns/IColumnImpl.h>
#include <Columns/ColumnVector.h>
#include <Core/Defines.h>
#include <Common/typeid_cast.h>

1
dbms/src/Columns/ColumnConst.cpp
View File

@ -2,6 +2,7 @@
#include <Columns/ColumnConst.h>
#include <Columns/ColumnsCommon.h>
#include <Common/PODArray.h>
#include <Common/typeid_cast.h>

1
dbms/src/Columns/ColumnDecimal.h
View File

@ -4,6 +4,7 @@
#include <Common/typeid_cast.h>
#include <Columns/IColumn.h>
#include <Columns/IColumnImpl.h>
#include <Columns/ColumnVectorHelper.h>
#include <Core/Field.h>

1
dbms/src/Columns/ColumnFixedString.h
View File

@ -5,6 +5,7 @@
#include <Common/typeid_cast.h>
#include <Common/assert_cast.h>
#include <Columns/IColumn.h>
#include <Columns/IColumnImpl.h>
#include <Columns/ColumnVectorHelper.h>
#include <Core/Field.h>

1
dbms/src/Columns/ColumnFunction.cpp
View File

@ -1,6 +1,7 @@
#include <Interpreters/ExpressionActions.h>
#include <Columns/ColumnFunction.h>
#include <Columns/ColumnsCommon.h>
#include <Common/PODArray.h>
#include <IO/WriteHelpers.h>
#include <Functions/IFunction.h>

1
dbms/src/Columns/ColumnNullable.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <Columns/IColumn.h>
#include <Columns/IColumnImpl.h>
#include <Columns/ColumnsNumber.h>
#include <Common/typeid_cast.h>
#include <Common/assert_cast.h>

1
dbms/src/Columns/ColumnString.h
View File

@ -4,6 +4,7 @@
#include <cassert>
#include <Columns/IColumn.h>
#include <Columns/IColumnImpl.h>
#include <Common/PODArray.h>
#include <Common/SipHash.h>
#include <Common/memcpySmall.h>

1
dbms/src/Columns/ColumnVector.h
View File

@ -2,6 +2,7 @@
#include <cmath>
#include <Columns/IColumn.h>
#include <Columns/IColumnImpl.h>
#include <Columns/ColumnVectorHelper.h>
#include <common/unaligned.h>
#include <Core/Field.h>

30
dbms/src/Columns/IColumn.h
View File

@ -1,10 +1,11 @@
#pragma once
#include <Common/COW.h>
#include <Common/PODArray.h>
#include <Common/PODArray_fwd.h>
#include <Common/Exception.h>
#include <Common/typeid_cast.h>
#include <common/StringRef.h>
#include <Core/Types.h>
class SipHash;
@ -373,32 +374,7 @@ protected:
/// Template is to devirtualize calls to insertFrom method.
/// In derived classes (that use final keyword), implement scatter method as call to scatterImpl.
template <typename Derived>
std::vector<MutablePtr> scatterImpl(ColumnIndex num_columns, const Selector & selector) const
{
size_t num_rows = size();
if (num_rows != selector.size())
throw Exception(
"Size of selector: " + std::to_string(selector.size()) + " doesn't match size of column: " + std::to_string(num_rows),
ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
std::vector<MutablePtr> columns(num_columns);
for (auto & column : columns)
column = cloneEmpty();
{
size_t reserve_size = num_rows * 1.1 / num_columns; /// 1.1 is just a guess. Better to use n-sigma rule.
if (reserve_size > 1)
for (auto & column : columns)
column->reserve(reserve_size);
}
for (size_t i = 0; i < num_rows; ++i)
static_cast<Derived &>(*columns[selector[i]]).insertFrom(*this, i);
return columns;
}
std::vector<MutablePtr> scatterImpl(ColumnIndex num_columns, const Selector & selector) const;
};
using ColumnPtr = IColumn::Ptr;

1
dbms/src/Columns/IColumnDummy.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <Common/Arena.h>
#include <Common/PODArray.h>
#include <Columns/IColumn.h>
#include <Columns/ColumnsCommon.h>

45
dbms/src/Columns/IColumnImpl.h Normal file
View File

@ -0,0 +1,45 @@
/**
* This file implements template methods of IColumn that depend on other types
* we don't want to include.
* Currently, this is only the scatterImpl method that depends on PODArray
* implementation.
*/
#pragma once
#include <Columns/IColumn.h>
#include <Common/PODArray.h>
namespace DB
{
template <typename Derived>
std::vector<IColumn::MutablePtr> IColumn::scatterImpl(ColumnIndex num_columns,
const Selector & selector) const
{
size_t num_rows = size();
if (num_rows != selector.size())
throw Exception(
"Size of selector: " + std::to_string(selector.size()) + " doesn't match size of column: " + std::to_string(num_rows),
ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
std::vector<MutablePtr> columns(num_columns);
for (auto & column : columns)
column = cloneEmpty();
{
size_t reserve_size = num_rows * 1.1 / num_columns; /// 1.1 is just a guess. Better to use n-sigma rule.
if (reserve_size > 1)
for (auto & column : columns)
column->reserve(reserve_size);
}
for (size_t i = 0; i < num_rows; ++i)
static_cast<Derived &>(*columns[selector[i]]).insertFrom(*this, i);
return columns;
}
}

6
dbms/src/Common/Allocator.h
View File

@ -30,6 +30,8 @@
#include <Common/Exception.h>
#include <Common/formatReadable.h>
#include <Common/Allocator_fwd.h>
/// Required for older Darwin builds, that lack definition of MAP_ANONYMOUS
#ifndef MAP_ANONYMOUS
@ -84,7 +86,7 @@ namespace ErrorCodes
* - random hint address for mmap
* - mmap_threshold for using mmap less or more
*/
template <bool clear_memory_, bool mmap_populate = false>
template <bool clear_memory_, bool mmap_populate>
class Allocator
{
public:
@ -270,7 +272,7 @@ private:
/** Allocator with optimization to place small memory ranges in automatic memory.
*/
template <typename Base, size_t N = 64, size_t Alignment = 1>
template <typename Base, size_t N, size_t Alignment>
class AllocatorWithStackMemory : private Base
{
private:

10
dbms/src/Common/Allocator_fwd.h Normal file
View File

@ -0,0 +1,10 @@
/**
* This file provides forward declarations for Allocator.
*/
#pragma once
template <bool clear_memory_, bool mmap_populate = false>
class Allocator;
template <typename Base, size_t N = 64, size_t Alignment = 1>
class AllocatorWithStackMemory;

2
dbms/src/Common/DiskSpaceMonitor.cpp
View File

@ -1,6 +1,6 @@
#include <Common/DiskSpaceMonitor.h>
#include <Common/escapeForFileName.h>
#include <IO/WriteHelpers.h>
#include <Common/quoteString.h>
#include <set>

21
dbms/src/Common/PODArray.h
View File

@ -21,6 +21,8 @@
#include <sys/mman.h>
#endif
#include <Common/PODArray_fwd.h>
namespace DB
{
@ -30,11 +32,6 @@ namespace ErrorCodes
extern const int CANNOT_MPROTECT;
}
inline constexpr size_t integerRoundUp(size_t value, size_t dividend)
{
return ((value + dividend - 1) / dividend) * dividend;
}
/** A dynamic array for POD types.
* Designed for a small number of large arrays (rather than a lot of small ones).
* To be more precise - for use in ColumnVector.
@ -258,7 +255,7 @@ public:
}
};
template <typename T, size_t initial_bytes = 4096, typename TAllocator = Allocator<false>, size_t pad_right_ = 0, size_t pad_left_ = 0>
template <typename T, size_t initial_bytes, typename TAllocator, size_t pad_right_, size_t pad_left_>
class PODArray : public PODArrayBase<sizeof(T), initial_bytes, TAllocator, pad_right_, pad_left_>
{
protected:
@ -625,17 +622,5 @@ void swap(PODArray<T, initial_bytes, TAllocator, pad_right_> & lhs, PODArray<T,
lhs.swap(rhs);
}
/** For columns. Padding is enough to read and write xmm-register at the address of the last element. */
template <typename T, size_t initial_bytes = 4096, typename TAllocator = Allocator<false>>
using PaddedPODArray = PODArray<T, initial_bytes, TAllocator, 15, 16>;
/** A helper for declaring PODArray that uses inline memory.
* The initial size is set to use all the inline bytes, since using less would
* only add some extra allocation calls.
*/
template <typename T, size_t inline_bytes,
size_t rounded_bytes = integerRoundUp(inline_bytes, sizeof(T))>
using PODArrayWithStackMemory = PODArray<T, rounded_bytes,
AllocatorWithStackMemory<Allocator<false>, rounded_bytes, alignof(T)>>;
}

35
dbms/src/Common/PODArray_fwd.h Normal file
View File

@ -0,0 +1,35 @@
/**
* This file contains some using-declarations that define various kinds of
* PODArray.
*/
#pragma once
#include <Common/Allocator_fwd.h>
namespace DB
{
inline constexpr size_t integerRoundUp(size_t value, size_t dividend)
{
return ((value + dividend - 1) / dividend) * dividend;
}
template <typename T, size_t initial_bytes = 4096,
typename TAllocator = Allocator<false>, size_t pad_right_ = 0,
size_t pad_left_ = 0>
class PODArray;
/** For columns. Padding is enough to read and write xmm-register at the address of the last element. */
template <typename T, size_t initial_bytes = 4096, typename TAllocator = Allocator<false>>
using PaddedPODArray = PODArray<T, initial_bytes, TAllocator, 15, 16>;
/** A helper for declaring PODArray that uses inline memory.
* The initial size is set to use all the inline bytes, since using less would
* only add some extra allocation calls.
*/
template <typename T, size_t inline_bytes,
size_t rounded_bytes = integerRoundUp(inline_bytes, sizeof(T))>
using PODArrayWithStackMemory = PODArray<T, rounded_bytes,
AllocatorWithStackMemory<Allocator<false>, rounded_bytes, alignof(T)>>;
}

7
dbms/src/Common/formatReadable.h
View File

@ -1,8 +1,13 @@
#pragma once
#include <string>
#include <IO/WriteBuffer.h>
namespace DB
{
class WriteBuffer;
}
/// Displays the passed size in bytes as 123.45 GiB.
void formatReadableSizeWithBinarySuffix(double value, DB::WriteBuffer & out, int precision = 2);

37
dbms/src/Common/quoteString.cpp Normal file
View File

@ -0,0 +1,37 @@
#include <Common/quoteString.h>
#include <IO/WriteHelpers.h>
#include <IO/WriteBufferFromString.h>
namespace DB
{
String quoteString(const StringRef & x)
{
String res(x.size, '\0');
WriteBufferFromString wb(res);
writeQuotedString(x, wb);
return res;
}
String backQuote(const StringRef & x)
{
String res(x.size, '\0');
{
WriteBufferFromString wb(res);
writeBackQuotedString(x, wb);
}
return res;
}
String backQuoteIfNeed(const StringRef & x)
{
String res(x.size, '\0');
{
WriteBufferFromString wb(res);
writeProbablyBackQuotedString(x, wb);
}
return res;
}
}

17
dbms/src/Common/quoteString.h Normal file
View File

@ -0,0 +1,17 @@
#pragma once
#include <Core/Types.h>
#include <common/StringRef.h>
namespace DB
{
/// Quote the string.
String quoteString(const StringRef & x);
/// Quote the identifier with backquotes.
String backQuote(const StringRef & x);
/// Quote the identifier with backquotes, if required.
String backQuoteIfNeed(const StringRef & x);
}

1
dbms/src/Compression/CompressionCodecMultiple.cpp
View File

@ -1,5 +1,6 @@
#include <Compression/CompressionCodecMultiple.h>
#include <Compression/CompressionInfo.h>
#include <Common/PODArray.h>
#include <common/unaligned.h>
#include <Compression/CompressionFactory.h>
#include <IO/ReadHelpers.h>

1
dbms/src/Compression/ICompressionCodec.h
View File

@ -4,7 +4,6 @@
#include <IO/ReadBuffer.h>
#include <IO/WriteBuffer.h>
#include <IO/BufferWithOwnMemory.h>
#include <Common/PODArray.h>
#include <DataTypes/IDataType.h>
#include <boost/noncopyable.hpp>
#include <IO/UncompressedCache.h>

4
dbms/src/Core/MySQLProtocol.h
View File

@ -919,10 +919,10 @@ public:
auto user = context.getUser(user_name);
if (user->password_double_sha1_hex.empty())
if (user->authentication.getType() != DB::Authentication::DOUBLE_SHA1_PASSWORD)
throw Exception("Cannot use " + getName() + " auth plugin for user " + user_name + " since its password isn't specified using double SHA1.", ErrorCodes::UNKNOWN_EXCEPTION);
Poco::SHA1Engine::Digest double_sha1_value = Poco::DigestEngine::digestFromHex(user->password_double_sha1_hex);
Poco::SHA1Engine::Digest double_sha1_value = user->authentication.getPasswordHashBinary();
assert(double_sha1_value.size() == Poco::SHA1Engine::DIGEST_SIZE);
Poco::SHA1Engine engine;

1
dbms/src/DataStreams/CheckConstraintsBlockOutputStream.cpp
View File

@ -4,6 +4,7 @@
#include <Columns/ColumnsCommon.h>
#include <Columns/ColumnsNumber.h>
#include <Common/assert_cast.h>
#include <Common/quoteString.h>
#include <Common/FieldVisitors.h>

85
dbms/src/DataStreams/CheckNonEmptySetBlockInputStream.cpp
View File

@ -1,85 +0,0 @@
#include <Interpreters/ExpressionActions.h>
#include <DataStreams/CheckNonEmptySetBlockInputStream.h>
#include <Interpreters/Set.h>
#include <Interpreters/Join.h>
#include <Columns/ColumnSet.h>
#include <Columns/ColumnConst.h>
namespace DB
{
CheckNonEmptySetBlockInputStream::CheckNonEmptySetBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_, const NameSet sets_)
: expression(expression_), sets(sets_)
{
children.push_back(input);
cached_header = children.back()->getHeader();
}
String CheckNonEmptySetBlockInputStream::getName() const { return "CheckNonEmptySet"; }
Block CheckNonEmptySetBlockInputStream::getTotals()
{
return children.back()->getTotals();
}
Block CheckNonEmptySetBlockInputStream::getHeader() const
{
return cached_header.cloneEmpty();
}
Block CheckNonEmptySetBlockInputStream::readImpl()
{
if (!initialized)
{
/// CheckNonEmptyBlockInputStream in the downstream with CreatingSetsBlockInputStream. So set has been created.
cached_result = inOrInnerRightJoinWithEmpty();
initialized = true;
}
Block res;
if (isCancelledOrThrowIfKilled() || cached_result)
return res;
return children.back()->read();
}
bool CheckNonEmptySetBlockInputStream::inOrInnerRightJoinWithEmpty() const
{
InOrInnerRightJoinWithEmpty checker;
for (const auto & action : expression->getActions())
{
if (action.type == ExpressionAction::ARRAY_JOIN)
{
return false;
}
else if (action.type == ExpressionAction::JOIN)
{
if (const auto * join = dynamic_cast<Join *>(action.join.get()))
{
checker.hasJoin = true;
checker.innerRightJoinWithEmpty &= join->getTotalRowCount() == 0 && isInnerOrRight(join->getKind());
}
}
else if (action.type == ExpressionAction::ADD_COLUMN)
{
if (!sets.count(action.result_name))
continue;
checker.hasIn = true;
ColumnPtr column_set_ptr = action.added_column;
const ColumnSet * column_set = typeid_cast<const ColumnSet *>(&*column_set_ptr);
checker.inWithEmpty &= column_set && column_set->getData()->getTotalRowCount() == 0;
}
}
/// Get the final result.
return checker.result();
}
}

63
dbms/src/DataStreams/CheckNonEmptySetBlockInputStream.h
View File

@ -1,63 +0,0 @@
#pragma once
#include <DataStreams/IBlockInputStream.h>
#include <Core/Names.h>
namespace DB
{
class CheckNonEmptySetBlockInputStream : public IBlockInputStream
{
private:
using ExpressionActionsPtr = std::shared_ptr<ExpressionActions>;
public:
CheckNonEmptySetBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_, const NameSet sets_);
String getName() const override;
Block getTotals() override;
Block getHeader() const override;
protected:
Block readImpl() override;
private:
Block cached_header;
ExpressionActionsPtr expression;
bool initialized = false;
bool cached_result = false;
NameSet sets;
bool inOrInnerRightJoinWithEmpty() const;
/**
* Used to determine if actions are IN OR INNER/RIGHT JOIN with empty.
*/
struct InOrInnerRightJoinWithEmpty
{
bool hasJoin = false;
bool hasIn = false;
bool innerRightJoinWithEmpty = true;
bool inWithEmpty = true;
bool result()
{
if (hasJoin && !hasIn)
return innerRightJoinWithEmpty;
else if (hasIn && !hasJoin)
return inWithEmpty;
else if (hasJoin && hasIn)
return innerRightJoinWithEmpty && inWithEmpty;
return false;
}
};
};
}

1
dbms/src/DataStreams/ConvertingBlockInputStream.cpp
View File

@ -2,6 +2,7 @@
#include <Interpreters/castColumn.h>
#include <Columns/ColumnConst.h>
#include <Common/assert_cast.h>
#include <Common/quoteString.h>
#include <Parsers/IAST.h>

8
dbms/src/DataStreams/ExpressionBlockInputStream.cpp
View File

@ -30,6 +30,14 @@ Block ExpressionBlockInputStream::getHeader() const
Block ExpressionBlockInputStream::readImpl()
{
if (!initialized)
{
if (expression->resultIsAlwaysEmpty())
return {};
initialized = true;
}
Block res = children.back()->read();
if (res)
expression->execute(res);

1
dbms/src/DataStreams/ExpressionBlockInputStream.h
View File

@ -31,6 +31,7 @@ protected:
private:
ExpressionActionsPtr expression;
Block cached_header;
bool initialized = false;
};
}

11
dbms/src/DataStreams/FilterBlockInputStream.cpp
View File

@ -17,9 +17,11 @@ namespace ErrorCodes
}
FilterBlockInputStream::FilterBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_,
const String & filter_column_name, bool remove_filter_)
: remove_filter(remove_filter_), expression(expression_)
FilterBlockInputStream::FilterBlockInputStream(const BlockInputStreamPtr & input, ExpressionActionsPtr expression_,
String filter_column_name_, bool remove_filter_)
: remove_filter(remove_filter_)
, expression(std::move(expression_))
, filter_column_name(std::move(filter_column_name_))
{
children.push_back(input);
@ -72,6 +74,9 @@ Block FilterBlockInputStream::readImpl()
if (constant_filter_description.always_false)
return removeFilterIfNeed(std::move(res));
if (expression->checkColumnIsAlwaysFalse(filter_column_name))
return {};
/// Until non-empty block after filtering or end of stream.
while (1)
{

5
dbms/src/DataStreams/FilterBlockInputStream.h
View File

@ -20,8 +20,8 @@ private:
using ExpressionActionsPtr = std::shared_ptr<ExpressionActions>;
public:
FilterBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_,
const String & filter_column_name_, bool remove_filter_ = false);
FilterBlockInputStream(const BlockInputStreamPtr & input, ExpressionActionsPtr expression_,
String filter_column_name_, bool remove_filter_ = false);
String getName() const override;
Block getTotals() override;
@ -35,6 +35,7 @@ protected:
private:
ExpressionActionsPtr expression;
Block header;
String filter_column_name;
ssize_t filter_column;
ConstantFilterDescription constant_filter_description;

1
dbms/src/DataStreams/LimitByBlockInputStream.cpp
View File

@ -1,4 +1,5 @@
#include <DataStreams/LimitByBlockInputStream.h>
#include <Common/PODArray.h>
#include <Common/SipHash.h>

2
dbms/src/DataStreams/ReverseBlockInputStream.cpp
View File

@ -1,5 +1,7 @@
#include "ReverseBlockInputStream.h"
#include <Common/PODArray.h>
namespace DB
{
ReverseBlockInputStream::ReverseBlockInputStream(const BlockInputStreamPtr & input)

1
dbms/src/Databases/DatabaseOnDisk.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <Common/escapeForFileName.h>
#include <Common/quoteString.h>
#include <Databases/DatabasesCommon.h>
#include <Interpreters/Context.h>
#include <Parsers/ASTCreateQuery.h>

51
dbms/src/Functions/FunctionFQDN.cpp Normal file
View File

@ -0,0 +1,51 @@
#include <Functions/IFunction.h>
#include <Functions/FunctionFactory.h>
#include <DataTypes/DataTypeString.h>
#include <Common/getFQDNOrHostName.h>
#include <Core/Field.h>
namespace DB
{
class FunctionFQDN : public IFunction
{
public:
static constexpr auto name = "FQDN";
static FunctionPtr create(const Context &)
{
return std::make_shared<FunctionFQDN>();
}
String getName() const override
{
return name;
}
bool isDeterministic() const override { return false; }
size_t getNumberOfArguments() const override
{
return 0;
}
DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override
{
return std::make_shared<DataTypeString>();
}
void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) override
{
block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst(
input_rows_count, getFQDNOrHostName())->convertToFullColumnIfConst();
}
};
void registerFunctionFQDN(FunctionFactory & factory)
{
factory.registerFunction<FunctionFQDN>(FunctionFactory::CaseInsensitive);
factory.registerFunction<FunctionFQDN>("fullHostName");
}
}

3
dbms/src/Functions/FunctionsBitmap.cpp
View File

@ -1,4 +1,7 @@
#include <Functions/FunctionFactory.h>
// TODO include this last because of a broken roaring header. See the comment
// inside.
#include <Functions/FunctionsBitmap.h>

4
dbms/src/Functions/FunctionsBitmap.h
View File

@ -1,7 +1,6 @@
#pragma once
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/AggregateFunctionGroupBitmapData.h>
#include <Columns/ColumnAggregateFunction.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnConst.h>
@ -15,6 +14,9 @@
#include <Common/typeid_cast.h>
#include <Common/assert_cast.h>
// TODO include this last because of a broken roaring header. See the comment
// inside.
#include <AggregateFunctions/AggregateFunctionGroupBitmapData.h>
namespace DB
{

2
dbms/src/Functions/FunctionsJSON.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <Functions/IFunction.h>
#include <Core/AccurateComparison.h>
#include <Functions/DummyJSONParser.h>
#include <Functions/SimdJSONParser.h>
#include <Functions/RapidJSONParser.h>
@ -8,7 +9,6 @@
#include <Common/CpuId.h>
#include <Common/typeid_cast.h>
#include <Common/assert_cast.h>
#include <Core/AccurateComparison.h>
#include <Core/Settings.h>
#include <Columns/ColumnConst.h>
#include <Columns/ColumnString.h>

2
dbms/src/Functions/registerFunctionsMiscellaneous.cpp
View File

@ -8,6 +8,7 @@ class FunctionFactory;
void registerFunctionCurrentDatabase(FunctionFactory &);
void registerFunctionCurrentUser(FunctionFactory &);
void registerFunctionHostName(FunctionFactory &);
void registerFunctionFQDN(FunctionFactory &);
void registerFunctionVisibleWidth(FunctionFactory &);
void registerFunctionToTypeName(FunctionFactory &);
void registerFunctionGetSizeOfEnumType(FunctionFactory &);
@ -61,6 +62,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory)
registerFunctionCurrentDatabase(factory);
registerFunctionCurrentUser(factory);
registerFunctionHostName(factory);
registerFunctionFQDN(factory);
registerFunctionVisibleWidth(factory);
registerFunctionToTypeName(factory);
registerFunctionGetSizeOfEnumType(factory);

22
dbms/src/IO/WriteHelpers.cpp
View File

@ -66,26 +66,4 @@ void writeException(const Exception & e, WriteBuffer & buf, bool with_stack_trac
if (has_nested)
writeException(Exception(Exception::CreateFromPoco, *e.nested()), buf, with_stack_trace);
}
String backQuoteIfNeed(const String & x)
{
String res(x.size(), '\0');
{
WriteBufferFromString wb(res);
writeProbablyBackQuotedString(x, wb);
}
return res;
}
String backQuote(const String & x)
{
String res(x.size(), '\0');
{
WriteBufferFromString wb(res);
writeBackQuotedString(x, wb);
}
return res;
}
}

35
dbms/src/IO/WriteHelpers.h
View File

@ -410,36 +410,36 @@ inline void writeQuotedString(const StringRef & ref, WriteBuffer & buf)
writeAnyQuotedString<'\''>(ref, buf);
}
inline void writeDoubleQuotedString(const String & s, WriteBuffer & buf)
inline void writeDoubleQuotedString(const StringRef & s, WriteBuffer & buf)
{
writeAnyQuotedString<'"'>(s, buf);
}
/// Outputs a string in backquotes.
inline void writeBackQuotedString(const String & s, WriteBuffer & buf)
inline void writeBackQuotedString(const StringRef & s, WriteBuffer & buf)
{
writeAnyQuotedString<'`'>(s, buf);
}
/// Outputs a string in backquotes for MySQL.
inline void writeBackQuotedStringMySQL(const String & s, WriteBuffer & buf)
inline void writeBackQuotedStringMySQL(const StringRef & s, WriteBuffer & buf)
{
writeChar('`', buf);
writeAnyEscapedString<'`', true>(s.data(), s.data() + s.size(), buf);
writeAnyEscapedString<'`', true>(s.data, s.data + s.size, buf);
writeChar('`', buf);
}
/// The same, but quotes apply only if there are characters that do not match the identifier without quotes.
template <typename F>
inline void writeProbablyQuotedStringImpl(const String & s, WriteBuffer & buf, F && write_quoted_string)
inline void writeProbablyQuotedStringImpl(const StringRef & s, WriteBuffer & buf, F && write_quoted_string)
{
if (s.empty() || !isValidIdentifierBegin(s[0]))
if (!s.size || !isValidIdentifierBegin(s.data[0]))
write_quoted_string(s, buf);
else
{
const char * pos = s.data() + 1;
const char * end = s.data() + s.size();
const char * pos = s.data + 1;
const char * end = s.data + s.size;
for (; pos < end; ++pos)
if (!isWordCharASCII(*pos))
break;
@ -450,19 +450,19 @@ inline void writeProbablyQuotedStringImpl(const String & s, WriteBuffer & buf, F
}
}
inline void writeProbablyBackQuotedString(const String & s, WriteBuffer & buf)
inline void writeProbablyBackQuotedString(const StringRef & s, WriteBuffer & buf)
{
writeProbablyQuotedStringImpl(s, buf, [](const String & s_, WriteBuffer & buf_) { return writeBackQuotedString(s_, buf_); });
writeProbablyQuotedStringImpl(s, buf, [](const StringRef & s_, WriteBuffer & buf_) { return writeBackQuotedString(s_, buf_); });
}
inline void writeProbablyDoubleQuotedString(const String & s, WriteBuffer & buf)
inline void writeProbablyDoubleQuotedString(const StringRef & s, WriteBuffer & buf)
{
writeProbablyQuotedStringImpl(s, buf, [](const String & s_, WriteBuffer & buf_) { return writeDoubleQuotedString(s_, buf_); });
writeProbablyQuotedStringImpl(s, buf, [](const StringRef & s_, WriteBuffer & buf_) { return writeDoubleQuotedString(s_, buf_); });
}
inline void writeProbablyBackQuotedStringMySQL(const String & s, WriteBuffer & buf)
inline void writeProbablyBackQuotedStringMySQL(const StringRef & s, WriteBuffer & buf)
{
writeProbablyQuotedStringImpl(s, buf, [](const String & s_, WriteBuffer & buf_) { return writeBackQuotedStringMySQL(s_, buf_); });
writeProbablyQuotedStringImpl(s, buf, [](const StringRef & s_, WriteBuffer & buf_) { return writeBackQuotedStringMySQL(s_, buf_); });
}
@ -905,11 +905,4 @@ inline String toString(const T & x)
writeText(x, buf);
return buf.str();
}
/// Quote the identifier with backquotes, if required.
String backQuoteIfNeed(const String & x);
/// Quote the identifier with backquotes.
String backQuote(const String & x);
}

1
dbms/src/Interpreters/ActionsVisitor.cpp
View File

@ -1,4 +1,5 @@
#include <Common/typeid_cast.h>
#include <Common/PODArray.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsMiscellaneous.h>

54
dbms/src/Interpreters/ExpressionActions.cpp
View File

@ -1,4 +1,5 @@
#include "config_core.h"
#include <Interpreters/Set.h>
#include <Common/ProfileEvents.h>
#include <Common/SipHash.h>
#include <Interpreters/ExpressionActions.h>
@ -13,6 +14,7 @@
#include <Functions/IFunction.h>
#include <set>
#include <optional>
#include <Columns/ColumnSet.h>
namespace ProfileEvents
@ -1167,6 +1169,58 @@ JoinPtr ExpressionActions::getTableJoinAlgo() const
}
bool ExpressionActions::resultIsAlwaysEmpty() const
{
/// Check that has join which returns empty result.
for (auto & action : actions)
{
if (action.type == action.JOIN && action.join && action.join->alwaysReturnsEmptySet())
return true;
}
return false;
}
bool ExpressionActions::checkColumnIsAlwaysFalse(const String & column_name) const
{
/// Check has column in (empty set).
String set_to_check;
for (auto & action : actions)
{
if (action.type == action.APPLY_FUNCTION && action.function_base)
{
auto name = action.function_base->getName();
if ((name == "in" || name == "globalIn")
&& action.result_name == column_name
&& action.argument_names.size() > 1)
{
set_to_check = action.argument_names[1];
}
}
}
if (!set_to_check.empty())
{
for (auto & action : actions)
{
if (action.type == action.ADD_COLUMN && action.result_name == set_to_check)
{
if (auto * column_set = typeid_cast<const ColumnSet *>(action.added_column.get()))
{
if (column_set->getData()->getTotalRowCount() == 0)
return true;
}
}
}
}
return false;
}
/// It is not important to calculate the hash of individual strings or their concatenation
UInt128 ExpressionAction::ActionHash::operator()(const ExpressionAction & action) const
{

7
dbms/src/Interpreters/ExpressionActions.h
View File

@ -239,6 +239,13 @@ public:
const Settings & getSettings() const { return settings; }
/// Check if result block has no rows. True if it's definite, false if we can't say for sure.
/// Call it only after subqueries for join were executed.
bool resultIsAlwaysEmpty() const;
/// Check if column is always zero. True if it's definite, false if we can't say for sure.
/// Call it only after subqueries for sets were executed.
bool checkColumnIsAlwaysFalse(const String & column_name) const;
struct ActionsHash
{

1
dbms/src/Interpreters/IJoin.h
View File

@ -30,6 +30,7 @@ public:
virtual void joinTotals(Block & block) const = 0;
virtual size_t getTotalRowCount() const = 0;
virtual bool alwaysReturnsEmptySet() const { return false; }
virtual BlockInputStreamPtr createStreamWithNonJoinedRows(const Block &, UInt64) const { return {}; }
};

1
dbms/src/Interpreters/InterpreterDropQuery.cpp
View File

@ -7,6 +7,7 @@
#include <Parsers/ASTDropQuery.h>
#include <Storages/IStorage.h>
#include <Common/escapeForFileName.h>
#include <Common/quoteString.h>
#include <Common/typeid_cast.h>

41
dbms/src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -25,7 +25,6 @@
#include <DataStreams/ConvertingBlockInputStream.h>
#include <DataStreams/ReverseBlockInputStream.h>
#include <DataStreams/FillingBlockInputStream.h>
#include <DataStreams/CheckNonEmptySetBlockInputStream.h>
#include <DataStreams/SquashingBlockInputStream.h>
#include <Parsers/ASTFunction.h>
@ -48,7 +47,6 @@
#include <Interpreters/JoinToSubqueryTransformVisitor.h>
#include <Interpreters/CrossToInnerJoinVisitor.h>
#include <Interpreters/AnalyzedJoin.h>
#include <Interpreters/Join.h>
#include <Storages/MergeTree/MergeTreeData.h>
#include <Storages/MergeTree/MergeTreeWhereOptimizer.h>
@ -714,35 +712,6 @@ InterpreterSelectQuery::analyzeExpressions(
return res;
}
BlockInputStreamPtr InterpreterSelectQuery::createCheckNonEmptySetIfNeed(BlockInputStreamPtr stream, const ExpressionActionsPtr & expression) const
{
for (const auto & action : expression->getActions())
{
if (action.type == ExpressionAction::JOIN)
{
const auto * join = dynamic_cast<Join *>(action.join.get());
if (!join)
continue;
if (isInnerOrRight(join->getKind()))
{
stream = std::make_shared<CheckNonEmptySetBlockInputStream>(stream, expression, syntax_analyzer_result->need_check_empty_sets);
break;
}
}
else if (action.type == ExpressionAction::ADD_COLUMN)
{
if (syntax_analyzer_result->need_check_empty_sets.count(action.result_name))
{
stream = std::make_shared<CheckNonEmptySetBlockInputStream>(stream, expression, syntax_analyzer_result->need_check_empty_sets);
break;
}
}
}
return stream;
}
static Field getWithFillFieldValue(const ASTPtr & node, const Context & context)
{
const auto & [field, type] = evaluateConstantExpression(node, context);
@ -998,7 +967,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
});
else
pipeline.streams.back() = std::make_shared<FilterBlockInputStream>(
createCheckNonEmptySetIfNeed(pipeline.streams.back(), expressions.prewhere_info->prewhere_actions), expressions.prewhere_info->prewhere_actions,
pipeline.streams.back(), expressions.prewhere_info->prewhere_actions,
expressions.prewhere_info->prewhere_column_name, expressions.prewhere_info->remove_prewhere_column);
// To remove additional columns in dry run
@ -1114,7 +1083,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
header_before_join = pipeline.firstStream()->getHeader();
/// Applies to all sources except stream_with_non_joined_data.
for (auto & stream : pipeline.streams)
stream = std::make_shared<ExpressionBlockInputStream>(createCheckNonEmptySetIfNeed(stream, expressions.before_join), expressions.before_join);
stream = std::make_shared<ExpressionBlockInputStream>(stream, expressions.before_join);
if (isMergeJoin(expressions.before_join->getTableJoinAlgo()) && settings.partial_merge_join_optimizations)
{
@ -1695,7 +1664,7 @@ void InterpreterSelectQuery::executeWhere(Pipeline & pipeline, const ExpressionA
{
pipeline.transform([&](auto & stream)
{
stream = std::make_shared<FilterBlockInputStream>(createCheckNonEmptySetIfNeed(stream, expression), expression, getSelectQuery().where()->getColumnName(), remove_fiter);
stream = std::make_shared<FilterBlockInputStream>(stream, expression, getSelectQuery().where()->getColumnName(), remove_fiter);
});
}
@ -1711,7 +1680,7 @@ void InterpreterSelectQuery::executeAggregation(Pipeline & pipeline, const Expre
{
pipeline.transform([&](auto & stream)
{
stream = std::make_shared<ExpressionBlockInputStream>(createCheckNonEmptySetIfNeed(stream, expression), expression);
stream = std::make_shared<ExpressionBlockInputStream>(stream, expression);
});
Names key_names;
@ -2077,7 +2046,7 @@ void InterpreterSelectQuery::executeExpression(Pipeline & pipeline, const Expres
{
pipeline.transform([&](auto & stream)
{
stream = std::make_shared<ExpressionBlockInputStream>(createCheckNonEmptySetIfNeed(stream, expression), expression);
stream = std::make_shared<ExpressionBlockInputStream>(stream, expression);
});
}

3
dbms/src/Interpreters/InterpreterSelectQuery.h
View File

@ -253,9 +253,6 @@ private:
*/
void initSettings();
/// Whether you need to check if the set is empty before ExpressionActions is executed. Create a CheckNonEmptySetBlockInputStream if needed.
BlockInputStreamPtr createCheckNonEmptySetIfNeed(BlockInputStreamPtr stream, const ExpressionActionsPtr & expression) const;
const SelectQueryOptions options;
ASTPtr query_ptr;
Context context;

3
dbms/src/Interpreters/Join.cpp
View File

@ -466,6 +466,9 @@ bool Join::addJoinedBlock(const Block & block)
size_t rows = block.rows();
if (rows)
has_no_rows_in_maps = false;
blocks.push_back(block);
Block * stored_block = &blocks.back();

5
dbms/src/Interpreters/Join.h
View File

@ -159,10 +159,12 @@ public:
BlockInputStreamPtr createStreamWithNonJoinedRows(const Block & left_sample_block, UInt64 max_block_size) const override;
/// Number of keys in all built JOIN maps.
size_t getTotalRowCount() const override;
size_t getTotalRowCount() const final;
/// Sum size in bytes of all buffers, used for JOIN maps and for all memory pools.
size_t getTotalByteCount() const;
bool alwaysReturnsEmptySet() const final { return isInnerOrRight(getKind()) && has_no_rows_in_maps; }
ASTTableJoin::Kind getKind() const { return kind; }
ASTTableJoin::Strictness getStrictness() const { return strictness; }
AsofRowRefs::Type getAsofType() const { return *asof_type; }
@ -299,6 +301,7 @@ private:
BlockNullmapList blocks_nullmaps;
MapsVariant maps;
bool has_no_rows_in_maps = true;
/// Additional data - strings for string keys and continuation elements of single-linked lists of references to rows.
Arena pool;

2
dbms/src/Interpreters/QueryAliasesVisitor.cpp
View File

@ -8,7 +8,7 @@
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/formatAST.h>
#include <Parsers/ASTSubquery.h>
#include <IO/WriteHelpers.h>
#include <Common/quoteString.h>
namespace DB
{

2
dbms/src/Interpreters/QueryNormalizer.cpp
View File

@ -10,7 +10,7 @@
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/typeid_cast.h>
#include <IO/WriteHelpers.h>
#include <Common/quoteString.h>
namespace DB
{

1
dbms/src/Interpreters/ReplaceQueryParameterVisitor.cpp
View File

@ -1,5 +1,6 @@
#include <boost/algorithm/string/replace.hpp>
#include <Common/typeid_cast.h>
#include <Common/quoteString.h>
#include <Columns/IColumn.h>
#include <Core/Field.h>
#include <DataTypes/IDataType.h>

34
dbms/src/Interpreters/SyntaxAnalyzer.cpp
View File

@ -615,39 +615,6 @@ std::vector<const ASTFunction *> getAggregates(const ASTPtr & query)
return {};
}
void collectCanShortCircuitSet(const ASTPtr & ast, NameSet & need_check_empty_sets)
{
if (const auto * function = ast->as<ASTFunction>())
{
if (function->name == "in" || function->name == "globalIn")
{
for (size_t i = 0; i < function->arguments->children.size(); ++i)
{
ASTPtr child = function->arguments->children[i];
if (const auto * subquery = child->as<ASTSubquery>())
need_check_empty_sets.insert(subquery->getColumnName());
}
}
else if (function->name != "or")
{
for (size_t i = 0; i < function->arguments->children.size(); ++i)
{
ASTPtr child = function->arguments->children[i];
collectCanShortCircuitSet(child, need_check_empty_sets);
}
}
}
else
{
for (auto & child : ast->children)
{
/// Do not go to FROM, JOIN, UNION.
if (!child->as<ASTTableExpression>() && !child->as<ASTSelectQuery>())
collectCanShortCircuitSet(child, need_check_empty_sets);
}
}
}
}
/// Calculate which columns are required to execute the expression.
@ -930,7 +897,6 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze(
setJoinStrictness(*select_query, settings.join_default_strictness, result.analyzed_join->table_join);
collectJoinedColumns(*result.analyzed_join, *select_query, source_columns_set, result.aliases);
collectCanShortCircuitSet(query, result.need_check_empty_sets);
}
result.aggregates = getAggregates(query);

3
dbms/src/Interpreters/SyntaxAnalyzer.h
View File

@ -40,9 +40,6 @@ struct SyntaxAnalyzerResult
/// Note: not used further.
NameToNameMap array_join_name_to_alias;
/// For sets created during query execution, check if they are empty after creation.
NameSet need_check_empty_sets;
/// Predicate optimizer overrides the sub queries
bool rewrite_subqueries = false;

298
dbms/src/Interpreters/Users.cpp
View File

@ -1,22 +1,10 @@
#include <string.h>
#include <Poco/RegularExpression.h>
#include <Poco/Net/IPAddress.h>
#include <Poco/Net/SocketAddress.h>
#include <Poco/Net/DNS.h>
#include <Poco/Util/Application.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <Poco/String.h>
#include <Common/Exception.h>
#include <IO/ReadHelpers.h>
#include <IO/HexWriteBuffer.h>
#include <IO/WriteBufferFromString.h>
#include <IO/WriteHelpers.h>
#include <common/SimpleCache.h>
#include <Common/StringUtils/StringUtils.h>
#include <Interpreters/Users.h>
#include <common/logger_useful.h>
#include <ext/scope_guard.h>
#include <Common/config.h>
namespace DB
@ -24,255 +12,12 @@ namespace DB
namespace ErrorCodes
{
extern const int DNS_ERROR;
extern const int UNKNOWN_ADDRESS_PATTERN_TYPE;
extern const int UNKNOWN_USER;
extern const int REQUIRED_PASSWORD;
extern const int WRONG_PASSWORD;
extern const int IP_ADDRESS_NOT_ALLOWED;
extern const int BAD_ARGUMENTS;
}
static Poco::Net::IPAddress toIPv6(const Poco::Net::IPAddress addr)
{
if (addr.family() == Poco::Net::IPAddress::IPv6)
return addr;
return Poco::Net::IPAddress("::FFFF:" + addr.toString());
}
/// IP-address or subnet mask. Example: 213.180.204.3 or 10.0.0.1/8 or 312.234.1.1/255.255.255.0
/// 2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/FFFF:FFFF:FFFF:FFFF::
class IPAddressPattern : public IAddressPattern
{
private:
/// Address of mask. Always transformed to IPv6.
Poco::Net::IPAddress mask_address;
/// Mask of net (ip form). Always transformed to IPv6.
Poco::Net::IPAddress subnet_mask;
public:
explicit IPAddressPattern(const String & str)
{
const char * pos = strchr(str.c_str(), '/');
if (nullptr == pos)
{
construct(Poco::Net::IPAddress(str));
}
else
{
String addr(str, 0, pos - str.c_str());
auto real_address = Poco::Net::IPAddress(addr);
String str_mask(str, addr.length() + 1, str.length() - addr.length() - 1);
if (isDigits(str_mask))
{
UInt8 prefix_bits = parse<UInt8>(pos + 1);
construct(prefix_bits, real_address.family() == Poco::Net::AddressFamily::IPv4);
}
else
{
subnet_mask = netmaskToIPv6(Poco::Net::IPAddress(str_mask));
}
mask_address = toIPv6(real_address);
}
}
bool contains(const Poco::Net::IPAddress & addr) const override
{
return prefixBitsEquals(addr, mask_address, subnet_mask);
}
private:
void construct(const Poco::Net::IPAddress & mask_address_)
{
mask_address = toIPv6(mask_address_);
subnet_mask = Poco::Net::IPAddress(128, Poco::Net::IPAddress::IPv6);
}
void construct(UInt8 prefix_bits, bool is_ipv4)
{
prefix_bits = is_ipv4 ? prefix_bits + 96 : prefix_bits;
subnet_mask = Poco::Net::IPAddress(prefix_bits, Poco::Net::IPAddress::IPv6);
}
static bool prefixBitsEquals(const Poco::Net::IPAddress & ip_address, const Poco::Net::IPAddress & net_address, const Poco::Net::IPAddress & mask)
{
return ((toIPv6(ip_address) & mask) == (toIPv6(net_address) & mask));
}
static bool isDigits(const std::string & str)
{
return std::all_of(str.begin(), str.end(), isNumericASCII);
}
static Poco::Net::IPAddress netmaskToIPv6(Poco::Net::IPAddress mask)
{
if (mask.family() == Poco::Net::IPAddress::IPv6)
return mask;
return Poco::Net::IPAddress(96, Poco::Net::IPAddress::IPv6) | toIPv6(mask);
}
};
/// Check that address equals to one of hostname addresses.
class HostExactPattern : public IAddressPattern
{
private:
String host;
static bool containsImpl(const String & host, const Poco::Net::IPAddress & addr)
{
Poco::Net::IPAddress addr_v6 = toIPv6(addr);
/// Resolve by hand, because Poco don't use AI_ALL flag but we need it.
addrinfo * ai = nullptr;
addrinfo hints;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
hints.ai_flags |= AI_V4MAPPED | AI_ALL;
int ret = getaddrinfo(host.c_str(), nullptr, &hints, &ai);
if (0 != ret)
throw Exception("Cannot getaddrinfo: " + std::string(gai_strerror(ret)), ErrorCodes::DNS_ERROR);
SCOPE_EXIT(
{
freeaddrinfo(ai);
});
for (; ai != nullptr; ai = ai->ai_next)
{
if (ai->ai_addrlen && ai->ai_addr)
{
if (ai->ai_family == AF_INET6)
{
if (addr_v6 == Poco::Net::IPAddress(
&reinterpret_cast<sockaddr_in6*>(ai->ai_addr)->sin6_addr, sizeof(in6_addr),
reinterpret_cast<sockaddr_in6*>(ai->ai_addr)->sin6_scope_id))
{
return true;
}
}
else if (ai->ai_family == AF_INET)
{
if (addr_v6 == toIPv6(Poco::Net::IPAddress(
&reinterpret_cast<sockaddr_in*>(ai->ai_addr)->sin_addr, sizeof(in_addr))))
{
return true;
}
}
}
}
return false;
}
public:
explicit HostExactPattern(const String & host_) : host(host_) {}
bool contains(const Poco::Net::IPAddress & addr) const override
{
static SimpleCache<decltype(containsImpl), &containsImpl> cache;
return cache(host, addr);
}
};
/// Check that PTR record for address match the regexp (and in addition, check that PTR record is resolved back to client address).
class HostRegexpPattern : public IAddressPattern
{
private:
Poco::RegularExpression host_regexp;
static String getDomain(const Poco::Net::IPAddress & addr)
{
Poco::Net::SocketAddress sock_addr(addr, 0);
/// Resolve by hand, because Poco library doesn't have such functionality.
char domain[1024];
int gai_errno = getnameinfo(sock_addr.addr(), sock_addr.length(), domain, sizeof(domain), nullptr, 0, NI_NAMEREQD);
if (0 != gai_errno)
throw Exception("Cannot getnameinfo: " + std::string(gai_strerror(gai_errno)), ErrorCodes::DNS_ERROR);
return domain;
}
public:
explicit HostRegexpPattern(const String & host_regexp_) : host_regexp(host_regexp_) {}
bool contains(const Poco::Net::IPAddress & addr) const override
{
static SimpleCache<decltype(getDomain), &getDomain> cache;
String domain = cache(addr);
Poco::RegularExpression::Match match;
if (host_regexp.match(domain, match) && HostExactPattern(domain).contains(addr))
return true;
return false;
}
};
bool AddressPatterns::contains(const Poco::Net::IPAddress & addr) const
{
for (size_t i = 0, size = patterns.size(); i < size; ++i)
{
/// If host cannot be resolved, skip it and try next.
try
{
if (patterns[i]->contains(addr))
return true;
}
catch (const DB::Exception & e)
{
LOG_WARNING(&Logger::get("AddressPatterns"),
"Failed to check if pattern contains address " << addr.toString() << ". " << e.displayText() << ", code = " << e.code());
if (e.code() == ErrorCodes::DNS_ERROR)
{
continue;
}
else
throw;
}
}
return false;
}
void AddressPatterns::addFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config)
{
Poco::Util::AbstractConfiguration::Keys config_keys;
config.keys(config_elem, config_keys);
for (Poco::Util::AbstractConfiguration::Keys::const_iterator it = config_keys.begin(); it != config_keys.end(); ++it)
{
Container::value_type pattern;
String value = config.getString(config_elem + "." + *it);
if (startsWith(*it, "ip"))
pattern = std::make_unique<IPAddressPattern>(value);
else if (startsWith(*it, "host_regexp"))
pattern = std::make_unique<HostRegexpPattern>(value);
else if (startsWith(*it, "host"))
pattern = std::make_unique<HostExactPattern>(value);
else
throw Exception("Unknown address pattern type: " + *it, ErrorCodes::UNKNOWN_ADDRESS_PATTERN_TYPE);
patterns.emplace_back(std::move(pattern));
}
}
User::User(const String & name_, const String & config_elem, const Poco::Util::AbstractConfiguration & config)
: name(name_)
{
@ -288,28 +33,43 @@ User::User(const String & name_, const String & config_elem, const Poco::Util::A
throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' must be specified for user " + name + ".", ErrorCodes::BAD_ARGUMENTS);
if (has_password)
password = config.getString(config_elem + ".password");
if (has_password_sha256_hex)
{
password_sha256_hex = Poco::toLower(config.getString(config_elem + ".password_sha256_hex"));
if (password_sha256_hex.size() != 64)
throw Exception("password_sha256_hex for user " + name + " has length " + toString(password_sha256_hex.size()) + " but must be exactly 64 symbols.", ErrorCodes::BAD_ARGUMENTS);
authentication = Authentication{Authentication::PLAINTEXT_PASSWORD};
authentication.setPassword(config.getString(config_elem + ".password"));
}
if (has_password_double_sha1_hex)
else if (has_password_sha256_hex)
{
password_double_sha1_hex = Poco::toLower(config.getString(config_elem + ".password_double_sha1_hex"));
if (password_double_sha1_hex.size() != 40)
throw Exception("password_double_sha1_hex for user " + name + " has length " + toString(password_double_sha1_hex.size()) + " but must be exactly 40 symbols.", ErrorCodes::BAD_ARGUMENTS);
authentication = Authentication{Authentication::SHA256_PASSWORD};
authentication.setPasswordHashHex(config.getString(config_elem + ".password_sha256_hex"));
}
else if (has_password_double_sha1_hex)
{
authentication = Authentication{Authentication::DOUBLE_SHA1_PASSWORD};
authentication.setPasswordHashHex(config.getString(config_elem + ".password_double_sha1_hex"));
}
profile = config.getString(config_elem + ".profile");
quota = config.getString(config_elem + ".quota");
addresses.addFromConfig(config_elem + ".networks", config);
/// Fill list of allowed hosts.
const auto config_networks = config_elem + ".networks";
if (config.has(config_networks))
{
Poco::Util::AbstractConfiguration::Keys config_keys;
config.keys(config_networks, config_keys);
for (Poco::Util::AbstractConfiguration::Keys::const_iterator it = config_keys.begin(); it != config_keys.end(); ++it)
{
String value = config.getString(config_networks + "." + *it);
if (startsWith(*it, "ip"))
allowed_client_hosts.addSubnet(value);
else if (startsWith(*it, "host_regexp"))
allowed_client_hosts.addHostRegexp(value);
else if (startsWith(*it, "host"))
allowed_client_hosts.addHostName(value);
else
throw Exception("Unknown address pattern type: " + *it, ErrorCodes::UNKNOWN_ADDRESS_PATTERN_TYPE);
}
}
/// Fill list of allowed databases.
const auto config_sub_elem = config_elem + ".allow_databases";

39
dbms/src/Interpreters/Users.h
View File

@ -1,20 +1,16 @@
#pragma once
#include <Core/Types.h>
#include <Access/Authentication.h>
#include <Access/AllowedClientHosts.h>
#include <memory>
#include <unordered_map>
#include <unordered_set>
#include <vector>
namespace Poco
{
namespace Net
{
class IPAddress;
}
namespace Util
{
class AbstractConfiguration;
@ -24,44 +20,19 @@ namespace Poco
namespace DB
{
/// Allow to check that address matches a pattern.
class IAddressPattern
{
public:
virtual bool contains(const Poco::Net::IPAddress & addr) const = 0;
virtual ~IAddressPattern() {}
};
class AddressPatterns
{
private:
using Container = std::vector<std::shared_ptr<IAddressPattern>>;
Container patterns;
public:
bool contains(const Poco::Net::IPAddress & addr) const;
void addFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config);
};
/** User and ACL.
*/
struct User
{
String name;
/// Required password. Could be stored in plaintext or in SHA256.
String password;
String password_sha256_hex;
String password_double_sha1_hex;
/// Required password.
Authentication authentication;
String profile;
String quota;
AddressPatterns addresses;
AllowedClientHosts allowed_client_hosts;
/// List of allowed databases.
using DatabaseSet = std::unordered_set<std::string>;

76
dbms/src/Interpreters/UsersManager.cpp
View File

@ -1,18 +1,7 @@
#include <Interpreters/UsersManager.h>
#include "config_core.h"
#include <Common/Exception.h>
#include <common/logger_useful.h>
#include <IO/HexWriteBuffer.h>
#include <IO/WriteBufferFromString.h>
#include <IO/WriteHelpers.h>
#include <Poco/Net/IPAddress.h>
#include <Poco/SHA1Engine.h>
#include <Poco/String.h>
#include <Poco/Util/AbstractConfiguration.h>
#if USE_SSL
# include <openssl/sha.h>
#endif
namespace DB
@ -20,14 +9,7 @@ namespace DB
namespace ErrorCodes
{
extern const int DNS_ERROR;
extern const int UNKNOWN_ADDRESS_PATTERN_TYPE;
extern const int UNKNOWN_USER;
extern const int REQUIRED_PASSWORD;
extern const int WRONG_PASSWORD;
extern const int IP_ADDRESS_NOT_ALLOWED;
extern const int BAD_ARGUMENTS;
extern const int SUPPORT_IS_DISABLED;
}
using UserPtr = UsersManager::UserPtr;
@ -58,62 +40,8 @@ UserPtr UsersManager::authorizeAndGetUser(
if (users.end() == it)
throw Exception("Unknown user " + user_name, ErrorCodes::UNKNOWN_USER);
if (!it->second->addresses.contains(address))
throw Exception("User " + user_name + " is not allowed to connect from address " + address.toString(), ErrorCodes::IP_ADDRESS_NOT_ALLOWED);
auto on_wrong_password = [&]()
{
if (password.empty())
throw Exception("Password required for user " + user_name, ErrorCodes::REQUIRED_PASSWORD);
else
throw Exception("Wrong password for user " + user_name, ErrorCodes::WRONG_PASSWORD);
};
if (!it->second->password_sha256_hex.empty())
{
#if USE_SSL
unsigned char hash[32];
SHA256_CTX ctx;
SHA256_Init(&ctx);
SHA256_Update(&ctx, reinterpret_cast<const unsigned char *>(password.data()), password.size());
SHA256_Final(hash, &ctx);
String hash_hex;
{
WriteBufferFromString buf(hash_hex);
HexWriteBuffer hex_buf(buf);
hex_buf.write(reinterpret_cast<const char *>(hash), sizeof(hash));
}
Poco::toLowerInPlace(hash_hex);
if (hash_hex != it->second->password_sha256_hex)
on_wrong_password();
#else
throw DB::Exception("SHA256 passwords support is disabled, because ClickHouse was built without SSL library", DB::ErrorCodes::SUPPORT_IS_DISABLED);
#endif
}
else if (!it->second->password_double_sha1_hex.empty())
{
Poco::SHA1Engine engine;
engine.update(password);
const auto & first_sha1 = engine.digest();
/// If it was MySQL compatibility server, then first_sha1 already contains double SHA1.
if (Poco::SHA1Engine::digestToHex(first_sha1) == it->second->password_double_sha1_hex)
return it->second;
engine.update(first_sha1.data(), first_sha1.size());
if (Poco::SHA1Engine::digestToHex(engine.digest()) != it->second->password_double_sha1_hex)
on_wrong_password();
}
else if (password != it->second->password)
{
on_wrong_password();
}
it->second->allowed_client_hosts.checkContains(address, user_name);
it->second->authentication.checkPassword(password, user_name);
return it->second;
}

1
dbms/src/Interpreters/executeQuery.cpp
View File

@ -1,4 +1,5 @@
#include <Common/formatReadable.h>
#include <Common/PODArray.h>
#include <Common/typeid_cast.h>
#include <IO/ConcatReadBuffer.h>

6
dbms/src/Parsers/ASTAlterQuery.cpp
View File

@ -1,6 +1,6 @@
#include <Parsers/ASTAlterQuery.h>
#include <iomanip>
#include <IO/WriteHelpers.h>
#include <Common/quoteString.h>
namespace DB
@ -183,9 +183,7 @@ void ASTAlterCommand::formatImpl(
settings.ostr << "VOLUME ";
break;
}
WriteBufferFromOwnString move_destination_name_buf;
writeQuoted(move_destination_name, move_destination_name_buf);
settings.ostr << move_destination_name_buf.str();
settings.ostr << quoteString(move_destination_name);
}
else if (type == ASTAlterCommand::REPLACE_PARTITION)
{

1
dbms/src/Parsers/ASTCheckQuery.h
View File

@ -2,6 +2,7 @@
#include <Parsers/ASTQueryWithTableAndOutput.h>
#include <Parsers/ASTPartition.h>
#include <Common/quoteString.h>
namespace DB

1
dbms/src/Parsers/ASTColumnDeclaration.cpp
View File

@ -1,4 +1,5 @@
#include <Parsers/ASTColumnDeclaration.h>
#include <Common/quoteString.h>
namespace DB

11
dbms/src/Parsers/ASTColumnsMatcher.cpp
View File

@ -1,9 +1,6 @@
#include "ASTColumnsMatcher.h"
#include <IO/WriteBuffer.h>
#include <IO/WriteHelpers.h>
#include <IO/WriteBufferFromString.h>
#include <Common/quoteString.h>
#include <re2/re2.h>
@ -22,10 +19,8 @@ void ASTColumnsMatcher::appendColumnName(WriteBuffer & ostr) const { writeString
void ASTColumnsMatcher::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const
{
WriteBufferFromOwnString pattern_quoted;
writeQuotedString(original_pattern, pattern_quoted);
settings.ostr << (settings.hilite ? hilite_keyword : "") << "COLUMNS" << (settings.hilite ? hilite_none : "") << "(" << pattern_quoted.str() << ")";
settings.ostr << (settings.hilite ? hilite_keyword : "") << "COLUMNS" << (settings.hilite ? hilite_none : "") << "("
<< quoteString(original_pattern) << ")";
}
void ASTColumnsMatcher::setPattern(String pattern)

2
dbms/src/Parsers/ASTConstraintDeclaration.cpp
View File

@ -1,4 +1,6 @@
#include <Parsers/ASTConstraintDeclaration.h>
#include <Common/quoteString.h>
namespace DB
{

1
dbms/src/Parsers/ASTCreateQuery.cpp
View File

@ -3,6 +3,7 @@
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ASTSetQuery.h>
#include <Common/quoteString.h>
namespace DB

2
dbms/src/Parsers/ASTDictionaryAttributeDeclaration.cpp
View File

@ -1,4 +1,6 @@
#include <Parsers/ASTDictionaryAttributeDeclaration.h>
#include <Common/quoteString.h>
namespace DB
{

1
dbms/src/Parsers/ASTDropQuery.cpp
View File

@ -1,4 +1,5 @@
#include <Parsers/ASTDropQuery.h>
#include <Common/quoteString.h>
namespace DB

5
dbms/src/Parsers/ASTIndexDeclaration.h
View File

@ -2,7 +2,8 @@
#include <Core/Field.h>
#include <Core/Types.h>
#include <IO/WriteHelpers.h>
#include <Common/FieldVisitors.h>
#include <Common/quoteString.h>
#include <Parsers/ASTExpressionList.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/IAST.h>
@ -52,7 +53,7 @@ public:
s.ostr << (s.hilite ? hilite_keyword : "") << " TYPE " << (s.hilite ? hilite_none : "");
type->formatImpl(s, state, frame);
s.ostr << (s.hilite ? hilite_keyword : "") << " GRANULARITY " << (s.hilite ? hilite_none : "");
s.ostr << toString(granularity);
s.ostr << granularity;
}
};

1
dbms/src/Parsers/ASTInsertQuery.cpp
View File

@ -1,6 +1,7 @@
#include <iomanip>
#include <Parsers/ASTInsertQuery.h>
#include <Parsers/ASTFunction.h>
#include <Common/quoteString.h>
namespace DB

1
dbms/src/Parsers/ASTNameTypePair.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <Parsers/IAST.h>
#include <Common/quoteString.h>
namespace DB

1
dbms/src/Parsers/ASTOptimizeQuery.cpp
View File

@ -1,4 +1,5 @@
#include <Parsers/ASTOptimizeQuery.h>
#include <Common/quoteString.h>
namespace DB
{

1
dbms/src/Parsers/ASTQueryParameter.cpp
View File

@ -1,5 +1,6 @@
#include <Parsers/ASTQueryParameter.h>
#include <IO/WriteHelpers.h>
#include <Common/quoteString.h>
namespace DB

1
dbms/src/Parsers/ASTQueryWithOnCluster.cpp
View File

@ -4,6 +4,7 @@
#include <Parsers/ExpressionElementParsers.h>
#include <Parsers/parseIdentifierOrStringLiteral.h>
#include <Common/typeid_cast.h>
#include <Common/quoteString.h>
#include <Interpreters/evaluateConstantExpression.h>

Some files were not shown because too many files have changed in this diff Show More