diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt new file mode 100644 index 00000000000..72959d3c1d8 --- /dev/null +++ b/cmake/autogenerated_versions.txt @@ -0,0 +1,9 @@ +# This strings autochanged from release_lib.sh: +SET(VERSION_REVISION 54435) +SET(VERSION_MAJOR 20) +SET(VERSION_MINOR 5) +SET(VERSION_PATCH 1) +SET(VERSION_GITHASH 91df18a906dcffdbee6816e5389df6c65f86e35f) +SET(VERSION_DESCRIBE v20.5.1.1-prestable) +SET(VERSION_STRING 20.5.1.1) +# end of autochange diff --git a/cmake/protobuf_generate_cpp.cmake b/cmake/protobuf_generate_cpp.cmake index 40ec3c8e681..7ee9e8d7c81 100644 --- a/cmake/protobuf_generate_cpp.cmake +++ b/cmake/protobuf_generate_cpp.cmake @@ -112,16 +112,21 @@ if (PROTOBUF_GENERATE_CPP_SCRIPT_MODE) set (intermediate_dir ${DIR}/intermediate) set (intermediate_output "${intermediate_dir}/${FILENAME}") - if (COMPILER_ID STREQUAL "GNU") + if (COMPILER_ID STREQUAL "Clang") + set (pragma_push "#pragma clang diagnostic push\n") + set (pragma_pop "#pragma clang diagnostic pop\n") + set (pragma_disable_warnings "#pragma clang diagnostic ignored \"-Weverything\"\n") + elseif (COMPILER_ID MATCHES "GNU") set (pragma_push "#pragma GCC diagnostic push\n") set (pragma_pop "#pragma GCC diagnostic pop\n") set (pragma_disable_warnings "#pragma GCC diagnostic ignored \"-Wall\"\n" "#pragma GCC diagnostic ignored \"-Wextra\"\n" - "#pragma GCC diagnostic ignored \"-Warray-bounds\"\n") - elseif (COMPILER_ID MATCHES "Clang") - set (pragma_push "#pragma clang diagnostic push\n") - set (pragma_pop "#pragma clang diagnostic pop\n") - set (pragma_disable_warnings "#pragma clang diagnostic ignored \"-Weverything\"\n") + "#pragma GCC diagnostic ignored \"-Warray-bounds\"\n" + "#pragma GCC diagnostic ignored \"-Wold-style-cast\"\n" + "#pragma GCC diagnostic ignored \"-Wshadow\"\n" + "#pragma GCC diagnostic ignored \"-Wsuggest-override\"\n" + "#pragma GCC diagnostic ignored \"-Wcast-qual\"\n" + "#pragma GCC diagnostic ignored \"-Wunused-parameter\"\n") endif() if (${FILENAME} MATCHES ".*\\.h") diff --git a/cmake/version.cmake b/cmake/version.cmake index 3f51f59cf0f..eea17f68c47 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -1,12 +1,4 @@ -# This strings autochanged from release_lib.sh: -set(VERSION_REVISION 54435) -set(VERSION_MAJOR 20) -set(VERSION_MINOR 5) -set(VERSION_PATCH 1) -set(VERSION_GITHASH 91df18a906dcffdbee6816e5389df6c65f86e35f) -set(VERSION_DESCRIBE v20.5.1.1-prestable) -set(VERSION_STRING 20.5.1.1) -# end of autochange +include(${CMAKE_SOURCE_DIR}/cmake/autogenerated_versions.txt) set(VERSION_EXTRA "" CACHE STRING "") set(VERSION_TWEAK "" CACHE STRING "") diff --git a/cmake/yandex/ya.make.versions.inc b/cmake/yandex/ya.make.versions.inc new file mode 100644 index 00000000000..3ac401cb108 --- /dev/null +++ b/cmake/yandex/ya.make.versions.inc @@ -0,0 +1,25 @@ +INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/autogenerated_versions.txt) + +# TODO: not sure if this is customizable per-binary +SET(VERSION_NAME "ClickHouse") + +# TODO: not quite sure how to replace dash with space in ya.make +SET(VERSION_FULL "${VERSION_NAME}-${VERSION_STRING}") + +CFLAGS (GLOBAL -DDBMS_NAME=\"ClickHouse\") +CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=${VERSION_MAJOR}) +CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=${VERSION_MINOR}) +CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=${VERSION_PATCH}) +CFLAGS (GLOBAL -DVERSION_FULL=\"\\\"${VERSION_FULL}\\\"\") +CFLAGS (GLOBAL -DVERSION_MAJOR=${VERSION_MAJOR}) +CFLAGS (GLOBAL -DVERSION_MINOR=${VERSION_MINOR}) +CFLAGS (GLOBAL -DVERSION_PATCH=${VERSION_PATCH}) + +# TODO: not supported yet, not sure if ya.make supports arithmetics. +CFLAGS (GLOBAL -DVERSION_INTEGER=0) + +CFLAGS (GLOBAL -DVERSION_NAME=\"\\\"${VERSION_NAME}\\\"\") +CFLAGS (GLOBAL -DVERSION_OFFICIAL=\"-arcadia\") +CFLAGS (GLOBAL -DVERSION_REVISION=${VERSION_REVISION}) +CFLAGS (GLOBAL -DVERSION_STRING=\"\\\"${VERSION_STRING}\\\"\") + diff --git a/contrib/jemalloc b/contrib/jemalloc index cd2931ad9bb..ea6b3e973b4 160000 --- a/contrib/jemalloc +++ b/contrib/jemalloc @@ -1 +1 @@ -Subproject commit cd2931ad9bbd78208565716ab102e86d858c2fff +Subproject commit ea6b3e973b477b8061e0076bb257dbd7f3faa756 diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 9081de593b8..63c5a5e66ea 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -17,7 +17,13 @@ if (ENABLE_JEMALLOC) # # By enabling percpu_arena number of arenas limited to number of CPUs and hence # this problem should go away. - set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu" CACHE STRING "Change default configuration string of JEMalloc" ) + set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0") + # CACHE variable is empty, to allow changing defaults without necessity + # to purge cache + set (JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE "" CACHE STRING "Change default configuration string of JEMalloc" ) + if (JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE) + set (JEMALLOC_CONFIG_MALLOC_CONF "${JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE}") + endif() message (STATUS "jemalloc malloc_conf: ${JEMALLOC_CONFIG_MALLOC_CONF}") set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/jemalloc") @@ -55,6 +61,7 @@ if (ENABLE_JEMALLOC) ${LIBRARY_DIR}/src/ticker.c ${LIBRARY_DIR}/src/tsd.c ${LIBRARY_DIR}/src/witness.c + ${LIBRARY_DIR}/src/safety_check.c ) if (OS_DARWIN) list(APPEND SRCS ${LIBRARY_DIR}/src/zone.c) @@ -89,6 +96,8 @@ if (ENABLE_JEMALLOC) endif () target_compile_options(jemalloc PRIVATE -Wno-redundant-decls) + # for RTLD_NEXT + target_compile_options(jemalloc PRIVATE -D_GNU_SOURCE) else () find_library(LIBRARY_JEMALLOC jemalloc) find_path(INCLUDE_JEMALLOC jemalloc/jemalloc.h) diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_defs.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h similarity index 84% rename from contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_defs.h rename to contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h index d1389237a77..6a03a231a0e 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_defs.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h @@ -5,6 +5,12 @@ /* Defined if alloc_size attribute is supported. */ #define JEMALLOC_HAVE_ATTR_ALLOC_SIZE +/* Defined if format_arg(...) attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_FORMAT_ARG + +/* Defined if format(gnu_printf, ...) attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF + /* Defined if format(printf, ...) attribute is supported. */ #define JEMALLOC_HAVE_ATTR_FORMAT_PRINTF diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h similarity index 87% rename from contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h rename to contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h index 7432f1cda53..34235894285 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_macros.h @@ -4,12 +4,13 @@ #include #include -#define JEMALLOC_VERSION "5.1.0-56-g41b7372eadee941b9164751b8d4963f915d3ceae" +#define JEMALLOC_VERSION "5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756" #define JEMALLOC_VERSION_MAJOR 5 -#define JEMALLOC_VERSION_MINOR 1 -#define JEMALLOC_VERSION_BUGFIX 0 -#define JEMALLOC_VERSION_NREV 56 -#define JEMALLOC_VERSION_GID "41b7372eadee941b9164751b8d4963f915d3ceae" +#define JEMALLOC_VERSION_MINOR 2 +#define JEMALLOC_VERSION_BUGFIX 1 +#define JEMALLOC_VERSION_NREV 0 +#define JEMALLOC_VERSION_GID "ea6b3e973b477b8061e0076bb257dbd7f3faa756" +#define JEMALLOC_VERSION_GID_IDENT ea6b3e973b477b8061e0076bb257dbd7f3faa756 #define MALLOCX_LG_ALIGN(la) ((int)(la)) #if LG_SIZEOF_PTR == 2 @@ -68,6 +69,7 @@ # define JEMALLOC_EXPORT __declspec(dllimport) # endif # endif +# define JEMALLOC_FORMAT_ARG(i) # define JEMALLOC_FORMAT_PRINTF(s, i) # define JEMALLOC_NOINLINE __declspec(noinline) # ifdef __cplusplus @@ -95,6 +97,11 @@ # ifndef JEMALLOC_EXPORT # define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) # endif +# ifdef JEMALLOC_HAVE_ATTR_FORMAT_ARG +# define JEMALLOC_FORMAT_ARG(i) JEMALLOC_ATTR(__format_arg__(3)) +# else +# define JEMALLOC_FORMAT_ARG(i) +# endif # ifdef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF # define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(gnu_printf, s, i)) # elif defined(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF) diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_protos.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos.h similarity index 100% rename from contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_protos.h rename to contrib/jemalloc-cmake/include/jemalloc/jemalloc_protos.h diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h index a2ea2dd3533..2e94f7a0cc3 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h @@ -17,6 +17,7 @@ # define je_malloc_stats_print malloc_stats_print # define je_malloc_usable_size malloc_usable_size # define je_mallocx mallocx +# define je_smallocx_ea6b3e973b477b8061e0076bb257dbd7f3faa756 smallocx_ea6b3e973b477b8061e0076bb257dbd7f3faa756 # define je_nallocx nallocx # define je_posix_memalign posix_memalign # define je_rallocx rallocx diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_typedefs.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h similarity index 88% rename from contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_typedefs.h rename to contrib/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h index 1a58874306e..5f94f16f937 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_typedefs.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_typedefs.h @@ -65,13 +65,13 @@ typedef bool (extent_merge_t)(extent_hooks_t *, void *, size_t, void *, size_t, bool, unsigned); struct extent_hooks_s { - extent_alloc_t *alloc; - extent_dalloc_t *dalloc; - extent_destroy_t *destroy; - extent_commit_t *commit; - extent_decommit_t *decommit; - extent_purge_t *purge_lazy; - extent_purge_t *purge_forced; - extent_split_t *split; - extent_merge_t *merge; + extent_alloc_t *alloc; + extent_dalloc_t *dalloc; + extent_destroy_t *destroy; + extent_commit_t *commit; + extent_decommit_t *decommit; + extent_purge_t *purge_lazy; + extent_purge_t *purge_forced; + extent_split_t *split; + extent_merge_t *merge; }; diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 089f1a773aa..cbd2740e1f1 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -1,12 +1,6 @@ /* include/jemalloc/internal/jemalloc_internal_defs.h. Generated from jemalloc_internal_defs.h.in by configure. */ #ifndef JEMALLOC_INTERNAL_DEFS_H_ #define JEMALLOC_INTERNAL_DEFS_H_ - - -#ifndef _GNU_SOURCE - #define _GNU_SOURCE -#endif - /* * If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all * public APIs to be prefixed. This makes it possible, with some care, to use @@ -25,7 +19,7 @@ #define JEMALLOC_OVERRIDE___LIBC_MEMALIGN #define JEMALLOC_OVERRIDE___LIBC_REALLOC #define JEMALLOC_OVERRIDE___LIBC_VALLOC -#define JEMALLOC_OVERRIDE___POSIX_MEMALIGN +/* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */ /* * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs. @@ -41,7 +35,7 @@ */ #define CPU_SPINWAIT /* 1 if CPU_SPINWAIT is defined, 0 otherwise. */ -#define HAVE_CPU_SPINWAIT 0 +#define HAVE_CPU_SPINWAIT 9 /* * Number of significant bits in virtual addresses. This may be less than the @@ -55,25 +49,13 @@ /* Defined if GCC __atomic atomics are available. */ #define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +/* and the 8-bit variant support. */ +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 /* Defined if GCC __sync atomics are available. */ #define JEMALLOC_GCC_SYNC_ATOMICS 1 - -/* - * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and - * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite - * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the - * functions are defined in libgcc instead of being inlines). - */ -/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 */ - -/* - * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and - * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite - * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the - * functions are defined in libgcc instead of being inlines). - */ -/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 */ +/* and the 8-bit variant support. */ +#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -85,19 +67,13 @@ */ /* #undef JEMALLOC_OS_UNFAIR_LOCK */ -/* - * Defined if OSSpin*() functions are available, as provided by Darwin, and - * documented in the spinlock(3) manual page. - */ -/* #undef JEMALLOC_OSSPIN */ - /* Defined if syscall(2) is usable. */ #define JEMALLOC_USE_SYSCALL /* * Defined if secure_getenv(3) is available. */ -#define JEMALLOC_HAVE_SECURE_GETENV +// #define JEMALLOC_HAVE_SECURE_GETENV /* * Defined if issetugid(2) is available. @@ -243,6 +219,12 @@ #define JEMALLOC_INTERNAL_FFSL __builtin_ffsl #define JEMALLOC_INTERNAL_FFS __builtin_ffs +/* + * popcount*() functions to use for bitmapping. + */ +#define JEMALLOC_INTERNAL_POPCOUNTL __builtin_popcountl +#define JEMALLOC_INTERNAL_POPCOUNT __builtin_popcount + /* * If defined, explicitly attempt to more uniformly distribute large allocation * pointer alignments across all cache indices. @@ -297,7 +279,7 @@ * MADV_FREE, though typically with higher * system overhead. */ -// #define JEMALLOC_PURGE_MADVISE_FREE +#define JEMALLOC_PURGE_MADVISE_FREE #define JEMALLOC_PURGE_MADVISE_DONTNEED #define JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS @@ -379,4 +361,7 @@ */ #define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE +/* Performs additional safety checks when defined. */ +/* #undef JEMALLOC_OPT_SAFETY_CHECKS */ + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_preamble.h b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_preamble.h index c150785fb4a..e5e34925b55 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_preamble.h +++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_preamble.h @@ -21,7 +21,7 @@ # include "jemalloc/jemalloc.h" #endif -#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) +#if defined(JEMALLOC_OSATOMIC) #include #endif @@ -161,7 +161,26 @@ static const bool config_log = false #endif ; -#ifdef JEMALLOC_HAVE_SCHED_GETCPU +/* + * Are extra safety checks enabled; things like checking the size of sized + * deallocations, double-frees, etc. + */ +static const bool config_opt_safety_checks = +#ifdef JEMALLOC_OPT_SAFETY_CHECKS + true +#elif defined(JEMALLOC_DEBUG) + /* + * This lets us only guard safety checks by one flag instead of two; fast + * checks can guard solely by config_opt_safety_checks and run in debug mode + * too. + */ + true +#else + false +#endif + ; + +#if defined(_WIN32) || defined(JEMALLOC_HAVE_SCHED_GETCPU) /* Currently percpu_arena depends on sched_getcpu. */ #define JEMALLOC_PERCPU_ARENA #endif diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_macros.h b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_macros.h deleted file mode 100644 index 79b13337fbb..00000000000 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_macros.h +++ /dev/null @@ -1,123 +0,0 @@ -#include -#include -#include -#include -#include - -#define JEMALLOC_VERSION "5.1.0-97-gcd2931ad9bbd78208565716ab102e86d858c2fff" -#define JEMALLOC_VERSION_MAJOR 5 -#define JEMALLOC_VERSION_MINOR 1 -#define JEMALLOC_VERSION_BUGFIX 0 -#define JEMALLOC_VERSION_NREV 97 -#define JEMALLOC_VERSION_GID "cd2931ad9bbd78208565716ab102e86d858c2fff" -#define JEMALLOC_VERSION_GID_IDENT cd2931ad9bbd78208565716ab102e86d858c2fff - -#define MALLOCX_LG_ALIGN(la) ((int)(la)) -#if LG_SIZEOF_PTR == 2 -# define MALLOCX_ALIGN(a) ((int)(ffs((int)(a))-1)) -#else -# define MALLOCX_ALIGN(a) \ - ((int)(((size_t)(a) < (size_t)INT_MAX) ? ffs((int)(a))-1 : \ - ffs((int)(((size_t)(a))>>32))+31)) -#endif -#define MALLOCX_ZERO ((int)0x40) -/* - * Bias tcache index bits so that 0 encodes "automatic tcache management", and 1 - * encodes MALLOCX_TCACHE_NONE. - */ -#define MALLOCX_TCACHE(tc) ((int)(((tc)+2) << 8)) -#define MALLOCX_TCACHE_NONE MALLOCX_TCACHE(-1) -/* - * Bias arena index bits so that 0 encodes "use an automatically chosen arena". - */ -#define MALLOCX_ARENA(a) ((((int)(a))+1) << 20) - -/* - * Use as arena index in "arena..{purge,decay,dss}" and - * "stats.arenas..*" mallctl interfaces to select all arenas. This - * definition is intentionally specified in raw decimal format to support - * cpp-based string concatenation, e.g. - * - * #define STRINGIFY_HELPER(x) #x - * #define STRINGIFY(x) STRINGIFY_HELPER(x) - * - * mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", NULL, NULL, NULL, - * 0); - */ -#define MALLCTL_ARENAS_ALL 4096 -/* - * Use as arena index in "stats.arenas..*" mallctl interfaces to select - * destroyed arenas. - */ -#define MALLCTL_ARENAS_DESTROYED 4097 - -#if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW) -# define JEMALLOC_CXX_THROW throw() -#else -# define JEMALLOC_CXX_THROW -#endif - -#if defined(_MSC_VER) -# define JEMALLOC_ATTR(s) -# define JEMALLOC_ALIGNED(s) __declspec(align(s)) -# define JEMALLOC_ALLOC_SIZE(s) -# define JEMALLOC_ALLOC_SIZE2(s1, s2) -# ifndef JEMALLOC_EXPORT -# ifdef DLLEXPORT -# define JEMALLOC_EXPORT __declspec(dllexport) -# else -# define JEMALLOC_EXPORT __declspec(dllimport) -# endif -# endif -# define JEMALLOC_FORMAT_PRINTF(s, i) -# define JEMALLOC_NOINLINE __declspec(noinline) -# ifdef __cplusplus -# define JEMALLOC_NOTHROW __declspec(nothrow) -# else -# define JEMALLOC_NOTHROW -# endif -# define JEMALLOC_SECTION(s) __declspec(allocate(s)) -# define JEMALLOC_RESTRICT_RETURN __declspec(restrict) -# if _MSC_VER >= 1900 && !defined(__EDG__) -# define JEMALLOC_ALLOCATOR __declspec(allocator) -# else -# define JEMALLOC_ALLOCATOR -# endif -#elif defined(JEMALLOC_HAVE_ATTR) -# define JEMALLOC_ATTR(s) __attribute__((s)) -# define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s)) -# ifdef JEMALLOC_HAVE_ATTR_ALLOC_SIZE -# define JEMALLOC_ALLOC_SIZE(s) JEMALLOC_ATTR(alloc_size(s)) -# define JEMALLOC_ALLOC_SIZE2(s1, s2) JEMALLOC_ATTR(alloc_size(s1, s2)) -# else -# define JEMALLOC_ALLOC_SIZE(s) -# define JEMALLOC_ALLOC_SIZE2(s1, s2) -# endif -# ifndef JEMALLOC_EXPORT -# define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) -# endif -# ifdef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF -# define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(gnu_printf, s, i)) -# elif defined(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF) -# define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(printf, s, i)) -# else -# define JEMALLOC_FORMAT_PRINTF(s, i) -# endif -# define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline) -# define JEMALLOC_NOTHROW JEMALLOC_ATTR(nothrow) -# define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s)) -# define JEMALLOC_RESTRICT_RETURN -# define JEMALLOC_ALLOCATOR -#else -# define JEMALLOC_ATTR(s) -# define JEMALLOC_ALIGNED(s) -# define JEMALLOC_ALLOC_SIZE(s) -# define JEMALLOC_ALLOC_SIZE2(s1, s2) -# define JEMALLOC_EXPORT -# define JEMALLOC_FORMAT_PRINTF(s, i) -# define JEMALLOC_NOINLINE -# define JEMALLOC_NOTHROW -# define JEMALLOC_SECTION(s) -# define JEMALLOC_RESTRICT_RETURN -# define JEMALLOC_ALLOCATOR -#endif diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_typedefs.h b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_typedefs.h deleted file mode 100644 index 1a58874306e..00000000000 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/jemalloc_typedefs.h +++ /dev/null @@ -1,77 +0,0 @@ -typedef struct extent_hooks_s extent_hooks_t; - -/* - * void * - * extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, - * size_t alignment, bool *zero, bool *commit, unsigned arena_ind); - */ -typedef void *(extent_alloc_t)(extent_hooks_t *, void *, size_t, size_t, bool *, - bool *, unsigned); - -/* - * bool - * extent_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, - * bool committed, unsigned arena_ind); - */ -typedef bool (extent_dalloc_t)(extent_hooks_t *, void *, size_t, bool, - unsigned); - -/* - * void - * extent_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, - * bool committed, unsigned arena_ind); - */ -typedef void (extent_destroy_t)(extent_hooks_t *, void *, size_t, bool, - unsigned); - -/* - * bool - * extent_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, - * size_t offset, size_t length, unsigned arena_ind); - */ -typedef bool (extent_commit_t)(extent_hooks_t *, void *, size_t, size_t, size_t, - unsigned); - -/* - * bool - * extent_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, - * size_t offset, size_t length, unsigned arena_ind); - */ -typedef bool (extent_decommit_t)(extent_hooks_t *, void *, size_t, size_t, - size_t, unsigned); - -/* - * bool - * extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size, - * size_t offset, size_t length, unsigned arena_ind); - */ -typedef bool (extent_purge_t)(extent_hooks_t *, void *, size_t, size_t, size_t, - unsigned); - -/* - * bool - * extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size, - * size_t size_a, size_t size_b, bool committed, unsigned arena_ind); - */ -typedef bool (extent_split_t)(extent_hooks_t *, void *, size_t, size_t, size_t, - bool, unsigned); - -/* - * bool - * extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, - * void *addr_b, size_t size_b, bool committed, unsigned arena_ind); - */ -typedef bool (extent_merge_t)(extent_hooks_t *, void *, size_t, void *, size_t, - bool, unsigned); - -struct extent_hooks_s { - extent_alloc_t *alloc; - extent_dalloc_t *dalloc; - extent_destroy_t *destroy; - extent_commit_t *commit; - extent_decommit_t *decommit; - extent_purge_t *purge_lazy; - extent_purge_t *purge_forced; - extent_split_t *split; - extent_merge_t *merge; -}; diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in index 63f7f765023..d5cf0e719ef 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -1,11 +1,6 @@ /* include/jemalloc/internal/jemalloc_internal_defs.h. Generated from jemalloc_internal_defs.h.in by configure. */ #ifndef JEMALLOC_INTERNAL_DEFS_H_ #define JEMALLOC_INTERNAL_DEFS_H_ - -#ifndef _GNU_SOURCE - #define _GNU_SOURCE -#endif - /* * If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all * public APIs to be prefixed. This makes it possible, with some care, to use @@ -24,7 +19,7 @@ #define JEMALLOC_OVERRIDE___LIBC_MEMALIGN #define JEMALLOC_OVERRIDE___LIBC_REALLOC #define JEMALLOC_OVERRIDE___LIBC_VALLOC -#define JEMALLOC_OVERRIDE___POSIX_MEMALIGN +/* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */ /* * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs. @@ -54,25 +49,13 @@ /* Defined if GCC __atomic atomics are available. */ #define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +/* and the 8-bit variant support. */ +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 /* Defined if GCC __sync atomics are available. */ #define JEMALLOC_GCC_SYNC_ATOMICS 1 - -/* - * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and - * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite - * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the - * functions are defined in libgcc instead of being inlines). - */ -/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 */ - -/* - * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and - * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite - * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the - * functions are defined in libgcc instead of being inlines). - */ -/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 */ +/* and the 8-bit variant support. */ +#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -84,20 +67,13 @@ */ /* #undef JEMALLOC_OS_UNFAIR_LOCK */ -/* - * Defined if OSSpin*() functions are available, as provided by Darwin, and - * documented in the spinlock(3) manual page. - */ -/* #undef JEMALLOC_OSSPIN */ - /* Defined if syscall(2) is usable. */ #define JEMALLOC_USE_SYSCALL /* * Defined if secure_getenv(3) is available. */ -// Don't want dependency on newer GLIBC -//#define JEMALLOC_HAVE_SECURE_GETENV +// #define JEMALLOC_HAVE_SECURE_GETENV /* * Defined if issetugid(2) is available. @@ -160,6 +136,9 @@ /* JEMALLOC_STATS enables statistics calculation. */ #define JEMALLOC_STATS +/* JEMALLOC_EXPERIMENTAL_SMALLOCX_API enables experimental smallocx API. */ +/* #undef JEMALLOC_EXPERIMENTAL_SMALLOCX_API */ + /* JEMALLOC_PROF enables allocation profiling. */ /* #undef JEMALLOC_PROF */ @@ -240,6 +219,12 @@ #define JEMALLOC_INTERNAL_FFSL __builtin_ffsl #define JEMALLOC_INTERNAL_FFS __builtin_ffs +/* + * popcount*() functions to use for bitmapping. + */ +#define JEMALLOC_INTERNAL_POPCOUNTL __builtin_popcountl +#define JEMALLOC_INTERNAL_POPCOUNT __builtin_popcount + /* * If defined, explicitly attempt to more uniformly distribute large allocation * pointer alignments across all cache indices. @@ -252,6 +237,12 @@ */ /* #undef JEMALLOC_LOG */ +/* + * If defined, use readlinkat() (instead of readlink()) to follow + * /etc/malloc_conf. + */ +/* #undef JEMALLOC_READLINKAT */ + /* * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. */ @@ -288,7 +279,7 @@ * MADV_FREE, though typically with higher * system overhead. */ -//#define JEMALLOC_PURGE_MADVISE_FREE +#define JEMALLOC_PURGE_MADVISE_FREE #define JEMALLOC_PURGE_MADVISE_DONTNEED #define JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS @@ -370,4 +361,7 @@ */ #define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE +/* Performs additional safety checks when defined. */ +/* #undef JEMALLOC_OPT_SAFETY_CHECKS */ + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h index c150785fb4a..e5e34925b55 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h @@ -21,7 +21,7 @@ # include "jemalloc/jemalloc.h" #endif -#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) +#if defined(JEMALLOC_OSATOMIC) #include #endif @@ -161,7 +161,26 @@ static const bool config_log = false #endif ; -#ifdef JEMALLOC_HAVE_SCHED_GETCPU +/* + * Are extra safety checks enabled; things like checking the size of sized + * deallocations, double-frees, etc. + */ +static const bool config_opt_safety_checks = +#ifdef JEMALLOC_OPT_SAFETY_CHECKS + true +#elif defined(JEMALLOC_DEBUG) + /* + * This lets us only guard safety checks by one flag instead of two; fast + * checks can guard solely by config_opt_safety_checks and run in debug mode + * too. + */ + true +#else + false +#endif + ; + +#if defined(_WIN32) || defined(JEMALLOC_HAVE_SCHED_GETCPU) /* Currently percpu_arena depends on sched_getcpu. */ #define JEMALLOC_PERCPU_ARENA #endif diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_defs.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_defs.h deleted file mode 100644 index d1389237a77..00000000000 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_defs.h +++ /dev/null @@ -1,43 +0,0 @@ -/* include/jemalloc/jemalloc_defs.h. Generated from jemalloc_defs.h.in by configure. */ -/* Defined if __attribute__((...)) syntax is supported. */ -#define JEMALLOC_HAVE_ATTR - -/* Defined if alloc_size attribute is supported. */ -#define JEMALLOC_HAVE_ATTR_ALLOC_SIZE - -/* Defined if format(printf, ...) attribute is supported. */ -#define JEMALLOC_HAVE_ATTR_FORMAT_PRINTF - -/* - * Define overrides for non-standard allocator-related functions if they are - * present on the system. - */ -#define JEMALLOC_OVERRIDE_MEMALIGN -#define JEMALLOC_OVERRIDE_VALLOC - -/* - * At least Linux omits the "const" in: - * - * size_t malloc_usable_size(const void *ptr); - * - * Match the operating system's prototype. - */ -#define JEMALLOC_USABLE_SIZE_CONST - -/* - * If defined, specify throw() for the public function prototypes when compiling - * with C++. The only justification for this is to match the prototypes that - * glibc defines. - */ -#define JEMALLOC_USE_CXX_THROW - -#ifdef _MSC_VER -# ifdef _WIN64 -# define LG_SIZEOF_PTR_WIN 3 -# else -# define LG_SIZEOF_PTR_WIN 2 -# endif -#endif - -/* sizeof(void *) == 2^LG_SIZEOF_PTR. */ -#define LG_SIZEOF_PTR 3 diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_protos.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_protos.h deleted file mode 100644 index ff025e30fa7..00000000000 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_protos.h +++ /dev/null @@ -1,66 +0,0 @@ -/* - * The je_ prefix on the following public symbol declarations is an artifact - * of namespace management, and should be omitted in application code unless - * JEMALLOC_NO_DEMANGLE is defined (see jemalloc_mangle.h). - */ -extern JEMALLOC_EXPORT const char *je_malloc_conf; -extern JEMALLOC_EXPORT void (*je_malloc_message)(void *cbopaque, - const char *s); - -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_malloc(size_t size) - JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_calloc(size_t num, size_t size) - JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2); -JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_posix_memalign(void **memptr, - size_t alignment, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(nonnull(1)); -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_aligned_alloc(size_t alignment, - size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) - JEMALLOC_ALLOC_SIZE(2); -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_realloc(void *ptr, size_t size) - JEMALLOC_CXX_THROW JEMALLOC_ALLOC_SIZE(2); -JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_free(void *ptr) - JEMALLOC_CXX_THROW; - -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_mallocx(size_t size, int flags) - JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_rallocx(void *ptr, size_t size, - int flags) JEMALLOC_ALLOC_SIZE(2); -JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_xallocx(void *ptr, size_t size, - size_t extra, int flags); -JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_sallocx(const void *ptr, - int flags) JEMALLOC_ATTR(pure); -JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_dallocx(void *ptr, int flags); -JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_sdallocx(void *ptr, size_t size, - int flags); -JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_nallocx(size_t size, int flags) - JEMALLOC_ATTR(pure); - -JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctl(const char *name, - void *oldp, size_t *oldlenp, void *newp, size_t newlen); -JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctlnametomib(const char *name, - size_t *mibp, size_t *miblenp); -JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctlbymib(const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen); -JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_malloc_stats_print( - void (*write_cb)(void *, const char *), void *je_cbopaque, - const char *opts); -JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_malloc_usable_size( - JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW; - -#ifdef JEMALLOC_OVERRIDE_MEMALIGN -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_memalign(size_t alignment, size_t size) - JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); -#endif - -#ifdef JEMALLOC_OVERRIDE_VALLOC -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN - void JEMALLOC_NOTHROW *je_valloc(size_t size) JEMALLOC_CXX_THROW - JEMALLOC_ATTR(malloc); -#endif diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 3ad20a9479c..93f192c3f3c 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -21,7 +21,7 @@ RUN apt-get update \ locales \ ca-certificates \ wget \ - tzata \ + tzdata \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/debconf \ diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index bf4599acb9a..209b36f59af 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -317,7 +317,7 @@ function report rm -r report ||: mkdir report report/tmp ||: -rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.tsv unstable-query-metrics.tsv changed-perf.tsv unstable-tests.tsv unstable-queries.tsv bad-tests.tsv slow-on-client.tsv all-queries.tsv ||: +rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.tsv unstable-query-metrics.tsv changed-perf.tsv unstable-tests.tsv unstable-queries.tsv bad-tests.tsv slow-on-client.tsv all-queries.tsv run-errors.tsv ||: build_log_column_definitions @@ -434,7 +434,7 @@ create table wall_clock engine Memory as select * from file('wall-clock-times.tsv', TSV, 'test text, real float, user float, system float'); create table slow_on_client_tsv engine File(TSV, 'report/slow-on-client.tsv') as - select client, server, floor(client/server, 3) p, query_display_name + select client, server, floor(client/server, 3) p, test, query_display_name from query_time left join query_display_names using (test, query_index) where p > 1.02 order by p desc; diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index b171603700d..866e78da098 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -189,7 +189,7 @@ if args.report == 'main': slow_on_client_rows = tsvRows('report/slow-on-client.tsv') error_tests += len(slow_on_client_rows) printSimpleTable('Slow on client', - ['Client time, s', 'Server time, s', 'Ratio', 'Query'], + ['Client time, s', 'Server time, s', 'Ratio', 'Test', 'Query'], slow_on_client_rows) def print_changes(): diff --git a/docker/test/test_runner.sh b/docker/test/test_runner.sh index 76e142e61f9..561117492b0 100755 --- a/docker/test/test_runner.sh +++ b/docker/test/test_runner.sh @@ -12,7 +12,7 @@ readonly CLICKHOUSE_PACKAGES_ARG="${2}" CLICKHOUSE_SERVER_IMAGE="${3}" if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then - readonly CLICKHOUSE_PACKAGES_DIR="$(realpath ${2})" # or --no-rebuild + readonly CLICKHOUSE_PACKAGES_DIR="$(realpath ${2})" # or --no-rebuild fi @@ -26,19 +26,19 @@ fi # TODO: optionally mount most recent clickhouse-test and queries directory from local machine if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then - docker build \ - -f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \ - --target clickhouse-test-runner-base \ - -t clickhouse-test-runner-base:preinstall \ - "${CLICKHOUSE_DOCKER_DIR}/test/stateless" + docker build \ + -f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \ + --target clickhouse-test-runner-base \ + -t clickhouse-test-runner-base:preinstall \ + "${CLICKHOUSE_DOCKER_DIR}/test/stateless" - docker rm -f clickhouse-test-runner-installing-packages || true - docker run \ - -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \ - --name clickhouse-test-runner-installing-packages \ - clickhouse-test-runner-base:preinstall - docker commit clickhouse-test-runner-installing-packages clickhouse-statelest-test-runner:local - docker rm -f clickhouse-test-runner-installing-packages || true + docker rm -f clickhouse-test-runner-installing-packages || true + docker run \ + -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \ + --name clickhouse-test-runner-installing-packages \ + clickhouse-test-runner-base:preinstall + docker commit clickhouse-test-runner-installing-packages clickhouse-statelest-test-runner:local + docker rm -f clickhouse-test-runner-installing-packages || true fi # # Create a bind-volume to the clickhouse-test script file @@ -47,38 +47,38 @@ fi # Build server image (optional) from local packages if [ -z "${CLICKHOUSE_SERVER_IMAGE}" ]; then - CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local" + CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local" - if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then - docker build \ - -f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \ - --target clickhouse-server-base \ - -t clickhouse-server-base:preinstall \ - "${CLICKHOUSE_DOCKER_DIR}/server" + if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then + docker build \ + -f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \ + --target clickhouse-server-base \ + -t clickhouse-server-base:preinstall \ + "${CLICKHOUSE_DOCKER_DIR}/server" - docker rm -f clickhouse_server_base_installing_server || true - docker run -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \ - --name clickhouse_server_base_installing_server \ - clickhouse-server-base:preinstall - docker commit clickhouse_server_base_installing_server clickhouse-server-base:postinstall + docker rm -f clickhouse_server_base_installing_server || true + docker run -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \ + --name clickhouse_server_base_installing_server \ + clickhouse-server-base:preinstall + docker commit clickhouse_server_base_installing_server clickhouse-server-base:postinstall - docker build \ - -f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \ - --target clickhouse-server \ - -t "${CLICKHOUSE_SERVER_IMAGE}" \ - "${CLICKHOUSE_DOCKER_DIR}/server" - fi + docker build \ + -f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \ + --target clickhouse-server \ + -t "${CLICKHOUSE_SERVER_IMAGE}" \ + "${CLICKHOUSE_DOCKER_DIR}/server" + fi fi docker rm -f test-runner || true docker-compose down CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \ - docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \ - create \ - --build --force-recreate + docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \ + create \ + --build --force-recreate CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \ - docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \ - run \ - --name test-runner \ - test-runner + docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \ + run \ + --name test-runner \ + test-runner diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 4697fbca7e1..a08f2db7149 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -37,6 +37,8 @@ The supported formats are: | [Avro](#data-format-avro) | ✔ | ✔ | | [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ | | [Parquet](#data-format-parquet) | ✔ | ✔ | +| [Arrow](#data-format-arrow) | ✔ | ✔ | +| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ | | [ORC](#data-format-orc) | ✔ | ✗ | | [RowBinary](#rowbinary) | ✔ | ✔ | | [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | @@ -985,9 +987,9 @@ See also [how to read/write length-delimited protobuf messages in popular langua ## Avro {#data-format-avro} -[Apache Avro](http://avro.apache.org/) is a row-oriented data serialization framework developed within Apache’s Hadoop project. +[Apache Avro](https://avro.apache.org/) is a row-oriented data serialization framework developed within Apache’s Hadoop project. -ClickHouse Avro format supports reading and writing [Avro data files](http://avro.apache.org/docs/current/spec.html#Object+Container+Files). +ClickHouse Avro format supports reading and writing [Avro data files](https://avro.apache.org/docs/current/spec.html#Object+Container+Files). ### Data Types Matching {#data_types-matching} @@ -1009,7 +1011,7 @@ The table below shows supported data types and how they match ClickHouse [data t | `long (timestamp-millis)` \* | [DateTime64(3)](../sql-reference/data-types/datetime.md) | `long (timestamp-millis)` \* | | `long (timestamp-micros)` \* | [DateTime64(6)](../sql-reference/data-types/datetime.md) | `long (timestamp-micros)` \* | -\* [Avro logical types](http://avro.apache.org/docs/current/spec.html#Logical+Types) +\* [Avro logical types](https://avro.apache.org/docs/current/spec.html#Logical+Types) Unsupported Avro data types: `record` (non-root), `map` @@ -1095,7 +1097,7 @@ SELECT * FROM topic1_stream; ## Parquet {#data-format-parquet} -[Apache Parquet](http://parquet.apache.org/) is a columnar storage format widespread in the Hadoop ecosystem. ClickHouse supports read and write operations for this format. +[Apache Parquet](https://parquet.apache.org/) is a columnar storage format widespread in the Hadoop ecosystem. ClickHouse supports read and write operations for this format. ### Data Types Matching {#data_types-matching-2} @@ -1141,6 +1143,16 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_ To exchange data with Hadoop, you can use [HDFS table engine](../engines/table-engines/integrations/hdfs.md). +## Arrow {#data-format-arrow} + +[Apache Arrow](https://arrow.apache.org/) comes with two built-in columnar storage formats. ClickHouse supports read and write operations for these formats. + +`Arrow` is Apache Arrow's "file mode" format. It is designed for in-memory random access. + +## ArrowStream {#data-format-arrow-stream} + +`ArrowStream` is Apache Arrow's "stream mode" format. It is designed for in-memory stream processing. + ## ORC {#data-format-orc} [Apache ORC](https://orc.apache.org/) is a columnar storage format widespread in the Hadoop ecosystem. You can only insert data in this format to ClickHouse. diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md index b7756f7fb79..20ebe28fe5b 100644 --- a/docs/ru/whats-new/extended-roadmap.md +++ b/docs/ru/whats-new/extended-roadmap.md @@ -73,10 +73,10 @@ Upd. Включено для системных таблиц. Q1. Закоммичено, но есть технический долг, который исправляется сейчас. Готово. Нет, не готово - там всё ещё технический долг. -### 1.9. Использование TTL для прореживания данных {#ispolzovanie-ttl-dlia-prorezhivaniia-dannykh} +### 1.9. + Использование TTL для прореживания данных {#ispolzovanie-ttl-dlia-prorezhivaniia-dannykh} Будет делать Сорокин Николай, ВШЭ и Яндекс. -Upd. Есть pull request. +Upd. Есть pull request. Upd. Сделано. Сейчас пользователь может задать в таблице выражение, которое определяет, сколько времени хранятся данные. Обычно это выражение задаётся относительно значения столбца с датой - например: удалять данные через три месяца. https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/\#table_engine-mergetree-ttl @@ -124,7 +124,7 @@ Q2. Upd. Олег будет делать только часть про HDFS. Upd. Реализация поверх S3 является рабочей на уровне PoC. -### 1.13. Ускорение запросов с FINAL {#uskorenie-zaprosov-s-final} +### 1.13. + Ускорение запросов с FINAL {#uskorenie-zaprosov-s-final} Требует 2.1. Делает [Николай Кочетов](https://github.com/KochetovNicolai). Нужно для Яндекс.Метрики. Q2. Upd: PR [#10463](https://github.com/ClickHouse/ClickHouse/pull/10463) @@ -203,10 +203,11 @@ Upd. SharedContext вынесен из Context. Upd. В очереди. Иван Лежанкин. -### 2.9. Логгировние в format-стиле {#loggirovnie-v-format-stile} +### 2.9. + Логгировние в format-стиле {#loggirovnie-v-format-stile} -Делает [Иван Лежанкин](https://github.com/abyss7). Низкий приоритет. -[\#6049](https://github.com/ClickHouse/ClickHouse/issues/6049#issuecomment-570836998) +[#6049](https://github.com/ClickHouse/ClickHouse/issues/6049#issuecomment-570836998) + +Сделано. ### 2.10. Запрашивать у таблиц не столбцы, а срезы {#zaprashivat-u-tablits-ne-stolbtsy-a-srezy} @@ -282,24 +283,20 @@ Upd. Сейчас обсуждается, как сделать другую з ### 4.3. Ограничение числа одновременных скачиваний с реплик {#ogranichenie-chisla-odnovremennykh-skachivanii-s-replik} -Дмитрий Григорьев, ВШЭ. Изначально делал Олег Алексеенков, но пока решение не готово, хотя там не так уж много доделывать. ### 4.4. Ограничение сетевой полосы при репликации {#ogranichenie-setevoi-polosy-pri-replikatsii} -Дмитрий Григорьев, ВШЭ. Нужно для Метрики. +Нужно для Метрики. ### 4.5. Возможность продолжить передачу куска данных при репликации после сбоя {#vozmozhnost-prodolzhit-peredachu-kuska-dannykh-pri-replikatsii-posle-sboia} -Дмитрий Григорьев, ВШЭ. - ### 4.6. p2p передача для GLOBAL подзапросов {#p2p-peredacha-dlia-global-podzaprosov} ### 4.7. Ленивая загрузка множеств для IN и JOIN с помощью k/v запросов {#lenivaia-zagruzka-mnozhestv-dlia-in-i-join-s-pomoshchiu-kv-zaprosov} ### 4.8. Разделить background pool для fetch и merge {#razdelit-background-pool-dlia-fetch-i-merge} -Дмитрий Григорьев, ВШЭ. В очереди. Исправить проблему, что восстанавливающаяся реплика перестаёт мержить. Частично компенсируется 4.3. @@ -329,6 +326,7 @@ Upd. Сделано. Эффективность работы под вопрос Метрика, БК, Маркет, Altinity уже используют более свежие версии чем LTS. Upd. Появилась вторая версия LTS - 20.3. + ## 6. Инструментирование {#instrumentirovanie} ### 6.1. + Исправления сэмплирующего профайлера запросов {#ispravleniia-sempliruiushchego-profailera-zaprosov} @@ -425,11 +423,11 @@ Upd. Рассмотрели все проверки подряд. UBSan включен в функциональных тестах, но не включен в интеграционных тестах. Требует 7.7. -### 7.11. Включение \*San в unit тестах {#vkliuchenie-san-v-unit-testakh} +### 7.11. + Включение \*San в unit тестах {#vkliuchenie-san-v-unit-testakh} У нас мало unit тестов по сравнению с функциональными тестами и их использование не обязательно. Но они всё-равно важны и нет причин не запускать их под всеми видами sanitizers. -Илья Яцишин. +Илья Яцишин. Сделано. ### 7.12. Показывать тестовое покрытие нового кода в PR {#pokazyvat-testovoe-pokrytie-novogo-koda-v-pr} @@ -528,6 +526,8 @@ Upd. Есть сборки, [пример](https://clickhouse-builds.s3.yandex.n Дарья Петрова, УрФУ. +Рабочий прототип: https://pulls-dashboard-demo.herokuapp.com/dashboard/ClickHouse/ClickHouse + Над ClickHouse одновременно работает большое количество разработчиков, которые оформляют свои изменения в виде pull requests. Когда непомерженных pull requests много, то возникает сложность с организацией работы - непонятно, на какой pull request смотреть в первую очередь. Предлагается реализовать простое одностраничное веб-приложение, в котором отображается список pull requests со следующей информацией: @@ -627,6 +627,7 @@ Upd. Готово (все директории кроме contrib). ### 7.32. Обфускация продакшен запросов {#obfuskatsiia-prodakshen-zaprosov} Роман Ильговский. Нужно для Яндекс.Метрики. +Есть pull request, почти готово: https://github.com/ClickHouse/ClickHouse/pull/10973 Имея SQL запрос, требуется вывести структуру таблиц, на которых этот запрос будет выполнен, и заполнить эти таблицы случайными данными, такими, что результат этого запроса зависит от выбора подмножества данных. @@ -1397,11 +1398,11 @@ Constraints позволяют задать выражение, истиннос Василий Морозов, Арслан Гумеров, Альберт Кидрачев, ВШЭ. В прошлом году задачу начинал делать другой человек, но не добился достаточного прогресса. -1. Оптимизация top sort. ++ 1. Оптимизация top sort. В ClickHouse используется неоптимальный вариант top sort. Суть его в том, что из каждого блока достаётся top N записей, а затем, все блоки мержатся. Но доставание top N записей у каждого следующего блока бессмысленно, если мы знаем, что из них в глобальный top N войдёт меньше. Конечно нужно реализовать вариацию на тему priority queue (heap) с быстрым пропуском целых блоков, если ни одна строка не попадёт в накопленный top. -2. Рекурсивный вариант сортировки по кортежам. ++ 2. Рекурсивный вариант сортировки по кортежам. Для сортировки по кортежам используется обычная сортировка с компаратором, который в цикле по элементам кортежа делает виртуальные вызовы `IColumn::compareAt`. Это неоптимально - как из-за короткого цикла по неизвестному в compile-time количеству элементов, так и из-за виртуальных вызовов. Чтобы обойтись без виртуальных вызовов, есть метод `IColumn::getPermutation`. Он используется в случае сортировки по одному столбцу. Есть вариант, что в случае сортировки по кортежу, что-то похожее тоже можно применить… например, сделать метод `updatePermutation`, принимающий аргументы offset и limit, и допереставляющий перестановку в диапазоне значений, в которых предыдущий столбец имел равные значения. diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index 4d4e9f98780..d9ea19ff389 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -21,7 +21,7 @@ mkdocs-htmlproofer-plugin==0.0.3 mkdocs-macros-plugin==0.4.9 nltk==3.5 nose==1.3.7 -protobuf==3.12.1 +protobuf==3.12.2 numpy==1.18.4 Pygments==2.5.2 pymdown-extensions==7.1 diff --git a/programs/client/CMakeLists.txt b/programs/client/CMakeLists.txt index 11ade559a8d..e273123afe0 100644 --- a/programs/client/CMakeLists.txt +++ b/programs/client/CMakeLists.txt @@ -6,14 +6,9 @@ set(CLICKHOUSE_CLIENT_SOURCES set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${Boost_PROGRAM_OPTIONS_LIBRARY}) -include(CheckSymbolExists) -check_symbol_exists(readpassphrase readpassphrase.h HAVE_READPASSPHRASE) -configure_file(config_client.h.in ${ConfigIncludePath}/config_client.h) - -if(NOT HAVE_READPASSPHRASE) - add_subdirectory(readpassphrase) - list(APPEND CLICKHOUSE_CLIENT_LINK PRIVATE readpassphrase) -endif() +# Always use internal readpassphrase +add_subdirectory(readpassphrase) +list(APPEND CLICKHOUSE_CLIENT_LINK PRIVATE readpassphrase) clickhouse_program_add(client) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index d6cac7a7b02..afc8f9a72b1 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -39,7 +39,6 @@ #include #include #include -#include #include #include #include @@ -77,6 +76,10 @@ #include #include +#if !defined(ARCADIA_BUILD) +# include +#endif + #ifndef __clang__ #pragma GCC optimize("-fno-var-tracking-assignments") #endif diff --git a/programs/client/ConnectionParameters.cpp b/programs/client/ConnectionParameters.cpp index f0ef3ae5694..d8b4d0f1add 100644 --- a/programs/client/ConnectionParameters.cpp +++ b/programs/client/ConnectionParameters.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include "readpassphrase/readpassphrase.h" namespace DB { diff --git a/programs/client/config_client.h.in b/programs/client/config_client.h.in deleted file mode 100644 index 5ad788ff54c..00000000000 --- a/programs/client/config_client.h.in +++ /dev/null @@ -1,3 +0,0 @@ -#pragma once - -#cmakedefine HAVE_READPASSPHRASE diff --git a/programs/client/readpassphrase/CMakeLists.txt b/programs/client/readpassphrase/CMakeLists.txt index a10b54c377d..dd1bf2c91b9 100644 --- a/programs/client/readpassphrase/CMakeLists.txt +++ b/programs/client/readpassphrase/CMakeLists.txt @@ -1,13 +1,7 @@ - # wget https://raw.githubusercontent.com/openssh/openssh-portable/master/openbsd-compat/readpassphrase.c # wget https://raw.githubusercontent.com/openssh/openssh-portable/master/openbsd-compat/readpassphrase.h -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-result -Wno-reserved-id-macro") +add_library(readpassphrase readpassphrase.c) -configure_file(includes.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/includes.h) -add_library(readpassphrase ${CMAKE_CURRENT_SOURCE_DIR}/readpassphrase.c) -set_target_properties(readpassphrase - PROPERTIES LINKER_LANGUAGE C - ) -# . to allow #include -target_include_directories(readpassphrase PUBLIC . ${CMAKE_CURRENT_BINARY_DIR}/include) +set_target_properties(readpassphrase PROPERTIES LINKER_LANGUAGE C) +target_compile_options(readpassphrase PRIVATE -Wno-unused-result -Wno-reserved-id-macro) diff --git a/programs/client/readpassphrase/includes.h.in b/programs/client/readpassphrase/includes.h similarity index 79% rename from programs/client/readpassphrase/includes.h.in rename to programs/client/readpassphrase/includes.h index 44580d1ed95..3ca5eb2bff8 100644 --- a/programs/client/readpassphrase/includes.h.in +++ b/programs/client/readpassphrase/includes.h @@ -1,6 +1,6 @@ #pragma once -#cmakedefine HAVE_READPASSPHRASE +/* #undef HAVE_READPASSPHRASE */ #if !defined(HAVE_READPASSPHRASE) # ifndef _PATH_TTY diff --git a/programs/client/readpassphrase/readpassphrase.c b/programs/client/readpassphrase/readpassphrase.c index 243701239bf..21b48e7efc3 100644 --- a/programs/client/readpassphrase/readpassphrase.c +++ b/programs/client/readpassphrase/readpassphrase.c @@ -25,13 +25,11 @@ #include "includes.h" -#ifndef HAVE_READPASSPHRASE - #include #include #include #include -#include +#include "readpassphrase.h" #include #include #include @@ -193,19 +191,7 @@ restart: } //DEF_WEAK(readpassphrase); -#if 0 -char * -getpass(const char *prompt) -{ - static char buf[_PASSWORD_LEN + 1]; - - return(readpassphrase(prompt, buf, sizeof(buf), RPP_ECHO_OFF)); -} -#endif - static void handler(int s) { - signo[s] = 1; } -#endif /* HAVE_READPASSPHRASE */ diff --git a/programs/client/readpassphrase/readpassphrase.h b/programs/client/readpassphrase/readpassphrase.h index 0782a1773ea..399eb7b062c 100644 --- a/programs/client/readpassphrase/readpassphrase.h +++ b/programs/client/readpassphrase/readpassphrase.h @@ -23,39 +23,22 @@ /* OPENBSD ORIGINAL: include/readpassphrase.h */ #pragma once -// #ifndef _READPASSPHRASE_H_ -// #define _READPASSPHRASE_H_ -//#include "includes.h" -#include "config_client.h" - -// Should not be included on BSD systems, but if it happen... -#ifdef HAVE_READPASSPHRASE -# include_next +#if defined(__cplusplus) +extern "C" { #endif -#ifndef HAVE_READPASSPHRASE -# ifdef __cplusplus -extern "C" { -# endif - - -# define RPP_ECHO_OFF 0x00 /* Turn off echo (default). */ -# define RPP_ECHO_ON 0x01 /* Leave echo on. */ -# define RPP_REQUIRE_TTY 0x02 /* Fail if there is no tty. */ -# define RPP_FORCELOWER 0x04 /* Force input to lower case. */ -# define RPP_FORCEUPPER 0x08 /* Force input to upper case. */ -# define RPP_SEVENBIT 0x10 /* Strip the high bit from input. */ -# define RPP_STDIN 0x20 /* Read from stdin, not /dev/tty */ +#define RPP_ECHO_OFF 0x00 /* Turn off echo (default). */ +#define RPP_ECHO_ON 0x01 /* Leave echo on. */ +#define RPP_REQUIRE_TTY 0x02 /* Fail if there is no tty. */ +#define RPP_FORCELOWER 0x04 /* Force input to lower case. */ +#define RPP_FORCEUPPER 0x08 /* Force input to upper case. */ +#define RPP_SEVENBIT 0x10 /* Strip the high bit from input. */ +#define RPP_STDIN 0x20 /* Read from stdin, not /dev/tty */ char * readpassphrase(const char *, char *, size_t, int); -# ifdef __cplusplus +#if defined(__cplusplus) } -# endif - - -#endif /* HAVE_READPASSPHRASE */ - -// #endif /* !_READPASSPHRASE_H_ */ +#endif diff --git a/programs/client/readpassphrase/ya.make b/programs/client/readpassphrase/ya.make new file mode 100644 index 00000000000..80ad197e5d4 --- /dev/null +++ b/programs/client/readpassphrase/ya.make @@ -0,0 +1,7 @@ +LIBRARY() + +SRCS( + readpassphrase.c +) + +END() diff --git a/programs/main.cpp b/programs/main.cpp index 2eb226d3c00..382a104b798 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -8,11 +8,8 @@ #include #include /// pair -#if __has_include("config_tools.h") -#include "config_tools.h" -#endif -#if __has_include("config_core.h") -#include "config_core.h" +#if !defined(ARCADIA_BUILD) +# include "config_tools.h" #endif #include @@ -22,31 +19,31 @@ /// Universal executable for various clickhouse applications -#if ENABLE_CLICKHOUSE_SERVER || !defined(ENABLE_CLICKHOUSE_SERVER) +#if ENABLE_CLICKHOUSE_SERVER int mainEntryClickHouseServer(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_CLIENT || !defined(ENABLE_CLICKHOUSE_CLIENT) +#if ENABLE_CLICKHOUSE_CLIENT int mainEntryClickHouseClient(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_LOCAL || !defined(ENABLE_CLICKHOUSE_LOCAL) +#if ENABLE_CLICKHOUSE_LOCAL int mainEntryClickHouseLocal(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_BENCHMARK || !defined(ENABLE_CLICKHOUSE_BENCHMARK) +#if ENABLE_CLICKHOUSE_BENCHMARK int mainEntryClickHouseBenchmark(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG || !defined(ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG) +#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG int mainEntryClickHouseExtractFromConfig(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_COMPRESSOR || !defined(ENABLE_CLICKHOUSE_COMPRESSOR) +#if ENABLE_CLICKHOUSE_COMPRESSOR int mainEntryClickHouseCompressor(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_FORMAT || !defined(ENABLE_CLICKHOUSE_FORMAT) +#if ENABLE_CLICKHOUSE_FORMAT int mainEntryClickHouseFormat(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_COPIER || !defined(ENABLE_CLICKHOUSE_COPIER) +#if ENABLE_CLICKHOUSE_COPIER int mainEntryClickHouseClusterCopier(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_OBFUSCATOR || !defined(ENABLE_CLICKHOUSE_OBFUSCATOR) +#if ENABLE_CLICKHOUSE_OBFUSCATOR int mainEntryClickHouseObfuscator(int argc, char ** argv); #endif @@ -60,31 +57,31 @@ using MainFunc = int (*)(int, char**); /// Add an item here to register new application std::pair clickhouse_applications[] = { -#if ENABLE_CLICKHOUSE_LOCAL || !defined(ENABLE_CLICKHOUSE_LOCAL) +#if ENABLE_CLICKHOUSE_LOCAL {"local", mainEntryClickHouseLocal}, #endif -#if ENABLE_CLICKHOUSE_CLIENT || !defined(ENABLE_CLICKHOUSE_CLIENT) +#if ENABLE_CLICKHOUSE_CLIENT {"client", mainEntryClickHouseClient}, #endif -#if ENABLE_CLICKHOUSE_BENCHMARK || !defined(ENABLE_CLICKHOUSE_BENCHMARK) +#if ENABLE_CLICKHOUSE_BENCHMARK {"benchmark", mainEntryClickHouseBenchmark}, #endif -#if ENABLE_CLICKHOUSE_SERVER || !defined(ENABLE_CLICKHOUSE_SERVER) +#if ENABLE_CLICKHOUSE_SERVER {"server", mainEntryClickHouseServer}, #endif -#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG || !defined(ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG) +#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG {"extract-from-config", mainEntryClickHouseExtractFromConfig}, #endif -#if ENABLE_CLICKHOUSE_COMPRESSOR || !defined(ENABLE_CLICKHOUSE_COMPRESSOR) +#if ENABLE_CLICKHOUSE_COMPRESSOR {"compressor", mainEntryClickHouseCompressor}, #endif -#if ENABLE_CLICKHOUSE_FORMAT || !defined(ENABLE_CLICKHOUSE_FORMAT) +#if ENABLE_CLICKHOUSE_FORMAT {"format", mainEntryClickHouseFormat}, #endif -#if ENABLE_CLICKHOUSE_COPIER || !defined(ENABLE_CLICKHOUSE_COPIER) +#if ENABLE_CLICKHOUSE_COPIER {"copier", mainEntryClickHouseClusterCopier}, #endif -#if ENABLE_CLICKHOUSE_OBFUSCATOR || !defined(ENABLE_CLICKHOUSE_OBFUSCATOR) +#if ENABLE_CLICKHOUSE_OBFUSCATOR {"obfuscator", mainEntryClickHouseObfuscator}, #endif }; @@ -127,9 +124,10 @@ enum class InstructionFail SSSE3 = 2, SSE4_1 = 3, SSE4_2 = 4, - AVX = 5, - AVX2 = 6, - AVX512 = 7 + POPCNT = 5, + AVX = 6, + AVX2 = 7, + AVX512 = 8 }; const char * instructionFailToString(InstructionFail fail) @@ -146,6 +144,8 @@ const char * instructionFailToString(InstructionFail fail) return "SSE4.1"; case InstructionFail::SSE4_2: return "SSE4.2"; + case InstructionFail::POPCNT: + return "POPCNT"; case InstructionFail::AVX: return "AVX"; case InstructionFail::AVX2: @@ -189,6 +189,16 @@ void checkRequiredInstructionsImpl(volatile InstructionFail & fail) __asm__ volatile ("pcmpgtq %%xmm0, %%xmm0" : : : "xmm0"); #endif + /// Defined by -msse4.2 +#if defined(__POPCNT__) + fail = InstructionFail::POPCNT; + { + uint64_t a = 0; + uint64_t b = 0; + __asm__ volatile ("popcnt %1, %0" : "=r"(a) :"r"(b) :); + } +#endif + #if defined(__AVX__) fail = InstructionFail::AVX; __asm__ volatile ("vaddpd %%ymm0, %%ymm0, %%ymm0" : : : "ymm0"); diff --git a/programs/ya.make b/programs/ya.make index 6c773c312b8..f4a61850212 100644 --- a/programs/ya.make +++ b/programs/ya.make @@ -1,3 +1,27 @@ -RECURSE( - server +PROGRAM(clickhouse) + +CFLAGS( + -DENABLE_CLICKHOUSE_CLIENT + -DENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG + -DENABLE_CLICKHOUSE_SERVER ) + +PEERDIR( + clickhouse/base/daemon + clickhouse/base/loggers + clickhouse/programs/client/readpassphrase + clickhouse/src +) + +SRCS( + main.cpp + + client/Client.cpp + client/ConnectionParameters.cpp + client/Suggest.cpp + extract-from-config/ExtractFromConfig.cpp + server/Server.cpp + server/MetricsTransmitter.cpp +) + +END() diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 9c8956e9253..4d49d205040 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -9,8 +9,8 @@ namespace DB { namespace ErrorCodes { - extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; + extern const int NOT_IMPLEMENTED; } Authentication::Digest Authentication::getPasswordDoubleSHA1() const @@ -40,8 +40,11 @@ Authentication::Digest Authentication::getPasswordDoubleSHA1() const case LDAP_PASSWORD: throw Exception("Cannot get password double SHA1 for user with 'LDAP_PASSWORD' authentication.", ErrorCodes::BAD_ARGUMENTS); + + case MAX_TYPE: + break; } - throw Exception("Unknown authentication type: " + std::to_string(static_cast(type)), ErrorCodes::LOGICAL_ERROR); + throw Exception("getPasswordDoubleSHA1(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED); } @@ -84,8 +87,11 @@ bool Authentication::isCorrectPassword(const String & password_, const String & LDAPSimpleAuthClient ldap_client(ldap_server_params); return ldap_client.check(); } + + case MAX_TYPE: + break; } - throw Exception("Unknown authentication type: " + std::to_string(static_cast(type)), ErrorCodes::LOGICAL_ERROR); + throw Exception("Cannot check if the password is correct for authentication type " + toString(type), ErrorCodes::NOT_IMPLEMENTED); } } diff --git a/src/Access/Authentication.h b/src/Access/Authentication.h index 903b8046a5d..65f45db499b 100644 --- a/src/Access/Authentication.h +++ b/src/Access/Authentication.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB @@ -14,6 +15,7 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; } class ExternalAuthenticators; @@ -39,6 +41,15 @@ public: /// Password is checked by a [remote] LDAP server. Connection will be made at each authentication attempt. LDAP_PASSWORD, + + MAX_TYPE, + }; + + struct TypeInfo + { + const char * const raw_name; + const String name; /// Lowercased with underscores, e.g. "sha256_password". + static const TypeInfo & get(Type type_); }; using Digest = std::vector; @@ -95,6 +106,53 @@ private: }; +inline const Authentication::TypeInfo & Authentication::TypeInfo::get(Type type_) +{ + static constexpr auto make_info = [](const char * raw_name_) + { + String init_name = raw_name_; + boost::to_lower(init_name); + return TypeInfo{raw_name_, std::move(init_name)}; + }; + + switch (type_) + { + case NO_PASSWORD: + { + static const auto info = make_info("NO_PASSWORD"); + return info; + } + case PLAINTEXT_PASSWORD: + { + static const auto info = make_info("PLAINTEXT_PASSWORD"); + return info; + } + case SHA256_PASSWORD: + { + static const auto info = make_info("SHA256_PASSWORD"); + return info; + } + case DOUBLE_SHA1_PASSWORD: + { + static const auto info = make_info("DOUBLE_SHA1_PASSWORD"); + return info; + } + case LDAP_PASSWORD: + { + static const auto info = make_info("LDAP"); + return info; + } + case MAX_TYPE: break; + } + throw Exception("Unknown authentication type: " + std::to_string(static_cast(type_)), ErrorCodes::LOGICAL_ERROR); +} + +inline String toString(Authentication::Type type_) +{ + return Authentication::TypeInfo::get(type_).raw_name; +} + + inline Authentication::Digest Authentication::encodeSHA256(const std::string_view & text [[maybe_unused]]) { #if USE_SSL @@ -135,8 +193,10 @@ inline void Authentication::setPassword(const String & password_) case LDAP_PASSWORD: throw Exception("Cannot specify password for the 'LDAP_PASSWORD' authentication type", ErrorCodes::LOGICAL_ERROR); + + case MAX_TYPE: break; } - throw Exception("Unknown authentication type: " + std::to_string(static_cast(type)), ErrorCodes::LOGICAL_ERROR); + throw Exception("setPassword(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED); } @@ -204,8 +264,10 @@ inline void Authentication::setPasswordHashBinary(const Digest & hash) case LDAP_PASSWORD: throw Exception("Cannot specify password for the 'LDAP_PASSWORD' authentication type", ErrorCodes::LOGICAL_ERROR); + + case MAX_TYPE: break; } - throw Exception("Unknown authentication type: " + std::to_string(static_cast(type)), ErrorCodes::LOGICAL_ERROR); + throw Exception("setPasswordHashBinary(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED); } inline const String & Authentication::getLDAPServerName() const diff --git a/src/Common/CurrentMetrics.h b/src/Common/CurrentMetrics.h index b87504ef49a..a3bac96a16c 100644 --- a/src/Common/CurrentMetrics.h +++ b/src/Common/CurrentMetrics.h @@ -94,6 +94,12 @@ namespace CurrentMetrics amount = new_amount; } + void sub(Value value = 1) + { + what->fetch_sub(value, std::memory_order_relaxed); + amount -= value; + } + /// Subtract value before destructor. void destroy() { diff --git a/src/Common/ya.make b/src/Common/ya.make index c0178f3d310..83a419212bd 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -18,17 +18,7 @@ PEERDIR( contrib/restricted/ryu ) -# TODO: stub for config_version.h -CFLAGS (GLOBAL -DDBMS_NAME=\"ClickHouse\") -CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=0) -CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=0) -CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=0) -CFLAGS (GLOBAL -DVERSION_FULL=\"ClickHouse\") -CFLAGS (GLOBAL -DVERSION_INTEGER=0) -CFLAGS (GLOBAL -DVERSION_NAME=\"ClickHouse\") -CFLAGS (GLOBAL -DVERSION_OFFICIAL=\"\\\(arcadia\\\)\") -CFLAGS (GLOBAL -DVERSION_REVISION=0) -CFLAGS (GLOBAL -DVERSION_STRING=\"Unknown\") +INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/yandex/ya.make.versions.inc) SRCS( ActionLock.cpp diff --git a/src/DataStreams/NullAndDoCopyBlockInputStream.h b/src/DataStreams/NullAndDoCopyBlockInputStream.h index 8fe05c387a3..8bfb3538f3a 100644 --- a/src/DataStreams/NullAndDoCopyBlockInputStream.h +++ b/src/DataStreams/NullAndDoCopyBlockInputStream.h @@ -21,19 +21,10 @@ class NullAndDoCopyBlockInputStream : public IBlockInputStream { public: NullAndDoCopyBlockInputStream(const BlockInputStreamPtr & input_, BlockOutputStreamPtr output_) + : input(std::move(input_)) + , output(std::move(output_)) { - input_streams.push_back(input_); - output_streams.push_back(output_); - - for (auto & input_stream : input_streams) - children.push_back(input_stream); - } - - NullAndDoCopyBlockInputStream(const BlockInputStreams & input_, BlockOutputStreams & output_) - : input_streams(input_), output_streams(output_) - { - for (auto & input_stream : input_) - children.push_back(input_stream); + children.push_back(input); } /// Suppress readPrefix and readSuffix, because they are called by copyData. @@ -53,16 +44,13 @@ protected: /// If query was cancelled, it will be processed by child streams. /// Part of the data will be processed. - if (input_streams.size() == 1 && output_streams.size() == 1) - copyData(*input_streams.at(0), *output_streams.at(0)); - else - copyData(input_streams, output_streams); + copyData(*input, *output); return Block(); } private: - BlockInputStreams input_streams; - BlockOutputStreams output_streams; + BlockInputStreamPtr input; + BlockOutputStreamPtr output; }; } diff --git a/src/DataStreams/copyData.cpp b/src/DataStreams/copyData.cpp index fd4bfab28d8..a0651999034 100644 --- a/src/DataStreams/copyData.cpp +++ b/src/DataStreams/copyData.cpp @@ -1,9 +1,6 @@ -#include #include #include #include -#include -#include #include @@ -55,79 +52,6 @@ void copyDataImpl(IBlockInputStream & from, IBlockOutputStream & to, TCancelCall inline void doNothing(const Block &) {} -namespace -{ - - -struct ParallelInsertsHandler -{ - using CencellationHook = std::function; - - explicit ParallelInsertsHandler(BlockOutputStreams & output_streams, CencellationHook cancellation_hook_, size_t num_threads) - : outputs(output_streams.size()), cancellation_hook(std::move(cancellation_hook_)) - { - exceptions.resize(num_threads); - - for (auto & output : output_streams) - outputs.push(output.get()); - } - - void onBlock(Block & block, size_t /*thread_num*/) - { - IBlockOutputStream * out = nullptr; - - outputs.pop(out); - out->write(block); - outputs.push(out); - } - - void onFinishThread(size_t /*thread_num*/) {} - void onFinish() {} - - void onException(std::exception_ptr & exception, size_t thread_num) - { - exceptions[thread_num] = exception; - cancellation_hook(); - } - - void rethrowFirstException() - { - for (auto & exception : exceptions) - if (exception) - std::rethrow_exception(exception); - } - - ConcurrentBoundedQueue outputs; - std::vector exceptions; - CencellationHook cancellation_hook; -}; - -} - -static void copyDataImpl(BlockInputStreams & inputs, BlockOutputStreams & outputs) -{ - for (auto & output : outputs) - output->writePrefix(); - - using Processor = ParallelInputsProcessor; - Processor * processor_ptr = nullptr; - - ParallelInsertsHandler handler(outputs, [&processor_ptr]() { processor_ptr->cancel(false); }, inputs.size()); - ParallelInputsProcessor processor(inputs, nullptr, inputs.size(), handler); - processor_ptr = &processor; - - processor.process(); - processor.wait(); - handler.rethrowFirstException(); - - /// readPrefix is called in ParallelInputsProcessor. - for (auto & input : inputs) - input->readSuffix(); - - for (auto & output : outputs) - output->writeSuffix(); -} - void copyData(IBlockInputStream & from, IBlockOutputStream & to, std::atomic * is_cancelled) { auto is_cancelled_pred = [is_cancelled] () @@ -138,11 +62,6 @@ void copyData(IBlockInputStream & from, IBlockOutputStream & to, std::atomic & is_cancelled) { copyDataImpl(from, to, is_cancelled, doNothing); diff --git a/src/DataStreams/copyData.h b/src/DataStreams/copyData.h index ae72dbd2421..f2bce8f411b 100644 --- a/src/DataStreams/copyData.h +++ b/src/DataStreams/copyData.h @@ -16,8 +16,6 @@ class Block; */ void copyData(IBlockInputStream & from, IBlockOutputStream & to, std::atomic * is_cancelled = nullptr); -void copyData(BlockInputStreams & inputs, BlockOutputStreams & outputs); - void copyData(IBlockInputStream & from, IBlockOutputStream & to, const std::function & is_cancelled); void copyData(IBlockInputStream & from, IBlockOutputStream & to, const std::function & is_cancelled, diff --git a/src/Formats/ProtobufReader.cpp b/src/Formats/ProtobufReader.cpp index d8302771002..d76b048c1e9 100644 --- a/src/Formats/ProtobufReader.cpp +++ b/src/Formats/ProtobufReader.cpp @@ -32,11 +32,12 @@ namespace BITS32 = 5, }; - // The following condition must always be true: - // any_cursor_position < min(END_OF_VARINT, END_OF_GROUP) - // This inequation helps to check conditions in SimpleReader. - constexpr UInt64 END_OF_VARINT = static_cast(-1); - constexpr UInt64 END_OF_GROUP = static_cast(-2); + // The following conditions must always be true: + // any_cursor_position > END_OF_VARINT + // any_cursor_position > END_OF_GROUP + // Those inequations helps checking conditions in ProtobufReader::SimpleReader. + constexpr Int64 END_OF_VARINT = -1; + constexpr Int64 END_OF_GROUP = -2; Int64 decodeZigZag(UInt64 n) { return static_cast((n >> 1) ^ (~(n & 1) + 1)); } @@ -77,7 +78,7 @@ void ProtobufReader::SimpleReader::endMessage(bool ignore_errors) if (!current_message_level) return; - UInt64 root_message_end = (current_message_level == 1) ? current_message_end : parent_message_ends.front(); + Int64 root_message_end = (current_message_level == 1) ? current_message_end : parent_message_ends.front(); if (cursor != root_message_end) { if (cursor < root_message_end) @@ -95,6 +96,9 @@ void ProtobufReader::SimpleReader::endMessage(bool ignore_errors) void ProtobufReader::SimpleReader::startNestedMessage() { assert(current_message_level >= 1); + if ((cursor > field_end) && (field_end != END_OF_GROUP)) + throwUnknownFormat(); + // Start reading a nested message which is located inside a length-delimited field // of another message. parent_message_ends.emplace_back(current_message_end); @@ -146,7 +150,7 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number) throwUnknownFormat(); } - if (cursor >= current_message_end) + if ((cursor >= current_message_end) && (current_message_end != END_OF_GROUP)) return false; UInt64 varint = readVarint(); @@ -196,11 +200,17 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number) bool ProtobufReader::SimpleReader::readUInt(UInt64 & value) { + if (field_end == END_OF_VARINT) + { + value = readVarint(); + field_end = cursor; + return true; + } + if (unlikely(cursor >= field_end)) return false; + value = readVarint(); - if (field_end == END_OF_VARINT) - field_end = cursor; return true; } @@ -227,6 +237,7 @@ bool ProtobufReader::SimpleReader::readFixed(T & value) { if (unlikely(cursor >= field_end)) return false; + readBinary(&value, sizeof(T)); return true; } diff --git a/src/Formats/ProtobufReader.h b/src/Formats/ProtobufReader.h index a50c2f202f0..7c3c19ba376 100644 --- a/src/Formats/ProtobufReader.h +++ b/src/Formats/ProtobufReader.h @@ -124,12 +124,12 @@ private: void ignoreGroup(); ReadBuffer & in; - UInt64 cursor; + Int64 cursor; size_t current_message_level; - UInt64 current_message_end; - std::vector parent_message_ends; - UInt64 field_end; - UInt64 last_string_pos; + Int64 current_message_end; + std::vector parent_message_ends; + Int64 field_end; + Int64 last_string_pos; }; class IConverter diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index f3b116e490c..b6efa5d6d46 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -28,6 +28,11 @@ #include #include #include +#include +#include +#include +#include +#include namespace DB @@ -117,8 +122,6 @@ BlockIO InterpreterInsertQuery::execute() if (!query.table_function) context.checkAccess(AccessType::INSERT, query.table_id, query_sample_block.getNames()); - BlockInputStreams in_streams; - BlockOutputStreams out_streams; bool is_distributed_insert_select = false; if (query.select && table->isRemote() && settings.parallel_distributed_insert_select) @@ -159,6 +162,8 @@ BlockIO InterpreterInsertQuery::execute() const auto & cluster = storage_src->getCluster(); const auto & shards_info = cluster->getShardsInfo(); + std::vector pipelines; + String new_query_str = queryToString(new_query); for (size_t shard_index : ext::range(0, shards_info.size())) { @@ -166,8 +171,7 @@ BlockIO InterpreterInsertQuery::execute() if (shard_info.isLocal()) { InterpreterInsertQuery interpreter(new_query, context); - auto block_io = interpreter.execute(); - in_streams.push_back(block_io.in); + pipelines.emplace_back(interpreter.execute().pipeline); } else { @@ -179,13 +183,20 @@ BlockIO InterpreterInsertQuery::execute() /// INSERT SELECT query returns empty block auto in_stream = std::make_shared(std::move(connections), new_query_str, Block{}, context); - in_streams.push_back(in_stream); + pipelines.emplace_back(); + pipelines.back().init(Pipe(std::make_shared(std::move(in_stream)))); + pipelines.back().setSinks([](const Block & header, QueryPipeline::StreamType) -> ProcessorPtr + { + return std::make_shared(header); + }); } - out_streams.push_back(std::make_shared(Block())); } + + res.pipeline.unitePipelines(std::move(pipelines), {}); } } + BlockOutputStreams out_streams; if (!is_distributed_insert_select || query.watch) { size_t out_streams_size = 1; @@ -193,27 +204,21 @@ BlockIO InterpreterInsertQuery::execute() { /// Passing 1 as subquery_depth will disable limiting size of intermediate result. InterpreterSelectWithUnionQuery interpreter_select{ query.select, context, SelectQueryOptions(QueryProcessingStage::Complete, 1)}; + res.pipeline = interpreter_select.executeWithProcessors(); if (table->supportsParallelInsert() && settings.max_insert_threads > 1) - { - in_streams = interpreter_select.executeWithMultipleStreams(res.pipeline); - out_streams_size = std::min(size_t(settings.max_insert_threads), in_streams.size()); - } + out_streams_size = std::min(size_t(settings.max_insert_threads), res.pipeline.getNumStreams()); + + if (out_streams_size == 1) + res.pipeline.addPipe({std::make_shared(res.pipeline.getHeader(), res.pipeline.getNumStreams())}); else - { - res = interpreter_select.execute(); - in_streams.emplace_back(res.in); - res.in = nullptr; - res.out = nullptr; - } + res.pipeline.resize(out_streams_size); } else if (query.watch) { InterpreterWatchQuery interpreter_watch{ query.watch, context }; res = interpreter_watch.execute(); - in_streams.emplace_back(res.in); - res.in = nullptr; - res.out = nullptr; + res.pipeline.init(Pipe(std::make_shared(std::move(res.in)))); } for (size_t i = 0; i < out_streams_size; i++) @@ -256,27 +261,35 @@ BlockIO InterpreterInsertQuery::execute() } /// What type of query: INSERT or INSERT SELECT or INSERT WATCH? - if (query.select || query.watch) + if (is_distributed_insert_select) { - for (auto & in_stream : in_streams) - { - in_stream = std::make_shared( - in_stream, out_streams.at(0)->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Position); - } + /// Pipeline was already built. + } + else if (query.select || query.watch) + { + const auto & header = out_streams.at(0)->getHeader(); - Block in_header = in_streams.at(0)->getHeader(); - if (in_streams.size() > 1) + res.pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr { - for (size_t i = 1; i < in_streams.size(); ++i) - assertBlocksHaveEqualStructure(in_streams[i]->getHeader(), in_header, query.select ? "INSERT SELECT" : "INSERT WATCH"); - } + return std::make_shared(in_header, header, + ConvertingTransform::MatchColumnsMode::Position); + }); - res.in = std::make_shared(in_streams, out_streams); + res.pipeline.setSinks([&](const Block &, QueryPipeline::StreamType type) -> ProcessorPtr + { + if (type != QueryPipeline::StreamType::Main) + return nullptr; + + auto stream = std::move(out_streams.back()); + out_streams.pop_back(); + + return std::make_shared(std::move(stream)); + }); if (!allow_materialized) { for (const auto & column : table->getColumns()) - if (column.default_desc.kind == ColumnDefaultKind::Materialized && in_header.has(column.name)) + if (column.default_desc.kind == ColumnDefaultKind::Materialized && header.has(column.name)) throw Exception("Cannot insert column " + column.name + ", because it is MATERIALIZED column.", ErrorCodes::ILLEGAL_COLUMN); } } @@ -288,6 +301,7 @@ BlockIO InterpreterInsertQuery::execute() } else res.out = std::move(out_streams.at(0)); + res.pipeline.addStorageHolder(table); return res; diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index 87e1265f793..e37c31aab22 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -56,10 +56,10 @@ namespace query->default_roles = user.default_roles.toASTWithNames(*manager); } - if (attach_mode && (user.authentication.getType() != Authentication::NO_PASSWORD)) + if (user.authentication.getType() != Authentication::NO_PASSWORD) { - /// We don't show password unless it's an ATTACH statement. query->authentication = user.authentication; + query->show_password = attach_mode; /// We don't show password unless it's an ATTACH statement. } if (!user.settings.empty()) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index c69631b2665..2cc6730b90d 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -363,6 +363,9 @@ static std::tuple executeQueryImpl( else res = interpreter->execute(); + if (res.pipeline.initialized()) + use_processors = true; + if (const auto * insert_interpreter = typeid_cast(&*interpreter)) { /// Save insertion table (not table function). TODO: support remote() table function. @@ -390,7 +393,7 @@ static std::tuple executeQueryImpl( /// Limits apply only to the final result. pipeline.setProgressCallback(context.getProgressCallback()); pipeline.setProcessListElement(context.getProcessListElement()); - if (stage == QueryProcessingStage::Complete) + if (stage == QueryProcessingStage::Complete && !pipeline.isCompleted()) { pipeline.resize(1); pipeline.addSimpleTransform([&](const Block & header) @@ -771,29 +774,36 @@ void executeQuery( if (ast_query_with_output && ast_query_with_output->settings_ast) InterpreterSetQuery(ast_query_with_output->settings_ast, context).executeForCurrentContext(); - pipeline.addSimpleTransform([](const Block & header) + if (!pipeline.isCompleted()) { - return std::make_shared(header); - }); + pipeline.addSimpleTransform([](const Block & header) + { + return std::make_shared(header); + }); - auto out = context.getOutputFormatProcessor(format_name, *out_buf, pipeline.getHeader()); - out->setAutoFlush(); + auto out = context.getOutputFormatProcessor(format_name, *out_buf, pipeline.getHeader()); + out->setAutoFlush(); - /// Save previous progress callback if any. TODO Do it more conveniently. - auto previous_progress_callback = context.getProgressCallback(); + /// Save previous progress callback if any. TODO Do it more conveniently. + auto previous_progress_callback = context.getProgressCallback(); - /// NOTE Progress callback takes shared ownership of 'out'. - pipeline.setProgressCallback([out, previous_progress_callback] (const Progress & progress) + /// NOTE Progress callback takes shared ownership of 'out'. + pipeline.setProgressCallback([out, previous_progress_callback] (const Progress & progress) + { + if (previous_progress_callback) + previous_progress_callback(progress); + out->onProgress(progress); + }); + + if (set_result_details) + set_result_details(context.getClientInfo().current_query_id, out->getContentType(), format_name, DateLUT::instance().getTimeZone()); + + pipeline.setOutputFormat(std::move(out)); + } + else { - if (previous_progress_callback) - previous_progress_callback(progress); - out->onProgress(progress); - }); - - if (set_result_details) - set_result_details(context.getClientInfo().current_query_id, out->getContentType(), format_name, DateLUT::instance().getTimeZone()); - - pipeline.setOutput(std::move(out)); + pipeline.setProgressCallback(context.getProgressCallback()); + } { auto executor = pipeline.execute(); diff --git a/src/Parsers/ASTCreateUserQuery.cpp b/src/Parsers/ASTCreateUserQuery.cpp index fdf2dfdf81f..9ad89fc02bd 100644 --- a/src/Parsers/ASTCreateUserQuery.cpp +++ b/src/Parsers/ASTCreateUserQuery.cpp @@ -6,6 +6,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + namespace { void formatRenameTo(const String & new_name, const IAST::FormatSettings & settings) @@ -15,31 +21,57 @@ namespace } - void formatAuthentication(const Authentication & authentication, const IAST::FormatSettings & settings) + void formatAuthentication(const Authentication & authentication, bool show_password, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH " << (settings.hilite ? IAST::hilite_none : ""); - switch (authentication.getType()) + auto authentication_type = authentication.getType(); + if (authentication_type == Authentication::NO_PASSWORD) { - case Authentication::Type::NO_PASSWORD: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "no_password" << (settings.hilite ? IAST::hilite_none : ""); - break; - case Authentication::Type::PLAINTEXT_PASSWORD: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "plaintext_password BY " << (settings.hilite ? IAST::hilite_none : "") - << quoteString(authentication.getPassword()); - break; - case Authentication::Type::SHA256_PASSWORD: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "sha256_hash BY " << (settings.hilite ? IAST::hilite_none : "") - << quoteString(authentication.getPasswordHashHex()); - break; - case Authentication::Type::DOUBLE_SHA1_PASSWORD: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "double_sha1_hash BY " << (settings.hilite ? IAST::hilite_none : "") - << quoteString(authentication.getPasswordHashHex()); - break; - case Authentication::Type::LDAP_PASSWORD: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ldap BY " << (settings.hilite ? IAST::hilite_none : "") - << quoteString(authentication.getLDAPServerName()); - break; + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NOT IDENTIFIED" + << (settings.hilite ? IAST::hilite_none : ""); + return; } + + String authentication_type_name = Authentication::TypeInfo::get(authentication_type).name; + std::optional password; + + if (show_password || authentication_type == Authentication::LDAP_PASSWORD) + { + switch (authentication_type) + { + case Authentication::PLAINTEXT_PASSWORD: + { + password = authentication.getPassword(); + break; + } + case Authentication::SHA256_PASSWORD: + { + authentication_type_name = "sha256_hash"; + password = authentication.getPasswordHashHex(); + break; + } + case Authentication::DOUBLE_SHA1_PASSWORD: + { + authentication_type_name = "double_sha1_hash"; + password = authentication.getPasswordHashHex(); + break; + } + case Authentication::LDAP_PASSWORD: + { + authentication_type_name = "ldap"; + password = authentication.getLDAPServerName(); + break; + } + + case Authentication::NO_PASSWORD: [[fallthrough]]; + case Authentication::MAX_TYPE: + throw Exception("AST: Unexpected authentication type " + toString(authentication_type), ErrorCodes::LOGICAL_ERROR); + } + } + + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH " << authentication_type_name + << (settings.hilite ? IAST::hilite_none : ""); + if (password) + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " BY " << quoteString(*password); } @@ -194,7 +226,7 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState & formatRenameTo(new_name, format); if (authentication) - formatAuthentication(*authentication, format); + formatAuthentication(*authentication, show_password, format); if (hosts) formatHosts(nullptr, *hosts, format); diff --git a/src/Parsers/ASTCreateUserQuery.h b/src/Parsers/ASTCreateUserQuery.h index e954adc7b53..42ec3fb877b 100644 --- a/src/Parsers/ASTCreateUserQuery.h +++ b/src/Parsers/ASTCreateUserQuery.h @@ -12,14 +12,14 @@ class ASTExtendedRoleSet; class ASTSettingsProfileElements; /** CREATE USER [IF NOT EXISTS | OR REPLACE] name - * [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH|LDAP}] BY {'password'|'hash'|'server_name'}] + * [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash|ldap}] BY {'password'|'hash'|'server_name'}] * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] - * [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD|LDAP}] BY {'password'|'hash'|'server_name'}] + * [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash|ldap}] BY {'password'|'hash'|'server_name'}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] @@ -38,6 +38,7 @@ public: String new_name; std::optional authentication; + bool show_password = true; /// formatImpl() will show the password or hash. std::optional hosts; std::optional add_hosts; diff --git a/src/Parsers/ParserCreateUserQuery.cpp b/src/Parsers/ParserCreateUserQuery.cpp index e07d76e6f32..f1b741a1056 100644 --- a/src/Parsers/ParserCreateUserQuery.cpp +++ b/src/Parsers/ParserCreateUserQuery.cpp @@ -35,111 +35,78 @@ namespace } - bool parseByPassword(IParserBase::Pos & pos, Expected & expected, String & password) - { - return IParserBase::wrapParseImpl(pos, [&] - { - if (!ParserKeyword{"BY"}.ignore(pos, expected)) - return false; - - ASTPtr ast; - if (!ParserStringLiteral{}.parse(pos, ast, expected)) - return false; - - password = ast->as().value.safeGet(); - return true; - }); - } - - bool parseAuthentication(IParserBase::Pos & pos, Expected & expected, std::optional & authentication) { return IParserBase::wrapParseImpl(pos, [&] { + if (ParserKeyword{"NOT IDENTIFIED"}.ignore(pos, expected)) + { + authentication = Authentication{Authentication::NO_PASSWORD}; + return true; + } + if (!ParserKeyword{"IDENTIFIED"}.ignore(pos, expected)) return false; - if (!ParserKeyword{"WITH"}.ignore(pos, expected)) - { - String password; - if (!parseByPassword(pos, expected, password)) - return false; + std::optional type; + bool expect_password = false; + bool expect_hash = false; + bool expect_server = false; - authentication = Authentication{Authentication::SHA256_PASSWORD}; - authentication->setPassword(password); - return true; + if (ParserKeyword{"WITH"}.ignore(pos, expected)) + { + for (auto check_type : ext::range(Authentication::MAX_TYPE)) + { + if (ParserKeyword{Authentication::TypeInfo::get(check_type).raw_name}.ignore(pos, expected)) + { + type = check_type; + expect_password = (check_type != Authentication::NO_PASSWORD && check_type != Authentication::LDAP_PASSWORD); + expect_server = (check_type == Authentication::LDAP_PASSWORD); + break; + } + } + + if (!type) + { + if (ParserKeyword{"SHA256_HASH"}.ignore(pos, expected)) + { + type = Authentication::SHA256_PASSWORD; + expect_hash = true; + } + else if (ParserKeyword{"DOUBLE_SHA1_HASH"}.ignore(pos, expected)) + { + type = Authentication::DOUBLE_SHA1_PASSWORD; + expect_hash = true; + } + else + return false; + } } - if (ParserKeyword{"PLAINTEXT_PASSWORD"}.ignore(pos, expected)) + if (!type) { - String password; - if (!parseByPassword(pos, expected, password)) - return false; - - authentication = Authentication{Authentication::PLAINTEXT_PASSWORD}; - authentication->setPassword(password); - return true; + type = Authentication::SHA256_PASSWORD; + expect_password = true; } - if (ParserKeyword{"SHA256_PASSWORD"}.ignore(pos, expected)) + String value; + if (expect_password || expect_hash || expect_server) { - String password; - if (!parseByPassword(pos, expected, password)) + ASTPtr ast; + if (!ParserKeyword{"BY"}.ignore(pos, expected) || !ParserStringLiteral{}.parse(pos, ast, expected)) return false; - authentication = Authentication{Authentication::SHA256_PASSWORD}; - authentication->setPassword(password); - return true; + value = ast->as().value.safeGet(); } - if (ParserKeyword{"SHA256_HASH"}.ignore(pos, expected)) - { - String hash; - if (!parseByPassword(pos, expected, hash)) - return false; + authentication = Authentication{*type}; + if (expect_password) + authentication->setPassword(value); + else if (expect_hash) + authentication->setPasswordHashHex(value); + else if (expect_server) + authentication->setLDAPServerName(value); - authentication = Authentication{Authentication::SHA256_PASSWORD}; - authentication->setPasswordHashHex(hash); - return true; - } - - if (ParserKeyword{"DOUBLE_SHA1_PASSWORD"}.ignore(pos, expected)) - { - String password; - if (!parseByPassword(pos, expected, password)) - return false; - - authentication = Authentication{Authentication::DOUBLE_SHA1_PASSWORD}; - authentication->setPassword(password); - return true; - } - - if (ParserKeyword{"DOUBLE_SHA1_HASH"}.ignore(pos, expected)) - { - String hash; - if (!parseByPassword(pos, expected, hash)) - return false; - - authentication = Authentication{Authentication::DOUBLE_SHA1_PASSWORD}; - authentication->setPasswordHashHex(hash); - return true; - } - - if (ParserKeyword{"LDAP"}.ignore(pos, expected)) - { - String server_name; - if (!parseByPassword(pos, expected, server_name)) - return false; - - authentication = Authentication{Authentication::LDAP_PASSWORD}; - authentication->setLDAPServerName(server_name); - return true; - } - - if (!ParserKeyword{"NO_PASSWORD"}.ignore(pos, expected)) - return false; - - authentication = Authentication{Authentication::NO_PASSWORD}; return true; }); } diff --git a/src/Parsers/ParserCreateUserQuery.h b/src/Parsers/ParserCreateUserQuery.h index e7235279584..b4e1918aaed 100644 --- a/src/Parsers/ParserCreateUserQuery.h +++ b/src/Parsers/ParserCreateUserQuery.h @@ -7,13 +7,13 @@ namespace DB { /** Parses queries like * CREATE USER [IF NOT EXISTS | OR REPLACE] name - * [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH|LDAP}] BY {'password'|'hash'|'server_name'}] + * [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash|ldap}] BY {'password'|'hash'|'server_name'}] * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] - * [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD|LDAP}] BY {'password'|'hash'|'server_name'}] + * [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash|ldap}] BY {'password'|'hash'|'server_name'}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] */ diff --git a/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp b/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp index 003508ab86f..49e85fcc6d3 100644 --- a/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp +++ b/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp @@ -14,9 +14,10 @@ struct PullingAsyncPipelineExecutor::Data { PipelineExecutorPtr executor; std::exception_ptr exception; - std::atomic_bool is_executed = false; + std::atomic_bool is_finished = false; std::atomic_bool has_exception = false; ThreadFromGlobalPool thread; + Poco::Event finish_event; ~Data() { @@ -36,8 +37,11 @@ struct PullingAsyncPipelineExecutor::Data PullingAsyncPipelineExecutor::PullingAsyncPipelineExecutor(QueryPipeline & pipeline_) : pipeline(pipeline_) { - lazy_format = std::make_shared(pipeline.getHeader()); - pipeline.setOutput(lazy_format); + if (!pipeline.isCompleted()) + { + lazy_format = std::make_shared(pipeline.getHeader()); + pipeline.setOutputFormat(lazy_format); + } } PullingAsyncPipelineExecutor::~PullingAsyncPipelineExecutor() @@ -54,7 +58,8 @@ PullingAsyncPipelineExecutor::~PullingAsyncPipelineExecutor() const Block & PullingAsyncPipelineExecutor::getHeader() const { - return lazy_format->getPort(IOutputFormat::PortKind::Main).getHeader(); + return lazy_format ? lazy_format->getPort(IOutputFormat::PortKind::Main).getHeader() + : pipeline.getHeader(); /// Empty. } static void threadFunction(PullingAsyncPipelineExecutor::Data & data, ThreadGroupStatusPtr thread_group, size_t num_threads) @@ -78,6 +83,9 @@ static void threadFunction(PullingAsyncPipelineExecutor::Data & data, ThreadGrou data.exception = std::current_exception(); data.has_exception = true; } + + data.is_finished = true; + data.finish_event.set(); } @@ -99,20 +107,33 @@ bool PullingAsyncPipelineExecutor::pull(Chunk & chunk, uint64_t milliseconds) if (data->has_exception) { /// Finish lazy format in case of exception. Otherwise thread.join() may hung. - lazy_format->finish(); + if (lazy_format) + lazy_format->finish(); + data->has_exception = false; std::rethrow_exception(std::move(data->exception)); } - if (lazy_format->isFinished()) + bool is_execution_finished = lazy_format ? lazy_format->isFinished() + : data->is_finished.load(); + + if (is_execution_finished) { - data->is_executed = true; + /// If lazy format is finished, we don't cancel pipeline but wait for main thread to be finished. + data->is_finished = true; /// Wait thread ant rethrow exception if any. cancel(); return false; } - chunk = lazy_format->getChunk(milliseconds); + if (lazy_format) + { + chunk = lazy_format->getChunk(milliseconds); + return true; + } + + chunk.clear(); + data->finish_event.tryWait(milliseconds); return true; } @@ -147,11 +168,11 @@ bool PullingAsyncPipelineExecutor::pull(Block & block, uint64_t milliseconds) void PullingAsyncPipelineExecutor::cancel() { /// Cancel execution if it wasn't finished. - if (data && !data->is_executed && data->executor) + if (data && !data->is_finished && data->executor) data->executor->cancel(); /// Finish lazy format. Otherwise thread.join() may hung. - if (!lazy_format->isFinished()) + if (lazy_format && !lazy_format->isFinished()) lazy_format->finish(); /// Join thread here to wait for possible exception. @@ -165,12 +186,14 @@ void PullingAsyncPipelineExecutor::cancel() Chunk PullingAsyncPipelineExecutor::getTotals() { - return lazy_format->getTotals(); + return lazy_format ? lazy_format->getTotals() + : Chunk(); } Chunk PullingAsyncPipelineExecutor::getExtremes() { - return lazy_format->getExtremes(); + return lazy_format ? lazy_format->getExtremes() + : Chunk(); } Block PullingAsyncPipelineExecutor::getTotalsBlock() @@ -197,7 +220,9 @@ Block PullingAsyncPipelineExecutor::getExtremesBlock() BlockStreamProfileInfo & PullingAsyncPipelineExecutor::getProfileInfo() { - return lazy_format->getProfileInfo(); + static BlockStreamProfileInfo profile_info; + return lazy_format ? lazy_format->getProfileInfo() + : profile_info; } } diff --git a/src/Processors/Executors/PullingPipelineExecutor.cpp b/src/Processors/Executors/PullingPipelineExecutor.cpp index 375f6c9ed0e..af061a373cc 100644 --- a/src/Processors/Executors/PullingPipelineExecutor.cpp +++ b/src/Processors/Executors/PullingPipelineExecutor.cpp @@ -9,7 +9,7 @@ namespace DB PullingPipelineExecutor::PullingPipelineExecutor(QueryPipeline & pipeline_) : pipeline(pipeline_) { pulling_format = std::make_shared(pipeline.getHeader(), has_data_flag); - pipeline.setOutput(pulling_format); + pipeline.setOutputFormat(pulling_format); } PullingPipelineExecutor::~PullingPipelineExecutor() diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp index df73c93fd55..2873a5417ea 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.cpp @@ -16,12 +16,12 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; + extern const int UNKNOWN_EXCEPTION; extern const int CANNOT_READ_ALL_DATA; } -ArrowBlockInputFormat::ArrowBlockInputFormat(ReadBuffer & in_, const Block & header_) - : IInputFormat(header_, in_) +ArrowBlockInputFormat::ArrowBlockInputFormat(ReadBuffer & in_, const Block & header_, bool stream_) + : IInputFormat(header_, in_), stream{stream_} { prepareReader(); } @@ -30,12 +30,23 @@ Chunk ArrowBlockInputFormat::generate() { Chunk res; const Block & header = getPort().getHeader(); - - if (record_batch_current >= record_batch_total) - return res; - std::vector> single_batch(1); - arrow::Status read_status = file_reader->ReadRecordBatch(record_batch_current, &single_batch[0]); + arrow::Status read_status; + + if (stream) + { + read_status = stream_reader->ReadNext(&single_batch[0]); + if (!single_batch[0]) + return res; + } + else + { + if (record_batch_current >= record_batch_total) + return res; + + read_status = file_reader->ReadRecordBatch(record_batch_current, &single_batch[0]); + } + if (!read_status.ok()) throw Exception{"Error while reading batch of Arrow data: " + read_status.ToString(), ErrorCodes::CANNOT_READ_ALL_DATA}; @@ -57,30 +68,54 @@ void ArrowBlockInputFormat::resetParser() { IInputFormat::resetParser(); - file_reader.reset(); + if (stream) + stream_reader.reset(); + else + file_reader.reset(); prepareReader(); } void ArrowBlockInputFormat::prepareReader() { - arrow::Status open_status = arrow::ipc::RecordBatchFileReader::Open(asArrowFile(in), &file_reader); - if (!open_status.ok()) - throw Exception(open_status.ToString(), ErrorCodes::BAD_ARGUMENTS); - record_batch_total = file_reader->num_record_batches(); + arrow::Status status; + + if (stream) + status = arrow::ipc::RecordBatchStreamReader::Open(asArrowFile(in), &stream_reader); + else + status = arrow::ipc::RecordBatchFileReader::Open(asArrowFile(in), &file_reader); + + if (!status.ok()) + throw Exception{"Error while opening a table: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION}; + + if (stream) + record_batch_total = -1; + else + record_batch_total = file_reader->num_record_batches(); + record_batch_current = 0; } void registerInputFormatProcessorArrow(FormatFactory &factory) { factory.registerInputFormatProcessor( - "Arrow", - [](ReadBuffer & buf, - const Block & sample, - const RowInputFormatParams & /* params */, - const FormatSettings & /* format_settings */) - { - return std::make_shared(buf, sample); - }); + "Arrow", + [](ReadBuffer & buf, + const Block & sample, + const RowInputFormatParams & /* params */, + const FormatSettings & /* format_settings */) + { + return std::make_shared(buf, sample, false); + }); + + factory.registerInputFormatProcessor( + "ArrowStream", + [](ReadBuffer & buf, + const Block & sample, + const RowInputFormatParams & /* params */, + const FormatSettings & /* format_settings */) + { + return std::make_shared(buf, sample, true); + }); } } diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h index 28ca033dd13..5ad112efde9 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h @@ -4,6 +4,7 @@ #include +namespace arrow { class RecordBatchReader; } namespace arrow::ipc { class RecordBatchFileReader; } namespace DB @@ -14,7 +15,7 @@ class ReadBuffer; class ArrowBlockInputFormat : public IInputFormat { public: - ArrowBlockInputFormat(ReadBuffer & in_, const Block & header_); + ArrowBlockInputFormat(ReadBuffer & in_, const Block & header_, bool stream_); void resetParser() override; @@ -24,12 +25,17 @@ protected: Chunk generate() override; private: - void prepareReader(); - -private: + // Whether to use ArrowStream format + bool stream; + // This field is only used for ArrowStream format + std::shared_ptr stream_reader; + // The following fields are used only for Arrow format std::shared_ptr file_reader; + int record_batch_total = 0; int record_batch_current = 0; + + void prepareReader(); }; } diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp index ed33eb08728..ef6bd315319 100644 --- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp @@ -15,8 +15,8 @@ namespace ErrorCodes extern const int UNKNOWN_EXCEPTION; } -ArrowBlockOutputFormat::ArrowBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_) - : IOutputFormat(header_, out_), format_settings{format_settings_}, arrow_ostream{std::make_shared(out_)} +ArrowBlockOutputFormat::ArrowBlockOutputFormat(WriteBuffer & out_, const Block & header_, bool stream_, const FormatSettings & format_settings_) + : IOutputFormat(header_, out_), stream{stream_}, format_settings{format_settings_}, arrow_ostream{std::make_shared(out_)} { } @@ -29,12 +29,7 @@ void ArrowBlockOutputFormat::consume(Chunk chunk) CHColumnToArrowColumn::chChunkToArrowTable(arrow_table, header, chunk, columns_num, "Arrow"); if (!writer) - { - // TODO: should we use arrow::ipc::IpcOptions::alignment? - auto status = arrow::ipc::RecordBatchFileWriter::Open(arrow_ostream.get(), arrow_table->schema(), &writer); - if (!status.ok()) - throw Exception{"Error while opening a table: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION}; - } + prepareWriter(arrow_table->schema()); // TODO: calculate row_group_size depending on a number of rows and table size auto status = writer->WriteTable(*arrow_table, format_settings.arrow.row_group_size); @@ -53,6 +48,20 @@ void ArrowBlockOutputFormat::finalize() } } +void ArrowBlockOutputFormat::prepareWriter(const std::shared_ptr & schema) +{ + arrow::Status status; + + // TODO: should we use arrow::ipc::IpcOptions::alignment? + if (stream) + status = arrow::ipc::RecordBatchStreamWriter::Open(arrow_ostream.get(), schema, &writer); + else + status = arrow::ipc::RecordBatchFileWriter::Open(arrow_ostream.get(), schema, &writer); + + if (!status.ok()) + throw Exception{"Error while opening a table writer: " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION}; +} + void registerOutputFormatProcessorArrow(FormatFactory & factory) { factory.registerOutputFormatProcessor( @@ -62,7 +71,17 @@ void registerOutputFormatProcessorArrow(FormatFactory & factory) FormatFactory::WriteCallback, const FormatSettings & format_settings) { - return std::make_shared(buf, sample, format_settings); + return std::make_shared(buf, sample, false, format_settings); + }); + + factory.registerOutputFormatProcessor( + "ArrowStream", + [](WriteBuffer & buf, + const Block & sample, + FormatFactory::WriteCallback, + const FormatSettings & format_settings) + { + return std::make_shared(buf, sample, true, format_settings); }); } diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h index 9ea84f83a0d..0cc6804705b 100644 --- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h @@ -6,6 +6,7 @@ #include #include "ArrowBufferedStreams.h" +namespace arrow { class Schema; } namespace arrow::ipc { class RecordBatchWriter; } namespace DB @@ -14,7 +15,7 @@ namespace DB class ArrowBlockOutputFormat : public IOutputFormat { public: - ArrowBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_); + ArrowBlockOutputFormat(WriteBuffer & out_, const Block & header_, bool stream_, const FormatSettings & format_settings_); String getName() const override { return "ArrowBlockOutputFormat"; } void consume(Chunk) override; @@ -23,9 +24,12 @@ public: String getContentType() const override { return "application/octet-stream"; } private: + bool stream; const FormatSettings format_settings; std::shared_ptr arrow_ostream; std::shared_ptr writer; + + void prepareWriter(const std::shared_ptr & schema); }; } diff --git a/src/Processors/Pipe.h b/src/Processors/Pipe.h index 984fa7605c6..ec5514915a7 100644 --- a/src/Processors/Pipe.h +++ b/src/Processors/Pipe.h @@ -23,7 +23,7 @@ public: /// Will connect pipes outputs with transform inputs automatically. Pipe(Pipes && pipes, ProcessorPtr transform); /// Create pipe from output port. If pipe was created that way, it possibly will not have tree shape. - Pipe(OutputPort * port); + explicit Pipe(OutputPort * port); Pipe(const Pipe & other) = delete; Pipe(Pipe && other) = default; diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index 13787a3fd3b..92c91a81b8a 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -34,6 +34,14 @@ void QueryPipeline::checkInitialized() throw Exception("QueryPipeline wasn't initialized.", ErrorCodes::LOGICAL_ERROR); } +void QueryPipeline::checkInitializedAndNotCompleted() +{ + checkInitialized(); + + if (streams.empty()) + throw Exception("QueryPipeline was already completed.", ErrorCodes::LOGICAL_ERROR); +} + void QueryPipeline::checkSource(const ProcessorPtr & source, bool can_have_totals) { if (!source->getInputs().empty()) @@ -194,11 +202,11 @@ static ProcessorPtr callProcessorGetter( template void QueryPipeline::addSimpleTransformImpl(const TProcessorGetter & getter) { - checkInitialized(); + checkInitializedAndNotCompleted(); Block header; - auto add_transform = [&](OutputPort *& stream, StreamType stream_type, size_t stream_num [[maybe_unused]] = IProcessor::NO_STREAM) + auto add_transform = [&](OutputPort *& stream, StreamType stream_type) { if (!stream) return; @@ -231,17 +239,14 @@ void QueryPipeline::addSimpleTransformImpl(const TProcessorGetter & getter) if (transform) { -// if (stream_type == StreamType::Main) -// transform->setStream(stream_num); - connect(*stream, transform->getInputs().front()); stream = &transform->getOutputs().front(); processors.emplace_back(std::move(transform)); } }; - for (size_t stream_num = 0; stream_num < streams.size(); ++stream_num) - add_transform(streams[stream_num], StreamType::Main, stream_num); + for (auto & stream : streams) + add_transform(stream, StreamType::Main); add_transform(totals_having_port, StreamType::Totals); add_transform(extremes_port, StreamType::Extremes); @@ -259,9 +264,50 @@ void QueryPipeline::addSimpleTransform(const ProcessorGetterWithStreamKind & get addSimpleTransformImpl(getter); } +void QueryPipeline::setSinks(const ProcessorGetterWithStreamKind & getter) +{ + checkInitializedAndNotCompleted(); + + auto add_transform = [&](OutputPort *& stream, StreamType stream_type) + { + if (!stream) + return; + + auto transform = getter(stream->getHeader(), stream_type); + + if (transform) + { + if (transform->getInputs().size() != 1) + throw Exception("Sink for query pipeline transform should have single input, " + "but " + transform->getName() + " has " + + toString(transform->getInputs().size()) + " inputs.", ErrorCodes::LOGICAL_ERROR); + + if (!transform->getOutputs().empty()) + throw Exception("Sink for query pipeline transform should have no outputs, " + "but " + transform->getName() + " has " + + toString(transform->getOutputs().size()) + " outputs.", ErrorCodes::LOGICAL_ERROR); + } + + if (!transform) + transform = std::make_shared(stream->getHeader()); + + connect(*stream, transform->getInputs().front()); + processors.emplace_back(std::move(transform)); + }; + + for (auto & stream : streams) + add_transform(stream, StreamType::Main); + + add_transform(totals_having_port, StreamType::Totals); + add_transform(extremes_port, StreamType::Extremes); + + streams.clear(); + current_header.clear(); +} + void QueryPipeline::addPipe(Processors pipe) { - checkInitialized(); + checkInitializedAndNotCompleted(); if (pipe.empty()) throw Exception("Can't add empty processors list to QueryPipeline.", ErrorCodes::LOGICAL_ERROR); @@ -298,7 +344,7 @@ void QueryPipeline::addPipe(Processors pipe) void QueryPipeline::addDelayedStream(ProcessorPtr source) { - checkInitialized(); + checkInitializedAndNotCompleted(); checkSource(source, false); assertBlocksHaveEqualStructure(current_header, source->getOutputs().front().getHeader(), "QueryPipeline"); @@ -313,7 +359,7 @@ void QueryPipeline::addDelayedStream(ProcessorPtr source) void QueryPipeline::resize(size_t num_streams, bool force, bool strict) { - checkInitialized(); + checkInitializedAndNotCompleted(); if (!force && num_streams == getNumStreams()) return; @@ -347,7 +393,7 @@ void QueryPipeline::enableQuotaForCurrentStreams() void QueryPipeline::addTotalsHavingTransform(ProcessorPtr transform) { - checkInitialized(); + checkInitializedAndNotCompleted(); if (!typeid_cast(transform.get())) throw Exception("TotalsHavingTransform expected for QueryPipeline::addTotalsHavingTransform.", @@ -370,7 +416,7 @@ void QueryPipeline::addTotalsHavingTransform(ProcessorPtr transform) void QueryPipeline::addDefaultTotals() { - checkInitialized(); + checkInitializedAndNotCompleted(); if (totals_having_port) throw Exception("Totals having transform was already added to pipeline.", ErrorCodes::LOGICAL_ERROR); @@ -392,7 +438,7 @@ void QueryPipeline::addDefaultTotals() void QueryPipeline::addTotals(ProcessorPtr source) { - checkInitialized(); + checkInitializedAndNotCompleted(); if (totals_having_port) throw Exception("Totals having transform was already added to pipeline.", ErrorCodes::LOGICAL_ERROR); @@ -423,7 +469,7 @@ void QueryPipeline::dropTotalsAndExtremes() void QueryPipeline::addExtremesTransform() { - checkInitialized(); + checkInitializedAndNotCompleted(); if (extremes_port) throw Exception("Extremes transform was already added to pipeline.", ErrorCodes::LOGICAL_ERROR); @@ -450,7 +496,7 @@ void QueryPipeline::addExtremesTransform() void QueryPipeline::addCreatingSetsTransform(ProcessorPtr transform) { - checkInitialized(); + checkInitializedAndNotCompleted(); if (!typeid_cast(transform.get())) throw Exception("CreatingSetsTransform expected for QueryPipeline::addExtremesTransform.", @@ -467,14 +513,14 @@ void QueryPipeline::addCreatingSetsTransform(ProcessorPtr transform) processors.emplace_back(std::move(concat)); } -void QueryPipeline::setOutput(ProcessorPtr output) +void QueryPipeline::setOutputFormat(ProcessorPtr output) { - checkInitialized(); + checkInitializedAndNotCompleted(); auto * format = dynamic_cast(output.get()); if (!format) - throw Exception("IOutputFormat processor expected for QueryPipeline::setOutput.", ErrorCodes::LOGICAL_ERROR); + throw Exception("IOutputFormat processor expected for QueryPipeline::setOutputFormat.", ErrorCodes::LOGICAL_ERROR); if (output_format) throw Exception("QueryPipeline already has output.", ErrorCodes::LOGICAL_ERROR); @@ -507,19 +553,25 @@ void QueryPipeline::setOutput(ProcessorPtr output) connect(*totals_having_port, totals); connect(*extremes_port, extremes); + streams.clear(); + current_header.clear(); + extremes_port = nullptr; + totals_having_port = nullptr; + initRowsBeforeLimit(); } void QueryPipeline::unitePipelines( std::vector && pipelines, const Block & common_header) { - checkInitialized(); - - addSimpleTransform([&](const Block & header) + if (initialized()) { - return std::make_shared( - header, common_header, ConvertingTransform::MatchColumnsMode::Position); - }); + addSimpleTransform([&](const Block & header) + { + return std::make_shared( + header, common_header, ConvertingTransform::MatchColumnsMode::Position); + }); + } std::vector extremes; std::vector totals; @@ -534,11 +586,14 @@ void QueryPipeline::unitePipelines( { pipeline.checkInitialized(); - pipeline.addSimpleTransform([&](const Block & header) + if (!pipeline.isCompleted()) { - return std::make_shared( - header, common_header, ConvertingTransform::MatchColumnsMode::Position); - }); + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared( + header, common_header, ConvertingTransform::MatchColumnsMode::Position); + }); + } if (pipeline.extremes_port) { @@ -703,6 +758,11 @@ void QueryPipeline::initRowsBeforeLimit() Pipe QueryPipeline::getPipe() && { resize(1); + return std::move(std::move(*this).getPipes()[0]); +} + +Pipes QueryPipeline::getPipes() && +{ Pipe pipe(std::move(processors), streams.at(0), totals_having_port, extremes_port); pipe.max_parallel_streams = streams.maxParallelStreams(); @@ -721,15 +781,19 @@ Pipe QueryPipeline::getPipe() && if (extremes_port) pipe.setExtremesPort(extremes_port); - return pipe; + Pipes pipes; + pipes.emplace_back(std::move(pipe)); + + for (size_t i = 1; i < streams.size(); ++i) + pipes.emplace_back(Pipe(streams[i])); + + return pipes; } PipelineExecutorPtr QueryPipeline::execute() { - checkInitialized(); - - if (!output_format) - throw Exception("Cannot execute pipeline because it doesn't have output.", ErrorCodes::LOGICAL_ERROR); + if (!isCompleted()) + throw Exception("Cannot execute pipeline because it is not completed.", ErrorCodes::LOGICAL_ERROR); return std::make_shared(processors, process_list_element); } diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index 45e38ffa715..129b7f5ae3c 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -28,6 +28,7 @@ private: { public: auto size() const { return data.size(); } + bool empty() const { return size() == 0; } auto begin() { return data.begin(); } auto end() { return data.end(); } auto & front() { return data.front(); } @@ -81,6 +82,7 @@ public: void init(Pipes pipes); void init(Pipe pipe); /// Simple init for single pipe bool initialized() { return !processors.empty(); } + bool isCompleted() { return initialized() && streams.empty(); } /// Type of logical data stream for simple transform. /// Sometimes it's important to know which part of pipeline we are working for. @@ -95,13 +97,23 @@ public: using ProcessorGetter = std::function; using ProcessorGetterWithStreamKind = std::function; + /// Add transform with simple input and simple output for each port. void addSimpleTransform(const ProcessorGetter & getter); void addSimpleTransform(const ProcessorGetterWithStreamKind & getter); + /// Add several processors. They must have same header for inputs and same for outputs. + /// Total number of inputs must be the same as the number of streams. Output ports will become new streams. void addPipe(Processors pipe); + /// Add TotalsHavingTransform. Resize pipeline to single input. Adds totals port. void addTotalsHavingTransform(ProcessorPtr transform); + /// Add transform which calculates extremes. This transform adds extremes port and doesn't change inputs number. void addExtremesTransform(); + /// Adds transform which creates sets. It will be executed before reading any data from input ports. void addCreatingSetsTransform(ProcessorPtr transform); - void setOutput(ProcessorPtr output); + /// Resize pipeline to single output and add IOutputFormat. Pipeline will be completed after this transformation. + void setOutputFormat(ProcessorPtr output); + /// Sink is a processor with single input port and no output ports. Creates sink for each output port. + /// Pipeline will be completed after this transformation. + void setSinks(const ProcessorGetterWithStreamKind & getter); /// Add totals which returns one chunk with single row with defaults. void addDefaultTotals(); @@ -118,6 +130,7 @@ public: /// Check if resize transform was used. (In that case another distinct transform will be added). bool hasMixedStreams() const { return has_resize || hasMoreThanOneStream(); } + /// Changes the number of input ports if needed. Adds ResizeTransform. void resize(size_t num_streams, bool force = false, bool strict = false); void enableQuotaForCurrentStreams(); @@ -155,8 +168,9 @@ public: /// Set upper limit for the recommend number of threads void setMaxThreads(size_t max_threads_) { max_threads = max_threads_; } - /// Convert query pipeline to single pipe. + /// Convert query pipeline to single or several pipes. Pipe getPipe() &&; + Pipes getPipes() &&; private: /// Destruction order: processors, header, locks, temporary storages, local contexts @@ -193,6 +207,7 @@ private: QueryStatus * process_list_element = nullptr; void checkInitialized(); + void checkInitializedAndNotCompleted(); static void checkSource(const ProcessorPtr & source, bool can_have_totals); template diff --git a/src/Processors/Transforms/PartialSortingTransform.cpp b/src/Processors/Transforms/PartialSortingTransform.cpp index 018614f0165..f68a415117b 100644 --- a/src/Processors/Transforms/PartialSortingTransform.cpp +++ b/src/Processors/Transforms/PartialSortingTransform.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { @@ -11,6 +12,38 @@ PartialSortingTransform::PartialSortingTransform( { } +static ColumnRawPtrs extractColumns(const Block & block, const SortDescription & description) +{ + size_t size = description.size(); + ColumnRawPtrs res; + res.reserve(size); + + for (size_t i = 0; i < size; ++i) + { + const IColumn * column = !description[i].column_name.empty() + ? block.getByName(description[i].column_name).column.get() + : block.safeGetByPosition(description[i].column_number).column.get(); + res.emplace_back(column); + } + + return res; +} + +bool less(const ColumnRawPtrs & lhs, UInt64 lhs_row_num, + const ColumnRawPtrs & rhs, UInt64 rhs_row_num, const SortDescription & description) +{ + size_t size = description.size(); + for (size_t i = 0; i < size; ++i) + { + int res = description[i].direction * lhs[i]->compareAt(lhs_row_num, rhs_row_num, *rhs[i], 1); + if (res < 0) + return true; + else if (res > 0) + return false; + } + return false; +} + void PartialSortingTransform::transform(Chunk & chunk) { if (read_rows) @@ -19,7 +52,42 @@ void PartialSortingTransform::transform(Chunk & chunk) auto block = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns()); chunk.clear(); + ColumnRawPtrs block_columns; + UInt64 rows_num = block.rows(); + + if (!threshold_block_columns.empty()) + { + IColumn::Filter filter(rows_num, 0); + block_columns = extractColumns(block, description); + size_t filtered_count = 0; + + for (UInt64 i = 0; i < rows_num; ++i) + { + if (less(threshold_block_columns, limit - 1, block_columns, i, description)) + { + ++filtered_count; + filter[i] = 1; + } + } + + if (filtered_count) + { + for (auto & column : block.getColumns()) + { + column = column->filter(filter, filtered_count); + } + } + } + sortBlock(block, description, limit); + + if (limit && limit < block.rows() && + (threshold_block_columns.empty() || less(block_columns, limit - 1, threshold_block_columns, limit - 1, description))) + { + threshold_block = block.cloneWithColumns(block.getColumns()); + threshold_block_columns = extractColumns(threshold_block, description); + } + chunk.setColumns(block.getColumns(), block.rows()); } diff --git a/src/Processors/Transforms/PartialSortingTransform.h b/src/Processors/Transforms/PartialSortingTransform.h index 47ac90c6904..d6749e4dfad 100644 --- a/src/Processors/Transforms/PartialSortingTransform.h +++ b/src/Processors/Transforms/PartialSortingTransform.h @@ -29,6 +29,8 @@ private: SortDescription description; UInt64 limit; RowsBeforeLimitCounterPtr read_rows; + Block threshold_block; + ColumnRawPtrs threshold_block_columns; }; } diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 190c549361f..b8f3660179d 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -262,8 +262,8 @@ void TCPHandler::runImpl() else if (state.need_receive_data_for_input) { /// It is special case for input(), all works for reading data from client will be done in callbacks. - /// state.io.in is NullAndDoCopyBlockInputStream so read it once. - state.io.in->read(); + auto executor = state.io.pipeline.execute(); + executor->execute(state.io.pipeline.getNumThreads()); state.io.onFinish(); } else if (state.io.pipeline.initialized()) diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index fdcbe8e69f9..4dd62db0965 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -110,8 +110,9 @@ void StorageDistributedDirectoryMonitor::flushAllData() { if (!quit) { + CurrentMetrics::Increment metric_pending_files{CurrentMetrics::DistributedFilesToInsert, 0}; std::unique_lock lock{mutex}; - processFiles(); + processFiles(metric_pending_files); } } @@ -131,6 +132,9 @@ void StorageDistributedDirectoryMonitor::run() { std::unique_lock lock{mutex}; + /// This metric will be updated with the number of pending files later. + CurrentMetrics::Increment metric_pending_files{CurrentMetrics::DistributedFilesToInsert, 0}; + bool do_sleep = false; while (!quit) { @@ -139,7 +143,7 @@ void StorageDistributedDirectoryMonitor::run() { try { - do_sleep = !processFiles(); + do_sleep = !processFiles(metric_pending_files); } catch (...) { @@ -222,7 +226,7 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri } -bool StorageDistributedDirectoryMonitor::processFiles() +bool StorageDistributedDirectoryMonitor::processFiles(CurrentMetrics::Increment & metric_pending_files) { std::map files; @@ -236,14 +240,16 @@ bool StorageDistributedDirectoryMonitor::processFiles() files[parse(file_path.getBaseName())] = file_path_str; } + /// Note: the value of this metric will be kept if this function will throw an exception. + /// This is needed, because in case of exception, files still pending. + metric_pending_files.changeTo(files.size()); + if (files.empty()) return false; - CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedFilesToInsert, CurrentMetrics::Value(files.size())}; - if (should_batch_inserts) { - processFilesWithBatching(files); + processFilesWithBatching(files, metric_pending_files); } else { @@ -252,14 +258,14 @@ bool StorageDistributedDirectoryMonitor::processFiles() if (quit) return true; - processFile(file.second); + processFile(file.second, metric_pending_files); } } return true; } -void StorageDistributedDirectoryMonitor::processFile(const std::string & file_path) +void StorageDistributedDirectoryMonitor::processFile(const std::string & file_path, CurrentMetrics::Increment & metric_pending_files) { LOG_TRACE(log, "Started processing `{}`", file_path); auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(storage.global_context->getSettingsRef()); @@ -289,6 +295,7 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa } Poco::File{file_path}.remove(); + metric_pending_files.sub(); LOG_TRACE(log, "Finished processing `{}`", file_path); } @@ -584,7 +591,9 @@ bool StorageDistributedDirectoryMonitor::scheduleAfter(size_t ms) return task_handle->scheduleAfter(ms, false); } -void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map & files) +void StorageDistributedDirectoryMonitor::processFilesWithBatching( + const std::map & files, + CurrentMetrics::Increment & metric_pending_files) { std::unordered_set file_indices_to_skip; @@ -596,6 +605,7 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map batch.readText(in); file_indices_to_skip.insert(batch.file_indices.begin(), batch.file_indices.end()); batch.send(); + metric_pending_files.sub(batch.file_indices.size()); } std::unordered_map header_to_batch; @@ -656,13 +666,17 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map batch.total_bytes += total_bytes; if (batch.isEnoughSize()) + { batch.send(); + metric_pending_files.sub(batch.file_indices.size()); + } } for (auto & kv : header_to_batch) { Batch & batch = kv.second; batch.send(); + metric_pending_files.sub(batch.file_indices.size()); } /// current_batch.txt will not exist if there was no send diff --git a/src/Storages/Distributed/DirectoryMonitor.h b/src/Storages/Distributed/DirectoryMonitor.h index 77abf35630c..e2a913ee1ef 100644 --- a/src/Storages/Distributed/DirectoryMonitor.h +++ b/src/Storages/Distributed/DirectoryMonitor.h @@ -9,6 +9,8 @@ #include +namespace CurrentMetrics { class Increment; } + namespace DB { @@ -37,9 +39,9 @@ public: bool scheduleAfter(size_t ms); private: void run(); - bool processFiles(); - void processFile(const std::string & file_path); - void processFilesWithBatching(const std::map & files); + bool processFiles(CurrentMetrics::Increment & metric_pending_files); + void processFile(const std::string & file_path, CurrentMetrics::Increment & metric_pending_files); + void processFilesWithBatching(const std::map & files, CurrentMetrics::Increment & metric_pending_files); static bool isFileBrokenErrorCode(int code); void markAsBroken(const std::string & file_path) const; diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index 636c7f9d64d..01c85c5238a 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -19,6 +19,8 @@ #include #include #include +#include +#include namespace DB @@ -62,29 +64,42 @@ Pipes StorageView::read( if (context.getSettings().enable_optimize_predicate_expression) current_inner_query = getRuntimeViewQuery(*query_info.query->as(), context); - QueryPipeline pipeline; InterpreterSelectWithUnionQuery interpreter(current_inner_query, context, {}, column_names); /// FIXME res may implicitly use some objects owned be pipeline, but them will be destructed after return if (query_info.force_tree_shaped_pipeline) { + QueryPipeline pipeline; BlockInputStreams streams = interpreter.executeWithMultipleStreams(pipeline); + + for (auto & stream : streams) + { + stream = std::make_shared(stream); + stream = std::make_shared(stream, getSampleBlockForColumns(column_names), + ConvertingBlockInputStream::MatchColumnsMode::Name); + } + for (auto & stream : streams) pipes.emplace_back(std::make_shared(std::move(stream))); } else - /// TODO: support multiple streams here. Need more general interface than pipes. - pipes.emplace_back(interpreter.executeWithProcessors().getPipe()); - - /// It's expected that the columns read from storage are not constant. - /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery. - for (auto & pipe : pipes) { - pipe.addSimpleTransform(std::make_shared(pipe.getHeader())); + auto pipeline = interpreter.executeWithProcessors(); + + /// It's expected that the columns read from storage are not constant. + /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery. + pipeline.addSimpleTransform([](const Block & header) + { + return std::make_shared(header); + }); /// And also convert to expected structure. - pipe.addSimpleTransform(std::make_shared( - pipe.getHeader(), getSampleBlockForColumns(column_names), - ConvertingTransform::MatchColumnsMode::Name)); + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, getSampleBlockForColumns(column_names), + ConvertingTransform::MatchColumnsMode::Name); + }); + + pipes = std::move(pipeline).getPipes(); } return pipes; diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index d0e042d054f..e0755fe59ab 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -15,12 +16,26 @@ namespace DB { +namespace +{ + DataTypeEnum8::Values getAuthenticationTypeEnumValues() + { + DataTypeEnum8::Values enum_values; + for (auto type : ext::range(Authentication::MAX_TYPE)) + enum_values.emplace_back(Authentication::TypeInfo::get(type).name, static_cast(type)); + return enum_values; + } +} + + NamesAndTypesList StorageSystemUsers::getNamesAndTypes() { NamesAndTypesList names_and_types{ {"name", std::make_shared()}, {"id", std::make_shared()}, {"storage", std::make_shared()}, + {"auth_type", std::make_shared(getAuthenticationTypeEnumValues())}, + {"auth_params", std::make_shared(std::make_shared())}, {"host_ip", std::make_shared(std::make_shared())}, {"host_names", std::make_shared(std::make_shared())}, {"host_names_regexp", std::make_shared(std::make_shared())}, @@ -43,6 +58,9 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context & auto & column_name = assert_cast(*res_columns[column_index++]); auto & column_id = assert_cast(*res_columns[column_index++]).getData(); auto & column_storage = assert_cast(*res_columns[column_index++]); + auto & column_auth_type = assert_cast(*res_columns[column_index++]).getData(); + auto & column_auth_params = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_auth_params_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_host_ip = assert_cast(assert_cast(*res_columns[column_index]).getData()); auto & column_host_ip_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_host_names = assert_cast(assert_cast(*res_columns[column_index]).getData()); @@ -60,12 +78,15 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context & auto add_row = [&](const String & name, const UUID & id, const String & storage_name, + const Authentication & authentication, const AllowedClientHosts & allowed_hosts, const ExtendedRoleSet & default_roles) { column_name.insertData(name.data(), name.length()); column_id.push_back(id); column_storage.insertData(storage_name.data(), storage_name.length()); + column_auth_type.push_back(static_cast(authentication.getType())); + column_auth_params_offsets.push_back(column_auth_params.size()); if (allowed_hosts.containsAnyHost()) { @@ -128,7 +149,7 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context & if (!storage) continue; - add_row(user->getName(), id, storage->getStorageName(), user->allowed_client_hosts, user->default_roles); + add_row(user->getName(), id, storage->getStorageName(), user->authentication, user->allowed_client_hosts, user->default_roles); } } diff --git a/tests/integration/test_disk_access_storage/test.py b/tests/integration/test_disk_access_storage/test.py index 315440b4358..a47af5ad5b8 100644 --- a/tests/integration/test_disk_access_storage/test.py +++ b/tests/integration/test_disk_access_storage/test.py @@ -39,7 +39,7 @@ def test_create(): def check(): assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n" - assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 HOST LOCAL DEFAULT ROLE rx\n" + assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE rx\n" assert instance.query("SHOW CREATE ROW POLICY p ON mydb.mytable") == "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a < 1000 TO u1, u2\n" assert instance.query("SHOW CREATE QUOTA q") == "CREATE QUOTA q KEYED BY \\'none\\' FOR INTERVAL 1 HOUR MAX QUERIES 100 TO ALL EXCEPT rx\n" assert instance.query("SHOW GRANTS FOR u1") == "" @@ -69,7 +69,7 @@ def test_alter(): def check(): assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n" - assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 HOST LOCAL DEFAULT ROLE ry\n" + assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE ry\n" assert instance.query("SHOW GRANTS FOR u1") == "GRANT SELECT ON mydb.mytable TO u1\n" assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx, ry TO u2\n" assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s2\n" diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index 8df68547f38..7054ce28e59 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -155,9 +155,9 @@ def test_introspection(): assert instance.query("SHOW ENABLED ROLES", user='A') == TSV([[ "R1", 0, 1, 1 ]]) assert instance.query("SHOW ENABLED ROLES", user='B') == TSV([[ "R2", 1, 1, 1 ]]) - assert instance.query("SELECT name, storage, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\ - TSV([[ "A", "disk", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ], - [ "B", "disk", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]]) + assert instance.query("SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\ + TSV([[ "A", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ], + [ "B", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]]) assert instance.query("SELECT name, storage from system.roles WHERE name IN ('R1', 'R2') ORDER BY name") ==\ TSV([[ "R1", "disk" ], diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py index a1884d059c7..71496c6dbf2 100644 --- a/tests/integration/test_row_policy/test.py +++ b/tests/integration/test_row_policy/test.py @@ -42,8 +42,13 @@ def started_cluster(): CREATE TABLE mydb.`.filtered_table4` (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; INSERT INTO mydb.`.filtered_table4` values (0, 0), (0, 1), (1, 0), (1, 1); + + CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; ''') + node.query("INSERT INTO mydb.local values (2, 0), (2, 1), (1, 0), (1, 1)") + node2.query("INSERT INTO mydb.local values (3, 0), (3, 1), (1, 0), (1, 1)") + yield cluster finally: @@ -122,6 +127,17 @@ def test_single_table_name(): assert node.query("SELECT a + b = 1 FROM mydb.filtered_table3") == TSV([[1], [1]]) +def test_policy_from_users_xml_affects_only_user_assigned(): + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1,0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table1", user="another") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) + + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table2", user="another") == TSV([[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) + + assert node.query("SELECT * FROM mydb.local") == TSV([[1,0], [1, 1], [2, 0], [2, 1]]) + assert node.query("SELECT * FROM mydb.local", user="another") == TSV([[1, 0], [1, 1]]) + + def test_custom_table_name(): copy_policy_xml('multiple_tags_with_table_names.xml') assert node.query("SELECT * FROM mydb.table") == TSV([[1, 0], [1, 1]]) @@ -286,9 +302,5 @@ def test_miscellaneous_engines(): # DistributedMergeTree node.query("DROP TABLE IF EXISTS mydb.not_filtered_table") node.query("CREATE TABLE mydb.not_filtered_table (a UInt8, b UInt8) ENGINE Distributed('test_local_cluster', mydb, local)") - node.query("CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a") - node2.query("CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a") - node.query("INSERT INTO mydb.local values (2, 0), (2, 1), (1, 0), (1, 1)") - node2.query("INSERT INTO mydb.local values (3, 0), (3, 1), (1, 0), (1, 1)") assert node.query("SELECT * FROM mydb.not_filtered_table", user="another") == TSV([[1, 0], [1, 1], [1, 0], [1, 1]]) assert node.query("SELECT sum(a), b FROM mydb.not_filtered_table GROUP BY b ORDER BY b", user="another") == TSV([[2, 0], [2, 1]]) diff --git a/tests/performance/leftpad.xml b/tests/performance/leftpad.xml index 4349c6ac0e5..199f9224b89 100644 --- a/tests/performance/leftpad.xml +++ b/tests/performance/leftpad.xml @@ -6,7 +6,6 @@ - diff --git a/tests/performance/sort_radix_trivial.xml b/tests/performance/sort_radix_trivial.xml index fb3dc838a9c..096c3dd0854 100644 --- a/tests/performance/sort_radix_trivial.xml +++ b/tests/performance/sort_radix_trivial.xml @@ -1,5 +1,5 @@ - SELECT rand32() AS x FROM numbers(1000000) ORDER BY x - SELECT rand64() AS x FROM numbers(1000000) ORDER BY x - SELECT 1 / rand64() AS x FROM numbers(1000000) ORDER BY x + SELECT rand32() AS x FROM numbers(1000000) ORDER BY x FORMAT Null + SELECT rand64() AS x FROM numbers(1000000) ORDER BY x FORMAT Null + SELECT 1 / rand64() AS x FROM numbers(1000000) ORDER BY x FORMAT Null diff --git a/tests/queries/0_stateless/00825_protobuf_format_input.reference b/tests/queries/0_stateless/00825_protobuf_format_input.reference index 884cc74c4e5..0c56bc4ebf0 100644 --- a/tests/queries/0_stateless/00825_protobuf_format_input.reference +++ b/tests/queries/0_stateless/00825_protobuf_format_input.reference @@ -8,3 +8,4 @@ a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +7495123456 0 0 2 4 3 9 +ok diff --git a/tests/queries/0_stateless/00825_protobuf_format_input.sh b/tests/queries/0_stateless/00825_protobuf_format_input.sh index d28b70bb002..1c915bc3f24 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_input.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_input.sh @@ -3,7 +3,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -set -e -o pipefail +set -eo pipefail # Run the client. $CLICKHOUSE_CLIENT --multiquery <<'EOF' @@ -48,5 +48,12 @@ source $CURDIR/00825_protobuf_format_input.insh $CLICKHOUSE_CLIENT --query "SELECT * FROM in_persons_00825 ORDER BY uuid;" $CLICKHOUSE_CLIENT --query "SELECT * FROM in_squares_00825 ORDER BY number;" +# Try to input malformed data. +set +eo pipefail +echo -ne '\xe0\x80\x3f\x0b' \ + | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'" 2>&1 \ + | grep -qF "Protobuf messages are corrupted" && echo "ok" || echo "fail" +set -eo pipefail + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_persons_00825;" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_squares_00825;" diff --git a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference index 59c88f013dd..03ed07cf1a4 100644 --- a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference +++ b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference @@ -1,4 +1,5 @@ Instruction check fail. The CPU does not support SSSE3 instruction set. Instruction check fail. The CPU does not support SSE4.1 instruction set. Instruction check fail. The CPU does not support SSE4.2 instruction set. +Instruction check fail. The CPU does not support POPCNT instruction set. 1 diff --git a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh index 15466e1889a..5ae4f8b3dd2 100755 --- a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh +++ b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh @@ -9,8 +9,13 @@ ${CLICKHOUSE_LOCAL} --query "SELECT max(value LIKE '%sanitize%') FROM system.bui command=$(command -v ${CLICKHOUSE_LOCAL}) -qemu-x86_64-static -cpu qemu64 $command --query "SELECT 1" 2>&1 | grep -v -F "warning: TCG doesn't support requested feature" ||: -qemu-x86_64-static -cpu qemu64,+ssse3 $command --query "SELECT 1" 2>&1 | grep -v -F "warning: TCG doesn't support requested feature" ||: -qemu-x86_64-static -cpu qemu64,+ssse3,+sse4.1 $command --query "SELECT 1" 2>&1 | grep -v -F "warning: TCG doesn't support requested feature" ||: -qemu-x86_64-static -cpu qemu64,+ssse3,+sse4.1,+sse4.2 $command --query "SELECT 1" 2>&1 | grep -v -F "warning: TCG doesn't support requested feature" ||: +function run_with_cpu() +{ + qemu-x86_64-static -cpu "$@" $command --query "SELECT 1" 2>&1 | grep -v -F "warning: TCG doesn't support requested feature" ||: +} +run_with_cpu qemu64 +run_with_cpu qemu64,+ssse3 +run_with_cpu qemu64,+ssse3,+sse4.1 +run_with_cpu qemu64,+ssse3,+sse4.1,+sse4.2 +run_with_cpu qemu64,+ssse3,+sse4.1,+sse4.2,+popcnt diff --git a/tests/queries/0_stateless/01273_arrow.reference b/tests/queries/0_stateless/01273_arrow.reference index 0f4be2c74a0..0dc503f65e4 100644 --- a/tests/queries/0_stateless/01273_arrow.reference +++ b/tests/queries/0_stateless/01273_arrow.reference @@ -31,8 +31,6 @@ 992 991 990 -ContextLock Number of times the lock of Context was acquired or tried to acquire. This is global lock. -Query Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries. original: -128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 -108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 diff --git a/tests/queries/0_stateless/01273_arrow.sh b/tests/queries/0_stateless/01273_arrow.sh index f659a81f118..deb0aa20ccc 100755 --- a/tests/queries/0_stateless/01273_arrow.sh +++ b/tests/queries/0_stateless/01273_arrow.sh @@ -34,14 +34,6 @@ ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_numbers ORDER BY number DESC L ${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_numbers" -${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_events" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_events (event String, value UInt64, description String) ENGINE = Memory" -${CLICKHOUSE_CLIENT} --query="SELECT * FROM system.events FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_events FORMAT Arrow" -${CLICKHOUSE_CLIENT} --query="SELECT event, description FROM arrow_events WHERE event IN ('ContextLock', 'Query') ORDER BY event" -${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_events" - - - ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types1" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types2" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types3" @@ -61,8 +53,6 @@ ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( -128, # max ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( 127, 255, 32767, 65535, 2147483647, 4294967295, 9223372036854775807, 9223372036854775807, -1.032, -1.064, 'string-2', 'fixedstring-2', '2004-06-07', '2004-02-03 04:05:06')" -# 'SELECT -127,-128,-129,126,127,128,255,256,257,-32767,-32768,-32769,32766,32767,32768,65535,65536,65537, -2147483647,-2147483648,-2147483649,2147483646,2147483647,2147483648,4294967295,4294967296,4294967297, -9223372036854775807,-9223372036854775808,9223372036854775806,9223372036854775807,9223372036854775808,18446744073709551615'; - ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types2 FORMAT Arrow" echo original: @@ -98,7 +88,6 @@ ${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types5 (int8 Nullable(Int ${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types6 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime)) ENGINE = Memory" ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types5 values ( NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 ORDER BY int8 FORMAT Arrow" > ${CLICKHOUSE_TMP}/arrow_all_types_5.arrow -#${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT Arrow" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT Arrow" ${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT Arrow" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT Arrow" echo dest from null: diff --git a/tests/queries/0_stateless/01273_arrow_stream.reference b/tests/queries/0_stateless/01273_arrow_stream.reference new file mode 100644 index 00000000000..0dc503f65e4 --- /dev/null +++ b/tests/queries/0_stateless/01273_arrow_stream.reference @@ -0,0 +1,60 @@ +9999 +9998 +9997 +9996 +9995 +9994 +9993 +9992 +9991 +9990 +99999 +99998 +99997 +99996 +99995 +99994 +99993 +99992 +99991 +99990 +2 +1 +0 +999 +998 +997 +996 +995 +994 +993 +992 +991 +990 +original: +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 +converted: +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 +diff: +dest: +79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 1970-01-01 06:29:04 +80 81 82 83 84 85 86 87 88 89 str02 fstr2\0\0\0\0\0\0\0\0\0\0 2005-03-04 2006-08-09 10:11:12 +min: +-128 0 0 0 0 0 0 0 -1 -1 string-1\0\0\0\0\0\0\0 fixedstring-1\0\0 2003-04-05 2003-02-03 +-108 108 8 92 -8 108 -40 -116 -1 -1 string-0\0\0\0\0\0\0\0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 +79 81 82 83 84 85 86 87 88 89 str01\0\0\0\0\0\0\0\0\0\0 fstr1\0\0\0\0\0\0\0\0\0\0 2003-03-04 2004-05-06 +127 -1 -1 -1 -1 -1 -1 -1 -1 -1 string-2\0\0\0\0\0\0\0 fixedstring-2\0\0 2004-06-07 2004-02-03 +max: +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1 -1 string-1 fixedstring-1\0\0 1970-01-01 06:22:27 2003-02-03 04:05:06 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1 -1 string-0 fixedstring\0\0\0\0 1970-01-01 06:09:16 2002-02-03 04:05:06 +80 81 82 83 84 85 86 87 88 89 str02 fstr2 2005-03-04 05:06:07 2006-08-09 10:11:12 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1 -1 string-2 fixedstring-2\0\0 1970-01-01 06:29:36 2004-02-03 04:05:06 +dest from null: +-128 0 -32768 0 -2147483648 0 -9223372036854775808 0 -1.032 -1.064 string-1 fixedstring-1\0\0 2003-04-05 2003-02-03 04:05:06 +-108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 +127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 +\N \N \N \N \N \N \N \N \N \N \N \N \N \N diff --git a/tests/queries/0_stateless/01273_arrow_stream.sh b/tests/queries/0_stateless/01273_arrow_stream.sh new file mode 100755 index 00000000000..a4bb403cb5a --- /dev/null +++ b/tests/queries/0_stateless/01273_arrow_stream.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash + +set -e + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CUR_DIR/../shell_config.sh + + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS contributors" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE contributors (name String) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM system.contributors ORDER BY name DESC FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO contributors FORMAT ArrowStream" +# random results +${CLICKHOUSE_CLIENT} --query="SELECT * FROM contributors LIMIT 10" > /dev/null +${CLICKHOUSE_CLIENT} --query="DROP TABLE contributors" + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_numbers" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_numbers (number UInt64) ENGINE = Memory" +# less than default block size (65k) +${CLICKHOUSE_CLIENT} --query="SELECT * FROM system.numbers LIMIT 10000 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_numbers FORMAT ArrowStream" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_numbers ORDER BY number DESC LIMIT 10" +${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE arrow_numbers" + +# More than default block size +${CLICKHOUSE_CLIENT} --query="SELECT * FROM system.numbers LIMIT 100000 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_numbers FORMAT ArrowStream" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_numbers ORDER BY number DESC LIMIT 10" +${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE arrow_numbers" + +${CLICKHOUSE_CLIENT} --max_block_size=2 --query="SELECT * FROM system.numbers LIMIT 3 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_numbers FORMAT ArrowStream" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_numbers ORDER BY number DESC LIMIT 10" + +${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE arrow_numbers" +${CLICKHOUSE_CLIENT} --max_block_size=1 --query="SELECT * FROM system.numbers LIMIT 1000 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_numbers FORMAT ArrowStream" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_numbers ORDER BY number DESC LIMIT 10" + +${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_numbers" + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types1" +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types2" +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types3" +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types4" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types1 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types2 (int8 Int8, uint8 UInt8, int16 Int16, uint16 UInt16, int32 Int32, uint32 UInt32, int64 Int64, uint64 UInt64, float32 Float32, float64 Float64, string String, fixedstring FixedString(15), date Date, datetime DateTime) ENGINE = Memory" +# convert min type +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types3 (int8 Int8, uint8 Int8, int16 Int8, uint16 Int8, int32 Int8, uint32 Int8, int64 Int8, uint64 Int8, float32 Int8, float64 Int8, string FixedString(15), fixedstring FixedString(15), date Date, datetime Date) ENGINE = Memory" +# convert max type +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types4 (int8 Int64, uint8 Int64, int16 Int64, uint16 Int64, int32 Int64, uint32 Int64, int64 Int64, uint64 Int64, float32 Int64, float64 Int64, string String, fixedstring String, date DateTime, datetime DateTime) ENGINE = Memory" + +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( -108, 108, -1016, 1116, -1032, 1132, -1064, 1164, -1.032, -1.064, 'string-0', 'fixedstring', '2001-02-03', '2002-02-03 04:05:06')" + +# min +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( -128, 0, -32768, 0, -2147483648, 0, -9223372036854775808, 0, -1.032, -1.064, 'string-1', 'fixedstring-1', '2003-04-05', '2003-02-03 04:05:06')" + +# max +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types1 values ( 127, 255, 32767, 65535, 2147483647, 4294967295, 9223372036854775807, 9223372036854775807, -1.032, -1.064, 'string-2', 'fixedstring-2', '2004-06-07', '2004-02-03 04:05:06')" + +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types2 FORMAT ArrowStream" + +echo original: +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8" | tee ${CLICKHOUSE_TMP}/arrow_all_types_1.dump +echo converted: +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types2 ORDER BY int8" | tee ${CLICKHOUSE_TMP}/arrow_all_types_2.dump +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT ArrowStream" > ${CLICKHOUSE_TMP}/arrow_all_types_1.arrow +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types2 ORDER BY int8 FORMAT ArrowStream" > ${CLICKHOUSE_TMP}/arrow_all_types_2.arrow +echo diff: +diff ${CLICKHOUSE_TMP}/arrow_all_types_1.dump ${CLICKHOUSE_TMP}/arrow_all_types_2.dump + +${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE arrow_types2" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types3 values ( 79, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str01', 'fstr1', '2003-03-04', '2004-05-06')" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types3 ORDER BY int8 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types2 FORMAT ArrowStream" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types3 FORMAT ArrowStream" + +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types4 values ( 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 'str02', 'fstr2', '2005-03-04 05:06:07', '2006-08-09 10:11:12')" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types4 ORDER BY int8 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types2 FORMAT ArrowStream" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types4 FORMAT ArrowStream" + +echo dest: +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types2 ORDER BY int8" +echo min: +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types3 ORDER BY int8" +echo max: +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types4 ORDER BY int8" + + +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types5" +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_types6" +${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE arrow_types2" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types5 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime)) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_types6 (int8 Nullable(Int8), uint8 Nullable(UInt8), int16 Nullable(Int16), uint16 Nullable(UInt16), int32 Nullable(Int32), uint32 Nullable(UInt32), int64 Nullable(Int64), uint64 Nullable(UInt64), float32 Nullable(Float32), float64 Nullable(Float64), string Nullable(String), fixedstring Nullable(FixedString(15)), date Nullable(Date), datetime Nullable(DateTime)) ENGINE = Memory" +${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types5 values ( NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 ORDER BY int8 FORMAT ArrowStream" > ${CLICKHOUSE_TMP}/arrow_all_types_5.arrow +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types5 ORDER BY int8 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT ArrowStream" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types1 ORDER BY int8 FORMAT ArrowStream" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_types6 FORMAT ArrowStream" +echo dest from null: +${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_types6 ORDER BY int8" + +${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_types5" +${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_types6" + + +${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_types1" +${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_types2" +${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_types3" +${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_types4" + diff --git a/tests/queries/0_stateless/01284_view_and_extremes_bug.reference b/tests/queries/0_stateless/01284_view_and_extremes_bug.reference new file mode 100644 index 00000000000..216e97ce082 --- /dev/null +++ b/tests/queries/0_stateless/01284_view_and_extremes_bug.reference @@ -0,0 +1 @@ +World diff --git a/tests/queries/0_stateless/01284_view_and_extremes_bug.sql b/tests/queries/0_stateless/01284_view_and_extremes_bug.sql new file mode 100644 index 00000000000..c444441a258 --- /dev/null +++ b/tests/queries/0_stateless/01284_view_and_extremes_bug.sql @@ -0,0 +1,4 @@ +drop table if exists view_bug_const; +CREATE VIEW view_bug_const AS SELECT 'World' AS hello FROM (SELECT number FROM system.numbers LIMIT 1) AS n1 JOIN (SELECT number FROM system.numbers LIMIT 1) AS n2 USING (number); +select * from view_bug_const; +drop table if exists view_bug_const; diff --git a/utils/ci/docker-multiarch/update.sh b/utils/ci/docker-multiarch/update.sh index 6e3e18047de..6abcf339607 100755 --- a/utils/ci/docker-multiarch/update.sh +++ b/utils/ci/docker-multiarch/update.sh @@ -37,27 +37,27 @@ fi # get the image if \ - wget -q --spider "$baseUrl/current" \ - && wget -q --spider "$baseUrl/current/$thisTar" \ - ; then - baseUrl+='/current' + wget -q --spider "$baseUrl/current" \ + && wget -q --spider "$baseUrl/current/$thisTar" \ + ; then + baseUrl+='/current' fi wget -qN "$baseUrl/"{{MD5,SHA{1,256}}SUMS{,.gpg},"$thisTarBase.manifest",'unpacked/build-info.txt'} || true wget -N "$baseUrl/$thisTar" # check checksum if [ -f SHA256SUMS ]; then - sha256sum="$(sha256sum "$thisTar" | cut -d' ' -f1)" - if ! grep -q "$sha256sum" SHA256SUMS; then - echo >&2 "error: '$thisTar' has invalid SHA256" - exit 1 - fi + sha256sum="$(sha256sum "$thisTar" | cut -d' ' -f1)" + if ! grep -q "$sha256sum" SHA256SUMS; then + echo >&2 "error: '$thisTar' has invalid SHA256" + exit 1 + fi fi cat > Dockerfile <<-EOF - FROM scratch - ADD $thisTar / - ENV ARCH=${ARCH} UBUNTU_SUITE=${VERSION} DOCKER_REPO=${DOCKER_REPO} + FROM scratch + ADD $thisTar / + ENV ARCH=${ARCH} UBUNTU_SUITE=${VERSION} DOCKER_REPO=${DOCKER_REPO} EOF # add qemu-user-static binary @@ -70,26 +70,26 @@ EOF fi cat >> Dockerfile <<-EOF - # a few minor docker-specific tweaks - # see https://github.com/docker/docker/blob/master/contrib/mkimage/debootstrap - RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d \\ - && echo 'exit 101' >> /usr/sbin/policy-rc.d \\ - && chmod +x /usr/sbin/policy-rc.d \\ - && dpkg-divert --local --rename --add /sbin/initctl \\ - && cp -a /usr/sbin/policy-rc.d /sbin/initctl \\ - && sed -i 's/^exit.*/exit 0/' /sbin/initctl \\ - && echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \\ - && echo 'DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' > /etc/apt/apt.conf.d/docker-clean \\ - && echo 'APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' >> /etc/apt/apt.conf.d/docker-clean \\ - && echo 'Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";' >> /etc/apt/apt.conf.d/docker-clean \\ - && echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/docker-no-languages \\ - && echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > /etc/apt/apt.conf.d/docker-gzip-indexes + # a few minor docker-specific tweaks + # see https://github.com/docker/docker/blob/master/contrib/mkimage/debootstrap + RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d \\ + && echo 'exit 101' >> /usr/sbin/policy-rc.d \\ + && chmod +x /usr/sbin/policy-rc.d \\ + && dpkg-divert --local --rename --add /sbin/initctl \\ + && cp -a /usr/sbin/policy-rc.d /sbin/initctl \\ + && sed -i 's/^exit.*/exit 0/' /sbin/initctl \\ + && echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \\ + && echo 'DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' > /etc/apt/apt.conf.d/docker-clean \\ + && echo 'APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' >> /etc/apt/apt.conf.d/docker-clean \\ + && echo 'Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";' >> /etc/apt/apt.conf.d/docker-clean \\ + && echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/docker-no-languages \\ + && echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > /etc/apt/apt.conf.d/docker-gzip-indexes - # enable the universe - RUN sed -i 's/^#\s*\(deb.*universe\)$/\1/g' /etc/apt/sources.list + # enable the universe + RUN sed -i 's/^#\s*\(deb.*universe\)$/\1/g' /etc/apt/sources.list - # overwrite this with 'CMD []' in a dependent Dockerfile - CMD ["/bin/bash"] + # overwrite this with 'CMD []' in a dependent Dockerfile + CMD ["/bin/bash"] EOF docker build -t "${DOCKER_REPO}:${TAG_ARCH}-${VERSION}" . diff --git a/utils/release/release_lib.sh b/utils/release/release_lib.sh index 2e700593523..569e1dfb9f2 100644 --- a/utils/release/release_lib.sh +++ b/utils/release/release_lib.sh @@ -12,10 +12,10 @@ function gen_version_string { function get_version { if [ -z "$VERSION_MAJOR" ] && [ -z "$VERSION_MINOR" ] && [ -z "$VERSION_PATCH" ]; then BASEDIR=$(dirname "${BASH_SOURCE[0]}")/../../ - VERSION_REVISION=`grep "set(VERSION_REVISION" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_REVISION \(.*\)$/\1/' | sed 's/[) ].*//'` - VERSION_MAJOR=`grep "set(VERSION_MAJOR" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_MAJOR \(.*\)/\1/' | sed 's/[) ].*//'` - VERSION_MINOR=`grep "set(VERSION_MINOR" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_MINOR \(.*\)/\1/' | sed 's/[) ].*//'` - VERSION_PATCH=`grep "set(VERSION_PATCH" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_PATCH \(.*\)/\1/' | sed 's/[) ].*//'` + VERSION_REVISION=`grep "SET(VERSION_REVISION" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_REVISION \(.*\)$/\1/' | sed 's/[) ].*//'` + VERSION_MAJOR=`grep "SET(VERSION_MAJOR" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_MAJOR \(.*\)/\1/' | sed 's/[) ].*//'` + VERSION_MINOR=`grep "SET(VERSION_MINOR" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_MINOR \(.*\)/\1/' | sed 's/[) ].*//'` + VERSION_PATCH=`grep "SET(VERSION_PATCH" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_PATCH \(.*\)/\1/' | sed 's/[) ].*//'` fi VERSION_PREFIX="${VERSION_PREFIX:-v}" VERSION_POSTFIX_TAG="${VERSION_POSTFIX:--testing}" @@ -90,28 +90,28 @@ function gen_revision_author { git_describe=`git describe` git_hash=`git rev-parse HEAD` - sed -i -e "s/set(VERSION_REVISION [^) ]*/set(VERSION_REVISION $VERSION_REVISION/g;" \ - -e "s/set(VERSION_DESCRIBE [^) ]*/set(VERSION_DESCRIBE $git_describe/g;" \ - -e "s/set(VERSION_GITHASH [^) ]*/set(VERSION_GITHASH $git_hash/g;" \ - -e "s/set(VERSION_MAJOR [^) ]*/set(VERSION_MAJOR $VERSION_MAJOR/g;" \ - -e "s/set(VERSION_MINOR [^) ]*/set(VERSION_MINOR $VERSION_MINOR/g;" \ - -e "s/set(VERSION_PATCH [^) ]*/set(VERSION_PATCH $VERSION_PATCH/g;" \ - -e "s/set(VERSION_STRING [^) ]*/set(VERSION_STRING $VERSION_STRING/g;" \ - cmake/version.cmake + sed -i -e "s/SET(VERSION_REVISION [^) ]*/SET(VERSION_REVISION $VERSION_REVISION/g;" \ + -e "s/SET(VERSION_DESCRIBE [^) ]*/SET(VERSION_DESCRIBE $git_describe/g;" \ + -e "s/SET(VERSION_GITHASH [^) ]*/SET(VERSION_GITHASH $git_hash/g;" \ + -e "s/SET(VERSION_MAJOR [^) ]*/SET(VERSION_MAJOR $VERSION_MAJOR/g;" \ + -e "s/SET(VERSION_MINOR [^) ]*/SET(VERSION_MINOR $VERSION_MINOR/g;" \ + -e "s/SET(VERSION_PATCH [^) ]*/SET(VERSION_PATCH $VERSION_PATCH/g;" \ + -e "s/SET(VERSION_STRING [^) ]*/SET(VERSION_STRING $VERSION_STRING/g;" \ + cmake/autogenerated_versions.txt gen_changelog "$VERSION_STRING" "" "$AUTHOR" "" gen_dockerfiles "$VERSION_STRING" src/Storages/System/StorageSystemContributors.sh ||: utils/list-versions/list-versions.sh > utils/list-versions/version_date.tsv - git commit -m "$auto_message [$VERSION_STRING] [$VERSION_REVISION]" cmake/version.cmake debian/changelog docker/*/Dockerfile src/Storages/System/StorageSystemContributors.generated.cpp utils/list-versions/version_date.tsv + git commit -m "$auto_message [$VERSION_STRING] [$VERSION_REVISION]" cmake/autogenerated_versions.txt debian/changelog docker/*/Dockerfile src/Storages/System/StorageSystemContributors.generated.cpp utils/list-versions/version_date.tsv if [ -z $NO_PUSH ]; then git push fi echo "Generated version: ${VERSION_STRING}, revision: ${VERSION_REVISION}." - # Second tag for correct version information in version.cmake inside tag + # Second tag for correct version information in autogenerated_versions.txt inside tag if git tag --force -a "$tag" -m "$tag" then if [ -z $NO_PUSH ]; then diff --git a/website/css/docs.css b/website/css/docs.css index 18bf40de13b..2c175b18db6 100644 --- a/website/css/docs.css +++ b/website/css/docs.css @@ -69,7 +69,6 @@ summary { #content code { color: #111; background: #eee; - padding: 2px; }