thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into ast

Browse Source
This commit is contained in:
Artem Zuikov 2020-08-24 14:09:30 +03:00
commit 69c77ff229
116 changed files with 4778 additions and 624 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitmodules vendored
View File

@ -180,3 +180,9 @@
[submodule "contrib/stats"]
path = contrib/stats
url = https://github.com/kthohr/stats.git
[submodule "contrib/krb5"]
path = contrib/krb5
url = https://github.com/krb5/krb5
[submodule "contrib/cyrus-sasl"]
path = contrib/cyrus-sasl
url = https://github.com/cyrusimap/cyrus-sasl

2
CMakeLists.txt
View File

@ -378,7 +378,9 @@ include (cmake/find/ltdl.cmake) # for odbc
# openssl, zlib before poco
include (cmake/find/sparsehash.cmake)
include (cmake/find/re2.cmake)
include (cmake/find/krb5.cmake)
include (cmake/find/libgsasl.cmake)
include (cmake/find/cyrus-sasl.cmake)
include (cmake/find/rdkafka.cmake)
include (cmake/find/amqpcpp.cmake)
include (cmake/find/capnp.cmake)

2
README.md
View File

@ -17,4 +17,4 @@ ClickHouse is an open-source column-oriented database management system that all
## Upcoming Events
* [ClickHouse at ByteDance (in Chinese)](https://mp.weixin.qq.com/s/Em-HjPylO8D7WPui4RREAQ) on August 14, 2020.
* [ClickHouse at ByteDance (in Chinese)](https://mp.weixin.qq.com/s/Em-HjPylO8D7WPui4RREAQ) on August 28, 2020.

23
cmake/find/cyrus-sasl.cmake Normal file
View File

@ -0,0 +1,23 @@
if (${ENABLE_LIBRARIES} AND ${ENABLE_KRB5})
set (DEFAULT_ENABLE_CYRUS_SASL 1)
else()
set (DEFAULT_ENABLE_CYRUS_SASL 0)
endif()
OPTION(ENABLE_CYRUS_SASL "Enable cyrus-sasl" ${DEFAULT_ENABLE_CYRUS_SASL})
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cyrus-sasl/README")
message (WARNING "submodule contrib/cyrus-sasl is missing. to fix try run: \n git submodule update --init --recursive")
set (ENABLE_CYRUS_SASL 0)
endif ()
if (ENABLE_CYRUS_SASL)
set (USE_CYRUS_SASL 1)
set (CYRUS_SASL_LIBRARY sasl2)
set (CYRUS_SASL_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/cyrus-sasl/include")
endif ()
message (STATUS "Using cyrus-sasl: krb5=${USE_KRB5}: ${CYRUS_SASL_INCLUDE_DIR} : ${CYRUS_SASL_LIBRARY}")

25
cmake/find/krb5.cmake Normal file
View File

@ -0,0 +1,25 @@
OPTION(ENABLE_KRB5 "Enable krb5" ${ENABLE_LIBRARIES})
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/krb5/README")
message (WARNING "submodule contrib/krb5 is missing. to fix try run: \n git submodule update --init --recursive")
set (ENABLE_KRB5 0)
endif ()
if (NOT CMAKE_SYSTEM_NAME MATCHES "Linux")
message (WARNING "krb5 disabled in non-Linux environments")
set (ENABLE_KRB5 0)
endif ()
if (ENABLE_KRB5)
set (USE_KRB5 1)
set (KRB5_LIBRARY krb5)
set (KRB5_INCLUDE_DIR
"${ClickHouse_SOURCE_DIR}/contrib/krb5/src/include"
"${ClickHouse_BINARY_DIR}/contrib/krb5-cmake/include"
)
endif ()
message (STATUS "Using krb5=${USE_KRB5}: ${KRB5_INCLUDE_DIR} : ${KRB5_LIBRARY}")

7
contrib/CMakeLists.txt vendored
View File

@ -311,3 +311,10 @@ if (USE_STATS)
add_subdirectory (stats-cmake)
add_subdirectory (gcem)
endif()
if (USE_KRB5)
add_subdirectory (krb5-cmake)
if (USE_CYRUS_SASL)
add_subdirectory (cyrus-sasl-cmake)
endif()
endif()

1
contrib/cyrus-sasl vendored Submodule

@ -0,0 +1 @@
Subproject commit 6054630889fd1cd8d0659573d69badcee1e23a00

69
contrib/cyrus-sasl-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,69 @@
set(CYRUS_SASL_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/cyrus-sasl)
add_library(${CYRUS_SASL_LIBRARY})
target_sources(${CYRUS_SASL_LIBRARY} PRIVATE
${CYRUS_SASL_SOURCE_DIR}/plugins/gssapi.c
# ${CYRUS_SASL_SOURCE_DIR}/plugins/gssapiv2_init.c
${CYRUS_SASL_SOURCE_DIR}/common/plugin_common.c
${CYRUS_SASL_SOURCE_DIR}/lib/common.c
${CYRUS_SASL_SOURCE_DIR}/lib/canonusr.c
${CYRUS_SASL_SOURCE_DIR}/lib/server.c
${CYRUS_SASL_SOURCE_DIR}/lib/config.c
${CYRUS_SASL_SOURCE_DIR}/lib/auxprop.c
${CYRUS_SASL_SOURCE_DIR}/lib/saslutil.c
${CYRUS_SASL_SOURCE_DIR}/lib/external.c
${CYRUS_SASL_SOURCE_DIR}/lib/seterror.c
${CYRUS_SASL_SOURCE_DIR}/lib/md5.c
${CYRUS_SASL_SOURCE_DIR}/lib/dlopen.c
${CYRUS_SASL_SOURCE_DIR}/lib/client.c
${CYRUS_SASL_SOURCE_DIR}/lib/checkpw.c
)
target_include_directories(${CYRUS_SASL_LIBRARY} PUBLIC
${CMAKE_CURRENT_BINARY_DIR}
)
target_include_directories(${CYRUS_SASL_LIBRARY} PRIVATE
${CMAKE_CURRENT_SOURCE_DIR} # for config.h
${CYRUS_SASL_SOURCE_DIR}/plugins
${CYRUS_SASL_SOURCE_DIR}
${CYRUS_SASL_SOURCE_DIR}/include
${CYRUS_SASL_SOURCE_DIR}/lib
${CYRUS_SASL_SOURCE_DIR}/sasldb
${CYRUS_SASL_SOURCE_DIR}/common
${CYRUS_SASL_SOURCE_DIR}/saslauthd
${CYRUS_SASL_SOURCE_DIR}/sample
${CYRUS_SASL_SOURCE_DIR}/utils
${CYRUS_SASL_SOURCE_DIR}/tests
)
target_compile_definitions(${CYRUS_SASL_LIBRARY} PUBLIC
HAVE_CONFIG_H
# PLUGINDIR="/usr/local/lib/sasl2"
PLUGINDIR=""
# PIC
OBSOLETE_CRAM_ATTR=1
# SASLAUTHD_CONF_FILE_DEFAULT="/usr/local/etc/saslauthd.conf"
SASLAUTHD_CONF_FILE_DEFAULT=""
# CONFIGDIR="/usr/local/lib/sasl2:/usr/local/etc/sasl2"
CONFIGDIR=""
OBSOLETE_DIGEST_ATTR=1
LIBSASL_EXPORTS=1
)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/sasl)
file(COPY
${CYRUS_SASL_SOURCE_DIR}/include/sasl.h
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/sasl
)
file(COPY
${CYRUS_SASL_SOURCE_DIR}/include/prop.h
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}
)
target_link_libraries(${CYRUS_SASL_LIBRARY}
PUBLIC ${KRB5_LIBRARY}
)

722
contrib/cyrus-sasl-cmake/config.h Normal file
View File

@ -0,0 +1,722 @@
/* config.h. Generated from config.h.in by configure. */
/* config.h.in. Generated from configure.ac by autoheader. */
/* acconfig.h - autoheader configuration input */
/*
* Copyright (c) 1998-2003 Carnegie Mellon University. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The name "Carnegie Mellon University" must not be used to
* endorse or promote products derived from this software without
* prior written permission. For permission or any other legal
* details, please contact
* Office of Technology Transfer
* Carnegie Mellon University
* 5000 Forbes Avenue
* Pittsburgh, PA 15213-3890
* (412) 268-4387, fax: (412) 268-7395
* tech-transfer@andrew.cmu.edu
*
* 4. Redistributions of any form whatsoever must retain the following
* acknowledgment:
* "This product includes software developed by Computing Services
* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
*
* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#ifndef CONFIG_H
#define CONFIG_H
/* Include SASLdb Support */
/* #undef AUTH_SASLDB */
/* Do we need a leading _ for dlsym? */
/* #undef DLSYM_NEEDS_UNDERSCORE */
/* Should we build a shared plugin (via dlopen) library? */
#define DO_DLOPEN /**/
/* should we support sasl_checkapop? */
#define DO_SASL_CHECKAPOP /**/
/* should we support setpass() for SRP? */
/* #undef DO_SRP_SETPASS */
/* Define if your getpwnam_r()/getspnam_r() functions take 5 arguments */
#define GETXXNAM_R_5ARG 1
/* should we mutex-wrap calls into the GSS library? */
#define GSS_USE_MUTEXES /**/
/* Enable 'alwaystrue' password verifier? */
/* #undef HAVE_ALWAYSTRUE */
/* Define to 1 if you have the `asprintf' function. */
#define HAVE_ASPRINTF 1
/* Include support for Courier's authdaemond? */
#define HAVE_AUTHDAEMON /**/
/* Define to 1 if you have the <crypt.h> header file. */
#define HAVE_CRYPT_H 1
/* Define to 1 if you have the <des.h> header file. */
/* #undef HAVE_DES_H */
/* Define to 1 if you have the <dirent.h> header file, and it defines `DIR'.
*/
#define HAVE_DIRENT_H 1
/* Define to 1 if you have the <dlfcn.h> header file. */
#define HAVE_DLFCN_H 1
/* Define to 1 if you have the `dns_lookup' function. */
/* #undef HAVE_DNS_LOOKUP */
/* Define to 1 if you have the `dn_expand' function. */
/* #undef HAVE_DN_EXPAND */
/* Define to 1 if you have the <fcntl.h> header file. */
#define HAVE_FCNTL_H 1
/* Do we have a getaddrinfo? */
#define HAVE_GETADDRINFO /**/
/* Define to 1 if you have the `getdomainname' function. */
#define HAVE_GETDOMAINNAME 1
/* Define to 1 if you have the `gethostname' function. */
#define HAVE_GETHOSTNAME 1
/* Do we have a getnameinfo() function? */
#define HAVE_GETNAMEINFO /**/
/* Define to 1 if you have the `getpassphrase' function. */
/* #undef HAVE_GETPASSPHRASE */
/* Define to 1 if you have the `getpwnam' function. */
#define HAVE_GETPWNAM 1
/* Define to 1 if you have the `getspnam' function. */
#define HAVE_GETSPNAM 1
/* do we have getsubopt()? */
#define HAVE_GETSUBOPT /**/
/* Define to 1 if you have the `gettimeofday' function. */
#define HAVE_GETTIMEOFDAY 1
/* Include GSSAPI/Kerberos 5 Support */
#define HAVE_GSSAPI /**/
/* Define to 1 if you have the <gssapi/gssapi_ext.h> header file. */
#define HAVE_GSSAPI_GSSAPI_EXT_H 1
/* Define if you have the gssapi/gssapi.h header file */
/* #undef HAVE_GSSAPI_GSSAPI_H */
/* Define to 1 if you have the <gssapi/gssapi_krb5.h> header file. */
#define HAVE_GSSAPI_GSSAPI_KRB5_H 1
/* Define if you have the gssapi.h header file */
#define HAVE_GSSAPI_H /**/
/* Define if your GSSAPI implementation defines
gsskrb5_register_acceptor_identity */
#define HAVE_GSSKRB5_REGISTER_ACCEPTOR_IDENTITY 1
/* Define if your GSSAPI implementation defines GSS_C_NT_HOSTBASED_SERVICE */
#define HAVE_GSS_C_NT_HOSTBASED_SERVICE /**/
/* Define if your GSSAPI implementation defines GSS_C_NT_USER_NAME */
#define HAVE_GSS_C_NT_USER_NAME /**/
/* Define if your GSSAPI implementation defines GSS_C_SEC_CONTEXT_SASL_SSF */
#define HAVE_GSS_C_SEC_CONTEXT_SASL_SSF /**/
/* Define to 1 if you have the `gss_decapsulate_token' function. */
#define HAVE_GSS_DECAPSULATE_TOKEN 1
/* Define to 1 if you have the `gss_encapsulate_token' function. */
#define HAVE_GSS_ENCAPSULATE_TOKEN 1
/* Define to 1 if you have the `gss_get_name_attribute' function. */
#define HAVE_GSS_GET_NAME_ATTRIBUTE 1
/* Define if your GSSAPI implementation defines gss_inquire_sec_context_by_oid
*/
#define HAVE_GSS_INQUIRE_SEC_CONTEXT_BY_OID 1
/* Define to 1 if you have the `gss_oid_equal' function. */
#define HAVE_GSS_OID_EQUAL 1
/* Define if your GSSAPI implementation supports SPNEGO */
#define HAVE_GSS_SPNEGO /**/
/* Include HTTP form Support */
/* #undef HAVE_HTTPFORM */
/* Define to 1 if you have the `inet_aton' function. */
#define HAVE_INET_ATON 1
/* Define to 1 if you have the <inttypes.h> header file. */
#define HAVE_INTTYPES_H 1
/* Define to 1 if you have the `jrand48' function. */
#define HAVE_JRAND48 1
/* Do we have Kerberos 4 Support? */
/* #undef HAVE_KRB */
/* Define to 1 if you have the <krb5.h> header file. */
#define HAVE_KRB5_H 1
/* Define to 1 if you have the `krb_get_err_text' function. */
/* #undef HAVE_KRB_GET_ERR_TEXT */
/* Define to 1 if you have the <lber.h> header file. */
/* #undef HAVE_LBER_H */
/* Support for LDAP? */
/* #undef HAVE_LDAP */
/* Define to 1 if you have the <ldap.h> header file. */
/* #undef HAVE_LDAP_H */
/* Define to 1 if you have the `resolv' library (-lresolv). */
#define HAVE_LIBRESOLV 1
/* Define to 1 if you have the <limits.h> header file. */
#define HAVE_LIMITS_H 1
/* Define to 1 if you have the <malloc.h> header file. */
#define HAVE_MALLOC_H 1
/* Define to 1 if you have the `memcpy' function. */
#define HAVE_MEMCPY 1
/* Define to 1 if you have the `memmem' function. */
#define HAVE_MEMMEM 1
/* Define to 1 if you have the <memory.h> header file. */
#define HAVE_MEMORY_H 1
/* Define to 1 if you have the `mkdir' function. */
#define HAVE_MKDIR 1
/* Do we have mysql support? */
/* #undef HAVE_MYSQL */
/* Define to 1 if you have the <ndir.h> header file, and it defines `DIR'. */
/* #undef HAVE_NDIR_H */
/* Do we have OpenSSL? */
#define HAVE_OPENSSL /**/
/* Use OPIE for server-side OTP? */
/* #undef HAVE_OPIE */
/* Support for PAM? */
/* #undef HAVE_PAM */
/* Define to 1 if you have the <paths.h> header file. */
#define HAVE_PATHS_H 1
/* Do we have Postgres support? */
/* #undef HAVE_PGSQL */
/* Include Support for pwcheck daemon? */
/* #undef HAVE_PWCHECK */
/* Include support for saslauthd? */
#define HAVE_SASLAUTHD /**/
/* Define to 1 if you have the `select' function. */
#define HAVE_SELECT 1
/* Do we have SHA512? */
#define HAVE_SHA512 /**/
/* Include SIA Support */
/* #undef HAVE_SIA */
/* Does the system have snprintf()? */
#define HAVE_SNPRINTF /**/
/* Does sockaddr have an sa_len? */
/* #undef HAVE_SOCKADDR_SA_LEN */
/* Define to 1 if you have the `socket' function. */
#define HAVE_SOCKET 1
/* Do we have a socklen_t? */
#define HAVE_SOCKLEN_T /**/
/* Do we have SQLite support? */
/* #undef HAVE_SQLITE */
/* Do we have SQLite3 support? */
/* #undef HAVE_SQLITE3 */
/* Is there an ss_family in sockaddr_storage? */
#define HAVE_SS_FAMILY /**/
/* Define to 1 if you have the <stdarg.h> header file. */
#define HAVE_STDARG_H 1
/* Define to 1 if you have the <stdint.h> header file. */
#define HAVE_STDINT_H 1
/* Define to 1 if you have the <stdlib.h> header file. */
#define HAVE_STDLIB_H 1
/* Define to 1 if you have the `strchr' function. */
#define HAVE_STRCHR 1
/* Define to 1 if you have the `strdup' function. */
#define HAVE_STRDUP 1
/* Define to 1 if you have the `strerror' function. */
#define HAVE_STRERROR 1
/* Define to 1 if you have the <strings.h> header file. */
#define HAVE_STRINGS_H 1
/* Define to 1 if you have the <string.h> header file. */
#define HAVE_STRING_H 1
/* Define to 1 if you have the `strlcat' function. */
/* #undef HAVE_STRLCAT */
/* Define to 1 if you have the `strlcpy' function. */
/* #undef HAVE_STRLCPY */
/* Define to 1 if you have the `strspn' function. */
#define HAVE_STRSPN 1
/* Define to 1 if you have the `strstr' function. */
#define HAVE_STRSTR 1
/* Define to 1 if you have the `strtol' function. */
#define HAVE_STRTOL 1
/* Do we have struct sockaddr_stroage? */
#define HAVE_STRUCT_SOCKADDR_STORAGE /**/
/* Define to 1 if you have the <sysexits.h> header file. */
#define HAVE_SYSEXITS_H 1
/* Define to 1 if you have the `syslog' function. */
#define HAVE_SYSLOG 1
/* Define to 1 if you have the <syslog.h> header file. */
#define HAVE_SYSLOG_H 1
/* Define to 1 if you have the <sys/dir.h> header file, and it defines `DIR'.
*/
/* #undef HAVE_SYS_DIR_H */
/* Define to 1 if you have the <sys/file.h> header file. */
#define HAVE_SYS_FILE_H 1
/* Define to 1 if you have the <sys/ndir.h> header file, and it defines `DIR'.
*/
/* #undef HAVE_SYS_NDIR_H */
/* Define to 1 if you have the <sys/param.h> header file. */
#define HAVE_SYS_PARAM_H 1
/* Define to 1 if you have the <sys/socket.h> header file. */
#define HAVE_SYS_SOCKET_H 1
/* Define to 1 if you have the <sys/stat.h> header file. */
#define HAVE_SYS_STAT_H 1
/* Define to 1 if you have the <sys/time.h> header file. */
#define HAVE_SYS_TIME_H 1
/* Define to 1 if you have the <sys/types.h> header file. */
#define HAVE_SYS_TYPES_H 1
/* Define to 1 if you have the <sys/uio.h> header file. */
#define HAVE_SYS_UIO_H 1
/* Define to 1 if you have <sys/wait.h> that is POSIX.1 compatible. */
#define HAVE_SYS_WAIT_H 1
/* Define to 1 if you have the <unistd.h> header file. */
#define HAVE_UNISTD_H 1
/* Define to 1 if you have the <varargs.h> header file. */
/* #undef HAVE_VARARGS_H */
/* Does the system have vsnprintf()? */
#define HAVE_VSNPRINTF /**/
/* Define to 1 if you have the <ws2tcpip.h> header file. */
/* #undef HAVE_WS2TCPIP_H */
/* Should we keep handle to DB open in SASLDB plugin? */
/* #undef KEEP_DB_OPEN */
/* Ignore IP Address in Kerberos 4 tickets? */
/* #undef KRB4_IGNORE_IP_ADDRESS */
/* Using Heimdal */
/* #undef KRB5_HEIMDAL */
/* Define to the sub-directory where libtool stores uninstalled libraries. */
#define LT_OBJDIR ".libs/"
/* Name of package */
#define PACKAGE "cyrus-sasl"
/* Define to the address where bug reports for this package should be sent. */
#define PACKAGE_BUGREPORT "https://github.com/cyrusimap/cyrus-sasl/issues"
/* Define to the full name of this package. */
#define PACKAGE_NAME "cyrus-sasl"
/* Define to the full name and version of this package. */
#define PACKAGE_STRING "cyrus-sasl 2.1.27"
/* Define to the one symbol short name of this package. */
#define PACKAGE_TARNAME "cyrus-sasl"
/* Define to the home page for this package. */
#define PACKAGE_URL "http://www.cyrusimap.org"
/* Define to the version of this package. */
#define PACKAGE_VERSION "2.1.27"
/* Where do we look for Courier authdaemond's socket? */
#define PATH_AUTHDAEMON_SOCKET "/dev/null"
/* Where do we look for saslauthd's socket? */
#define PATH_SASLAUTHD_RUNDIR "/var/state/saslauthd"
/* Force a preferred mechanism */
/* #undef PREFER_MECH */
/* Location of pwcheck socket */
/* #undef PWCHECKDIR */
/* Define as the return type of signal handlers (`int' or `void'). */
#define RETSIGTYPE void
/* Use BerkeleyDB for SASLdb */
/* #undef SASL_BERKELEYDB */
/* Path to default SASLdb database */
#define SASL_DB_PATH "/etc/sasldb2"
/* File to use for source of randomness */
#define SASL_DEV_RANDOM "/dev/urandom"
/* Use GDBM for SASLdb */
/* #undef SASL_GDBM */
/* Use LMDB for SASLdb */
/* #undef SASL_LMDB */
/* Use NDBM for SASLdb */
/* #undef SASL_NDBM */
/* The size of `long', as computed by sizeof. */
#define SIZEOF_LONG 8
/* Link ANONYMOUS Statically */
// #define STATIC_ANONYMOUS /**/
/* Link CRAM-MD5 Statically */
// #define STATIC_CRAMMD5 /**/
/* Link DIGEST-MD5 Statically */
// #define STATIC_DIGESTMD5 /**/
/* Link GSSAPI Statically */
#define STATIC_GSSAPIV2 /**/
/* User KERBEROS_V4 Staticly */
/* #undef STATIC_KERBEROS4 */
/* Link ldapdb plugin Statically */
/* #undef STATIC_LDAPDB */
/* Link LOGIN Statically */
/* #undef STATIC_LOGIN */
/* Link NTLM Statically */
/* #undef STATIC_NTLM */
/* Link OTP Statically */
// #define STATIC_OTP /**/
/* Link PASSDSS Statically */
/* #undef STATIC_PASSDSS */
/* Link PLAIN Staticly */
// #define STATIC_PLAIN /**/
/* Link SASLdb Staticly */
// #define STATIC_SASLDB /**/
/* Link SCRAM Statically */
// #define STATIC_SCRAM /**/
/* Link SQL plugin statically */
/* #undef STATIC_SQL */
/* Link SRP Statically */
/* #undef STATIC_SRP */
/* Define to 1 if you have the ANSI C header files. */
#define STDC_HEADERS 1
/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */
#define TIME_WITH_SYS_TIME 1
/* Should we try to dlopen() plugins while statically compiled? */
/* #undef TRY_DLOPEN_WHEN_STATIC */
/* use the doors IPC API for saslauthd? */
/* #undef USE_DOORS */
/* Enable extensions on AIX 3, Interix. */
#ifndef _ALL_SOURCE
# define _ALL_SOURCE 1
#endif
/* Enable GNU extensions on systems that have them. */
#ifndef _GNU_SOURCE
# define _GNU_SOURCE 1
#endif
/* Enable threading extensions on Solaris. */
#ifndef _POSIX_PTHREAD_SEMANTICS
# define _POSIX_PTHREAD_SEMANTICS 1
#endif
/* Enable extensions on HP NonStop. */
#ifndef _TANDEM_SOURCE
# define _TANDEM_SOURCE 1
#endif
/* Enable general extensions on Solaris. */
#ifndef __EXTENSIONS__
# define __EXTENSIONS__ 1
#endif
/* Version number of package */
#define VERSION "2.1.27"
/* Use DES */
#define WITH_DES /**/
/* Linking against dmalloc? */
/* #undef WITH_DMALLOC */
/* Use RC4 */
#define WITH_RC4 /**/
/* Use OpenSSL DES Implementation */
#define WITH_SSL_DES /**/
/* Define to 1 if on MINIX. */
/* #undef _MINIX */
/* Define to 2 if the system does not provide POSIX.1 features except with
this defined. */
/* #undef _POSIX_1_SOURCE */
/* Define to 1 if you need to in order for `stat' and other things to work. */
/* #undef _POSIX_SOURCE */
/* Define to empty if `const' does not conform to ANSI C. */
/* #undef const */
/* Define to `__inline__' or `__inline' if that's what the C compiler
calls it, or to nothing if 'inline' is not supported under any name. */
#ifndef __cplusplus
/* #undef inline */
#endif
/* Define to `int' if <sys/types.h> does not define. */
/* #undef mode_t */
/* Define to `int' if <sys/types.h> does not define. */
/* #undef pid_t */
/* Create a struct iovec if we need one */
#if !defined(_WIN32)
#if !defined(HAVE_SYS_UIO_H)
/* (win32 is handled in sasl.h) */
struct iovec {
char *iov_base;
long iov_len;
};
#else
#include <sys/types.h>
#include <sys/uio.h>
#endif
#endif
/* location of the random number generator */
#ifdef DEV_RANDOM
/* #undef DEV_RANDOM */
#endif
#define DEV_RANDOM SASL_DEV_RANDOM
/* if we've got krb_get_err_txt, we might as well use it;
especially since krb_err_txt isn't in some newer distributions
(MIT Kerb for Mac 4 being a notable example). If we don't have
it, we fall back to the krb_err_txt array */
#ifdef HAVE_KRB_GET_ERR_TEXT
#define get_krb_err_txt krb_get_err_text
#else
#define get_krb_err_txt(X) (krb_err_txt[(X)])
#endif
/* Make Solaris happy... */
#ifndef __EXTENSIONS__
#define __EXTENSIONS__ 1
#endif
/* Make Linux happy... */
#ifndef _GNU_SOURCE
#define _GNU_SOURCE 1
#endif
#define SASL_PATH_ENV_VAR "SASL_PATH"
#define SASL_CONF_PATH_ENV_VAR "SASL_CONF_PATH"
#include <stdlib.h>
#include <sys/types.h>
#ifndef WIN32
# include <sys/socket.h>
# include <netdb.h>
# include <netinet/in.h>
# ifdef HAVE_SYS_PARAM_H
# include <sys/param.h>
# endif
#else /* WIN32 */
# include <winsock2.h>
#endif /* WIN32 */
#include <string.h>
#ifndef HAVE_SOCKLEN_T
typedef unsigned int socklen_t;
#endif /* HAVE_SOCKLEN_T */
#if !defined(HAVE_STRUCT_SOCKADDR_STORAGE) && !defined(WIN32)
#define _SS_MAXSIZE 128 /* Implementation specific max size */
#define _SS_PADSIZE (_SS_MAXSIZE - sizeof (struct sockaddr))
struct sockaddr_storage {
struct sockaddr ss_sa;
char __ss_pad2[_SS_PADSIZE];
};
# define ss_family ss_sa.sa_family
#endif /* !HAVE_STRUCT_SOCKADDR_STORAGE */
#ifndef AF_INET6
/* Define it to something that should never appear */
#define AF_INET6 AF_MAX
#endif
#ifndef HAVE_GETADDRINFO
#define getaddrinfo sasl_getaddrinfo
#define freeaddrinfo sasl_freeaddrinfo
#define gai_strerror sasl_gai_strerror
#endif
#ifndef HAVE_GETNAMEINFO
#define getnameinfo sasl_getnameinfo
#endif
#if !defined(HAVE_GETNAMEINFO) || !defined(HAVE_GETADDRINFO)
#include "gai.h"
#endif
#ifndef AI_NUMERICHOST /* support glibc 2.0.x */
#define AI_NUMERICHOST 4
#define NI_NUMERICHOST 2
#define NI_NAMEREQD 4
#define NI_NUMERICSERV 8
#endif
#ifndef HAVE_SYSEXITS_H
#include "exits.h"
#else
#include "sysexits.h"
#endif
/* Get the correct time.h */
#if TIME_WITH_SYS_TIME
# include <sys/time.h>
# include <time.h>
#else
# if HAVE_SYS_TIME_H
# include <sys/time.h>
# else
# include <time.h>
# endif
#endif
#ifndef HIER_DELIMITER
#define HIER_DELIMITER '/'
#endif
#ifdef WIN32
#define SASL_ROOT_KEY "SOFTWARE\\Carnegie Mellon\\Project Cyrus\\SASL Library"
#define SASL_PLUGIN_PATH_ATTR "SearchPath"
#define SASL_CONF_PATH_ATTR "ConfFile"
#include <windows.h>
inline static unsigned int sleep(unsigned int seconds) {
Sleep(seconds * 1000);
return 0;
}
#endif
/* handy string manipulation functions */
#ifndef HAVE_STRLCPY
extern size_t saslauthd_strlcpy(char *dst, const char *src, size_t len);
#define strlcpy(x,y,z) saslauthd_strlcpy((x),(y),(z))
#endif
#ifndef HAVE_STRLCAT
extern size_t saslauthd_strlcat(char *dst, const char *src, size_t len);
#define strlcat(x,y,z) saslauthd_strlcat((x),(y),(z))
#endif
#ifndef HAVE_ASPRINTF
extern int asprintf(char **str, const char *fmt, ...);
#endif
#endif /* CONFIG_H */
#if defined __GNUC__ && __GNUC__ > 6
#define GCC_FALLTHROUGH __attribute__((fallthrough));
#else
#define GCC_FALLTHROUGH /* fall through */
#endif

1
contrib/krb5 vendored Submodule

@ -0,0 +1 @@
Subproject commit 99f7ad2831a01f264c07eed42a0a3a9336b86184

670
contrib/krb5-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,670 @@
find_program(AWK_PROGRAM awk)
if(NOT AWK_PROGRAM)
message(FATAL_ERROR "You need the awk program to build ClickHouse with krb5 enabled.")
endif()
set(KRB5_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/krb5/src)
set(ALL_SRCS
${KRB5_SOURCE_DIR}/util/et/et_name.c
${KRB5_SOURCE_DIR}/util/et/com_err.c
${KRB5_SOURCE_DIR}/util/et/error_message.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_inq_names.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_rel_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_unwrap_aead.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_set_name_attr.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_glue.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_imp_cred.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/gssd_pname_to_uid.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_authorize_localname.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_prf.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_acquire_cred_with_pw.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_set_cred_option.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_map_name_to_any.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_inq_cred.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_rel_cred.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_seal.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_delete_sec_context.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_context_time.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_get_name_attr.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_mech_invoke.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_unwrap_iov.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_exp_sec_context.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_init_sec_context.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_accept_sec_context.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_verify.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_sign.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_mechname.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_mechattr.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_complete_auth_token.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_wrap_aead.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_inq_cred_oid.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_rel_buffer.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_initialize.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_export_name_comp.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_set_context_option.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_acquire_cred.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_acquire_cred_imp_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_imp_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_inq_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_set_neg_mechs.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_inq_context.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_export_cred.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_oid_ops.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_inq_context_oid.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_del_name_attr.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_decapsulate_token.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_compare_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_rel_name_mapping.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_imp_sec_context.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_dup_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_export_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_wrap_iov.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_rel_oid_set.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_unseal.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_store_cred.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_buffer_set.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_canon_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_dsp_status.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_dsp_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_dsp_name_ext.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_saslname.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_process_context.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_encapsulate_token.c
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue/g_negoex.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/delete_sec_context.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/lucid_context.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/duplicate_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/get_tkt_flags.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/set_allowable_enctypes.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/k5sealiov.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/gssapi_err_krb5.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/canon_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/inq_cred.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/export_sec_context.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/inq_names.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/prf.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/k5sealv3iov.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/store_cred.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/import_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/export_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/naming_exts.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/s4u_gss_glue.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/rel_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/k5unsealiov.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/gssapi_krb5.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/disp_status.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/import_cred.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/k5seal.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/accept_sec_context.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/import_sec_context.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/process_context_token.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/disp_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/wrap_size_limit.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/krb5_gss_glue.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/util_crypt.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/set_ccache.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/export_cred.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/rel_oid.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/val_cred.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/context_time.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/cred_store.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/iakerb.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/copy_ccache.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/init_sec_context.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/indicate_mechs.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/inq_context.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/util_seed.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/util_seqnum.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/compare_name.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/ser_sctx.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/k5sealv3.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/acquire_cred.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/k5unseal.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/rel_cred.c
${KRB5_SOURCE_DIR}/lib/gssapi/krb5/util_cksum.c
${KRB5_SOURCE_DIR}/lib/gssapi/generic/disp_com_err_status.c
${KRB5_SOURCE_DIR}/lib/gssapi/generic/gssapi_generic.c
${KRB5_SOURCE_DIR}/lib/gssapi/generic/rel_oid_set.c
${KRB5_SOURCE_DIR}/lib/gssapi/generic/oid_ops.c
${KRB5_SOURCE_DIR}/lib/gssapi/generic/util_buffer.c
${KRB5_SOURCE_DIR}/lib/gssapi/generic/util_buffer_set.c
${KRB5_SOURCE_DIR}/lib/gssapi/generic/util_set.c
${KRB5_SOURCE_DIR}/lib/gssapi/generic/util_token.c
${KRB5_SOURCE_DIR}/lib/gssapi/generic/gssapi_err_generic.c
${KRB5_SOURCE_DIR}/lib/gssapi/generic/disp_major_status.c
${KRB5_SOURCE_DIR}/lib/gssapi/generic/util_seqstate.c
${KRB5_SOURCE_DIR}/lib/gssapi/generic/util_errmap.c
${KRB5_SOURCE_DIR}/lib/gssapi/generic/rel_buffer.c
${KRB5_SOURCE_DIR}/lib/gssapi/spnego/spnego_mech.c
${KRB5_SOURCE_DIR}/lib/gssapi/spnego/negoex_util.c
${KRB5_SOURCE_DIR}/lib/gssapi/spnego/negoex_ctx.c
# ${KRB5_SOURCE_DIR}/lib/gssapi/spnego/negoex_trace.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/prng.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/enc_dk_cmac.c
# ${KRB5_SOURCE_DIR}/lib/crypto/krb/crc32.c
# ${KRB5_SOURCE_DIR}/lib/crypto/krb/checksum_cbc.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/enctype_util.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/enc_etm.c
# ${KRB5_SOURCE_DIR}/lib/crypto/krb/combine_keys.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/default_state.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/decrypt_iov.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/checksum_dk_cmac.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/etypes.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/old_api_glue.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/cksumtypes.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/prf_cmac.c
# ${KRB5_SOURCE_DIR}/lib/crypto/krb/enc_old.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/decrypt.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/prf_dk.c
# ${KRB5_SOURCE_DIR}/lib/crypto/krb/s2k_des.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/checksum_unkeyed.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/crypto_length.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/block_size.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/string_to_key.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/verify_checksum.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/crypto_libinit.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/derive.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/random_to_key.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/verify_checksum_iov.c
# ${KRB5_SOURCE_DIR}/lib/crypto/krb/checksum_confounder.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/checksum_length.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/enc_dk_hmac.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/make_checksum.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/prf_des.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/prf.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/coll_proof_cksum.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/enc_rc4.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/cf2.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/aead.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/encrypt_iov.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/cksumtype_to_string.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/key.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/enc_raw.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/keylengths.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/checksum_hmac_md5.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/keyed_cksum.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/keyed_checksum_types.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/prf_aes2.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/state.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/checksum_dk_hmac.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/encrypt.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/checksum_etm.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/make_random_key.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/string_to_cksumtype.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/mandatory_sumtype.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/make_checksum_iov.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/s2k_rc4.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/valid_cksumtype.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/nfold.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/prng_fortuna.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/encrypt_length.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/cmac.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/keyblocks.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/prf_rc4.c
${KRB5_SOURCE_DIR}/lib/crypto/krb/s2k_pbkdf2.c
${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/aes.c
# ${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/des.c
${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/rc4.c
${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/des3.c
${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/camellia.c
${KRB5_SOURCE_DIR}/lib/crypto/openssl/sha256.c
${KRB5_SOURCE_DIR}/lib/crypto/openssl/hmac.c
${KRB5_SOURCE_DIR}/lib/crypto/openssl/pbkdf2.c
${KRB5_SOURCE_DIR}/lib/crypto/openssl/init.c
${KRB5_SOURCE_DIR}/lib/crypto/openssl/stubs.c
# ${KRB5_SOURCE_DIR}/lib/crypto/openssl/hash_provider/hash_crc32.c
${KRB5_SOURCE_DIR}/lib/crypto/openssl/hash_provider/hash_evp.c
${KRB5_SOURCE_DIR}/lib/crypto/openssl/des/des_keys.c
${KRB5_SOURCE_DIR}/util/support/fake-addrinfo.c
${KRB5_SOURCE_DIR}/util/support/k5buf.c
${KRB5_SOURCE_DIR}/util/support/hex.c
${KRB5_SOURCE_DIR}/util/support/threads.c
${KRB5_SOURCE_DIR}/util/support/utf8.c
${KRB5_SOURCE_DIR}/util/support/hashtab.c
${KRB5_SOURCE_DIR}/util/support/dir_filenames.c
${KRB5_SOURCE_DIR}/util/support/base64.c
${KRB5_SOURCE_DIR}/util/support/strerror_r.c
${KRB5_SOURCE_DIR}/util/support/plugins.c
${KRB5_SOURCE_DIR}/util/support/path.c
${KRB5_SOURCE_DIR}/util/support/init-addrinfo.c
${KRB5_SOURCE_DIR}/util/support/json.c
${KRB5_SOURCE_DIR}/util/support/errors.c
${KRB5_SOURCE_DIR}/util/support/utf8_conv.c
${KRB5_SOURCE_DIR}/util/support/strlcpy.c
${KRB5_SOURCE_DIR}/util/support/gmt_mktime.c
${KRB5_SOURCE_DIR}/util/support/zap.c
${KRB5_SOURCE_DIR}/util/support/bcmp.c
${KRB5_SOURCE_DIR}/util/support/secure_getenv.c
${KRB5_SOURCE_DIR}/util/profile/prof_tree.c
${KRB5_SOURCE_DIR}/util/profile/prof_file.c
${KRB5_SOURCE_DIR}/util/profile/prof_parse.c
${KRB5_SOURCE_DIR}/util/profile/prof_get.c
${KRB5_SOURCE_DIR}/util/profile/prof_set.c
${KRB5_SOURCE_DIR}/util/profile/prof_err.c
${KRB5_SOURCE_DIR}/util/profile/prof_init.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/fwd_tgt.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/conv_creds.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/fast.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/ser_adata.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/copy_tick.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/enc_keyhelper.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/ser_actx.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/init_ctx.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/preauth2.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/copy_princ.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/parse_host_string.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/pr_to_salt.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/rd_req.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/pac_sign.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/copy_addrs.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/conv_princ.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/rd_rep.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/str_conv.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/gic_opt.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/recvauth.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/copy_cksum.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/ai_authdata.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/ser_ctx.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/appdefault.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/bld_princ.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/in_tkt_sky.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/copy_creds.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/auth_con.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/copy_key.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/kdc_rep_dc.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/mk_cred.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/gic_keytab.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/rd_req_dec.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/set_realm.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/preauth_sam2.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/libdef_parse.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/privsafe.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/ser_auth.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/val_renew.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/addr_order.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/authdata_dec.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/walk_rtree.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/gen_subkey.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/copy_auth.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/chpw.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/mk_req.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/allow_weak.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/mk_rep.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/mk_priv.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/s4u_authdata.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/preauth_otp.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/init_keyblock.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/ser_addr.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/encrypt_tk.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/s4u_creds.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/srv_dec_tkt.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/rd_priv.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/authdata_enc.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/authdata_exp.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/decode_kdc.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/decrypt_tk.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/enc_helper.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/mk_req_ext.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/ser_key.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/preauth_encts.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/send_tgs.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/ser_cksum.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/tgtname.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/encode_kdc.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/rd_cred.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/rd_safe.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/preauth_pkinit.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/srv_rcache.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/chk_trans.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/etype_list.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/get_creds.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/ser_princ.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/gic_pwd.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/authdata.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/gen_save_subkey.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/vfy_increds.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/addr_comp.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/kfree.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/response_items.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/serialize.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/cammac_util.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/gc_via_tkt.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/copy_ctx.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/sendauth.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/addr_srch.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/mk_safe.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/preauth_ec.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/bld_pr_ext.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/random_str.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/sname_match.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/princ_comp.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/get_in_tkt.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/gen_seqnum.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/cp_key_cnt.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/mk_error.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/copy_athctr.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/deltat.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/get_etype_info.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/plugin.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/kerrs.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/vic_opt.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/unparse.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/parse.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/rd_error.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/pac.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/valid_times.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/copy_data.c
${KRB5_SOURCE_DIR}/lib/krb5/krb/padata.c
${KRB5_SOURCE_DIR}/lib/krb5/os/hostrealm.c
${KRB5_SOURCE_DIR}/lib/krb5/os/thread_safe.c
${KRB5_SOURCE_DIR}/lib/krb5/os/krbfileio.c
${KRB5_SOURCE_DIR}/lib/krb5/os/toffset.c
${KRB5_SOURCE_DIR}/lib/krb5/os/hostaddr.c
${KRB5_SOURCE_DIR}/lib/krb5/os/ustime.c
${KRB5_SOURCE_DIR}/lib/krb5/os/timeofday.c
${KRB5_SOURCE_DIR}/lib/krb5/os/ccdefname.c
${KRB5_SOURCE_DIR}/lib/krb5/os/full_ipadr.c
${KRB5_SOURCE_DIR}/lib/krb5/os/read_pwd.c
${KRB5_SOURCE_DIR}/lib/krb5/os/trace.c
${KRB5_SOURCE_DIR}/lib/krb5/os/localauth_k5login.c
${KRB5_SOURCE_DIR}/lib/krb5/os/localauth_rule.c
${KRB5_SOURCE_DIR}/lib/krb5/os/localaddr.c
${KRB5_SOURCE_DIR}/lib/krb5/os/hostrealm_dns.c
${KRB5_SOURCE_DIR}/lib/krb5/os/hostrealm_domain.c
${KRB5_SOURCE_DIR}/lib/krb5/os/sn2princ.c
${KRB5_SOURCE_DIR}/lib/krb5/os/net_write.c
${KRB5_SOURCE_DIR}/lib/krb5/os/gen_rname.c
${KRB5_SOURCE_DIR}/lib/krb5/os/net_read.c
${KRB5_SOURCE_DIR}/lib/krb5/os/accessor.c
${KRB5_SOURCE_DIR}/lib/krb5/os/hostrealm_profile.c
${KRB5_SOURCE_DIR}/lib/krb5/os/c_ustime.c
${KRB5_SOURCE_DIR}/lib/krb5/os/expand_path.c
${KRB5_SOURCE_DIR}/lib/krb5/os/port2ip.c
${KRB5_SOURCE_DIR}/lib/krb5/os/changepw.c
${KRB5_SOURCE_DIR}/lib/krb5/os/unlck_file.c
${KRB5_SOURCE_DIR}/lib/krb5/os/gen_port.c
${KRB5_SOURCE_DIR}/lib/krb5/os/localauth_an2ln.c
${KRB5_SOURCE_DIR}/lib/krb5/os/genaddrs.c
${KRB5_SOURCE_DIR}/lib/krb5/os/init_os_ctx.c
${KRB5_SOURCE_DIR}/lib/krb5/os/localauth.c
${KRB5_SOURCE_DIR}/lib/krb5/os/locate_kdc.c
${KRB5_SOURCE_DIR}/lib/krb5/os/prompter.c
${KRB5_SOURCE_DIR}/lib/krb5/os/ktdefname.c
${KRB5_SOURCE_DIR}/lib/krb5/os/realm_dom.c
${KRB5_SOURCE_DIR}/lib/krb5/os/dnssrv.c
${KRB5_SOURCE_DIR}/lib/krb5/os/mk_faddr.c
# ${KRB5_SOURCE_DIR}/lib/krb5/os/dnsglue.c
${KRB5_SOURCE_DIR}/lib/krb5/os/sendto_kdc.c
${KRB5_SOURCE_DIR}/lib/krb5/os/hostrealm_registry.c
${KRB5_SOURCE_DIR}/lib/krb5/os/write_msg.c
${KRB5_SOURCE_DIR}/lib/krb5/os/localauth_names.c
${KRB5_SOURCE_DIR}/lib/krb5/os/read_msg.c
${KRB5_SOURCE_DIR}/lib/krb5/os/lock_file.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/ccselect.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/ccselect_realm.c
# ${KRB5_SOURCE_DIR}/lib/krb5/ccache/ser_cc.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/ccdefops.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/cc_retr.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/ccselect_k5identity.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/cccopy.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/ccfns.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/cc_file.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/ccbase.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/cccursor.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/ccdefault.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/cc_memory.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/ccmarshal.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/ccselect_hostname.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/cc_dir.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/cc_keyring.c
${KRB5_SOURCE_DIR}/lib/krb5/ccache/cc_kcm.c
${KRB5_SOURCE_DIR}/lib/krb5/keytab/ktadd.c
${KRB5_SOURCE_DIR}/lib/krb5/keytab/ktbase.c
${KRB5_SOURCE_DIR}/lib/krb5/keytab/ktdefault.c
${KRB5_SOURCE_DIR}/lib/krb5/keytab/kt_memory.c
${KRB5_SOURCE_DIR}/lib/krb5/keytab/ktfns.c
${KRB5_SOURCE_DIR}/lib/krb5/keytab/ktremove.c
${KRB5_SOURCE_DIR}/lib/krb5/keytab/read_servi.c
${KRB5_SOURCE_DIR}/lib/krb5/keytab/kt_file.c
${KRB5_SOURCE_DIR}/lib/krb5/keytab/read_servi.c
${KRB5_SOURCE_DIR}/lib/krb5/keytab/ktfr_entry.c
${KRB5_SOURCE_DIR}/lib/krb5/error_tables/k5e1_err.c
${KRB5_SOURCE_DIR}/lib/krb5/error_tables/kdb5_err.c
${KRB5_SOURCE_DIR}/lib/krb5/error_tables/asn1_err.c
${KRB5_SOURCE_DIR}/lib/krb5/error_tables/krb5_err.c
${KRB5_SOURCE_DIR}/lib/krb5/error_tables/krb524_err.c
${KRB5_SOURCE_DIR}/lib/krb5/error_tables/kv5m_err.c
${KRB5_SOURCE_DIR}/lib/krb5/rcache/rc_base.c
${KRB5_SOURCE_DIR}/lib/krb5/rcache/rc_dfl.c
${KRB5_SOURCE_DIR}/lib/krb5/rcache/rc_file2.c
${KRB5_SOURCE_DIR}/lib/krb5/rcache/rc_none.c
${KRB5_SOURCE_DIR}/lib/krb5/rcache/memrcache.c
${KRB5_SOURCE_DIR}/lib/krb5/unicode/ucdata/ucdata.c
${KRB5_SOURCE_DIR}/lib/krb5/unicode/ucstr.c
${KRB5_SOURCE_DIR}/lib/krb5/asn.1/asn1_encode.c
${KRB5_SOURCE_DIR}/lib/krb5/asn.1/asn1_k_encode.c
${KRB5_SOURCE_DIR}/lib/krb5/asn.1/ldap_key_seq.c
${KRB5_SOURCE_DIR}/lib/krb5/krb5_libinit.c
)
add_custom_command(
OUTPUT ${KRB5_SOURCE_DIR}/util/et/compile_et
COMMAND /bin/sh
./config_script
./compile_et.sh
"/usr/local/share/et"
${AWK_PROGRAM}
sed
>
compile_et
DEPENDS ${KRB5_SOURCE_DIR}/util/et/compile_et.sh ${KRB5_SOURCE_DIR}/util/et/config_script
WORKING_DIRECTORY "${KRB5_SOURCE_DIR}/util/et"
)
add_custom_target(
CREATE_COMPILE_ET ALL
DEPENDS ${KRB5_SOURCE_DIR}/util/et/compile_et
COMMENT "creating compile_et"
VERBATIM
)
file(GLOB_RECURSE ET_FILES
"${KRB5_SOURCE_DIR}/*.et"
)
function(preprocess_et out_var)
set(result)
foreach(in_f ${ARGN})
string(REPLACE
.et
.c
F_C
${in_f}
)
string(REPLACE
.et
.h
F_H
${in_f}
)
get_filename_component(ET_PATH ${in_f} DIRECTORY)
add_custom_command(OUTPUT ${F_C} ${F_H}
COMMAND perl ${KRB5_SOURCE_DIR}/util/et/compile_et -d "${KRB5_SOURCE_DIR}/util/et" ${in_f}
DEPENDS ${in_f} ${KRB5_SOURCE_DIR}/util/et/compile_et
WORKING_DIRECTORY ${ET_PATH}
COMMENT "Creating preprocessed file ${F_C}"
VERBATIM
)
list(APPEND result ${F_C})
endforeach()
set(${out_var} "${result}" PARENT_SCOPE)
endfunction()
add_custom_command(
OUTPUT ${KRB5_SOURCE_DIR}/lib/gssapi/krb5/error_map.h
COMMAND perl
-I../../../util
../../../util/gen-map.pl
-oerror_map.h
NAME=gsserrmap
KEY=OM_uint32
VALUE=char*
COMPARE=compare_OM_uint32
FREEVALUE=free_string
WORKING_DIRECTORY "${KRB5_SOURCE_DIR}/lib/gssapi/krb5"
)
add_custom_target(
ERROR_MAP_H ALL
DEPENDS ${KRB5_SOURCE_DIR}/lib/gssapi/krb5/error_map.h
COMMENT "generating error_map.h"
VERBATIM
)
add_custom_command(
OUTPUT ${KRB5_SOURCE_DIR}/lib/gssapi/generic/errmap.h
COMMAND perl -w -I../../../util ../../../util/gen.pl bimap errmap.h NAME=mecherrmap LEFT=OM_uint32 RIGHT=struct\ mecherror LEFTPRINT=print_OM_uint32 RIGHTPRINT=mecherror_print LEFTCMP=cmp_OM_uint32 RIGHTCMP=mecherror_cmp
WORKING_DIRECTORY "${KRB5_SOURCE_DIR}/lib/gssapi/generic"
)
add_custom_target(
ERRMAP_H ALL
DEPENDS ${KRB5_SOURCE_DIR}/lib/gssapi/generic/errmap.h
COMMENT "generating errmap.h"
VERBATIM
)
add_custom_target(
KRB_5_H ALL
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/include/krb5/krb5.h
COMMENT "generating krb5.h"
VERBATIM
)
add_library(${KRB5_LIBRARY})
add_dependencies(
${KRB5_LIBRARY}
ERRMAP_H
ERROR_MAP_H
KRB_5_H
)
preprocess_et(processed_et_files ${ET_FILES})
add_custom_command(
OUTPUT ${KRB5_SOURCE_DIR}/lib/gssapi/generic/errmap.h
COMMAND perl -w -I../../../util ../../../util/gen.pl bimap errmap.h NAME=mecherrmap LEFT=OM_uint32 RIGHT=struct\ mecherror LEFTPRINT=print_OM_uint32 RIGHTPRINT=mecherror_print LEFTCMP=cmp_OM_uint32 RIGHTCMP=mecherror_cmp
WORKING_DIRECTORY "${KRB5_SOURCE_DIR}/lib/gssapi/generic"
)
target_sources(${KRB5_LIBRARY} PRIVATE
${ALL_SRCS}
)
file(MAKE_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR}/include/gssapi
)
file(GLOB GSSAPI_GENERIC_HEADERS
${KRB5_SOURCE_DIR}/lib/gssapi/generic/*.h
${KRB5_SOURCE_DIR}/lib/gssapi/generic/gssapi.hin
)
file(COPY ${GSSAPI_GENERIC_HEADERS}
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/include/gssapi/
)
file(RENAME
${CMAKE_CURRENT_BINARY_DIR}/include/gssapi/gssapi.hin
${CMAKE_CURRENT_BINARY_DIR}/include/gssapi/gssapi.h
)
file(COPY ${KRB5_SOURCE_DIR}/lib/gssapi/krb5/gssapi_krb5.h
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/include/gssapi/
)
file(COPY ${KRB5_SOURCE_DIR}/util/et/com_err.h
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/include/
)
file(MAKE_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR}/include/krb5
)
SET(KRBHDEP
${KRB5_SOURCE_DIR}/include/krb5/krb5.hin
${KRB5_SOURCE_DIR}/lib/krb5/error_tables/krb5_err.h
${KRB5_SOURCE_DIR}/lib/krb5/error_tables/k5e1_err.h
${KRB5_SOURCE_DIR}/lib/krb5/error_tables/kdb5_err.h
${KRB5_SOURCE_DIR}/lib/krb5/error_tables/kv5m_err.h
${KRB5_SOURCE_DIR}/lib/krb5/error_tables/krb524_err.h
${KRB5_SOURCE_DIR}/lib/krb5/error_tables/asn1_err.h
)
# cmake < 3.18 does not have 'cat' command
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/include/krb5/krb5.h
COMMAND cat ${KRBHDEP} > ${CMAKE_CURRENT_BINARY_DIR}/include/krb5/krb5.h
DEPENDS ${KRBHDEP}
)
target_include_directories(${KRB5_LIBRARY} PUBLIC
${KRB5_SOURCE_DIR}/include
${CMAKE_CURRENT_BINARY_DIR}/include
)
target_include_directories(${KRB5_LIBRARY} PRIVATE
${CMAKE_CURRENT_SOURCE_DIR} #for autoconf.h
${KRB5_SOURCE_DIR}
${KRB5_SOURCE_DIR}/include
${KRB5_SOURCE_DIR}/lib/gssapi/mechglue
${KRB5_SOURCE_DIR}/lib/
${KRB5_SOURCE_DIR}/lib/gssapi
${KRB5_SOURCE_DIR}/lib/gssapi/generic
${KRB5_SOURCE_DIR}/lib/gssapi/krb5
${KRB5_SOURCE_DIR}/lib/gssapi/spnego
${KRB5_SOURCE_DIR}/util/et
${KRB5_SOURCE_DIR}/lib/crypto/openssl
${KRB5_SOURCE_DIR}/lib/crypto/krb
${KRB5_SOURCE_DIR}/util/profile
${KRB5_SOURCE_DIR}/lib/krb5/ccache/ccapi
${KRB5_SOURCE_DIR}/lib/krb5/ccache
${KRB5_SOURCE_DIR}/lib/krb5/keytab
${KRB5_SOURCE_DIR}/lib/krb5/rcache
${KRB5_SOURCE_DIR}/lib/krb5/unicode
${KRB5_SOURCE_DIR}/lib/krb5/os
# ${OPENSSL_INCLUDE_DIR}
)
target_compile_definitions(${KRB5_LIBRARY} PRIVATE
KRB5_PRIVATE
_GSS_STATIC_LINK=1
KRB5_DEPRECATED=1
LOCALEDIR="/usr/local/share/locale"
BINDIR="/usr/local/bin"
SBINDIR="/usr/local/sbin"
LIBDIR="/usr/local/lib"
)
target_link_libraries(${KRB5_LIBRARY}
PRIVATE ${OPENSSL_CRYPTO_LIBRARY}
)

764
contrib/krb5-cmake/autoconf.h Normal file
View File

@ -0,0 +1,764 @@
/* include/autoconf.h. Generated from autoconf.h.in by configure. */
/* include/autoconf.h.in. Generated from configure.in by autoheader. */
#ifndef KRB5_AUTOCONF_H
#define KRB5_AUTOCONF_H
/* Define if AES-NI support is enabled */
/* #undef AESNI */
/* Define if socket can't be bound to 0.0.0.0 */
/* #undef BROKEN_STREAMS_SOCKETS */
/* Define if va_list objects can be simply copied by assignment. */
/* #undef CAN_COPY_VA_LIST */
/* Define to reduce code size even if it means more cpu usage */
/* #undef CONFIG_SMALL */
/* Define if __attribute__((constructor)) works */
#define CONSTRUCTOR_ATTR_WORKS 1
/* Define to default ccache name */
#define DEFCCNAME "FILE:/tmp/krb5cc_%{uid}"
/* Define to default client keytab name */
#define DEFCKTNAME "FILE:/etc/krb5/user/%{euid}/client.keytab"
/* Define to default keytab name */
#define DEFKTNAME "FILE:/etc/krb5.keytab"
/* Define if library initialization should be delayed until first use */
#define DELAY_INITIALIZER 1
/* Define if __attribute__((destructor)) works */
#define DESTRUCTOR_ATTR_WORKS 1
/* Define to disable PKINIT plugin support */
/* #undef DISABLE_PKINIT */
/* Define if LDAP KDB support within the Kerberos library (mainly ASN.1 code)
should be enabled. */
/* #undef ENABLE_LDAP */
/* Define if translation functions should be used. */
#define ENABLE_NLS 1
/* Define if thread support enabled */
#define ENABLE_THREADS 1
/* Define as return type of endrpcent */
#define ENDRPCENT_TYPE void
/* Define if Fortuna PRNG is selected */
#define FORTUNA 1
/* Define to the type of elements in the array set by `getgroups'. Usually
this is either `int' or `gid_t'. */
#define GETGROUPS_T gid_t
/* Define if gethostbyname_r returns int rather than struct hostent * */
#define GETHOSTBYNAME_R_RETURNS_INT 1
/* Type of getpeername second argument. */
#define GETPEERNAME_ARG3_TYPE GETSOCKNAME_ARG3_TYPE
/* Define if getpwnam_r exists but takes only 4 arguments (e.g., POSIX draft 6
implementations like some Solaris releases). */
/* #undef GETPWNAM_R_4_ARGS */
/* Define if getpwnam_r returns an int */
#define GETPWNAM_R_RETURNS_INT 1
/* Define if getpwuid_r exists but takes only 4 arguments (e.g., POSIX draft 6
implementations like some Solaris releases). */
/* #undef GETPWUID_R_4_ARGS */
/* Define if getservbyname_r returns int rather than struct servent * */
#define GETSERVBYNAME_R_RETURNS_INT 1
/* Type of pointer target for argument 3 to getsockname */
#define GETSOCKNAME_ARG3_TYPE socklen_t
/* Define if gmtime_r returns int instead of struct tm pointer, as on old
HP-UX systems. */
/* #undef GMTIME_R_RETURNS_INT */
/* Define if va_copy macro or function is available. */
#define HAS_VA_COPY 1
/* Define to 1 if you have the `access' function. */
#define HAVE_ACCESS 1
/* Define to 1 if you have the <alloca.h> header file. */
#define HAVE_ALLOCA_H 1
/* Define to 1 if you have the <arpa/inet.h> header file. */
#define HAVE_ARPA_INET_H 1
/* Define to 1 if you have the `bswap16' function. */
/* #undef HAVE_BSWAP16 */
/* Define to 1 if you have the `bswap64' function. */
/* #undef HAVE_BSWAP64 */
/* Define to 1 if bswap_16 is available via byteswap.h */
#define HAVE_BSWAP_16 1
/* Define to 1 if bswap_64 is available via byteswap.h */
#define HAVE_BSWAP_64 1
/* Define if bt_rseq is available, for recursive btree traversal. */
#define HAVE_BT_RSEQ 1
/* Define to 1 if you have the <byteswap.h> header file. */
#define HAVE_BYTESWAP_H 1
/* Define to 1 if you have the `chmod' function. */
#define HAVE_CHMOD 1
/* Define if cmocka library is available. */
/* #undef HAVE_CMOCKA */
/* Define to 1 if you have the `compile' function. */
/* #undef HAVE_COMPILE */
/* Define if com_err has compatible gettext support */
#define HAVE_COM_ERR_INTL 1
/* Define to 1 if you have the <cpuid.h> header file. */
/* #undef HAVE_CPUID_H */
/* Define to 1 if you have the `daemon' function. */
#define HAVE_DAEMON 1
/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you
don't. */
#define HAVE_DECL_STRERROR_R 1
/* Define to 1 if you have the <dirent.h> header file, and it defines `DIR'.
*/
#define HAVE_DIRENT_H 1
/* Define to 1 if you have the <dlfcn.h> header file. */
#define HAVE_DLFCN_H 1
/* Define to 1 if you have the `dn_skipname' function. */
#define HAVE_DN_SKIPNAME 0
/* Define to 1 if you have the <endian.h> header file. */
#define HAVE_ENDIAN_H 1
/* Define to 1 if you have the <errno.h> header file. */
#define HAVE_ERRNO_H 1
/* Define to 1 if you have the `fchmod' function. */
#define HAVE_FCHMOD 1
/* Define to 1 if you have the <fcntl.h> header file. */
#define HAVE_FCNTL_H 1
/* Define to 1 if you have the `flock' function. */
#define HAVE_FLOCK 1
/* Define to 1 if you have the `fnmatch' function. */
#define HAVE_FNMATCH 1
/* Define to 1 if you have the <fnmatch.h> header file. */
#define HAVE_FNMATCH_H 1
/* Define if you have the getaddrinfo function */
#define HAVE_GETADDRINFO 1
/* Define to 1 if you have the `getcwd' function. */
#define HAVE_GETCWD 1
/* Define to 1 if you have the `getenv' function. */
#define HAVE_GETENV 1
/* Define to 1 if you have the `geteuid' function. */
#define HAVE_GETEUID 1
/* Define if gethostbyname_r exists and its return type is known */
#define HAVE_GETHOSTBYNAME_R 1
/* Define to 1 if you have the `getnameinfo' function. */
#define HAVE_GETNAMEINFO 1
/* Define if system getopt should be used. */
#define HAVE_GETOPT 1
/* Define if system getopt_long should be used. */
#define HAVE_GETOPT_LONG 1
/* Define if getpwnam_r is available and useful. */
#define HAVE_GETPWNAM_R 1
/* Define if getpwuid_r is available and useful. */
#define HAVE_GETPWUID_R 1
/* Define if getservbyname_r exists and its return type is known */
#define HAVE_GETSERVBYNAME_R 1
/* Have the gettimeofday function */
#define HAVE_GETTIMEOFDAY 1
/* Define to 1 if you have the `getusershell' function. */
#define HAVE_GETUSERSHELL 1
/* Define to 1 if you have the `gmtime_r' function. */
#define HAVE_GMTIME_R 1
/* Define to 1 if you have the <ifaddrs.h> header file. */
#define HAVE_IFADDRS_H 1
/* Define to 1 if you have the `inet_ntop' function. */
#define HAVE_INET_NTOP 1
/* Define to 1 if you have the `inet_pton' function. */
#define HAVE_INET_PTON 1
/* Define to 1 if the system has the type `int16_t'. */
#define HAVE_INT16_T 1
/* Define to 1 if the system has the type `int32_t'. */
#define HAVE_INT32_T 1
/* Define to 1 if the system has the type `int8_t'. */
#define HAVE_INT8_T 1
/* Define to 1 if you have the <inttypes.h> header file. */
#define HAVE_INTTYPES_H 1
/* Define to 1 if you have the <keyutils.h> header file. */
/* #undef HAVE_KEYUTILS_H */
/* Define to 1 if you have the <lber.h> header file. */
/* #undef HAVE_LBER_H */
/* Define to 1 if you have the <ldap.h> header file. */
/* #undef HAVE_LDAP_H */
/* Define to 1 if you have the `crypto' library (-lcrypto). */
#define HAVE_LIBCRYPTO 1
/* Define if building with libedit. */
/* #undef HAVE_LIBEDIT */
/* Define to 1 if you have the `nsl' library (-lnsl). */
/* #undef HAVE_LIBNSL */
/* Define to 1 if you have the `resolv' library (-lresolv). */
#define HAVE_LIBRESOLV 1
/* Define to 1 if you have the `socket' library (-lsocket). */
/* #undef HAVE_LIBSOCKET */
/* Define if the util library is available */
#define HAVE_LIBUTIL 1
/* Define to 1 if you have the <limits.h> header file. */
#define HAVE_LIMITS_H 1
/* Define to 1 if you have the `localtime_r' function. */
#define HAVE_LOCALTIME_R 1
/* Define to 1 if you have the <machine/byte_order.h> header file. */
/* #undef HAVE_MACHINE_BYTE_ORDER_H */
/* Define to 1 if you have the <machine/endian.h> header file. */
/* #undef HAVE_MACHINE_ENDIAN_H */
/* Define to 1 if you have the <memory.h> header file. */
#define HAVE_MEMORY_H 1
/* Define to 1 if you have the `mkstemp' function. */
#define HAVE_MKSTEMP 1
/* Define to 1 if you have the <ndir.h> header file, and it defines `DIR'. */
/* #undef HAVE_NDIR_H */
/* Define to 1 if you have the <netdb.h> header file. */
#define HAVE_NETDB_H 1
/* Define if netdb.h declares h_errno */
#define HAVE_NETDB_H_H_ERRNO 1
/* Define to 1 if you have the <netinet/in.h> header file. */
#define HAVE_NETINET_IN_H 1
/* Define to 1 if you have the `ns_initparse' function. */
#define HAVE_NS_INITPARSE 0
/* Define to 1 if you have the `ns_name_uncompress' function. */
#define HAVE_NS_NAME_UNCOMPRESS 0
/* Define if OpenSSL supports cms. */
#define HAVE_OPENSSL_CMS 1
/* Define to 1 if you have the <paths.h> header file. */
#define HAVE_PATHS_H 1
/* Define if persistent keyrings are supported */
/* #undef HAVE_PERSISTENT_KEYRING */
/* Define to 1 if you have the <poll.h> header file. */
#define HAVE_POLL_H 1
/* Define if #pragma weak references work */
#define HAVE_PRAGMA_WEAK_REF 1
/* Define if you have POSIX threads libraries and header files. */
#define HAVE_PTHREAD 1
/* Define to 1 if you have the `pthread_once' function. */
/* #undef HAVE_PTHREAD_ONCE */
/* Have PTHREAD_PRIO_INHERIT. */
#define HAVE_PTHREAD_PRIO_INHERIT 1
/* Define to 1 if you have the `pthread_rwlock_init' function. */
/* #undef HAVE_PTHREAD_RWLOCK_INIT */
/* Define if pthread_rwlock_init is provided in the thread library. */
#define HAVE_PTHREAD_RWLOCK_INIT_IN_THREAD_LIB 1
/* Define to 1 if you have the <pwd.h> header file. */
#define HAVE_PWD_H 1
/* Define if building with GNU Readline. */
/* #undef HAVE_READLINE */
/* Define if regcomp exists and functions */
#define HAVE_REGCOMP 1
/* Define to 1 if you have the `regexec' function. */
#define HAVE_REGEXEC 1
/* Define to 1 if you have the <regexpr.h> header file. */
/* #undef HAVE_REGEXPR_H */
/* Define to 1 if you have the <regex.h> header file. */
#define HAVE_REGEX_H 1
/* Define to 1 if you have the `res_nclose' function. */
#define HAVE_RES_NCLOSE 1
/* Define to 1 if you have the `res_ndestroy' function. */
/* #undef HAVE_RES_NDESTROY */
/* Define to 1 if you have the `res_ninit' function. */
#define HAVE_RES_NINIT 1
/* Define to 1 if you have the `res_nsearch' function. */
#define HAVE_RES_NSEARCH 0
/* Define to 1 if you have the `res_search' function */
#define HAVE_RES_SEARCH 1
/* Define to 1 if you have the `re_comp' function. */
#define HAVE_RE_COMP 1
/* Define to 1 if you have the `re_exec' function. */
#define HAVE_RE_EXEC 1
/* Define to 1 if you have the <sasl/sasl.h> header file. */
/* #undef HAVE_SASL_SASL_H */
/* Define if struct sockaddr contains sa_len */
/* #undef HAVE_SA_LEN */
/* Define to 1 if you have the `setegid' function. */
#define HAVE_SETEGID 1
/* Define to 1 if you have the `setenv' function. */
#define HAVE_SETENV 1
/* Define to 1 if you have the `seteuid' function. */
#define HAVE_SETEUID 1
/* Define if setluid provided in OSF/1 security library */
/* #undef HAVE_SETLUID */
/* Define to 1 if you have the `setregid' function. */
#define HAVE_SETREGID 1
/* Define to 1 if you have the `setresgid' function. */
#define HAVE_SETRESGID 1
/* Define to 1 if you have the `setresuid' function. */
#define HAVE_SETRESUID 1
/* Define to 1 if you have the `setreuid' function. */
#define HAVE_SETREUID 1
/* Define to 1 if you have the `setsid' function. */
#define HAVE_SETSID 1
/* Define to 1 if you have the `setvbuf' function. */
#define HAVE_SETVBUF 1
/* Define if there is a socklen_t type. If not, probably use size_t */
#define HAVE_SOCKLEN_T 1
/* Define to 1 if you have the `srand' function. */
#define HAVE_SRAND 1
/* Define to 1 if you have the `srand48' function. */
#define HAVE_SRAND48 1
/* Define to 1 if you have the `srandom' function. */
#define HAVE_SRANDOM 1
/* Define to 1 if the system has the type `ssize_t'. */
#define HAVE_SSIZE_T 1
/* Define to 1 if you have the `stat' function. */
#define HAVE_STAT 1
/* Define to 1 if you have the <stddef.h> header file. */
#define HAVE_STDDEF_H 1
/* Define to 1 if you have the <stdint.h> header file. */
#define HAVE_STDINT_H 1
/* Define to 1 if you have the <stdlib.h> header file. */
#define HAVE_STDLIB_H 1
/* Define to 1 if you have the `step' function. */
/* #undef HAVE_STEP */
/* Define to 1 if you have the `strchr' function. */
#define HAVE_STRCHR 1
/* Define to 1 if you have the `strdup' function. */
#define HAVE_STRDUP 1
/* Define to 1 if you have the `strerror' function. */
#define HAVE_STRERROR 1
/* Define to 1 if you have the `strerror_r' function. */
#define HAVE_STRERROR_R 1
/* Define to 1 if you have the <strings.h> header file. */
#define HAVE_STRINGS_H 1
/* Define to 1 if you have the <string.h> header file. */
#define HAVE_STRING_H 1
/* Define to 1 if you have the `strlcpy' function. */
/* #undef HAVE_STRLCPY */
/* Define to 1 if you have the `strptime' function. */
#define HAVE_STRPTIME 1
/* Define to 1 if the system has the type `struct cmsghdr'. */
#define HAVE_STRUCT_CMSGHDR 1
/* Define if there is a struct if_laddrconf. */
/* #undef HAVE_STRUCT_IF_LADDRCONF */
/* Define to 1 if the system has the type `struct in6_pktinfo'. */
#define HAVE_STRUCT_IN6_PKTINFO 1
/* Define to 1 if the system has the type `struct in_pktinfo'. */
#define HAVE_STRUCT_IN_PKTINFO 1
/* Define if there is a struct lifconf. */
/* #undef HAVE_STRUCT_LIFCONF */
/* Define to 1 if the system has the type `struct rt_msghdr'. */
/* #undef HAVE_STRUCT_RT_MSGHDR */
/* Define to 1 if the system has the type `struct sockaddr_storage'. */
#define HAVE_STRUCT_SOCKADDR_STORAGE 1
/* Define to 1 if `st_mtimensec' is a member of `struct stat'. */
/* #undef HAVE_STRUCT_STAT_ST_MTIMENSEC */
/* Define to 1 if `st_mtimespec.tv_nsec' is a member of `struct stat'. */
/* #undef HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC */
/* Define to 1 if `st_mtim.tv_nsec' is a member of `struct stat'. */
#define HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC 1
/* Define to 1 if you have the <sys/bswap.h> header file. */
/* #undef HAVE_SYS_BSWAP_H */
/* Define to 1 if you have the <sys/dir.h> header file, and it defines `DIR'.
*/
/* #undef HAVE_SYS_DIR_H */
/* Define if sys_errlist in libc */
#define HAVE_SYS_ERRLIST 1
/* Define to 1 if you have the <sys/file.h> header file. */
#define HAVE_SYS_FILE_H 1
/* Define to 1 if you have the <sys/filio.h> header file. */
/* #undef HAVE_SYS_FILIO_H */
/* Define to 1 if you have the <sys/ndir.h> header file, and it defines `DIR'.
*/
/* #undef HAVE_SYS_NDIR_H */
/* Define to 1 if you have the <sys/param.h> header file. */
#define HAVE_SYS_PARAM_H 1
/* Define to 1 if you have the <sys/select.h> header file. */
#define HAVE_SYS_SELECT_H 1
/* Define to 1 if you have the <sys/socket.h> header file. */
#define HAVE_SYS_SOCKET_H 1
/* Define to 1 if you have the <sys/sockio.h> header file. */
/* #undef HAVE_SYS_SOCKIO_H */
/* Define to 1 if you have the <sys/stat.h> header file. */
#define HAVE_SYS_STAT_H 1
/* Define to 1 if you have the <sys/time.h> header file. */
#define HAVE_SYS_TIME_H 1
/* Define to 1 if you have the <sys/types.h> header file. */
#define HAVE_SYS_TYPES_H 1
/* Define to 1 if you have the <sys/uio.h> header file. */
#define HAVE_SYS_UIO_H 1
/* Define if tcl.h found */
/* #undef HAVE_TCL_H */
/* Define if tcl/tcl.h found */
/* #undef HAVE_TCL_TCL_H */
/* Define to 1 if you have the `timegm' function. */
#define HAVE_TIMEGM 1
/* Define to 1 if you have the <time.h> header file. */
#define HAVE_TIME_H 1
/* Define to 1 if you have the <unistd.h> header file. */
#define HAVE_UNISTD_H 1
/* Define to 1 if you have the `unsetenv' function. */
#define HAVE_UNSETENV 1
/* Define to 1 if the system has the type `u_char'. */
#define HAVE_U_CHAR 1
/* Define to 1 if the system has the type `u_int'. */
#define HAVE_U_INT 1
/* Define to 1 if the system has the type `u_int16_t'. */
#define HAVE_U_INT16_T 1
/* Define to 1 if the system has the type `u_int32_t'. */
#define HAVE_U_INT32_T 1
/* Define to 1 if the system has the type `u_int8_t'. */
#define HAVE_U_INT8_T 1
/* Define to 1 if the system has the type `u_long'. */
#define HAVE_U_LONG 1
/* Define to 1 if you have the `vasprintf' function. */
#define HAVE_VASPRINTF 1
/* Define to 1 if you have the `vsnprintf' function. */
#define HAVE_VSNPRINTF 1
/* Define to 1 if you have the `vsprintf' function. */
#define HAVE_VSPRINTF 1
/* Define to 1 if the system has the type `__int128_t'. */
#define HAVE___INT128_T 1
/* Define to 1 if the system has the type `__uint128_t'. */
#define HAVE___UINT128_T 1
/* Define if errno.h declares perror */
/* #undef HDR_HAS_PERROR */
/* May need to be defined to enable IPv6 support, for example on IRIX */
/* #undef INET6 */
/* Define if MIT Project Athena default configuration should be used */
/* #undef KRB5_ATHENA_COMPAT */
/* Define for DNS support of locating realms and KDCs */
#undef KRB5_DNS_LOOKUP
/* Define to enable DNS lookups of Kerberos realm names */
/* #undef KRB5_DNS_LOOKUP_REALM */
/* Define if the KDC should return only vague error codes to clients */
/* #undef KRBCONF_VAGUE_ERRORS */
/* define if the system header files are missing prototype for daemon() */
/* #undef NEED_DAEMON_PROTO */
/* Define if in6addr_any is not defined in libc */
/* #undef NEED_INSIXADDR_ANY */
/* define if the system header files are missing prototype for
ss_execute_command() */
/* #undef NEED_SS_EXECUTE_COMMAND_PROTO */
/* define if the system header files are missing prototype for strptime() */
/* #undef NEED_STRPTIME_PROTO */
/* define if the system header files are missing prototype for swab() */
/* #undef NEED_SWAB_PROTO */
/* Define if need to declare sys_errlist */
/* #undef NEED_SYS_ERRLIST */
/* define if the system header files are missing prototype for vasprintf() */
/* #undef NEED_VASPRINTF_PROTO */
/* Define if the KDC should use no lookaside cache */
/* #undef NOCACHE */
/* Define if references to pthread routines should be non-weak. */
/* #undef NO_WEAK_PTHREADS */
/* Define if lex produes code with yylineno */
/* #undef NO_YYLINENO */
/* Define to the address where bug reports for this package should be sent. */
#define PACKAGE_BUGREPORT "krb5-bugs@mit.edu"
/* Define to the full name of this package. */
#define PACKAGE_NAME "Kerberos 5"
/* Define to the full name and version of this package. */
#define PACKAGE_STRING "Kerberos 5 1.17.1"
/* Define to the one symbol short name of this package. */
#define PACKAGE_TARNAME "krb5"
/* Define to the home page for this package. */
#define PACKAGE_URL ""
/* Define to the version of this package. */
#define PACKAGE_VERSION "1.17.1"
/* Define if setjmp indicates POSIX interface */
/* #undef POSIX_SETJMP */
/* Define if POSIX signal handling is used */
#define POSIX_SIGNALS 1
/* Define if POSIX signal handlers are used */
#define POSIX_SIGTYPE 1
/* Define if termios.h exists and tcsetattr exists */
#define POSIX_TERMIOS 1
/* Define to necessary symbol if this constant uses a non-standard name on
your system. */
/* #undef PTHREAD_CREATE_JOINABLE */
/* Define as the return type of signal handlers (`int' or `void'). */
#define RETSIGTYPE void
/* Define as return type of setrpcent */
#define SETRPCENT_TYPE void
/* The size of `size_t', as computed by sizeof. */
#define SIZEOF_SIZE_T 8
/* The size of `time_t', as computed by sizeof. */
#define SIZEOF_TIME_T 8
/* Define to use OpenSSL for SPAKE preauth */
#define SPAKE_OPENSSL 1
/* Define for static plugin linkage */
/* #undef STATIC_PLUGINS */
/* Define to 1 if you have the ANSI C header files. */
#define STDC_HEADERS 1
/* Define to 1 if strerror_r returns char *. */
#define STRERROR_R_CHAR_P 1
/* Define if sys_errlist is defined in errno.h */
#define SYS_ERRLIST_DECLARED 1
/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */
#define TIME_WITH_SYS_TIME 1
/* Define if no TLS implementation is selected */
/* #undef TLS_IMPL_NONE */
/* Define if TLS implementation is OpenSSL */
#define TLS_IMPL_OPENSSL 1
/* Define if you have dirent.h functionality */
#define USE_DIRENT_H 1
/* Define if dlopen should be used */
#define USE_DLOPEN 1
/* Define if the keyring ccache should be enabled */
/* #undef USE_KEYRING_CCACHE */
/* Define if link-time options for library finalization will be used */
/* #undef USE_LINKER_FINI_OPTION */
/* Define if link-time options for library initialization will be used */
/* #undef USE_LINKER_INIT_OPTION */
/* Define if sigprocmask should be used */
#define USE_SIGPROCMASK 1
/* Define if wait takes int as a argument */
#define WAIT_USES_INT 1
/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a
`char[]'. */
#define YYTEXT_POINTER 1
/* Define to enable extensions in glibc */
#define _GNU_SOURCE 1
/* Define to enable C11 extensions */
#define __STDC_WANT_LIB_EXT1__ 1
/* Define to empty if `const' does not conform to ANSI C. */
/* #undef const */
/* Define to `int' if <sys/types.h> doesn't define. */
/* #undef gid_t */
/* Define to `__inline__' or `__inline' if that's what the C compiler
calls it, or to nothing if 'inline' is not supported under any name. */
#ifndef __cplusplus
/* #undef inline */
#endif
/* Define krb5_sigtype to type of signal handler */
#define krb5_sigtype void
/* Define to `int' if <sys/types.h> does not define. */
/* #undef mode_t */
/* Define to `long int' if <sys/types.h> does not define. */
/* #undef off_t */
/* Define to `long' if <sys/types.h> does not define. */
/* #undef time_t */
/* Define to `int' if <sys/types.h> doesn't define. */
/* #undef uid_t */
#if defined(__GNUC__) && !defined(inline)
/* Silence gcc pedantic warnings about ANSI C. */
# define inline __inline__
#endif
#endif /* KRB5_AUTOCONF_H */

141
contrib/krb5-cmake/osconf.h Normal file
View File

@ -0,0 +1,141 @@
/* -*- mode: c; c-basic-offset: 4; indent-tabs-mode: nil -*- */
/*
* Copyright 1990,1991,2008 by the Massachusetts Institute of Technology.
* All Rights Reserved.
*
* Export of this software from the United States of America may
* require a specific license from the United States Government.
* It is the responsibility of any person or organization contemplating
* export to obtain such a license before exporting.
*
* WITHIN THAT CONSTRAINT, permission to use, copy, modify, and
* distribute this software and its documentation for any purpose and
* without fee is hereby granted, provided that the above copyright
* notice appear in all copies and that both that copyright notice and
* this permission notice appear in supporting documentation, and that
* the name of M.I.T. not be used in advertising or publicity pertaining
* to distribution of the software without specific, written prior
* permission. Furthermore if you modify this software you must label
* your software as modified software and not distribute it in such a
* fashion that it might be confused with the original M.I.T. software.
* M.I.T. makes no representations about the suitability of
* this software for any purpose. It is provided "as is" without express
* or implied warranty.
*/
/* Site- and OS- dependent configuration */
#ifndef KRB5_OSCONF__
#define KRB5_OSCONF__
#if !defined(_WIN32)
/* Don't try to pull in autoconf.h for Windows, since it's not used */
#ifndef KRB5_AUTOCONF__
#define KRB5_AUTOCONF__
#include "autoconf.h"
#endif
#endif
#if defined(__MACH__) && defined(__APPLE__)
# include <TargetConditionals.h>
#endif
#if defined(_WIN32)
#define DEFAULT_PROFILE_FILENAME "krb5.ini"
#else /* !_WINDOWS */
#if TARGET_OS_MAC
#define DEFAULT_SECURE_PROFILE_PATH "/Library/Preferences/edu.mit.Kerberos:/etc/krb5.conf:/usr/local/etc/krb5.conf"
#define DEFAULT_PROFILE_PATH ("~/Library/Preferences/edu.mit.Kerberos" ":" DEFAULT_SECURE_PROFILE_PATH)
#define KRB5_PLUGIN_BUNDLE_DIR "/System/Library/KerberosPlugins/KerberosFrameworkPlugins"
#define KDB5_PLUGIN_BUNDLE_DIR "/System/Library/KerberosPlugins/KerberosDatabasePlugins"
#define KRB5_AUTHDATA_PLUGIN_BUNDLE_DIR "/System/Library/KerberosPlugins/KerberosAuthDataPlugins"
#else
#define DEFAULT_SECURE_PROFILE_PATH "/etc/krb5.conf:/usr/local/etc/krb5.conf"
#define DEFAULT_PROFILE_PATH DEFAULT_SECURE_PROFILE_PATH
#endif
#endif /* _WINDOWS */
#ifdef _WIN32
#define DEFAULT_PLUGIN_BASE_DIR "%{LIBDIR}\\plugins"
#else
#define DEFAULT_PLUGIN_BASE_DIR "/usr/local/lib/krb5/plugins"
#endif
#if defined(_WIN64)
#define PLUGIN_EXT "64.dll"
#elif defined(_WIN32)
#define PLUGIN_EXT "32.dll"
#else
#define PLUGIN_EXT ".so"
#endif
#define KDC_DIR "/usr/local/var/krb5kdc"
#define KDC_RUN_DIR "/run/krb5kdc"
#define DEFAULT_KDB_FILE KDC_DIR "/principal"
#define DEFAULT_KEYFILE_STUB KDC_DIR "/.k5."
#define KRB5_DEFAULT_ADMIN_ACL KDC_DIR "/krb5_adm.acl"
/* Used by old admin server */
#define DEFAULT_ADMIN_ACL KDC_DIR "/kadm_old.acl"
/* Location of KDC profile */
#define DEFAULT_KDC_PROFILE KDC_DIR "/kdc.conf"
#define KDC_PROFILE_ENV "KRB5_KDC_PROFILE"
#if TARGET_OS_MAC
#define DEFAULT_KDB_LIB_PATH { KDB5_PLUGIN_BUNDLE_DIR, "/usr/local/lib/krb5/plugins/kdb", NULL }
#else
#define DEFAULT_KDB_LIB_PATH { "/usr/local/lib/krb5/plugins/kdb", NULL }
#endif
#define DEFAULT_KDC_ENCTYPE ENCTYPE_AES256_CTS_HMAC_SHA1_96
#define KDCRCACHE "dfl:krb5kdc_rcache"
#define KDC_PORTNAME "kerberos" /* for /etc/services or equiv. */
#define KRB5_DEFAULT_PORT 88
#define DEFAULT_KPASSWD_PORT 464
#define DEFAULT_KDC_UDP_PORTLIST "88"
#define DEFAULT_KDC_TCP_PORTLIST "88"
#define DEFAULT_TCP_LISTEN_BACKLOG 5
/*
* Defaults for the KADM5 admin system.
*/
#define DEFAULT_KADM5_KEYTAB KDC_DIR "/kadm5.keytab"
#define DEFAULT_KADM5_ACL_FILE KDC_DIR "/kadm5.acl"
#define DEFAULT_KADM5_PORT 749 /* assigned by IANA */
#define KRB5_DEFAULT_SUPPORTED_ENCTYPES \
"aes256-cts-hmac-sha1-96:normal " \
"aes128-cts-hmac-sha1-96:normal"
#define MAX_DGRAM_SIZE 65536
#define RCTMPDIR "/var/tmp" /* directory to store replay caches */
#define KRB5_PATH_TTY "/dev/tty"
#define KRB5_PATH_LOGIN "/usr/local/sbin/login.krb5"
#define KRB5_PATH_RLOGIN "/usr/local/bin/rlogin"
#define KRB5_ENV_CCNAME "KRB5CCNAME"
/*
* krb5 replica support follows
*/
#define KPROP_DEFAULT_FILE KDC_DIR "/replica_datatrans"
#define KPROPD_DEFAULT_FILE KDC_DIR "/from_master"
#define KPROPD_DEFAULT_KDB5_UTIL "/usr/local/sbin/kdb5_util"
#define KPROPD_DEFAULT_KPROP "/usr/local/sbin/kprop"
#define KPROPD_DEFAULT_KRB_DB DEFAULT_KDB_FILE
#define KPROPD_ACL_FILE KDC_DIR "/kpropd.acl"
/*
* GSS mechglue
*/
#define MECH_CONF "/usr/local/etc/gss/mech"
#define MECH_LIB_PREFIX "/usr/local/lib/gss/"
#endif /* KRB5_OSCONF__ */

2
contrib/krb5-cmake/profile.h Normal file
View File

@ -0,0 +1,2 @@
#include "util/profile/profile.hin"
#include "util/profile/prof_err.h"

27
contrib/librdkafka-cmake/CMakeLists.txt
View File

@ -49,7 +49,6 @@ set(SRCS
${RDKAFKA_SOURCE_DIR}/rdkafka_request.c
${RDKAFKA_SOURCE_DIR}/rdkafka_roundrobin_assignor.c
${RDKAFKA_SOURCE_DIR}/rdkafka_sasl.c
# ${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_cyrus.c # needed to support Kerberos, requires cyrus-sasl
${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_oauthbearer.c
${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_plain.c
${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_scram.c
@ -77,12 +76,34 @@ set(SRCS
${RDKAFKA_SOURCE_DIR}/rdgz.c
)
if(${ENABLE_CYRUS_SASL})
message (STATUS "librdkafka with SASL support")
set(SRCS
${SRCS}
${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_cyrus.c # needed to support Kerberos, requires cyrus-sasl
)
endif()
add_library(rdkafka ${SRCS})
target_compile_options(rdkafka PRIVATE -fno-sanitize=undefined)
target_include_directories(rdkafka SYSTEM PUBLIC include)
# target_include_directories(rdkafka SYSTEM PUBLIC include)
target_include_directories(rdkafka SYSTEM PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) # for "librdkafka/rdkafka.h"
target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR}) # Because weird logic with "include_next" is used.
target_include_directories(rdkafka SYSTEM PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/auxdir) # for "../config.h"
target_include_directories(rdkafka SYSTEM PRIVATE ${ZSTD_INCLUDE_DIR}/common) # Because wrong path to "zstd_errors.h" is used.
target_link_libraries(rdkafka PRIVATE lz4 ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LIBGSASL_LIBRARY})
target_link_libraries(rdkafka PRIVATE lz4 ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY})
if(OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY)
target_link_libraries(rdkafka PRIVATE ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
endif()
if(${ENABLE_CYRUS_SASL})
target_link_libraries(rdkafka PRIVATE ${CYRUS_SASL_LIBRARY})
set(WITH_SASL_CYRUS 1)
endif()
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/auxdir)
configure_file(
"${CMAKE_CURRENT_SOURCE_DIR}/config.h.in"
"${CMAKE_CURRENT_BINARY_DIR}/config.h"
IMMEDIATE @ONLY
)

3
contrib/librdkafka-cmake/config.h → contrib/librdkafka-cmake/config.h.in
View File

@ -1,4 +1,4 @@
// Automatically generated by ./configure
// Originally generated by ./configure
#ifndef _CONFIG_H_
#define _CONFIG_H_
#define ARCH "x86_64"
@ -65,6 +65,7 @@
#define WITH_SASL_SCRAM 1
// WITH_SASL_OAUTHBEARER
#define WITH_SASL_OAUTHBEARER 1
#cmakedefine WITH_SASL_CYRUS 1
// crc32chw
#if !defined(__PPC__)
#define WITH_CRC32C_HW 1

5
docker/images.json
View File

@ -127,5 +127,10 @@
"docker/test/integration/postgresql_java_client": {
"name": "yandex/clickhouse-postgresql-java-client",
"dependent": []
},
"docker/test/base": {
"name": "yandex/clickhouse-test-base",
"dependent": [
]
}
}

51
docker/test/base/Dockerfile Normal file
View File

@ -0,0 +1,51 @@
# docker build -t yandex/clickhouse-test-base .
FROM ubuntu:19.10
RUN apt-get --allow-unauthenticated update -y && apt-get install --yes wget gnupg
RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 main" >> /etc/apt/sources.list
# initial packages
RUN apt-get --allow-unauthenticated update -y \
&& env DEBIAN_FRONTEND=noninteractive \
apt-get --allow-unauthenticated install --yes --no-install-recommends \
apt-transport-https \
bash \
ca-certificates \
curl \
fakeroot \
gnupg \
software-properties-common
# Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able
# to compress files using pigz (https://zlib.net/pigz/) instead of gzip.
# Significantly increase deb packaging speed and compatible with old systems
RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb
RUN chmod +x dpkg-deb
RUN cp dpkg-deb /usr/bin
RUN apt-get --allow-unauthenticated update -y \
&& env DEBIAN_FRONTEND=noninteractive \
apt-get --allow-unauthenticated install --yes --no-install-recommends \
clang-10 \
debhelper \
devscripts \
gdb \
git \
gperf \
lcov \
llvm-10 \
moreutils \
perl \
perl \
pigz \
pkg-config \
tzdata
# Sanitizer options
RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment; \
echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \
echo "MSAN_OPTIONS='abort_on_error=1'" >> /etc/environment; \
ln -s /usr/lib/llvm-10/bin/llvm-symbolizer /usr/bin/llvm-symbolizer;
CMD sleep 1

59
docker/test/performance-comparison/compare.sh
View File

@ -1,5 +1,5 @@
#!/bin/bash
set -ex
set -exu
set -o pipefail
trap "exit" INT TERM
trap 'kill $(jobs -pr) ||:' EXIT
@ -7,6 +7,29 @@ trap 'kill $(jobs -pr) ||:' EXIT
stage=${stage:-}
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
function wait_for_server # port, pid
{
for _ in {1..60}
do
if clickhouse-client --port "$1" --query "select 1" || ! kill -0 "$2"
then
break
fi
sleep 1
done
if ! clickhouse-client --port "$1" --query "select 1"
then
echo "Cannot connect to ClickHouse server at $1"
return 1
fi
if ! kill -0 "$2"
then
echo "Server pid '$2' is not running"
return 1
fi
}
function configure
{
@ -27,8 +50,9 @@ function configure
kill -0 $left_pid
disown $left_pid
set +m
while ! clickhouse-client --port 9001 --query "select 1" && kill -0 $left_pid ; do echo . ; sleep 1 ; done
echo server for setup started
wait_for_server 9001 $left_pid
echo Server for setup started
clickhouse-client --port 9001 --query "create database test" ||:
clickhouse-client --port 9001 --query "rename table datasets.hits_v1 to test.hits" ||:
@ -67,9 +91,10 @@ function restart
set +m
while ! clickhouse-client --port 9001 --query "select 1" && kill -0 $left_pid ; do echo . ; sleep 1 ; done
wait_for_server 9001 $left_pid
echo left ok
while ! clickhouse-client --port 9002 --query "select 1" && kill -0 $right_pid ; do echo . ; sleep 1 ; done
wait_for_server 9002 $right_pid
echo right ok
clickhouse-client --port 9001 --query "select * from system.tables where database != 'system'"
@ -117,6 +142,7 @@ function run_tests
if [ -v CHPC_TEST_GREP ]
then
# Run only explicitly specified tests, if any.
# shellcheck disable=SC2010
test_files=$(ls "$test_prefix" | grep "$CHPC_TEST_GREP" | xargs -I{} -n1 readlink -f "$test_prefix/{}")
elif [ "$changed_test_files" != "" ]
then
@ -130,7 +156,7 @@ function run_tests
# Determine which concurrent benchmarks to run. For now, the only test
# we run as a concurrent benchmark is 'website'. Run it as benchmark if we
# are also going to run it as a normal test.
for test in $test_files; do echo $test; done | sed -n '/website/p' > benchmarks-to-run.txt
for test in $test_files; do echo "$test"; done | sed -n '/website/p' > benchmarks-to-run.txt
# Delete old report files.
for x in {test-times,wall-clock-times}.tsv
@ -178,7 +204,7 @@ function run_benchmark
mkdir benchmark ||:
# The list is built by run_tests.
for file in $(cat benchmarks-to-run.txt)
while IFS= read -r file
do
name=$(basename "$file" ".xml")
@ -190,7 +216,7 @@ function run_benchmark
"${command[@]}" --port 9001 --json "benchmark/$name-left.json" < "benchmark/$name-queries.txt"
"${command[@]}" --port 9002 --json "benchmark/$name-right.json" < "benchmark/$name-queries.txt"
done
done < benchmarks-to-run.txt
}
function get_profiles_watchdog
@ -273,8 +299,7 @@ mkdir analyze analyze/tmp ||:
build_log_column_definitions
# Split the raw test output into files suitable for analysis.
IFS=$'\n'
for test_file in $(find . -maxdepth 1 -name "*-raw.tsv" -print)
for test_file in *-raw.tsv
do
test_name=$(basename "$test_file" "-raw.tsv")
sed -n "s/^query\t/$test_name\t/p" < "$test_file" >> "analyze/query-runs.tsv"
@ -285,7 +310,6 @@ do
sed -n "s/^short\t/$test_name\t/p" < "$test_file" >> "analyze/marked-short-queries.tsv"
sed -n "s/^partial\t/$test_name\t/p" < "$test_file" >> "analyze/partial-queries.tsv"
done
unset IFS
# for each query run, prepare array of metrics from query log
clickhouse-local --query "
@ -394,7 +418,7 @@ create table query_run_metric_names engine File(TSV, 'analyze/query-run-metric-n
IFS=$'\n'
for prefix in $(cut -f1,2 "analyze/query-run-metrics-for-stats.tsv" | sort | uniq)
do
file="analyze/tmp/$(echo "$prefix" | sed 's/\t/_/g').tsv"
file="analyze/tmp/${prefix// /_}.tsv"
grep "^$prefix " "analyze/query-run-metrics-for-stats.tsv" > "$file" &
printf "%s\0\n" \
"clickhouse-local \
@ -831,15 +855,13 @@ wait
unset IFS
# Create differential flamegraphs.
IFS=$'\n'
for query_file in $(cat report/query-files.txt)
while IFS= read -r query_file
do
~/fg/difffolded.pl "report/tmp/$query_file.stacks.left.tsv" \
"report/tmp/$query_file.stacks.right.tsv" \
| tee "report/tmp/$query_file.stacks.diff.tsv" \
| ~/fg/flamegraph.pl > "$query_file.diff.svg" &
done
unset IFS
done < report/query-files.txt
wait
# Create per-query files with metrics. Note that the key is different from flamegraphs.
@ -906,8 +928,7 @@ create table changes engine File(TSV, 'metrics/changes.tsv') as
)
order by diff desc
;
"
2> >(tee -a metrics/errors.log 1>&2)
" 2> >(tee -a metrics/errors.log 1>&2)
IFS=$'\n'
for prefix in $(cut -f1 "metrics/metrics.tsv" | sort | uniq)
@ -981,7 +1002,7 @@ case "$stage" in
# to collect the logs. Prefer not to restart, because addresses might change
# and we won't be able to process trace_log data. Start in a subshell, so that
# it doesn't interfere with the watchdog through `wait`.
( get_profiles || restart && get_profiles ||: )
( get_profiles || restart && get_profiles ) ||:
# Kill the whole process group, because somehow when the subshell is killed,
# the sleep inside remains alive and orphaned.

10
docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml
View File

@ -6,6 +6,16 @@
<allow_introspection_functions>1</allow_introspection_functions>
<log_queries>1</log_queries>
<metrics_perf_events_enabled>1</metrics_perf_events_enabled>
<!--
If a test takes too long by mistake, the entire test task can
time out and the author won't get a proper message. Put some cap
on query execution time to prevent this. Test query run time is
limited to about 2 seconds, but this limit applies to all queries,
including fill/create and maintenance such as downloading trace
logs, so it must be generous enough. As a second line of defense,
we might also add time check to perf.py script.
-->
<max_execution_time>300</max_execution_time>
</default>
</profiles>
</yandex>

4
docker/test/stress/run.sh
View File

@ -17,8 +17,8 @@ function wait_server()
then
echo "Cannot start clickhouse-server"
cat /var/log/clickhouse-server/stdout.log
cat /var/log/clickhouse-server/stderr.log
cat /var/log/clickhouse-server/clickhouse-server.err.log
tail -n1000 /var/log/clickhouse-server/stderr.log
tail -n1000 /var/log/clickhouse-server/clickhouse-server.err.log
break
fi
sleep 0.5

17
docs/en/development/build.md
View File

@ -146,6 +146,14 @@ $ cd ClickHouse
$ ./release
```
## Faster builds for development
Normally all tools of the ClickHouse bundle, such as `clickhouse-server`, `clickhouse-client` etc., are linked into a single static executable, `clickhouse`. This executable must be re-linked on every change, which might be slow. Two common ways to improve linking time are to use `lld` linker, and use the 'split' build configuration, which builds a separate binary for every tool, and further splits the code into serveral shared libraries. To enable these tweaks, pass the following flags to `cmake`:
```
-DCMAKE_C_FLAGS="-fuse-ld=lld" -DCMAKE_CXX_FLAGS="-fuse-ld=lld" -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1
```
## You Don’t Have to Build ClickHouse {#you-dont-have-to-build-clickhouse}
ClickHouse is available in pre-built binaries and packages. Binaries are portable and can be run on any Linux flavour.
@ -154,4 +162,13 @@ They are built for stable, prestable and testing releases as long as for every c
To find the freshest build from `master`, go to [commits page](https://github.com/ClickHouse/ClickHouse/commits/master), click on the first green checkmark or red cross near commit, and click to the “Details” link right after “ClickHouse Build Check”.
## Split build configuration {#split-build}
Normally ClickHouse is statically linked into a single static `clickhouse` binary with minimal dependencies. This is convenient for distribution, but it means that on every change the entire binary is linked again, which is slow and may be inconvenient for development. There is an alternative configuration which creates dynamically loaded shared libraries instead, allowing faster incremental builds. To use it, add the following flags to your `cmake` invocation:
```
-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1
```
Note that in this configuration there is no single `clickhouse` binary, and you have to run `clickhouse-server`, `clickhouse-client` etc.
[Original article](https://clickhouse.tech/docs/en/development/build/) <!--hide-->

215
docs/en/development/continuous-integration.md Normal file
View File

@ -0,0 +1,215 @@
---
toc_priority: 62
toc_title: Continuous Integration Checks
---
# Continuous Integration Checks
When you submit a pull request, some automated checks are ran for your code by
the ClickHouse [continuous integration (CI) system](tests.md#test-automation).
This happens after a repository maintainer (someone from ClickHouse team) has
screened your code and added the `can be tested` label to your pull request.
The results of the checks are listed on the GitHub pull request page as
described in the [GitHub checks
documentation](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/about-status-checks).
If a check is failing, you might be required to fix it. This page gives an
overview of checks you may encounter, and what you can do to fix them.
If it looks like the check failure is not related to your changes, it may be
some transient failure or an infrastructure problem. Push an empty commit to
the pull request to restart the CI checks:
```
git reset
git commit --allow-empty
git push
```
If you are not sure what to do, ask a maintainer for help.
## Merge With Master
Verifies that the PR can be merged to master. If not, it will fail with the
message 'Cannot fetch mergecommit'. To fix this check, resolve the conflict as
described in the [GitHub
documentation](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/resolving-a-merge-conflict-on-github),
or merge the `master` branch to your pull request branch using git.
## Docs check
Tries to build the ClickHouse documentation website. It can fail if you changed
something in the documentation. Most probable reason is that some cross-link in
the documentation is wrong. Go to the check report and look for `ERROR` and `WARNING` messages.
### Report Details
- [Status page example](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html)
- `docs_output.txt` contains the building log. [Successful result example](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check/docs_output.txt)
## Description Check
Check that the description of your pull request conforms to the template
[PULL_REQUEST_TEMPLATE.md](https://github.com/ClickHouse/ClickHouse/blob/master/.github/PULL_REQUEST_TEMPLATE.md).
You have to specify a changelog category for your change (e.g., Bug Fix), and
write a user-readable message describing the change for [CHANGELOG.md](../whats-new/changelog/index.md)
## Push To Dockerhub
Builds docker images used for build and tests, then pushes them to DockerHub.
## Marker Check
This check means that the CI system started to process the pull request. When it has 'pending' status, it means that not all checks have been started yet. After all checks have been started, it changes status to 'success'.
## Style Check
Performs some simple regex-based checks of code style, using the [`utils/check-style/check-style`](https://github.com/ClickHouse/ClickHouse/blob/master/utils/check-style/check-style) binary (note that it can be run locally).
If it fails, fix the style errors following the [code style guide](style.md).
### Report Details
- [Status page example](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check.html)
- `output.txt` contains the check resulting errors (invalid tabulation etc), blank page means no errors. [Successful result example](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check/output.txt).
## PVS Check
Check the code with [PVS-studio](https://www.viva64.com/en/pvs-studio/), a static analysis tool. Look at the report to see the exact errors. Fix them if you can, if not -- ask a ClickHouse maintainer for help.
### Report Details
- [Status page example](https://clickhouse-test-reports.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/pvs_check.html)
- `test_run.txt.out.log` contains the building and analyzing log file. It includes only parsing or not-found errors.
- `HTML report` contains the analysis results. For its description visit PVS's [official site](https://www.viva64.com/en/m/0036/#ID14E9A2B2CD).
## Fast Test
Normally this is the first check that is ran for a PR. It builds ClickHouse and
runs most of [stateless functional tests](tests.md#functional-tests), omitting
some. If it fails, further checks are not started until it is fixed. Look at
the report to see which tests fail, then reproduce the failure locally as
described [here](tests.md#functional-test-locally).
### Report Details
[Status page example](https://clickhouse-test-reports.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/fast_test.html)
#### Status Page Files
- `runlog.out.log` is the general log that includes all other logs.
- `test_log.txt`
- `submodule_log.txt` contains the messages about cloning and checkouting needed submodules.
- `stderr.log`
- `stdout.log`
- `clickhouse-server.log`
- `clone_log.txt`
- `install_log.txt`
- `clickhouse-server.err.log`
- `build_log.txt`
- `cmake_log.txt` contains messages about the C/C++ and Linux flags check.
#### Status Page Columns
- *Test name* contains the name of the test (without the path e.g. all types of tests will be stripped to the name).
- *Test status* -- one of _Skipped_, _Success_, or _Fail_.
- *Test time, sec.* -- empty on this test.
## Build Check {#build-check}
Builds ClickHouse in various configurations for use in further steps. You have to fix the builds that fail. Build logs often has enough information to fix the error, but you might have to reproduce the failure locally. The `cmake` options can be found in the build log, grepping for `cmake`. Use these options and follow the [general build process](build.md).
### Report Details
[Status page example](https://clickhouse-builds.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/clickhouse_build_check/report.html).
- **Compiler**: `gcc-9` or `clang-10` (or `clang-10-xx` for other architectures e.g. `clang-10-freebsd`).
- **Build type**: `Debug` or `RelWithDebInfo` (cmake).
- **Sanitizer**: `none` (without sanitizers), `address` (ASan), `memory` (MSan), `undefined` (UBSan), or `thread` (TSan).
- **Bundled**: `bundled` build uses system libraries, and `unbundled` build uses libraries from `contrib` folder.
- **Splitted** `splitted` is a [split build](build.md#split-build)
- **Status**: `success` or `fail`
- **Build log**: link to the building and files copying log, useful when build failed.
- **Build time**.
- **Artifacts**: build result files (with `XXX` being the server version e.g. `20.8.1.4344`).
- `clickhouse-client_XXX_all.deb`
- `clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb`
- `clickhouse-common-staticXXX_amd64.deb`
- `clickhouse-server_XXX_all.deb`
- `clickhouse-test_XXX_all.deb`
- `clickhouse_XXX_amd64.buildinfo`
- `clickhouse_XXX_amd64.changes`
- `clickhouse`: Main built binary.
- `clickhouse-odbc-bridge`
- `unit_tests_dbms`: GoogleTest binary with ClickHouse unit tests.
- `shared_build.tgz`: build with shared libraries.
- `performance.tgz`: Special package for performance tests.
## Special Build Check
Performs static analysis and code style checks using `clang-tidy`. The report is similar to the [build check](#build-check). Fix the errors found in the build log.
## Functional Stateless Tests
Runs [stateless functional tests](tests.md#functional-tests) for ClickHouse
binaries built in various configurations -- release, debug, with sanitizers,
etc. Look at the report to see which tests fail, then reproduce the failure
locally as described [here](tests.md#functional-test-locally). Note that you
have to use the correct build configuration to reproduce -- a test might fail
under AddressSanitizer but pass in Debug. Download the binary from [CI build
checks page](build.md#you-dont-have-to-build-clickhouse), or build it locally.
## Functional Stateful Tests
Runs [stateful functional tests](tests.md#functional-tests). Treat them in the same way as the functional stateless tests. The difference is that they require `hits` and `visits` tables from the [Yandex.Metrica dataset](../getting-started/example-datasets/metrica.md) to run.
## Integration Tests
Runs [integration tests](tests.md#integration-tests).
## Testflows Check
Runs some tests using Testflows test system. See [here](https://github.com/ClickHouse/ClickHouse/tree/master/tests/testflows#running-tests-locally) how to run them locally.
## Stress Test
Runs stateless functional tests concurrently from several clients to detect
concurrency-related errors. If it fails:
* Fix all other test failures first;
* Look at the report to find the server logs and check them for possible causes
of error.
## Split Build Smoke Test
Checks that the server build in [split build](build.md#split-build)
configuration can start and run simple queries. If it fails:
* Fix other test errors first;
* Build the server in [split build](build.md#split-build) configuration
locally and check whether it can start and run `select 1`.
## Compatibility Check
Checks that `clickhouse` binary runs on distributions with old libc versions. If it fails, ask a maintainer for help.
## AST Fuzzer
Runs randomly generated queries to catch program errors. If it fails, ask a maintainer for help.
## Performance Tests
Measure changes in query performance. This is the longest check that takes just below 6 hours to run. The performance test report is described in detail [here](https://github.com/ClickHouse/ClickHouse/tree/master/docker/test/performance-comparison#how-to-read-the-report).
# QA
> What is a `Task (private network)` item on status pages?
It's a link to the Yandex's internal job system. Yandex employees can see the check's start time and its more verbose status.
> Where the tests are run
Somewhere on Yandex internal infrastructure.

2
docs/en/development/style.md
View File

@ -703,7 +703,7 @@ But other things being equal, cross-platform or portable code is preferred.
**3.** Compiler: `gcc`. At this time (August 2020), the code is compiled using version 9.3. (It can also be compiled using `clang 8`.)
The standard library is used (`libstdc++` or `libc++`).
The standard library is used (`libc++`).
**4.**OS: Linux Ubuntu, not older than Precise.

14
docs/en/development/tests.md
View File

@ -27,6 +27,18 @@ If you want to use distributed queries in functional tests, you can leverage `re
Some tests are marked with `zookeeper`, `shard` or `long` in their names. `zookeeper` is for tests that are using ZooKeeper. `shard` is for tests that requires server to listen `127.0.0.*`; `distributed` or `global` have the same meaning. `long` is for tests that run slightly longer that one second. You can disable these groups of tests using `--no-zookeeper`, `--no-shard` and `--no-long` options, respectively.
### Running a particular test locally {#functional-test-locally}
Start the ClickHouse server locally, listening on the default port (9000). To
run, for example, the test `01428_hash_set_nan_key`, change to the repository
folder and run the following command:
```
PATH=$PATH:<path to clickhouse-client> tests/clickhouse-test 01428_hash_set_nan_key
```
For more options, see `tests/clickhouse-test --help`.
## Known Bugs {#known-bugs}
If we know some bugs that can be easily reproduced by functional tests, we place prepared functional tests in `tests/queries/bugs` directory. These tests will be moved to `tests/queries/0_stateless` when bugs are fixed.
@ -168,7 +180,7 @@ Main ClickHouse code (that is located in `dbms` directory) is built with `-Wall
Clang has even more useful warnings - you can look for them with `-Weverything` and pick something to default build.
For production builds, gcc is used (it still generates slightly more efficient code than clang). For development, clang is usually more convenient to use. You can build on your own machine with debug mode (to save battery of your laptop), but please note that compiler is able to generate more warnings with `-O3` due to better control flow and inter-procedure analysis. When building with clang, `libc++` is used instead of `libstdc++` and when building with debug mode, debug version of `libc++` is used that allows to catch more errors at runtime.
For production builds, gcc is used (it still generates slightly more efficient code than clang). For development, clang is usually more convenient to use. You can build on your own machine with debug mode (to save battery of your laptop), but please note that compiler is able to generate more warnings with `-O3` due to better control flow and inter-procedure analysis. When building with clang in debug mode, debug version of `libc++` is used that allows to catch more errors at runtime.
## Sanitizers {#sanitizers}

3
docs/en/introduction/adopters.md
View File

@ -15,6 +15,7 @@ toc_title: Adopters
| <a href="https://amadeus.com/" class="favicon">Amadeus</a> | Travel | Analytics | — | — | [Press Release, April 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) |
| <a href="https://www.appsflyer.com" class="favicon">Appsflyer</a> | Mobile analytics | Main product | — | — | [Talk in Russian, July 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) |
| <a href="https://arenadata.tech/" class="favicon">ArenaData</a> | Data Platform | Main product | — | — | [Slides in Russian, December 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) |
| <a href="https://avito.ru/" class="favicon">Avito</a> | Classifieds | Monitoring | — | — | [Meetup, April 2020](https://www.youtube.com/watch?v=n1tm4j4W8ZQ) |
| <a href="https://badoo.com" class="favicon">Badoo</a> | Dating | Timeseries | — | — | [Slides in Russian, December 2019](https://presentations.clickhouse.tech/meetup38/forecast.pdf) |
| <a href="https://www.benocs.com/" class="favicon">Benocs</a> | Network Telemetry and Analytics | Main Product | — | — | [Slides in English, October 2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) |
| <a href="https://www.bloomberg.com/" class="favicon">Bloomberg</a> | Finance, Media | Monitoring | 102 servers | — | [Slides, May 2018](https://www.slideshare.net/Altinity/http-analytics-for-6m-requests-per-second-using-clickhouse-by-alexander-bocharov) |
@ -47,6 +48,7 @@ toc_title: Adopters
| <a href="https://www.instana.com" class="favicon">Instana</a> | APM Platform | Main product | — | — | [Twitter post](https://twitter.com/mieldonkers/status/1248884119158882304) |
| <a href="https://integros.com" class="favicon">Integros</a> | Platform for video services | Analytics | — | — | [Slides in Russian, May 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup22/strategies.pdf) |
| <a href="https://ippon.tech" class="favicon">Ippon Technologies</a> | Technology Consulting | — | — | — | [Talk in English, July 2020](https://youtu.be/GMiXCMFDMow?t=205) |
| <a href="https://www.ivi.ru/" class="favicon">Ivi</a> | Online Cinema | Analytics, Monitoring | — | — | [Article in Russian, Jan 2018](https://habr.com/en/company/ivi/blog/347408/) |
| <a href="https://jinshuju.net" class="favicon">Jinshuju 金数据</a> | BI Analytics | Main product | — | — | [Slides in Chinese, October 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/3.%20金数据数据架构调整方案Public.pdf) |
| <a href="https://www.kodiakdata.com/" class="favicon">Kodiak Data</a> | Clouds | Main product | — | — | [Slides in Engish, April 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup13/kodiak_data.pdf) |
| <a href="https://kontur.ru" class="favicon">Kontur</a> | Software Development | Metrics | — | — | [Talk in Russian, November 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) |
@ -57,6 +59,7 @@ toc_title: Adopters
| <a href="https://www.messagebird.com" class="favicon">MessageBird</a> | Telecommunications | Statistics | — | — | [Slides in English, November 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) |
| <a href="https://www.mindsdb.com/" class="favicon">MindsDB</a> | Machine Learning | Main Product | — | — | [Official Website](https://www.mindsdb.com/blog/machine-learning-models-as-tables-in-ch) |
| <a href="https://www.mgid.com/" class="favicon">MGID</a> | Ad network | Web-analytics | — | — | [Blog post in Russian, April 2020](http://gs-studio.com/news-about-it/32777----clickhouse---c) |
| <a href="https://getnoc.com/" class="favicon">NOC Project</a> | Network Monitoring | Analytics | Main Product | — | [Official Website](https://getnoc.com/features/big-data/) |
| <a href="https://www.nuna.com/" class="favicon">Nuna Inc.</a> | Health Data Analytics | — | — | — | [Talk in English, July 2020](https://youtu.be/GMiXCMFDMow?t=170) |
| <a href="https://www.oneapm.com/" class="favicon">OneAPM</a> | Monitorings and Data Analysis | Main product | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) |
| <a href="https://www.percent.cn/" class="favicon">Percent 百分点</a> | Analytics | Main Product | — | — | [Slides in Chinese, June 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) |

56
docs/en/sql-reference/aggregate-functions/reference/grouparraysample.md
View File

@ -9,18 +9,14 @@ Creates an array of sample argument values. The size of the resulting array is l
**Syntax**
``` sql
groupArraySample(max_size)(x)
```
or
``` sql
groupArraySample(max_size, seed)(x)
groupArraySample(max_size[, seed])(x)
```
**Parameters**
- `max_size` — Maximum size of the resulting array. Positive [UInt64](../../data-types/int-uint.md).
- `seed` — Seed for the random number generator. Optional, can be omitted. Positive [UInt64](../../data-types/int-uint.md). Default value: `123456`.
- `x` — Argument name. [String](../../data-types/string.md).
- `max_size` — Maximum size of the resulting array. [UInt64](../../data-types/int-uint.md).
- `seed` — Seed for the random number generator. Optional. [UInt64](../../data-types/int-uint.md). Default value: `123456`.
- `x` — Argument (column name or expression).
**Returned values**
@ -42,44 +38,44 @@ Consider table `colors`:
└────┴────────┘
```
Select `id`-s query:
Query with column name as argument:
``` sql
SELECT groupArraySample(3)(id) FROM colors;
```
Result:
``` text
┌─groupArraySample(3)(id)─┐
│ [1,2,4] │
└─────────────────────────┘
```
Select `color`-s query:
``` sql
SELECT groupArraySample(3)(color) FROM colors;
SELECT groupArraySample(3)(color) as newcolors FROM colors;
```
Result:
```text
┌─groupArraySample(3)(color)─┐
┌─newcolors──────────────────┐
│ ['white','blue','green'] │
└────────────────────────────┘
```
Select `color`-s query with different seed:
Query with column name and different seed:
``` sql
SELECT groupArraySample(3, 987654321)(color) FROM colors;
SELECT groupArraySample(3, 987654321)(color) as newcolors FROM colors;
```
Result:
```text
┌─groupArraySample(3, 987654321)(color)─┐
│ ['red','orange','green'] │
└───────────────────────────────────────┘
┌─newcolors──────────────────┐
│ ['red','orange','green'] │
└────────────────────────────┘
```
Query with expression as argument:
``` sql
SELECT groupArraySample(3)(concat('light-', color)) as newcolors FROM colors;
```
Result:
```text
┌─newcolors───────────────────────────────────┐
│ ['light-blue','light-orange','light-green'] │
└─────────────────────────────────────────────┘
```

26
docs/en/sql-reference/functions/string-search-functions.md
View File

@ -21,15 +21,16 @@ For a case-insensitive search, use the function [positionCaseInsensitive](#posit
**Syntax**
``` sql
position(haystack, needle)
position(haystack, needle[, start_pos])
```
Alias: `locate(haystack, needle)`.
Alias: `locate(haystack, needle[, start_pos])`.
**Parameters**
- `haystack` — string, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Returned values**
@ -56,6 +57,18 @@ Result:
└────────────────────────────────┘
```
``` sql
SELECT
position('Hello, world!', 'o', 1),
position('Hello, world!', 'o', 7)
```
``` text
┌─position('Hello, world!', 'o', 1)─┬─position('Hello, world!', 'o', 7)─┐
│ 5 │ 9 │
└───────────────────────────────────┴───────────────────────────────────┘
```
The same phrase in Russian contains characters which can’t be represented using a single byte. The function returns some unexpected result (use [positionUTF8](#positionutf8) function for multi-byte encoded text):
Query:
@ -81,13 +94,14 @@ Works under the assumption that the string contains a set of bytes representing
**Syntax**
``` sql
positionCaseInsensitive(haystack, needle)
positionCaseInsensitive(haystack, needle[, start_pos])
```
**Parameters**
- `haystack` — string, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Returned values**
@ -123,13 +137,14 @@ For a case-insensitive search, use the function [positionCaseInsensitiveUTF8](#p
**Syntax**
``` sql
positionUTF8(haystack, needle)
positionUTF8(haystack, needle[, start_pos])
```
**Parameters**
- `haystack` — string, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Returned values**
@ -195,13 +210,14 @@ Works under the assumption that the string contains a set of bytes representing
**Syntax**
``` sql
positionCaseInsensitiveUTF8(haystack, needle)
positionCaseInsensitiveUTF8(haystack, needle[, start_pos])
```
**Parameters**
- `haystack` — string, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Returned value**

2
docs/es/development/style.md
View File

@ -705,7 +705,7 @@ Pero en igualdad de condiciones, se prefiere el código multiplataforma o portá
**3.** Compilación: `gcc`. En este momento (agosto 2020), el código se compila utilizando la versión 9.3. (También se puede compilar usando `clang 8`.)
Se utiliza la biblioteca estándar (`libstdc++` o `libc++`).
Se utiliza la biblioteca estándar (`libc++`).
**4.**OS: Linux Ubuntu, no más viejo que Precise.

2
docs/es/development/tests.md
View File

@ -175,7 +175,7 @@ Código principal de ClickHouse (que se encuentra en `dbms` directorio) se const
Clang tiene advertencias aún más útiles: puedes buscarlas con `-Weverything` y elige algo para la compilación predeterminada.
Para las compilaciones de producción, se usa gcc (todavía genera un código ligeramente más eficiente que clang). Para el desarrollo, el clang suele ser más conveniente de usar. Puede construir en su propia máquina con el modo de depuración (para ahorrar batería de su computadora portátil), pero tenga en cuenta que el compilador puede generar más advertencias con `-O3` debido a un mejor flujo de control y análisis entre procedimientos. Al construir con clang, `libc++` se utiliza en lugar de `libstdc++` y al construir con el modo de depuración, la versión de depuración de `libc++` se utiliza que permite detectar más errores en tiempo de ejecución.
Para las compilaciones de producción, se usa gcc (todavía genera un código ligeramente más eficiente que clang). Para el desarrollo, el clang suele ser más conveniente de usar. Puede construir en su propia máquina con el modo de depuración (para ahorrar batería de su computadora portátil), pero tenga en cuenta que el compilador puede generar más advertencias con `-O3` debido a un mejor flujo de control y análisis entre procedimientos. Al construir con clang con el modo de depuración, la versión de depuración de `libc++` se utiliza que permite detectar más errores en tiempo de ejecución.
## Desinfectantes {#sanitizers}

14
docs/es/sql-reference/functions/string-search-functions.md
View File

@ -20,15 +20,16 @@ Para una búsqueda sin distinción de mayúsculas y minúsculas, utilice la func
**Sintaxis**
``` sql
position(haystack, needle)
position(haystack, needle[, start_pos])
```
Apodo: `locate(haystack, needle)`.
Apodo: `locate(haystack, needle[, start_pos])`.
**Parámetros**
- `haystack` — string, in which substring will to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Valores devueltos**
@ -80,13 +81,14 @@ Funciona bajo el supuesto de que la cadena contiene un conjunto de bytes que rep
**Sintaxis**
``` sql
positionCaseInsensitive(haystack, needle)
positionCaseInsensitive(haystack, needle[, start_pos])
```
**Parámetros**
- `haystack` — string, in which substring will to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Valores devueltos**
@ -122,13 +124,14 @@ Para una búsqueda sin distinción de mayúsculas y minúsculas, utilice la func
**Sintaxis**
``` sql
positionUTF8(haystack, needle)
positionUTF8(haystack, needle[, start_pos])
```
**Parámetros**
- `haystack` — string, in which substring will to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Valores devueltos**
@ -194,13 +197,14 @@ Funciona bajo el supuesto de que la cadena contiene un conjunto de bytes que rep
**Sintaxis**
``` sql
positionCaseInsensitiveUTF8(haystack, needle)
positionCaseInsensitiveUTF8(haystack, needle[, start_pos])
```
**Parámetros**
- `haystack` — string, in which substring will to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Valor devuelto**

2
docs/fa/development/style.md
View File

@ -706,7 +706,7 @@ auto s = std::string{"Hello"};
**3.** کامپایلر: `gcc`. در این زمان (اوت 2020), کد با استفاده از نسخه وارد شده 9.3. (همچنین می تواند با استفاده از وارد شود `clang 8`.)
کتابخانه استاندارد استفاده شده است (`libstdc++` یا `libc++`).
کتابخانه استاندارد استفاده شده است (`libc++`).
**4.**سیستم عامل: لینوکس اوبونتو, مسن تر از دقیق نیست.

14
docs/fa/sql-reference/functions/string-search-functions.md
View File

@ -21,15 +21,16 @@ toc_title: "\u0628\u0631\u0627\u06CC \u062C\u0633\u062A\u062C\u0648\u06CC \u0631
**نحو**
``` sql
position(haystack, needle)
position(haystack, needle[, start_pos])
```
نام مستعار: `locate(haystack, needle)`.
نام مستعار: `locate(haystack, needle[, start_pos])`.
**پارامترها**
- `haystack` — string, in which substring will to be searched. [رشته](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [رشته](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**مقادیر بازگشتی**
@ -81,13 +82,14 @@ SELECT position('Привет, мир!', '!')
**نحو**
``` sql
positionCaseInsensitive(haystack, needle)
positionCaseInsensitive(haystack, needle[, start_pos])
```
**پارامترها**
- `haystack` — string, in which substring will to be searched. [رشته](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [رشته](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**مقادیر بازگشتی**
@ -123,13 +125,14 @@ SELECT positionCaseInsensitive('Hello, world!', 'hello')
**نحو**
``` sql
positionUTF8(haystack, needle)
positionUTF8(haystack, needle[, start_pos])
```
**پارامترها**
- `haystack` — string, in which substring will to be searched. [رشته](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [رشته](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**مقادیر بازگشتی**
@ -195,13 +198,14 @@ SELECT positionUTF8('Salut, étudiante!', '!')
**نحو**
``` sql
positionCaseInsensitiveUTF8(haystack, needle)
positionCaseInsensitiveUTF8(haystack, needle[, start_pos])
```
**پارامترها**
- `haystack` — string, in which substring will to be searched. [رشته](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [رشته](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**مقدار بازگشتی**

2
docs/fr/development/style.md
View File

@ -705,7 +705,7 @@ Mais toutes choses étant égales par ailleurs, le code multi-plateforme ou port
**3.** Compilateur: `gcc`. En ce moment (août 2020), le code est compilé en utilisant la version 9.3. (Il peut également être compilé en utilisant `clang 8`.)
La bibliothèque standard est utilisée (`libstdc++` ou `libc++`).
La bibliothèque standard est utilisée (`libc++`).
**4.**OS: Linux Ubuntu, pas plus vieux que précis.

2
docs/fr/development/tests.md
View File

@ -175,7 +175,7 @@ Code ClickHouse principal (qui est situé dans `dbms` annuaire) est construit av
Clang a des avertissements encore plus utiles - vous pouvez les chercher avec `-Weverything` et choisissez quelque chose à construire par défaut.
Pour les builds de production, gcc est utilisé (il génère toujours un code légèrement plus efficace que clang). Pour le développement, clang est généralement plus pratique à utiliser. Vous pouvez construire sur votre propre machine avec le mode débogage (pour économiser la batterie de votre ordinateur portable), mais veuillez noter que le compilateur est capable de générer plus d'Avertissements avec `-O3` grâce à une meilleure analyse du flux de contrôle et de l'inter-procédure. Lors de la construction avec clang, `libc++` est utilisé au lieu de `libstdc++` et lors de la construction avec le mode débogage, la version de débogage de `libc++` est utilisé qui permet d'attraper plus d'erreurs à l'exécution.
Pour les builds de production, gcc est utilisé (il génère toujours un code légèrement plus efficace que clang). Pour le développement, clang est généralement plus pratique à utiliser. Vous pouvez construire sur votre propre machine avec le mode débogage (pour économiser la batterie de votre ordinateur portable), mais veuillez noter que le compilateur est capable de générer plus d'Avertissements avec `-O3` grâce à une meilleure analyse du flux de contrôle et de l'inter-procédure. Lors de la construction avec clang avec le mode débogage, la version de débogage de `libc++` est utilisé qui permet d'attraper plus d'erreurs à l'exécution.
## Désinfectant {#sanitizers}

2
docs/ja/development/style.md
View File

@ -705,7 +705,7 @@ auto s = std::string{"Hello"};
**3.** コンパイラ: `gcc`. 2020年現在、コードはバージョン9.3を使用してコンパイルされている。 （以下を使ってコンパイルできます `clang 8`.)
標準ライブラリが使用されます (`libstdc++` または `libc++`).
標準ライブラリが使用されます (`libc++`).
**4.**OS：LinuxのUbuntuの、正確よりも古いではありません。

14
docs/ja/sql-reference/functions/string-search-functions.md
View File

@ -20,15 +20,16 @@ toc_title: "\u6587\u5B57\u5217\u3092\u691C\u7D22\u3059\u308B\u5834\u5408"
**構文**
``` sql
position(haystack, needle)
position(haystack, needle[, start_pos])
```
別名: `locate(haystack, needle)`.
別名: `locate(haystack, needle[, start_pos])`.
**パラメータ**
- `haystack` — string, in which substring will to be searched. [文字列](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [文字列](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**戻り値**
@ -80,13 +81,14 @@ SELECT position('Привет, мир!', '!')
**構文**
``` sql
positionCaseInsensitive(haystack, needle)
positionCaseInsensitive(haystack, needle[, start_pos])
```
**パラメータ**
- `haystack` — string, in which substring will to be searched. [文字列](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [文字列](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**戻り値**
@ -122,13 +124,14 @@ SELECT positionCaseInsensitive('Hello, world!', 'hello')
**構文**
``` sql
positionUTF8(haystack, needle)
positionUTF8(haystack, needle[, start_pos])
```
**パラメータ**
- `haystack` — string, in which substring will to be searched. [文字列](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [文字列](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**戻り値**
@ -194,13 +197,14 @@ SELECT positionUTF8('Salut, étudiante!', '!')
**構文**
``` sql
positionCaseInsensitiveUTF8(haystack, needle)
positionCaseInsensitiveUTF8(haystack, needle[, start_pos])
```
**パラメータ**
- `haystack` — string, in which substring will to be searched. [文字列](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [文字列](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**戻り値**

80
docs/ru/sql-reference/aggregate-functions/reference/grouparraysample.md Normal file
View File

@ -0,0 +1,80 @@
---
toc_priority: 114
---
# groupArraySample {#grouparraysample}
Создает массив из случайно выбранных значений аргумента. Количество элементов в массиве ограничено значением параметра `max_size`. Элементы добавляются в результирующий массив в случайном порядке.
**Синтаксис**
``` sql
groupArraySample(max_size[, seed])(x)
```
**Параметры**
- `max_size` — максимальное количество элементов в возвращаемом массиве. [UInt64](../../data-types/int-uint.md).
- `seed` — состояние генератора случайных чисел. Необязательный параметр. [UInt64](../../data-types/int-uint.md). Значение по умолчанию: `123456`.
- `x` — аргумент (название колонки таблицы или выражение).
**Возвращаемые значения**
- Массив случайно выбранных значений аргумента `x`.
Тип: [Массив](../../data-types/array.md).
**Примеры**
Рассмотрим таблицу `colors`:
``` text
┌─id─┬─color──┐
│ 1 │ red │
│ 2 │ blue │
│ 3 │ green │
│ 4 │ white │
│ 5 │ orange │
└────┴────────┘
```
Запрос с названием колонки таблицы в качестве аргумента:
``` sql
SELECT groupArraySample(3)(color) as newcolors FROM colors;
```
Результат:
```text
┌─newcolors──────────────────┐
│ ['white','blue','green'] │
└────────────────────────────┘
```
Запрос с названием колонки и другим состоянием генератора случайных чисел:
``` sql
SELECT groupArraySample(3, 987654321)(color) as newcolors FROM colors;
```
Результат:
```text
┌─newcolors─────────────────────────────┐
│ ['red','orange','green'] │
└───────────────────────────────────────┘
```
Запрос с выражением в качестве аргумента:
``` sql
SELECT groupArraySample(3)(concat('light-', color)) as newcolors FROM colors;
```
Результат:
```text
┌─newcolors───────────────────────────────────┐
│ ['light-blue','light-orange','light-green'] │
└─────────────────────────────────────────────┘
```

14
docs/ru/sql-reference/functions/string-search-functions.md
View File

@ -15,15 +15,16 @@
**Синтаксис**
``` sql
position(haystack, needle)
position(haystack, needle[, start_pos])
```
Алиас: `locate(haystack, needle)`.
Алиас: `locate(haystack, needle[, start_pos])`.
**Параметры**
- `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal).
- `needle` — подстрока, которую необходимо найти. [Строка](../syntax.md#syntax-string-literal).
- `start_pos` – Опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md)
**Возвращаемые значения**
@ -75,13 +76,14 @@ SELECT position('Привет, мир!', '!')
**Синтаксис**
``` sql
positionCaseInsensitive(haystack, needle)
positionCaseInsensitive(haystack, needle[, start_pos])
```
**Параметры**
- `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal).
- `needle` — подстрока, которую необходимо найти. [Строка](../syntax.md#syntax-string-literal).
- `start_pos` – Опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md)
**Возвращаемые значения**
@ -117,13 +119,14 @@ SELECT positionCaseInsensitive('Hello, world!', 'hello')
**Синтаксис**
``` sql
positionUTF8(haystack, needle)
positionUTF8(haystack, needle[, start_pos])
```
**Параметры**
- `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal).
- `needle` — подстрока, которую необходимо найти. [Строка](../syntax.md#syntax-string-literal).
- `start_pos` – Опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md)
**Возвращаемые значения**
@ -189,13 +192,14 @@ SELECT positionUTF8('Salut, étudiante!', '!')
**Синтаксис**
``` sql
positionCaseInsensitiveUTF8(haystack, needle)
positionCaseInsensitiveUTF8(haystack, needle[, start_pos])
```
**Параметры**
- `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal).
- `needle` — подстрока, которую необходимо найти. [Строка](../syntax.md#syntax-string-literal).
- `start_pos` – Опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md)
**Возвращаемые значения**

2
docs/tr/development/style.md
View File

@ -705,7 +705,7 @@ Ama diğer şeyler eşit olmak, çapraz platform veya taşınabilir kod tercih e
**3.** Derleyici: `gcc`. Şu anda (Ağustos 2020), kod sürüm 9.3 kullanılarak derlenmiştir. (Ayrıca kullanılarak derlenebilir `clang 8`.)
Standart kütüphane kullanılır (`libstdc++` veya `libc++`).
Standart kütüphane kullanılır (`libc++`).
**4.**OS: Linux UB .untu, daha eski değil.

2
docs/tr/development/tests.md
View File

@ -176,7 +176,7 @@ Ana ClickHouse kodu (bu `dbms` dizin) ile inşa edilmiştir `-Wall -Wextra -Werr
Clang daha yararlı uyarılar vardır-Sen ile onları arayabilirsiniz `-Weverything` ve varsayılan oluşturmak için bir şey seçin.
Üretim yapıları için gcc kullanılır (hala clang'dan biraz daha verimli kod üretir). Geliştirme için, clang genellikle kullanımı daha uygundur. Hata ayıklama modu ile kendi makinenizde inşa edebilirsiniz (dizüstü bilgisayarınızın pilinden tasarruf etmek için), ancak derleyicinin daha fazla uyarı üretebileceğini lütfen unutmayın `-O3` daha iyi kontrol akışı ve prosedürler arası analiz nedeniyle. Clang ile inşa ederken, `libc++` yerine kullanılır `libstdc++` ve hata ayıklama modu ile oluştururken, hata ayıklama sürümü `libc++` çalışma zamanında daha fazla hata yakalamak için izin verir kullanılır.
Üretim yapıları için gcc kullanılır (hala clang'dan biraz daha verimli kod üretir). Geliştirme için, clang genellikle kullanımı daha uygundur. Hata ayıklama modu ile kendi makinenizde inşa edebilirsiniz (dizüstü bilgisayarınızın pilinden tasarruf etmek için), ancak derleyicinin daha fazla uyarı üretebileceğini lütfen unutmayın `-O3` daha iyi kontrol akışı ve prosedürler arası analiz nedeniyle. Clang ile inşa ederken ayıklama modu ile oluştururken, hata ayıklama sürümü `libc++` çalışma zamanında daha fazla hata yakalamak için izin verir kullanılır.
## Dezenfektanlar {#sanitizers}

14
docs/tr/sql-reference/functions/string-search-functions.md
View File

@ -20,15 +20,16 @@ Büyük / küçük harf duyarsız arama için işlevi kullanın [positionCaseİn
**Sözdizimi**
``` sql
position(haystack, needle)
position(haystack, needle[, start_pos])
```
Takma ad: `locate(haystack, needle)`.
Takma ad: `locate(haystack, needle[, start_pos])`.
**Parametre**
- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Döndürülen değerler**
@ -80,13 +81,14 @@ Dize, tek baytlık kodlanmış bir metni temsil eden bir bayt kümesi içerdiği
**Sözdizimi**
``` sql
positionCaseInsensitive(haystack, needle)
positionCaseInsensitive(haystack, needle[, start_pos])
```
**Parametre**
- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Döndürülen değerler**
@ -122,13 +124,14 @@ Büyük / küçük harf duyarsız arama için işlevi kullanın [positionCaseİn
**Sözdizimi**
``` sql
positionUTF8(haystack, needle)
positionUTF8(haystack, needle[, start_pos])
```
**Parametre**
- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Döndürülen değerler**
@ -194,13 +197,14 @@ Dizenin UTF-8 kodlanmış bir metni temsil eden bir bayt kümesi içerdiği vars
**Sözdizimi**
``` sql
positionCaseInsensitiveUTF8(haystack, needle)
positionCaseInsensitiveUTF8(haystack, needle[, start_pos])
```
**Parametre**
- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal).
- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Döndürülen değer**

2
docs/zh/development/style.md
View File

@ -698,7 +698,7 @@ auto s = std::string{"Hello"};
**3.** 编译器： `gcc`。 此时（2020年08月），代码使用9.3版编译。（它也可以使用`clang 8` 编译）
使用标准库 (`libstdc++` 或 `libc++`)。
使用标准库 (`libc++`)。
**4.** 操作系统：Linux Ubuntu，不比 Precise 早。

3
programs/client/Client.cpp
View File

@ -1640,7 +1640,8 @@ private:
return false;
default:
throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_SERVER, "Unknown packet {} from server {}",
packet.type, connection->getDescription());
}
}

3
programs/client/Suggest.cpp
View File

@ -156,7 +156,8 @@ void Suggest::fetch(Connection & connection, const ConnectionTimeouts & timeouts
return;
default:
throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_SERVER, "Unknown packet {} from server {}",
packet.type, connection.getDescription());
}
}
}

3
src/AggregateFunctions/AggregateFunctionResample.h
View File

@ -18,7 +18,8 @@ template <typename Key>
class AggregateFunctionResample final : public IAggregateFunctionHelper<AggregateFunctionResample<Key>>
{
private:
const size_t MAX_ELEMENTS = 4096;
/// Sanity threshold to avoid creation of too large arrays. The choice of this number is arbitrary.
const size_t MAX_ELEMENTS = 1048576;
AggregateFunctionPtr nested_function;

10
src/CMakeLists.txt
View File

@ -262,6 +262,8 @@ target_link_libraries(clickhouse_common_io
roaring
)
if (USE_RDKAFKA)
dbms_target_link_libraries(PRIVATE ${CPPKAFKA_LIBRARY} ${RDKAFKA_LIBRARY})
if(NOT USE_INTERNAL_RDKAFKA_LIBRARY)
@ -273,6 +275,14 @@ if (USE_AMQPCPP)
dbms_target_link_libraries(PUBLIC amqp-cpp)
endif()
if (USE_CYRUS_SASL)
dbms_target_link_libraries(PRIVATE ${CYRUS_SASL_LIBRARY})
endif()
if (USE_KRB5)
dbms_target_link_libraries(PRIVATE ${KRB5_LIBRARY})
endif()
if(RE2_INCLUDE_DIR)
target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${RE2_INCLUDE_DIR})
endif()

9
src/Client/ConnectionPoolWithFailover.cpp
View File

@ -100,13 +100,14 @@ Int64 ConnectionPoolWithFailover::getPriority() const
ConnectionPoolWithFailover::Status ConnectionPoolWithFailover::getStatus() const
{
const Base::PoolStates states = getPoolStates();
const Base::NestedPools pools = nested_pools;
const auto [states, pools, error_decrease_time] = getPoolExtendedStates();
// NOTE: to avoid data races do not touch any data of ConnectionPoolWithFailover or PoolWithFailoverBase in the code below.
assert(states.size() == pools.size());
ConnectionPoolWithFailover::Status result;
result.reserve(states.size());
const time_t since_last_error_decrease = time(nullptr) - last_error_decrease_time;
const time_t since_last_error_decrease = time(nullptr) - error_decrease_time;
for (size_t i = 0; i < states.size(); ++i)
{
@ -114,7 +115,7 @@ ConnectionPoolWithFailover::Status ConnectionPoolWithFailover::getStatus() const
const auto seconds_to_zero_errors = std::max(static_cast<time_t>(0), rounds_to_zero_errors * decrease_error_period - since_last_error_decrease);
result.emplace_back(NestedPoolStatus{
pools[i].get(),
pools[i],
states[i].error_count,
std::chrono::seconds{seconds_to_zero_errors}
});

2
src/Client/ConnectionPoolWithFailover.h
View File

@ -72,7 +72,7 @@ public:
struct NestedPoolStatus
{
const IConnectionPool * pool;
const Base::NestedPoolPtr pool;
size_t error_count;
std::chrono::seconds estimated_recovery_time;
};

15
src/Common/PoolWithFailoverBase.h
View File

@ -124,7 +124,12 @@ protected:
/// This function returns a copy of pool states to avoid race conditions when modifying shared pool states.
PoolStates updatePoolStates(size_t max_ignored_errors);
PoolStates getPoolStates() const;
auto getPoolExtendedStates() const
{
std::lock_guard lock(pool_states_mutex);
return std::make_tuple(shared_pool_states, nested_pools, last_error_decrease_time);
}
NestedPools nested_pools;
@ -382,11 +387,3 @@ PoolWithFailoverBase<TNestedPool>::updatePoolStates(size_t max_ignored_errors)
return result;
}
template <typename TNestedPool>
typename PoolWithFailoverBase<TNestedPool>::PoolStates
PoolWithFailoverBase<TNestedPool>::getPoolStates() const
{
std::lock_guard lock(pool_states_mutex);
return shared_pool_states;
}

14
src/DataStreams/RemoteQueryExecutor.cpp
View File

@ -237,7 +237,9 @@ Block RemoteQueryExecutor::read()
default:
got_unknown_packet_from_replica = true;
throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_SERVER, "Unknown packet {} from one of the following replicas: {}",
toString(packet.type),
multiplexed_connections->dumpAddresses());
}
}
}
@ -269,6 +271,12 @@ void RemoteQueryExecutor::finish()
finished = true;
break;
case Protocol::Server::Log:
/// Pass logs from remote server to client
if (auto log_queue = CurrentThread::getInternalTextLogsQueue())
log_queue->pushBlock(std::move(packet.block));
break;
case Protocol::Server::Exception:
got_exception_from_replica = true;
packet.exception->rethrow();
@ -276,7 +284,9 @@ void RemoteQueryExecutor::finish()
default:
got_unknown_packet_from_replica = true;
throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_SERVER, "Unknown packet {} from one of the following replicas: {}",
toString(packet.type),
multiplexed_connections->dumpAddresses());
}
}

70
src/Dictionaries/CacheDictionary.cpp
View File

@ -109,6 +109,22 @@ CacheDictionary::~CacheDictionary()
update_pool.wait();
}
size_t CacheDictionary::getBytesAllocated() const
{
/// In case of existing string arena we check the size of it.
/// But the same appears in setAttributeValue() function, which is called from update() function
/// which in turn is called from another thread.
const ProfilingScopedReadRWLock read_lock{rw_lock, ProfileEvents::DictCacheLockReadNs};
return bytes_allocated + (string_arena ? string_arena->size() : 0);
}
const IDictionarySource * CacheDictionary::getSource() const
{
/// Mutex required here because of the getSourceAndUpdateIfNeeded() function
/// which is used from another thread.
std::lock_guard lock(source_mutex);
return source_ptr.get();
}
void CacheDictionary::toParent(const PaddedPODArray<Key> & ids, PaddedPODArray<Key> & out) const
{
@ -750,53 +766,29 @@ void CacheDictionary::updateThreadFunction()
setThreadName("AsyncUpdater");
while (!finished)
{
UpdateUnitPtr first_popped;
update_queue.pop(first_popped);
UpdateUnitPtr popped;
update_queue.pop(popped);
if (finished)
break;
/// Here we pop as many unit pointers from update queue as we can.
/// We fix current size to avoid livelock (or too long waiting),
/// when this thread pops from the queue and other threads push to the queue.
const size_t current_queue_size = update_queue.size();
if (current_queue_size > 0)
LOG_TRACE(log, "Performing bunch of keys update in cache dictionary with {} keys", current_queue_size + 1);
std::vector<UpdateUnitPtr> update_request;
update_request.reserve(current_queue_size + 1);
update_request.emplace_back(first_popped);
UpdateUnitPtr current_unit_ptr;
while (update_request.size() < current_queue_size + 1 && update_queue.tryPop(current_unit_ptr))
update_request.emplace_back(std::move(current_unit_ptr));
BunchUpdateUnit bunch_update_unit(update_request);
try
{
/// Update a bunch of ids.
update(bunch_update_unit);
update(popped);
/// Notify all threads about finished updating the bunch of ids
/// Notify thread about finished updating the bunch of ids
/// where their own ids were included.
std::unique_lock<std::mutex> lock(update_mutex);
for (auto & unit_ptr: update_request)
unit_ptr->is_done = true;
popped->is_done = true;
is_update_finished.notify_all();
}
catch (...)
{
std::unique_lock<std::mutex> lock(update_mutex);
/// It is a big trouble, because one bad query can make other threads fail with not relative exception.
/// So at this point all threads (and queries) will receive the same exception.
for (auto & unit_ptr: update_request)
unit_ptr->current_exception = std::current_exception();
popped->current_exception = std::current_exception();
is_update_finished.notify_all();
}
}
@ -844,13 +836,13 @@ void CacheDictionary::tryPushToUpdateQueueOrThrow(UpdateUnitPtr & update_unit_pt
std::to_string(update_queue.size()));
}
void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const
void CacheDictionary::update(UpdateUnitPtr & update_unit_ptr) const
{
CurrentMetrics::Increment metric_increment{CurrentMetrics::DictCacheRequests};
ProfileEvents::increment(ProfileEvents::DictCacheKeysRequested, bunch_update_unit.getRequestedIds().size());
ProfileEvents::increment(ProfileEvents::DictCacheKeysRequested, update_unit_ptr->requested_ids.size());
std::unordered_map<Key, UInt8> remaining_ids{bunch_update_unit.getRequestedIds().size()};
for (const auto id : bunch_update_unit.getRequestedIds())
std::unordered_map<Key, UInt8> remaining_ids{update_unit_ptr->requested_ids.size()};
for (const auto id : update_unit_ptr->requested_ids)
remaining_ids.insert({id, 0});
const auto now = std::chrono::system_clock::now();
@ -864,7 +856,7 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const
Stopwatch watch;
BlockInputStreamPtr stream = current_source_ptr->loadIds(bunch_update_unit.getRequestedIds());
BlockInputStreamPtr stream = current_source_ptr->loadIds(update_unit_ptr->requested_ids);
stream->readPrefix();
@ -917,7 +909,7 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const
else
cell.setExpiresAt(std::chrono::time_point<std::chrono::system_clock>::max());
bunch_update_unit.informCallersAboutPresentId(id, cell_idx);
update_unit_ptr->getPresentIdHandler()(id, cell_idx);
/// mark corresponding id as found
remaining_ids[id] = 1;
}
@ -979,9 +971,9 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const
if (was_default)
cell.setDefault();
if (was_default)
bunch_update_unit.informCallersAboutAbsentId(id, cell_idx);
update_unit_ptr->getAbsentIdHandler()(id, cell_idx);
else
bunch_update_unit.informCallersAboutPresentId(id, cell_idx);
update_unit_ptr->getPresentIdHandler()(id, cell_idx);
continue;
}
/// We don't have expired data for that `id` so all we can do is to rethrow `last_exception`.
@ -1013,7 +1005,7 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const
setDefaultAttributeValue(attribute, cell_idx);
/// inform caller that the cell has not been found
bunch_update_unit.informCallersAboutAbsentId(id, cell_idx);
update_unit_ptr->getAbsentIdHandler()(id, cell_idx);
}
ProfileEvents::increment(ProfileEvents::DictCacheKeysRequestedMiss, not_found_num);

144
src/Dictionaries/CacheDictionary.h
View File

@ -66,7 +66,7 @@ public:
std::string getTypeName() const override { return "Cache"; }
size_t getBytesAllocated() const override { return bytes_allocated + (string_arena ? string_arena->size() : 0); }
size_t getBytesAllocated() const override;
size_t getQueryCount() const override { return query_count.load(std::memory_order_relaxed); }
@ -97,7 +97,7 @@ public:
max_threads_for_updates);
}
const IDictionarySource * getSource() const override { return source_ptr.get(); }
const IDictionarySource * getSource() const override;
const DictionaryLifetime & getLifetime() const override { return dict_lifetime; }
@ -388,15 +388,28 @@ private:
PresentIdHandler present_id_handler_,
AbsentIdHandler absent_id_handler_) :
requested_ids(std::move(requested_ids_)),
alive_keys(CurrentMetrics::CacheDictionaryUpdateQueueKeys, requested_ids.size()),
present_id_handler(present_id_handler_),
absent_id_handler(absent_id_handler_),
alive_keys(CurrentMetrics::CacheDictionaryUpdateQueueKeys, requested_ids.size()){}
absent_id_handler(absent_id_handler_){}
explicit UpdateUnit(std::vector<Key> requested_ids_) :
requested_ids(std::move(requested_ids_)),
alive_keys(CurrentMetrics::CacheDictionaryUpdateQueueKeys, requested_ids.size()),
present_id_handler([](Key, size_t){}),
absent_id_handler([](Key, size_t){}),
alive_keys(CurrentMetrics::CacheDictionaryUpdateQueueKeys, requested_ids.size()){}
absent_id_handler([](Key, size_t){}){}
PresentIdHandler getPresentIdHandler()
{
std::lock_guard lock(callback_mutex);
return can_use_callback ? present_id_handler : PresentIdHandler{};
}
AbsentIdHandler getAbsentIdHandler()
{
std::lock_guard lock(callback_mutex);
return can_use_callback ? absent_id_handler : AbsentIdHandler{};
}
std::vector<Key> requested_ids;
@ -405,130 +418,21 @@ private:
std::mutex callback_mutex;
bool can_use_callback{true};
PresentIdHandler present_id_handler;
AbsentIdHandler absent_id_handler;
std::atomic<bool> is_done{false};
std::exception_ptr current_exception{nullptr};
/// While UpdateUnit is alive, it is accounted in update_queue size.
CurrentMetrics::Increment alive_batch{CurrentMetrics::CacheDictionaryUpdateQueueBatches};
CurrentMetrics::Increment alive_keys;
private:
PresentIdHandler present_id_handler;
AbsentIdHandler absent_id_handler;
};
using UpdateUnitPtr = std::shared_ptr<UpdateUnit>;
using UpdateQueue = ConcurrentBoundedQueue<UpdateUnitPtr>;
/*
* This class is used to concatenate requested_keys.
*
* Imagine that we have several UpdateUnit with different vectors of keys and callbacks for that keys.
* We concatenate them into a long vector of keys that looks like:
*
* a1...ak_a b1...bk_2 c1...ck_3,
*
* where a1...ak_a are requested_keys from the first UpdateUnit.
* In addition we have the same number (three in this case) of callbacks.
* This class helps us to find a callback (or many callbacks) for a special key.
* */
class BunchUpdateUnit
{
public:
explicit BunchUpdateUnit(std::vector<UpdateUnitPtr> & update_request)
{
/// Here we prepare total count of all requested ids
/// not to do useless allocations later.
size_t total_requested_keys_count = 0;
for (auto & unit_ptr: update_request)
{
total_requested_keys_count += unit_ptr->requested_ids.size();
if (helper.empty())
helper.push_back(unit_ptr->requested_ids.size());
else
helper.push_back(unit_ptr->requested_ids.size() + helper.back());
present_id_handlers.emplace_back(unit_ptr->present_id_handler);
absent_id_handlers.emplace_back(unit_ptr->absent_id_handler);
update_units.emplace_back(unit_ptr);
}
concatenated_requested_ids.reserve(total_requested_keys_count);
for (auto & unit_ptr: update_request)
std::for_each(std::begin(unit_ptr->requested_ids), std::end(unit_ptr->requested_ids),
[&] (const Key & key) {concatenated_requested_ids.push_back(key);});
}
const std::vector<Key> & getRequestedIds()
{
return concatenated_requested_ids;
}
void informCallersAboutPresentId(Key id, size_t cell_idx)
{
for (size_t position = 0; position < concatenated_requested_ids.size(); ++position)
{
if (concatenated_requested_ids[position] == id)
{
auto unit_number = getUpdateUnitNumberForRequestedIdPosition(position);
auto lock = getLockToCurrentUnit(unit_number);
if (canUseCallback(unit_number))
getPresentIdHandlerForPosition(unit_number)(id, cell_idx);
}
}
}
void informCallersAboutAbsentId(Key id, size_t cell_idx)
{
for (size_t position = 0; position < concatenated_requested_ids.size(); ++position)
if (concatenated_requested_ids[position] == id)
{
auto unit_number = getUpdateUnitNumberForRequestedIdPosition(position);
auto lock = getLockToCurrentUnit(unit_number);
if (canUseCallback(unit_number))
getAbsentIdHandlerForPosition(unit_number)(id, cell_idx);
}
}
private:
/// Needed for control the usage of callback to avoid SEGFAULTs.
bool canUseCallback(size_t unit_number)
{
return update_units[unit_number].get()->can_use_callback;
}
std::unique_lock<std::mutex> getLockToCurrentUnit(size_t unit_number)
{
return std::unique_lock<std::mutex>(update_units[unit_number].get()->callback_mutex);
}
PresentIdHandler & getPresentIdHandlerForPosition(size_t unit_number)
{
return update_units[unit_number].get()->present_id_handler;
}
AbsentIdHandler & getAbsentIdHandlerForPosition(size_t unit_number)
{
return update_units[unit_number].get()->absent_id_handler;
}
size_t getUpdateUnitNumberForRequestedIdPosition(size_t position)
{
return std::lower_bound(helper.begin(), helper.end(), position) - helper.begin();
}
std::vector<Key> concatenated_requested_ids;
std::vector<PresentIdHandler> present_id_handlers;
std::vector<AbsentIdHandler> absent_id_handlers;
std::vector<std::reference_wrapper<UpdateUnitPtr>> update_units;
std::vector<size_t> helper;
};
mutable UpdateQueue update_queue;
ThreadPool update_pool;
@ -548,7 +452,7 @@ private:
*
*/
void updateThreadFunction();
void update(BunchUpdateUnit & bunch_update_unit) const;
void update(UpdateUnitPtr & update_unit_ptr) const;
void tryPushToUpdateQueueOrThrow(UpdateUnitPtr & update_unit_ptr) const;

77
src/Functions/FunctionsStringSearch.h
View File

@ -10,7 +10,7 @@
#include <Functions/FunctionHelpers.h>
#include <Functions/IFunctionImpl.h>
#include <Interpreters/Context.h>
#include <IO/WriteHelpers.h>
namespace DB
{
@ -38,8 +38,9 @@ namespace DB
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int ILLEGAL_COLUMN;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
template <typename Impl, typename Name>
@ -51,19 +52,33 @@ public:
String getName() const override { return name; }
size_t getNumberOfArguments() const override { return 2; }
bool isVariadic() const override { return Impl::supports_start_pos; }
size_t getNumberOfArguments() const override
{
if (Impl::supports_start_pos)
return 0;
return 2;
}
bool useDefaultImplementationForConstants() const override { return Impl::use_default_implementation_for_constants; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override
{
return Impl::use_default_implementation_for_constants
? ColumnNumbers{1, 2}
: ColumnNumbers{};
if (!Impl::use_default_implementation_for_constants)
return ColumnNumbers{};
if (!Impl::supports_start_pos)
return ColumnNumbers{1, 2};
return ColumnNumbers{1, 2, 3};
}
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (arguments.size() < 2 || 3 < arguments.size())
throw Exception("Number of arguments for function " + String(Name::name) + " doesn't match: passed "
+ toString(arguments.size()) + ", should be 2 or 3.",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
if (!isStringOrFixedString(arguments[0]))
throw Exception(
"Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
@ -72,6 +87,13 @@ public:
throw Exception(
"Illegal type " + arguments[1]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
if (arguments.size() >= 3)
{
if (!isUnsignedInteger(arguments[2]))
throw Exception(
"Illegal type " + arguments[2]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
return std::make_shared<DataTypeNumber<typename Impl::ResultType>>();
}
@ -82,17 +104,34 @@ public:
const ColumnPtr & column_haystack = block.getByPosition(arguments[0]).column;
const ColumnPtr & column_needle = block.getByPosition(arguments[1]).column;
ColumnPtr column_start_pos = nullptr;
if (arguments.size() >= 3)
column_start_pos = block.getByPosition(arguments[2]).column;
const ColumnConst * col_haystack_const = typeid_cast<const ColumnConst *>(&*column_haystack);
const ColumnConst * col_needle_const = typeid_cast<const ColumnConst *>(&*column_needle);
if constexpr (!Impl::use_default_implementation_for_constants)
{
bool is_col_start_pos_const = column_start_pos == nullptr || isColumnConst(*column_start_pos);
if (col_haystack_const && col_needle_const)
{
ResultType res{};
Impl::constantConstant(col_haystack_const->getValue<String>(), col_needle_const->getValue<String>(), res);
block.getByPosition(result).column
= block.getByPosition(result).type->createColumnConst(col_haystack_const->size(), toField(res));
auto col_res = ColumnVector<ResultType>::create();
typename ColumnVector<ResultType>::Container & vec_res = col_res->getData();
vec_res.resize(is_col_start_pos_const ? 1 : column_start_pos->size());
Impl::constantConstant(
col_haystack_const->getValue<String>(),
col_needle_const->getValue<String>(),
column_start_pos,
vec_res);
if (is_col_start_pos_const)
block.getByPosition(result).column
= block.getByPosition(result).type->createColumnConst(col_haystack_const->size(), toField(vec_res[0]));
else
block.getByPosition(result).column = std::move(col_res);
return;
}
}
@ -112,16 +151,28 @@ public:
col_haystack_vector->getOffsets(),
col_needle_vector->getChars(),
col_needle_vector->getOffsets(),
column_start_pos,
vec_res);
else if (col_haystack_vector && col_needle_const)
Impl::vectorConstant(
col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), col_needle_const->getValue<String>(), vec_res);
col_haystack_vector->getChars(),
col_haystack_vector->getOffsets(),
col_needle_const->getValue<String>(),
column_start_pos,
vec_res);
else if (col_haystack_vector_fixed && col_needle_const)
Impl::vectorFixedConstant(
col_haystack_vector_fixed->getChars(), col_haystack_vector_fixed->getN(), col_needle_const->getValue<String>(), vec_res);
col_haystack_vector_fixed->getChars(),
col_haystack_vector_fixed->getN(),
col_needle_const->getValue<String>(),
vec_res);
else if (col_haystack_const && col_needle_vector)
Impl::constantVector(
col_haystack_const->getValue<String>(), col_needle_vector->getChars(), col_needle_vector->getOffsets(), vec_res);
col_haystack_const->getValue<String>(),
col_needle_vector->getChars(),
col_needle_vector->getOffsets(),
column_start_pos,
vec_res);
else
throw Exception(
"Illegal columns " + block.getByPosition(arguments[0]).column->getName() + " and "

10
src/Functions/FunctionsVisitParam.h
View File

@ -36,6 +36,7 @@ namespace DB
namespace ErrorCodes
{
extern const int ILLEGAL_COLUMN;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
@ -79,12 +80,19 @@ struct ExtractParamImpl
using ResultType = typename ParamExtractor::ResultType;
static constexpr bool use_default_implementation_for_constants = true;
static constexpr bool supports_start_pos = false;
/// It is assumed that `res` is the correct size and initialized with zeros.
static void vectorConstant(const ColumnString::Chars & data, const ColumnString::Offsets & offsets,
static void vectorConstant(
const ColumnString::Chars & data,
const ColumnString::Offsets & offsets,
std::string needle,
const ColumnPtr & start_pos,
PaddedPODArray<ResultType> & res)
{
if (start_pos != nullptr)
throw Exception("Functions 'visitParamHas' and 'visitParamExtract*' doesn't support start_pos argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
/// We are looking for a parameter simply as a substring of the form "name"
needle = "\"" + needle + "\":";

11
src/Functions/HasTokenImpl.h
View File

@ -9,6 +9,7 @@ namespace DB
namespace ErrorCodes
{
extern const int ILLEGAL_COLUMN;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
/** Token search the string, means that needle must be surrounded by some separator chars, like whitespace or puctuation.
@ -19,10 +20,18 @@ struct HasTokenImpl
using ResultType = UInt8;
static constexpr bool use_default_implementation_for_constants = true;
static constexpr bool supports_start_pos = false;
static void vectorConstant(
const ColumnString::Chars & data, const ColumnString::Offsets & offsets, const std::string & pattern, PaddedPODArray<UInt8> & res)
const ColumnString::Chars & data,
const ColumnString::Offsets & offsets,
const std::string & pattern,
const ColumnPtr & start_pos,
PaddedPODArray<UInt8> & res)
{
if (start_pos != nullptr)
throw Exception("Function 'hasToken' does not support start_pos argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
if (offsets.empty())
return;

14
src/Functions/MatchImpl.h
View File

@ -25,6 +25,7 @@ namespace DB
namespace ErrorCodes
{
extern const int ILLEGAL_COLUMN;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
@ -76,6 +77,7 @@ template <bool like, bool revert = false, bool case_insensitive = false>
struct MatchImpl
{
static constexpr bool use_default_implementation_for_constants = true;
static constexpr bool supports_start_pos = false;
using ResultType = UInt8;
@ -84,8 +86,15 @@ struct MatchImpl
VolnitskyUTF8>;
static void vectorConstant(
const ColumnString::Chars & data, const ColumnString::Offsets & offsets, const std::string & pattern, PaddedPODArray<UInt8> & res)
const ColumnString::Chars & data,
const ColumnString::Offsets & offsets,
const std::string & pattern,
const ColumnPtr & start_pos,
PaddedPODArray<UInt8> & res)
{
if (start_pos != nullptr)
throw Exception("Functions 'like' and 'match' don't support start_pos argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
if (offsets.empty())
return;
@ -238,7 +247,8 @@ struct MatchImpl
/// Very carefully crafted copy-paste.
static void vectorFixedConstant(
const ColumnString::Chars & data, size_t n, const std::string & pattern, PaddedPODArray<UInt8> & res)
const ColumnString::Chars & data, size_t n, const std::string & pattern,
PaddedPODArray<UInt8> & res)
{
if (data.empty())
return;

2
src/Functions/MultiMatchAnyImpl.h
View File

@ -120,7 +120,7 @@ struct MultiMatchAnyImpl
memset(accum.data(), 0, accum.size());
for (size_t j = 0; j < needles.size(); ++j)
{
MatchImpl<false, false>::vectorConstant(haystack_data, haystack_offsets, needles[j].toString(), accum);
MatchImpl<false, false>::vectorConstant(haystack_data, haystack_offsets, needles[j].toString(), nullptr, accum);
for (size_t i = 0; i < res.size(); ++i)
{
if constexpr (FindAny)

149
src/Functions/PositionImpl.h
View File

@ -1,11 +1,11 @@
#include "FunctionsStringSearch.h"
#include <algorithm>
#include <string>
#include <vector>
#include <Poco/UTF8String.h>
#include <Common/Volnitsky.h>
namespace DB
{
@ -42,6 +42,11 @@ struct PositionCaseSensitiveASCII
return MultiSearcherInBigHaystack(needles);
}
static const char * advancePos(const char * pos, const char * end, size_t n)
{
return std::min(pos + n, end);
}
/// Number of code points between 'begin' and 'end' (this has different behaviour for ASCII and UTF-8).
static size_t countChars(const char * begin, const char * end) { return end - begin; }
@ -73,6 +78,11 @@ struct PositionCaseInsensitiveASCII
return MultiSearcherInBigHaystack(needles);
}
static const char * advancePos(const char * pos, const char * end, size_t n)
{
return std::min(pos + n, end);
}
static size_t countChars(const char * begin, const char * end) { return end - begin; }
static void toLowerIfNeed(std::string & s) { std::transform(std::begin(s), std::end(s), std::begin(s), tolower); }
@ -100,6 +110,20 @@ struct PositionCaseSensitiveUTF8
return MultiSearcherInBigHaystack(needles);
}
static const char * advancePos(const char * pos, const char * end, size_t n)
{
for (auto it = pos; it != end; ++it)
{
if (!UTF8::isContinuationOctet(static_cast<UInt8>(*it)))
{
if (n == 0)
return it;
n--;
}
}
return end;
}
static size_t countChars(const char * begin, const char * end)
{
size_t res = 0;
@ -134,13 +158,16 @@ struct PositionCaseInsensitiveUTF8
return MultiSearcherInBigHaystack(needles);
}
static const char * advancePos(const char * pos, const char * end, size_t n)
{
// reuse implementation that doesn't depend on case
return PositionCaseSensitiveUTF8::advancePos(pos, end, n);
}
static size_t countChars(const char * begin, const char * end)
{
size_t res = 0;
for (auto it = begin; it != end; ++it)
if (!UTF8::isContinuationOctet(static_cast<UInt8>(*it)))
++res;
return res;
// reuse implementation that doesn't depend on case
return PositionCaseSensitiveUTF8::countChars(begin, end);
}
static void toLowerIfNeed(std::string & s) { Poco::UTF8::toLowerInPlace(s); }
@ -151,12 +178,17 @@ template <typename Impl>
struct PositionImpl
{
static constexpr bool use_default_implementation_for_constants = false;
static constexpr bool supports_start_pos = true;
using ResultType = UInt64;
/// Find one substring in many strings.
static void vectorConstant(
const ColumnString::Chars & data, const ColumnString::Offsets & offsets, const std::string & needle, PaddedPODArray<UInt64> & res)
const ColumnString::Chars & data,
const ColumnString::Offsets & offsets,
const std::string & needle,
const ColumnPtr & start_pos,
PaddedPODArray<UInt64> & res)
{
const UInt8 * begin = data.data();
const UInt8 * pos = begin;
@ -176,13 +208,26 @@ struct PositionImpl
res[i] = 0;
++i;
}
auto start = start_pos != nullptr ? start_pos->getUInt(i) : 0;
/// We check that the entry does not pass through the boundaries of strings.
if (pos + needle.size() < begin + offsets[i])
res[i] = 1 + Impl::countChars(reinterpret_cast<const char *>(begin + offsets[i - 1]), reinterpret_cast<const char *>(pos));
{
auto res_pos = 1 + Impl::countChars(reinterpret_cast<const char *>(begin + offsets[i - 1]), reinterpret_cast<const char *>(pos));
if (res_pos < start)
{
pos = reinterpret_cast<const UInt8 *>(Impl::advancePos(
reinterpret_cast<const char *>(pos),
reinterpret_cast<const char *>(begin + offsets[i]),
start - res_pos));
continue;
}
res[i] = res_pos;
}
else
{
res[i] = 0;
}
pos = begin + offsets[i];
++i;
}
@ -192,24 +237,68 @@ struct PositionImpl
}
/// Search for substring in string.
static void constantConstant(std::string data, std::string needle, UInt64 & res)
static void constantConstantScalar(
std::string data,
std::string needle,
UInt64 start_pos,
UInt64 & res)
{
Impl::toLowerIfNeed(data);
Impl::toLowerIfNeed(needle);
auto start = std::max(start_pos, UInt64(1));
res = data.find(needle);
if (needle.size() == 0)
{
size_t haystack_size = Impl::countChars(data.data(), data.data() + data.size());
res = start <= haystack_size + 1 ? start : 0;
return;
}
size_t start_byte = Impl::advancePos(data.data(), data.data() + data.size(), start - 1) - data.data();
res = data.find(needle, start_byte);
if (res == std::string::npos)
res = 0;
else
res = 1 + Impl::countChars(data.data(), data.data() + res);
}
/// Search for substring in string starting from different positions.
static void constantConstant(
std::string data,
std::string needle,
const ColumnPtr & start_pos,
PaddedPODArray<UInt64> & res)
{
Impl::toLowerIfNeed(data);
Impl::toLowerIfNeed(needle);
if (start_pos == nullptr)
{
constantConstantScalar(data, needle, 0, res[0]);
return;
}
size_t haystack_size = Impl::countChars(data.data(), data.data() + data.size());
size_t size = start_pos != nullptr ? start_pos->size() : 0;
for (size_t i = 0; i < size; ++i)
{
auto start = start_pos->getUInt(i);
if (start > haystack_size + 1)
{
res[i] = 0;
continue;
}
constantConstantScalar(data, needle, start, res[i]);
}
}
/// Search each time for a different single substring inside each time different string.
static void vectorVector(
const ColumnString::Chars & haystack_data,
const ColumnString::Offsets & haystack_offsets,
const ColumnString::Chars & needle_data,
const ColumnString::Offsets & needle_offsets,
const ColumnPtr & start_pos,
PaddedPODArray<UInt64> & res)
{
ColumnString::Offset prev_haystack_offset = 0;
@ -222,10 +311,16 @@ struct PositionImpl
size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
size_t haystack_size = haystack_offsets[i] - prev_haystack_offset - 1;
if (0 == needle_size)
auto start = start_pos != nullptr ? std::max(start_pos->getUInt(i), UInt64(1)) : UInt64(1);
if (start > haystack_size + 1)
{
/// An empty string is always at the very beginning of `haystack`.
res[i] = 1;
res[i] = 0;
}
else if (0 == needle_size)
{
/// An empty string is always at any position in `haystack`.
res[i] = start;
}
else
{
@ -234,8 +329,12 @@ struct PositionImpl
reinterpret_cast<const char *>(&needle_data[prev_needle_offset]),
needle_offsets[i] - prev_needle_offset - 1); /// zero byte at the end
const char * beg = Impl::advancePos(
reinterpret_cast<const char *>(&haystack_data[prev_haystack_offset]),
reinterpret_cast<const char *>(&haystack_data[haystack_offsets[i] - 1]),
start - 1);
/// searcher returns a pointer to the found substring or to the end of `haystack`.
size_t pos = searcher.search(&haystack_data[prev_haystack_offset], &haystack_data[haystack_offsets[i] - 1])
size_t pos = searcher.search(reinterpret_cast<const UInt8 *>(beg), &haystack_data[haystack_offsets[i] - 1])
- &haystack_data[prev_haystack_offset];
if (pos != haystack_size)
@ -259,9 +358,10 @@ struct PositionImpl
const String & haystack,
const ColumnString::Chars & needle_data,
const ColumnString::Offsets & needle_offsets,
const ColumnPtr & start_pos,
PaddedPODArray<UInt64> & res)
{
// NOTE You could use haystack indexing. But this is a rare case.
/// NOTE You could use haystack indexing. But this is a rare case.
ColumnString::Offset prev_needle_offset = 0;
@ -271,17 +371,24 @@ struct PositionImpl
{
size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
if (0 == needle_size)
auto start = start_pos != nullptr ? std::max(start_pos->getUInt(i), UInt64(1)) : UInt64(1);
if (start > haystack.size() + 1)
{
res[i] = 1;
res[i] = 0;
}
else if (0 == needle_size)
{
res[i] = start;
}
else
{
typename Impl::SearcherInSmallHaystack searcher = Impl::createSearcherInSmallHaystack(
reinterpret_cast<const char *>(&needle_data[prev_needle_offset]), needle_offsets[i] - prev_needle_offset - 1);
const char * beg = Impl::advancePos(haystack.data(), haystack.data() + haystack.size(), start - 1);
size_t pos = searcher.search(
reinterpret_cast<const UInt8 *>(haystack.data()),
reinterpret_cast<const UInt8 *>(beg),
reinterpret_cast<const UInt8 *>(haystack.data()) + haystack.size())
- reinterpret_cast<const UInt8 *>(haystack.data());

17
src/Functions/lgamma.cpp
View File

@ -1,11 +1,26 @@
#include <Functions/FunctionMathUnary.h>
#include <Functions/FunctionFactory.h>
#if defined(OS_DARWIN)
extern "C"
{
/// Is defined in libglibc-compatibility.a
double lgamma_r(double x, int * signgamp);
}
#endif
namespace DB
{
/// Use wrapper and use lgamma_r version because std::lgamma is not threadsafe.
static Float64 lgamma_wrapper(Float64 arg)
{
int signp;
return lgamma_r(arg, &signp);
}
struct LGammaName { static constexpr auto name = "lgamma"; };
using FunctionLGamma = FunctionMathUnary<UnaryFunctionPlain<LGammaName, std::lgamma>>;
using FunctionLGamma = FunctionMathUnary<UnaryFunctionPlain<LGammaName, lgamma_wrapper>>;
void registerFunctionLGamma(FunctionFactory & factory)
{

17
src/Interpreters/ActionsVisitor.cpp
View File

@ -1,6 +1,5 @@
#include "Common/quoteString.h"
#include <Common/quoteString.h>
#include <Common/typeid_cast.h>
#include <Common/PODArray.h>
#include <Core/Row.h>
#include <Functions/FunctionFactory.h>
@ -9,7 +8,6 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <DataTypes/DataTypeSet.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeFunction.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeTuple.h>
@ -21,7 +19,6 @@
#include <Columns/ColumnSet.h>
#include <Columns/ColumnConst.h>
#include <Columns/ColumnsNumber.h>
#include <Storages/StorageSet.h>
@ -546,10 +543,14 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
if (!data.only_consts)
{
String result_name = column_name.get(ast);
data.addAction(ExpressionAction::copyColumn(arg->getColumnName(), result_name));
NameSet joined_columns;
joined_columns.insert(result_name);
data.addAction(ExpressionAction::arrayJoin(joined_columns, false, data.context));
/// Here we copy argument because arrayJoin removes source column.
/// It makes possible to remove source column before arrayJoin if it won't be needed anymore.
/// It could have been possible to implement arrayJoin which keeps source column,
/// but in this case it will always be replicated (as many arrays), which is expensive.
String tmp_name = data.getUniqueName("_array_join_" + arg->getColumnName());
data.addAction(ExpressionAction::copyColumn(arg->getColumnName(), tmp_name));
data.addAction(ExpressionAction::arrayJoin(tmp_name, result_name));
}
return;

31
src/Interpreters/ArrayJoinAction.cpp
View File

@ -48,7 +48,7 @@ void ArrayJoinAction::prepare(Block & sample_block)
}
}
void ArrayJoinAction::execute(Block & block, bool dry_run)
void ArrayJoinAction::execute(Block & block)
{
if (columns.empty())
throw Exception("No arrays to join", ErrorCodes::LOGICAL_ERROR);
@ -105,7 +105,7 @@ void ArrayJoinAction::execute(Block & block, bool dry_run)
Block tmp_block{src_col, {{}, src_col.type, {}}};
function_builder->build({src_col})->execute(tmp_block, {0}, 1, src_col.column->size(), dry_run);
function_builder->build({src_col})->execute(tmp_block, {0}, 1, src_col.column->size());
non_empty_array_columns[name] = tmp_block.safeGetByPosition(1).column;
}
@ -140,31 +140,4 @@ void ArrayJoinAction::execute(Block & block, bool dry_run)
}
}
void ArrayJoinAction::finalize(NameSet & needed_columns, NameSet & unmodified_columns, NameSet & final_columns)
{
/// Do not ARRAY JOIN columns that are not used anymore.
/// Usually, such columns are not used until ARRAY JOIN, and therefore are ejected further in this function.
/// We will not remove all the columns so as not to lose the number of rows.
for (auto it = columns.begin(); it != columns.end();)
{
bool need = needed_columns.count(*it);
if (!need && columns.size() > 1)
{
columns.erase(it++);
}
else
{
needed_columns.insert(*it);
unmodified_columns.erase(*it);
/// If no ARRAY JOIN results are used, forcibly leave an arbitrary column at the output,
/// so you do not lose the number of rows.
if (!need)
final_columns.insert(*it);
++it;
}
}
}
}

8
src/Interpreters/ArrayJoinAction.h
View File

@ -12,8 +12,9 @@ class Context;
class IFunctionOverloadResolver;
using FunctionOverloadResolverPtr = std::shared_ptr<IFunctionOverloadResolver>;
struct ArrayJoinAction
class ArrayJoinAction
{
public:
NameSet columns;
bool is_left = false;
bool is_unaligned = false;
@ -28,8 +29,9 @@ struct ArrayJoinAction
ArrayJoinAction(const NameSet & array_joined_columns_, bool array_join_is_left, const Context & context);
void prepare(Block & sample_block);
void execute(Block & block, bool dry_run);
void finalize(NameSet & needed_columns, NameSet & unmodified_columns, NameSet & final_columns);
void execute(Block & block);
};
using ArrayJoinActionPtr = std::shared_ptr<ArrayJoinAction>;
}

343
src/Interpreters/ExpressionActions.cpp
View File

@ -1,19 +1,19 @@
#include <Interpreters/Set.h>
#include <Common/ProfileEvents.h>
#include <Common/SipHash.h>
#include <Interpreters/ArrayJoinAction.h>
#include <Interpreters/ExpressionActions.h>
#include <Interpreters/ExpressionJIT.h>
#include <Interpreters/TableJoin.h>
#include <Interpreters/Context.h>
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnArray.h>
#include <Common/typeid_cast.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypesNumber.h>
#include <Functions/FunctionFactory.h>
#include <Functions/IFunction.h>
#include <optional>
#include <Columns/ColumnSet.h>
#include <Functions/FunctionHelpers.h>
#if !defined(ARCADIA_BUILD)
# include "config_core.h"
@ -37,6 +37,7 @@ namespace ErrorCodes
extern const int NOT_FOUND_COLUMN_IN_BLOCK;
extern const int TOO_MANY_TEMPORARY_COLUMNS;
extern const int TOO_MANY_TEMPORARY_NON_CONST_COLUMNS;
extern const int TYPE_MISMATCH;
}
/// Read comment near usage
@ -47,9 +48,6 @@ Names ExpressionAction::getNeededColumns() const
{
Names res = argument_names;
if (array_join)
res.insert(res.end(), array_join->columns.begin(), array_join->columns.end());
if (table_join)
res.insert(res.end(), table_join->keyNamesLeft().begin(), table_join->keyNamesLeft().end());
@ -143,11 +141,15 @@ ExpressionAction ExpressionAction::addAliases(const NamesWithAliases & aliased_c
return a;
}
ExpressionAction ExpressionAction::arrayJoin(const NameSet & array_joined_columns, bool array_join_is_left, const Context & context)
ExpressionAction ExpressionAction::arrayJoin(std::string source_name, std::string result_name)
{
if (source_name == result_name)
throw Exception("ARRAY JOIN action should have different source and result names", ErrorCodes::LOGICAL_ERROR);
ExpressionAction a;
a.type = ARRAY_JOIN;
a.array_join = std::make_shared<ArrayJoinAction>(array_joined_columns, array_join_is_left, context);
a.source_name = std::move(source_name);
a.result_name = std::move(result_name);
return a;
}
@ -243,7 +245,18 @@ void ExpressionAction::prepare(Block & sample_block, const Settings & settings,
case ARRAY_JOIN:
{
array_join->prepare(sample_block);
ColumnWithTypeAndName current = sample_block.getByName(source_name);
sample_block.erase(source_name);
const DataTypeArray * array_type = typeid_cast<const DataTypeArray *>(&*current.type);
if (!array_type)
throw Exception("ARRAY JOIN requires array argument", ErrorCodes::TYPE_MISMATCH);
current.name = result_name;
current.type = array_type->getNestedType();
current.column = nullptr; /// Result is never const
sample_block.insert(std::move(current));
break;
}
@ -369,7 +382,23 @@ void ExpressionAction::execute(Block & block, bool dry_run) const
case ARRAY_JOIN:
{
array_join->execute(block, dry_run);
auto source = block.getByName(source_name);
block.erase(source_name);
source.column = source.column->convertToFullColumnIfConst();
const ColumnArray * array = typeid_cast<const ColumnArray *>(source.column.get());
if (!array)
throw Exception("ARRAY JOIN of not array: " + source_name, ErrorCodes::TYPE_MISMATCH);
for (auto & column : block)
column.column = column.column->replicate(array->getOffsets());
source.column = array->getDataPtr();
source.type = assert_cast<const DataTypeArray &>(*source.type).getNestedType();
source.name = result_name;
block.insert(std::move(source));
break;
}
@ -478,13 +507,7 @@ std::string ExpressionAction::toString() const
break;
case ARRAY_JOIN:
ss << (array_join->is_left ? "LEFT " : "") << "ARRAY JOIN ";
for (NameSet::const_iterator it = array_join->columns.begin(); it != array_join->columns.end(); ++it)
{
if (it != array_join->columns.begin())
ss << ", ";
ss << *it;
}
ss << "ARRAY JOIN " << source_name << " -> " << result_name;
break;
case JOIN:
@ -597,9 +620,6 @@ void ExpressionActions::addImpl(ExpressionAction action, Names & new_names)
if (!action.result_name.empty())
new_names.push_back(action.result_name);
if (action.array_join)
new_names.insert(new_names.end(), action.array_join->columns.begin(), action.array_join->columns.end());
/// Compiled functions are custom functions and they don't need building
if (action.type == ExpressionAction::APPLY_FUNCTION && !action.is_function_compiled)
{
@ -631,51 +651,6 @@ void ExpressionActions::prependProjectInput()
actions.insert(actions.begin(), ExpressionAction::project(getRequiredColumns()));
}
void ExpressionActions::prependArrayJoin(const ExpressionAction & action, const Block & sample_block_before)
{
if (action.type != ExpressionAction::ARRAY_JOIN)
throw Exception("ARRAY_JOIN action expected", ErrorCodes::LOGICAL_ERROR);
NameSet array_join_set(action.array_join->columns.begin(), action.array_join->columns.end());
for (auto & it : input_columns)
{
if (array_join_set.count(it.name))
{
array_join_set.erase(it.name);
it.type = std::make_shared<DataTypeArray>(it.type);
}
}
for (const std::string & name : array_join_set)
{
input_columns.emplace_back(name, sample_block_before.getByName(name).type);
actions.insert(actions.begin(), ExpressionAction::removeColumn(name));
}
actions.insert(actions.begin(), action);
optimizeArrayJoin();
}
bool ExpressionActions::popUnusedArrayJoin(const Names & required_columns, ExpressionAction & out_action)
{
if (actions.empty() || actions.back().type != ExpressionAction::ARRAY_JOIN)
return false;
NameSet required_set(required_columns.begin(), required_columns.end());
for (const std::string & name : actions.back().array_join->columns)
{
if (required_set.count(name))
return false;
}
for (const std::string & name : actions.back().array_join->columns)
{
DataTypePtr & type = sample_block.getByName(name).type;
type = std::make_shared<DataTypeArray>(type);
}
out_action = actions.back();
actions.pop_back();
return true;
}
void ExpressionActions::execute(Block & block, bool dry_run) const
{
for (const auto & action : actions)
@ -809,7 +784,18 @@ void ExpressionActions::finalize(const Names & output_columns)
}
else if (action.type == ExpressionAction::ARRAY_JOIN)
{
action.array_join->finalize(needed_columns, unmodified_columns, final_columns);
/// We need source anyway, in order to calculate number of rows correctly.
needed_columns.insert(action.source_name);
unmodified_columns.erase(action.result_name);
needed_columns.erase(action.result_name);
/// Note: technically, if result of arrayJoin is not needed,
/// we may remove all the columns and loose the number of rows here.
/// However, I cannot imagine how it is possible.
/// For "big" ARRAY JOIN it could have happened in query like
/// SELECT count() FROM table ARRAY JOIN x
/// Now, "big" ARRAY JOIN is moved to separate pipeline step,
/// and arrayJoin(x) is an expression which result can't be lost.
}
else
{
@ -946,7 +932,7 @@ void ExpressionActions::finalize(const Names & output_columns)
auto process = [&] (const String & name)
{
auto refcount = --columns_refcount[name];
if (refcount <= 0)
if (refcount <= 0 && action.type != ExpressionAction::ARRAY_JOIN)
{
new_actions.push_back(ExpressionAction::removeColumn(name));
if (sample_block.has(name))
@ -1046,8 +1032,6 @@ void ExpressionActions::optimizeArrayJoin()
if (!actions[i].result_name.empty())
array_joined_columns.insert(actions[i].result_name);
if (actions[i].array_join)
array_joined_columns.insert(actions[i].array_join->columns.begin(), actions[i].array_join->columns.end());
array_join_dependencies.insert(needed.begin(), needed.end());
}
@ -1077,6 +1061,134 @@ void ExpressionActions::optimizeArrayJoin()
}
}
ExpressionActionsPtr ExpressionActions::splitActionsBeforeArrayJoin(const NameSet & array_joined_columns)
{
/// Create new actions.
/// Copy from this because we don't have context.
/// TODO: remove context from constructor?
auto split_actions = std::make_shared<ExpressionActions>(*this);
split_actions->actions.clear();
split_actions->sample_block.clear();
split_actions->input_columns.clear();
/// Expected chain:
/// Expression (this) -> ArrayJoin (array_joined_columns) -> Expression (split_actions)
/// We are going to move as many actions as we can from this to split_actions.
/// We can move all inputs which are not depend on array_joined_columns
/// (with some exceptions to PROJECT and REMOVE_COLUMN
/// Use the same inputs for split_actions, except array_joined_columns.
for (const auto & input_column : input_columns)
{
if (array_joined_columns.count(input_column.name) == 0)
{
split_actions->input_columns.emplace_back(input_column);
split_actions->sample_block.insert(ColumnWithTypeAndName(nullptr, input_column.type, input_column.name));
}
}
/// Do not split action if input depends only on array joined columns.
if (split_actions->input_columns.empty())
return nullptr;
/// Actions which depend on ARRAY JOIN result.
NameSet array_join_dependent_columns = array_joined_columns;
/// Arguments of actions which depend on ARRAY JOIN result.
/// This columns can't be deleted in split_actions.
NameSet array_join_dependent_columns_arguments;
/// We create new_actions list for `this`. Current actions are moved to new_actions nor added to split_actions.
Actions new_actions;
for (const auto & action : actions)
{
/// Exception for PROJECT.
/// It removes columns, so it will remove split_actions output which may be needed for actions from `this`.
/// So, we replace it ADD_ALIASES.
/// Usually, PROJECT is added to begin of actions in order to remove unused output of prev actions.
/// We skip it now, but will prependProjectInput at the end.
if (action.type == ExpressionAction::PROJECT)
{
/// Each alias has separate dependencies, so we split this action into two parts.
NamesWithAliases split_aliases;
NamesWithAliases depend_aliases;
for (const auto & pair : action.projection)
{
/// Skip if is not alias.
if (pair.second.empty())
continue;
if (array_join_dependent_columns.count(pair.first))
{
array_join_dependent_columns.insert(pair.second);
depend_aliases.emplace_back(std::move(pair));
}
else
split_aliases.emplace_back(std::move(pair));
}
if (!split_aliases.empty())
split_actions->add(ExpressionAction::addAliases(split_aliases));
if (!depend_aliases.empty())
new_actions.emplace_back(ExpressionAction::addAliases(depend_aliases));
continue;
}
bool depends_on_array_join = false;
for (auto & column : action.getNeededColumns())
if (array_join_dependent_columns.count(column) != 0)
depends_on_array_join = true;
if (depends_on_array_join)
{
/// Add result of this action to array_join_dependent_columns too.
if (!action.result_name.empty())
array_join_dependent_columns.insert(action.result_name);
/// Add arguments of this action to array_join_dependent_columns_arguments.
auto needed = action.getNeededColumns();
array_join_dependent_columns_arguments.insert(needed.begin(), needed.end());
new_actions.emplace_back(action);
}
else if (action.type == ExpressionAction::REMOVE_COLUMN)
{
/// Exception for REMOVE_COLUMN.
/// We cannot move it to split_actions if any argument from `this` needed that column.
if (array_join_dependent_columns_arguments.count(action.source_name))
new_actions.emplace_back(action);
else
split_actions->add(action);
}
else
split_actions->add(action);
}
/// Return empty actions if nothing was separated. Keep `this` unchanged.
if (split_actions->getActions().empty())
return nullptr;
std::swap(actions, new_actions);
/// Collect inputs from ARRAY JOIN.
NamesAndTypesList inputs_from_array_join;
for (auto & column : input_columns)
if (array_joined_columns.count(column.name))
inputs_from_array_join.emplace_back(std::move(column));
/// Fix inputs for `this`.
/// It is output of split_actions + inputs from ARRAY JOIN.
input_columns = split_actions->getSampleBlock().getNamesAndTypesList();
input_columns.insert(input_columns.end(), inputs_from_array_join.begin(), inputs_from_array_join.end());
/// Remove not needed columns.
if (!actions.empty())
prependProjectInput();
return split_actions;
}
JoinPtr ExpressionActions::getTableJoinAlgo() const
{
@ -1178,9 +1290,8 @@ UInt128 ExpressionAction::ActionHash::operator()(const ExpressionAction & action
hash.update(arg_name);
break;
case ARRAY_JOIN:
hash.update(action.array_join->is_left);
for (const auto & col : action.array_join->columns)
hash.update(col);
hash.update(action.result_name);
hash.update(action.source_name);
break;
case JOIN:
for (const auto & col : action.table_join->columnsAddedByJoin())
@ -1236,15 +1347,9 @@ bool ExpressionAction::operator==(const ExpressionAction & other) const
return false;
}
bool same_array_join = !array_join && !other.array_join;
if (array_join && other.array_join)
same_array_join = (array_join->columns == other.array_join->columns) &&
(array_join->is_left == other.array_join->is_left);
return source_name == other.source_name
&& result_name == other.result_name
&& argument_names == other.argument_names
&& same_array_join
&& TableJoin::sameJoin(table_join.get(), other.table_join.get())
&& projection == other.projection
&& is_function_compiled == other.is_function_compiled;
@ -1255,8 +1360,8 @@ void ExpressionActionsChain::addStep()
if (steps.empty())
throw Exception("Cannot add action to empty ExpressionActionsChain", ErrorCodes::LOGICAL_ERROR);
ColumnsWithTypeAndName columns = steps.back().actions->getSampleBlock().getColumnsWithTypeAndName();
steps.push_back(Step(std::make_shared<ExpressionActions>(columns, context)));
ColumnsWithTypeAndName columns = steps.back()->getResultColumns();
steps.push_back(std::make_unique<ExpressionActionsStep>(std::make_shared<ExpressionActions>(columns, context)));
}
void ExpressionActionsChain::finalize()
@ -1264,16 +1369,16 @@ void ExpressionActionsChain::finalize()
/// Finalize all steps. Right to left to define unnecessary input columns.
for (int i = static_cast<int>(steps.size()) - 1; i >= 0; --i)
{
Names required_output = steps[i].required_output;
Names required_output = steps[i]->required_output;
std::unordered_map<String, size_t> required_output_indexes;
for (size_t j = 0; j < required_output.size(); ++j)
required_output_indexes[required_output[j]] = j;
auto & can_remove_required_output = steps[i].can_remove_required_output;
auto & can_remove_required_output = steps[i]->can_remove_required_output;
if (i + 1 < static_cast<int>(steps.size()))
{
const NameSet & additional_input = steps[i + 1].additional_input;
for (const auto & it : steps[i + 1].actions->getRequiredColumnsWithTypes())
const NameSet & additional_input = steps[i + 1]->additional_input;
for (const auto & it : steps[i + 1]->getRequiredColumns())
{
if (additional_input.count(it.name) == 0)
{
@ -1285,27 +1390,19 @@ void ExpressionActionsChain::finalize()
}
}
}
steps[i].actions->finalize(required_output);
}
/// When possible, move the ARRAY JOIN from earlier steps to later steps.
for (size_t i = 1; i < steps.size(); ++i)
{
ExpressionAction action;
if (steps[i - 1].actions->popUnusedArrayJoin(steps[i - 1].required_output, action))
steps[i].actions->prependArrayJoin(action, steps[i - 1].actions->getSampleBlock());
steps[i]->finalize(required_output);
}
/// Adding the ejection of unnecessary columns to the beginning of each step.
for (size_t i = 1; i < steps.size(); ++i)
{
size_t columns_from_previous = steps[i - 1].actions->getSampleBlock().columns();
size_t columns_from_previous = steps[i - 1]->getResultColumns().size();
/// If unnecessary columns are formed at the output of the previous step, we'll add them to the beginning of this step.
/// Except when we drop all the columns and lose the number of rows in the block.
if (!steps[i].actions->getRequiredColumnsWithTypes().empty()
&& columns_from_previous > steps[i].actions->getRequiredColumnsWithTypes().size())
steps[i].actions->prependProjectInput();
if (!steps[i]->getResultColumns().empty()
&& columns_from_previous > steps[i]->getRequiredColumns().size())
steps[i]->prependProjectInput();
}
}
@ -1317,12 +1414,62 @@ std::string ExpressionActionsChain::dumpChain() const
{
ss << "step " << i << "\n";
ss << "required output:\n";
for (const std::string & name : steps[i].required_output)
for (const std::string & name : steps[i]->required_output)
ss << name << "\n";
ss << "\n" << steps[i].actions->dumpActions() << "\n";
ss << "\n" << steps[i]->dump() << "\n";
}
return ss.str();
}
ExpressionActionsChain::ArrayJoinStep::ArrayJoinStep(ArrayJoinActionPtr array_join_, ColumnsWithTypeAndName required_columns_, Names required_output_)
: Step(std::move(required_output_))
, array_join(std::move(array_join_))
, result_columns(std::move(required_columns_))
{
for (auto & column : result_columns)
{
required_columns.emplace_back(NameAndTypePair(column.name, column.type));
if (array_join->columns.count(column.name) > 0)
{
const auto * array = typeid_cast<const DataTypeArray *>(column.type.get());
column.type = array->getNestedType();
/// Arrays are materialized
column.column = nullptr;
}
}
}
void ExpressionActionsChain::ArrayJoinStep::finalize(const Names & required_output_)
{
NamesAndTypesList new_required_columns;
ColumnsWithTypeAndName new_result_columns;
NameSet names(required_output_.begin(), required_output_.end());
for (const auto & column : result_columns)
{
if (array_join->columns.count(column.name) != 0 || names.count(column.name) != 0)
new_result_columns.emplace_back(column);
}
for (const auto & column : required_columns)
{
if (array_join->columns.count(column.name) != 0 || names.count(column.name) != 0)
new_required_columns.emplace_back(column);
}
std::swap(required_columns, new_required_columns);
std::swap(result_columns, new_result_columns);
}
ExpressionActionsPtr & ExpressionActionsChain::Step::actions()
{
return typeid_cast<ExpressionActionsStep *>(this)->actions;
}
const ExpressionActionsPtr & ExpressionActionsChain::Step::actions() const
{
return typeid_cast<const ExpressionActionsStep *>(this)->actions;
}
}

123
src/Interpreters/ExpressionActions.h
View File

@ -9,7 +9,9 @@
#include <unordered_map>
#include <unordered_set>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Interpreters/ArrayJoinAction.h>
#include <DataTypes/DataTypeArray.h>
#include <variant>
#if !defined(ARCADIA_BUILD)
# include "config_core.h"
@ -44,6 +46,9 @@ using DataTypePtr = std::shared_ptr<const IDataType>;
class ExpressionActions;
class CompiledExpressionCache;
class ArrayJoinAction;
using ArrayJoinActionPtr = std::shared_ptr<ArrayJoinAction>;
/** Action on the block.
*/
struct ExpressionAction
@ -59,10 +64,9 @@ public:
APPLY_FUNCTION,
/** Replaces the specified columns with arrays into columns with elements.
* Duplicates the values in the remaining columns by the number of elements in the arrays.
* Arrays must be parallel (have the same lengths).
*/
/// Replaces the source column with array into column with elements.
/// Duplicates the values in the remaining columns by the number of elements in the arrays.
/// Source column is removed from block.
ARRAY_JOIN,
JOIN,
@ -75,7 +79,7 @@ public:
Type type{};
/// For ADD/REMOVE/COPY_COLUMN.
/// For ADD/REMOVE/ARRAY_JOIN/COPY_COLUMN.
std::string source_name;
std::string result_name;
DataTypePtr result_type;
@ -97,9 +101,6 @@ public:
Names argument_names;
bool is_function_compiled = false;
/// For ARRAY JOIN
std::shared_ptr<ArrayJoinAction> array_join;
/// For JOIN
std::shared_ptr<const TableJoin> table_join;
JoinPtr join;
@ -117,7 +118,7 @@ public:
static ExpressionAction project(const NamesWithAliases & projected_columns_);
static ExpressionAction project(const Names & projected_columns_);
static ExpressionAction addAliases(const NamesWithAliases & aliased_columns_);
static ExpressionAction arrayJoin(const NameSet & array_joined_columns, bool array_join_is_left, const Context & context);
static ExpressionAction arrayJoin(std::string source_name, std::string result_name);
static ExpressionAction ordinaryJoin(std::shared_ptr<TableJoin> table_join, JoinPtr join);
/// Which columns necessary to perform this action.
@ -143,6 +144,8 @@ private:
void execute(Block & block, bool dry_run) const;
};
class ExpressionActions;
using ExpressionActionsPtr = std::shared_ptr<ExpressionActions>;
/** Contains a sequence of actions on the block.
*/
@ -174,13 +177,9 @@ public:
/// Adds to the beginning the removal of all extra columns.
void prependProjectInput();
/// Add the specified ARRAY JOIN action to the beginning. Change the appropriate input types to arrays.
/// If there are unknown columns in the ARRAY JOIN list, take their types from sample_block, and immediately after ARRAY JOIN remove them.
void prependArrayJoin(const ExpressionAction & action, const Block & sample_block_before);
/// If the last action is ARRAY JOIN, and it does not affect the columns from required_columns, discard and return it.
/// Change the corresponding output types to arrays.
bool popUnusedArrayJoin(const Names & required_columns, ExpressionAction & out_action);
/// Splits actions into two parts. Returned half may be swapped with ARRAY JOIN.
/// Returns nullptr if no actions may be moved before ARRAY JOIN.
ExpressionActionsPtr splitActionsBeforeArrayJoin(const NameSet & array_joined_columns);
/// - Adds actions to delete all but the specified columns.
/// - Removes unused input columns.
@ -196,8 +195,8 @@ public:
Names getRequiredColumns() const
{
Names names;
for (NamesAndTypesList::const_iterator it = input_columns.begin(); it != input_columns.end(); ++it)
names.push_back(it->name);
for (const auto & input : input_columns)
names.push_back(input.name);
return names;
}
@ -274,8 +273,6 @@ private:
void optimizeArrayJoin();
};
using ExpressionActionsPtr = std::shared_ptr<ExpressionActions>;
/** The sequence of transformations over the block.
* It is assumed that the result of each step is fed to the input of the next step.
@ -288,11 +285,14 @@ using ExpressionActionsPtr = std::shared_ptr<ExpressionActions>;
*/
struct ExpressionActionsChain
{
ExpressionActionsChain(const Context & context_)
: context(context_) {}
explicit ExpressionActionsChain(const Context & context_) : context(context_) {}
struct Step
{
ExpressionActionsPtr actions;
virtual ~Step() = default;
explicit Step(Names required_output_) : required_output(std::move(required_output_)) {}
/// Columns were added to the block before current step in addition to prev step output.
NameSet additional_input;
/// Columns which are required in the result of current step.
@ -302,11 +302,72 @@ struct ExpressionActionsChain
/// If not empty, has the same size with required_output; is filled in finalize().
std::vector<bool> can_remove_required_output;
Step(const ExpressionActionsPtr & actions_ = nullptr, const Names & required_output_ = Names())
: actions(actions_), required_output(required_output_) {}
virtual const NamesAndTypesList & getRequiredColumns() const = 0;
virtual const ColumnsWithTypeAndName & getResultColumns() const = 0;
/// Remove unused result and update required columns
virtual void finalize(const Names & required_output_) = 0;
/// Add projections to expression
virtual void prependProjectInput() const = 0;
virtual std::string dump() const = 0;
/// Only for ExpressionActionsStep
ExpressionActionsPtr & actions();
const ExpressionActionsPtr & actions() const;
};
using Steps = std::vector<Step>;
struct ExpressionActionsStep : public Step
{
ExpressionActionsPtr actions;
explicit ExpressionActionsStep(ExpressionActionsPtr actions_, Names required_output_ = Names())
: Step(std::move(required_output_))
, actions(std::move(actions_))
{
}
const NamesAndTypesList & getRequiredColumns() const override
{
return actions->getRequiredColumnsWithTypes();
}
const ColumnsWithTypeAndName & getResultColumns() const override
{
return actions->getSampleBlock().getColumnsWithTypeAndName();
}
void finalize(const Names & required_output_) override
{
actions->finalize(required_output_);
}
void prependProjectInput() const override
{
actions->prependProjectInput();
}
std::string dump() const override
{
return actions->dumpActions();
}
};
struct ArrayJoinStep : public Step
{
ArrayJoinActionPtr array_join;
NamesAndTypesList required_columns;
ColumnsWithTypeAndName result_columns;
ArrayJoinStep(ArrayJoinActionPtr array_join_, ColumnsWithTypeAndName required_columns_, Names required_output_);
const NamesAndTypesList & getRequiredColumns() const override { return required_columns; }
const ColumnsWithTypeAndName & getResultColumns() const override { return result_columns; }
void finalize(const Names & required_output_) override;
void prependProjectInput() const override {} /// TODO: remove unused columns before ARRAY JOIN ?
std::string dump() const override { return "ARRAY JOIN"; }
};
using StepPtr = std::unique_ptr<Step>;
using Steps = std::vector<StepPtr>;
const Context & context;
Steps steps;
@ -329,7 +390,7 @@ struct ExpressionActionsChain
throw Exception("Empty ExpressionActionsChain", ErrorCodes::LOGICAL_ERROR);
}
return steps.back().actions;
return steps.back()->actions();
}
Step & getLastStep()
@ -337,14 +398,14 @@ struct ExpressionActionsChain
if (steps.empty())
throw Exception("Empty ExpressionActionsChain", ErrorCodes::LOGICAL_ERROR);
return steps.back();
return *steps.back();
}
Step & lastStep(const NamesAndTypesList & columns)
{
if (steps.empty())
steps.emplace_back(std::make_shared<ExpressionActions>(columns, context));
return steps.back();
steps.emplace_back(std::make_unique<ExpressionActionsStep>(std::make_shared<ExpressionActions>(columns, context)));
return *steps.back();
}
std::string dumpChain() const;

142
src/Interpreters/ExpressionAnalyzer.cpp
View File

@ -1,30 +1,20 @@
#include <Poco/Util/Application.h>
#include <Poco/String.h>
#include <Core/Block.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTQualifiedAsterisk.h>
#include <Parsers/ASTExpressionList.h>
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ASTSubquery.h>
#include <Parsers/ASTOrderByElement.h>
#include <Parsers/formatAST.h>
#include <Parsers/DumpASTNode.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/NestedUtils.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeLowCardinality.h>
#include <Columns/IColumn.h>
#include <Interpreters/ExpressionAnalyzer.h>
#include <Interpreters/ExpressionActions.h>
#include <Interpreters/LogicalExpressionsOptimizer.h>
#include <Interpreters/ArrayJoinAction.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/Set.h>
#include <Interpreters/TableJoin.h>
@ -42,20 +32,14 @@
#include <Storages/StorageJoin.h>
#include <DataStreams/copyData.h>
#include <DataStreams/IBlockInputStream.h>
#include <Dictionaries/IDictionary.h>
#include <Dictionaries/DictionaryStructure.h>
#include <Common/typeid_cast.h>
#include <Common/StringUtils/StringUtils.h>
#include <ext/range.h>
#include <DataTypes/DataTypeFactory.h>
#include <Functions/FunctionsMiscellaneous.h>
#include <Parsers/ExpressionListParsers.h>
#include <Parsers/parseQuery.h>
#include <Parsers/queryToString.h>
#include <Interpreters/interpretSubquery.h>
#include <Interpreters/DatabaseAndTableWithAlias.h>
#include <Interpreters/misc.h>
@ -173,25 +157,37 @@ void ExpressionAnalyzer::analyzeAggregation()
if (select_query)
{
NamesAndTypesList array_join_columns;
bool is_array_join_left;
ASTPtr array_join_expression_list = select_query->arrayJoinExpressionList(is_array_join_left);
if (array_join_expression_list)
if (ASTPtr array_join_expression_list = select_query->arrayJoinExpressionList(is_array_join_left))
{
getRootActionsNoMakeSet(array_join_expression_list, true, temp_actions, false);
addMultipleArrayJoinAction(temp_actions, is_array_join_left);
if (auto array_join = addMultipleArrayJoinAction(temp_actions, is_array_join_left))
{
auto sample_block = temp_actions->getSampleBlock();
array_join->prepare(sample_block);
temp_actions = std::make_shared<ExpressionActions>(sample_block.getColumnsWithTypeAndName(), context);
}
array_join_columns.clear();
for (auto & column : temp_actions->getSampleBlock().getNamesAndTypesList())
if (syntax->array_join_result_to_source.count(column.name))
array_join_columns.emplace_back(column);
}
columns_after_array_join = sourceColumns();
columns_after_array_join.insert(columns_after_array_join.end(), array_join_columns.begin(), array_join_columns.end());
const ASTTablesInSelectQueryElement * join = select_query->join();
if (join)
{
getRootActionsNoMakeSet(analyzedJoin().leftKeysList(), true, temp_actions, false);
addJoinAction(temp_actions);
}
columns_after_join = columns_after_array_join;
const auto & added_by_join = analyzedJoin().columnsAddedByJoin();
columns_after_join.insert(columns_after_join.end(), added_by_join.begin(), added_by_join.end());
}
has_aggregation = makeAggregateDescriptions(temp_actions);
@ -281,16 +277,6 @@ void ExpressionAnalyzer::initGlobalSubqueriesAndExternalTables(bool do_global)
}
NamesAndTypesList ExpressionAnalyzer::sourceWithJoinedColumns() const
{
auto result_columns = sourceColumns();
result_columns.insert(result_columns.end(), array_join_columns.begin(), array_join_columns.end());
result_columns.insert(result_columns.end(),
analyzedJoin().columnsAddedByJoin().begin(), analyzedJoin().columnsAddedByJoin().end());
return result_columns;
}
void SelectQueryExpressionAnalyzer::tryMakeSetForIndexFromSubquery(const ASTPtr & subquery_or_table_name)
{
auto set_key = PreparedSetKey::forSubquery(*subquery_or_table_name);
@ -374,7 +360,7 @@ void SelectQueryExpressionAnalyzer::makeSetsForIndex(const ASTPtr & node)
}
else
{
ExpressionActionsPtr temp_actions = std::make_shared<ExpressionActions>(sourceWithJoinedColumns(), context);
ExpressionActionsPtr temp_actions = std::make_shared<ExpressionActions>(columns_after_join, context);
getRootActions(left_in_operand, true, temp_actions);
Block sample_block_with_calculated_columns = temp_actions->getSampleBlock();
@ -455,7 +441,7 @@ const ASTSelectQuery * SelectQueryExpressionAnalyzer::getAggregatingQuery() cons
}
/// "Big" ARRAY JOIN.
void ExpressionAnalyzer::addMultipleArrayJoinAction(ExpressionActionsPtr & actions, bool array_join_is_left) const
ArrayJoinActionPtr ExpressionAnalyzer::addMultipleArrayJoinAction(ExpressionActionsPtr & actions, bool array_join_is_left) const
{
NameSet result_columns;
for (const auto & result_source : syntax->array_join_result_to_source)
@ -468,25 +454,32 @@ void ExpressionAnalyzer::addMultipleArrayJoinAction(ExpressionActionsPtr & actio
result_columns.insert(result_source.first);
}
actions->add(ExpressionAction::arrayJoin(result_columns, array_join_is_left, context));
return std::make_shared<ArrayJoinAction>(result_columns, array_join_is_left, context);
}
bool SelectQueryExpressionAnalyzer::appendArrayJoin(ExpressionActionsChain & chain, bool only_types)
ArrayJoinActionPtr SelectQueryExpressionAnalyzer::appendArrayJoin(ExpressionActionsChain & chain, ExpressionActionsPtr & before_array_join, bool only_types)
{
const auto * select_query = getSelectQuery();
bool is_array_join_left;
ASTPtr array_join_expression_list = select_query->arrayJoinExpressionList(is_array_join_left);
if (!array_join_expression_list)
return false;
return nullptr;
ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns());
getRootActions(array_join_expression_list, only_types, step.actions);
getRootActions(array_join_expression_list, only_types, step.actions());
addMultipleArrayJoinAction(step.actions, is_array_join_left);
before_array_join = chain.getLastActions();
auto array_join = addMultipleArrayJoinAction(step.actions(), is_array_join_left);
return true;
chain.steps.push_back(std::make_unique<ExpressionActionsChain::ArrayJoinStep>(
array_join, step.getResultColumns(),
Names())); /// Required output is empty because all array joined columns are kept by step.
chain.addStep();
return array_join;
}
void ExpressionAnalyzer::addJoinAction(ExpressionActionsPtr & actions, JoinPtr join) const
@ -496,9 +489,9 @@ void ExpressionAnalyzer::addJoinAction(ExpressionActionsPtr & actions, JoinPtr j
bool SelectQueryExpressionAnalyzer::appendJoinLeftKeys(ExpressionActionsChain & chain, bool only_types)
{
ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns());
ExpressionActionsChain::Step & step = chain.lastStep(columns_after_array_join);
getRootActions(analyzedJoin().leftKeysList(), only_types, step.actions);
getRootActions(analyzedJoin().leftKeysList(), only_types, step.actions());
return true;
}
@ -506,9 +499,9 @@ bool SelectQueryExpressionAnalyzer::appendJoin(ExpressionActionsChain & chain)
{
JoinPtr table_join = makeTableJoin(*syntax->ast_join);
ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns());
ExpressionActionsChain::Step & step = chain.lastStep(columns_after_array_join);
addJoinAction(step.actions, table_join);
addJoinAction(step.actions(), table_join);
return true;
}
@ -632,12 +625,12 @@ bool SelectQueryExpressionAnalyzer::appendPrewhere(
return false;
auto & step = chain.lastStep(sourceColumns());
getRootActions(select_query->prewhere(), only_types, step.actions);
getRootActions(select_query->prewhere(), only_types, step.actions());
String prewhere_column_name = select_query->prewhere()->getColumnName();
step.required_output.push_back(prewhere_column_name);
step.can_remove_required_output.push_back(true);
auto filter_type = step.actions->getSampleBlock().getByName(prewhere_column_name).type;
auto filter_type = step.actions()->getSampleBlock().getByName(prewhere_column_name).type;
if (!filter_type->canBeUsedInBooleanContext())
throw Exception("Invalid type for filter in PREWHERE: " + filter_type->getName(),
ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER);
@ -661,7 +654,7 @@ bool SelectQueryExpressionAnalyzer::appendPrewhere(
}
}
auto names = step.actions->getSampleBlock().getNames();
auto names = step.actions()->getSampleBlock().getNames();
NameSet name_set(names.begin(), names.end());
for (const auto & column : sourceColumns())
@ -669,7 +662,7 @@ bool SelectQueryExpressionAnalyzer::appendPrewhere(
name_set.erase(column.name);
Names required_output(name_set.begin(), name_set.end());
step.actions->finalize(required_output);
step.actions()->finalize(required_output);
}
{
@ -680,8 +673,8 @@ bool SelectQueryExpressionAnalyzer::appendPrewhere(
/// 2. Store side columns which were calculated during prewhere actions execution if they are used.
/// Example: select F(A) prewhere F(A) > 0. F(A) can be saved from prewhere step.
/// 3. Check if we can remove filter column at prewhere step. If we can, action will store single REMOVE_COLUMN.
ColumnsWithTypeAndName columns = step.actions->getSampleBlock().getColumnsWithTypeAndName();
auto required_columns = step.actions->getRequiredColumns();
ColumnsWithTypeAndName columns = step.actions()->getSampleBlock().getColumnsWithTypeAndName();
auto required_columns = step.actions()->getRequiredColumns();
NameSet prewhere_input_names(required_columns.begin(), required_columns.end());
NameSet unused_source_columns;
@ -694,8 +687,9 @@ bool SelectQueryExpressionAnalyzer::appendPrewhere(
}
}
chain.steps.emplace_back(std::make_shared<ExpressionActions>(std::move(columns), context));
chain.steps.back().additional_input = std::move(unused_source_columns);
chain.steps.emplace_back(std::make_unique<ExpressionActionsChain::ExpressionActionsStep>(
std::make_shared<ExpressionActions>(std::move(columns), context)));
chain.steps.back()->additional_input = std::move(unused_source_columns);
}
return true;
@ -706,7 +700,7 @@ void SelectQueryExpressionAnalyzer::appendPreliminaryFilter(ExpressionActionsCha
ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns());
// FIXME: assert(filter_info);
step.actions = std::move(actions);
step.actions() = std::move(actions);
step.required_output.push_back(std::move(column_name));
step.can_remove_required_output = {true};
@ -720,15 +714,15 @@ bool SelectQueryExpressionAnalyzer::appendWhere(ExpressionActionsChain & chain,
if (!select_query->where())
return false;
ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns());
ExpressionActionsChain::Step & step = chain.lastStep(columns_after_join);
auto where_column_name = select_query->where()->getColumnName();
step.required_output.push_back(where_column_name);
step.can_remove_required_output = {true};
getRootActions(select_query->where(), only_types, step.actions);
getRootActions(select_query->where(), only_types, step.actions());
auto filter_type = step.actions->getSampleBlock().getByName(where_column_name).type;
auto filter_type = step.actions()->getSampleBlock().getByName(where_column_name).type;
if (!filter_type->canBeUsedInBooleanContext())
throw Exception("Invalid type for filter in WHERE: " + filter_type->getName(),
ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER);
@ -744,21 +738,20 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain
if (!select_query->groupBy())
return false;
ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns());
ExpressionActionsChain::Step & step = chain.lastStep(columns_after_join);
ASTs asts = select_query->groupBy()->children;
for (const auto & ast : asts)
{
step.required_output.emplace_back(ast->getColumnName());
getRootActions(ast, only_types, step.actions);
getRootActions(ast, only_types, step.actions());
}
if (optimize_aggregation_in_order)
{
auto all_columns = sourceWithJoinedColumns();
for (auto & child : asts)
{
group_by_elements_actions.emplace_back(std::make_shared<ExpressionActions>(all_columns, context));
group_by_elements_actions.emplace_back(std::make_shared<ExpressionActions>(columns_after_join, context));
getRootActions(child, only_types, group_by_elements_actions.back());
}
}
@ -770,7 +763,7 @@ void SelectQueryExpressionAnalyzer::appendAggregateFunctionsArguments(Expression
{
const auto * select_query = getAggregatingQuery();
ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns());
ExpressionActionsChain::Step & step = chain.lastStep(columns_after_join);
for (const auto & desc : aggregate_descriptions)
for (const auto & name : desc.argument_names)
@ -791,7 +784,7 @@ void SelectQueryExpressionAnalyzer::appendAggregateFunctionsArguments(Expression
/// TODO: data.aggregates -> aggregates()
for (const ASTFunction * node : data.aggregates)
for (auto & argument : node->arguments->children)
getRootActions(argument, only_types, step.actions);
getRootActions(argument, only_types, step.actions());
}
bool SelectQueryExpressionAnalyzer::appendHaving(ExpressionActionsChain & chain, bool only_types)
@ -804,7 +797,7 @@ bool SelectQueryExpressionAnalyzer::appendHaving(ExpressionActionsChain & chain,
ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns);
step.required_output.push_back(select_query->having()->getColumnName());
getRootActions(select_query->having(), only_types, step.actions);
getRootActions(select_query->having(), only_types, step.actions());
return true;
}
@ -815,7 +808,7 @@ void SelectQueryExpressionAnalyzer::appendSelect(ExpressionActionsChain & chain,
ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns);
getRootActions(select_query->select(), only_types, step.actions);
getRootActions(select_query->select(), only_types, step.actions());
for (const auto & child : select_query->select()->children)
step.required_output.push_back(child->getColumnName());
@ -831,7 +824,7 @@ bool SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChain & chain
ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns);
getRootActions(select_query->orderBy(), only_types, step.actions);
getRootActions(select_query->orderBy(), only_types, step.actions());
for (auto & child : select_query->orderBy()->children)
{
@ -844,10 +837,9 @@ bool SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChain & chain
if (optimize_read_in_order)
{
auto all_columns = sourceWithJoinedColumns();
for (auto & child : select_query->orderBy()->children)
{
order_by_elements_actions.emplace_back(std::make_shared<ExpressionActions>(all_columns, context));
order_by_elements_actions.emplace_back(std::make_shared<ExpressionActions>(columns_after_join, context));
getRootActions(child, only_types, order_by_elements_actions.back());
}
}
@ -863,7 +855,7 @@ bool SelectQueryExpressionAnalyzer::appendLimitBy(ExpressionActionsChain & chain
ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns);
getRootActions(select_query->limitBy(), only_types, step.actions);
getRootActions(select_query->limitBy(), only_types, step.actions());
NameSet aggregated_names;
for (const auto & column : aggregated_columns)
@ -928,14 +920,14 @@ void SelectQueryExpressionAnalyzer::appendProjectResult(ExpressionActionsChain &
}
}
step.actions->add(ExpressionAction::project(result_columns));
step.actions()->add(ExpressionAction::project(result_columns));
}
void ExpressionAnalyzer::appendExpression(ExpressionActionsChain & chain, const ASTPtr & expr, bool only_types)
{
ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns());
getRootActions(expr, only_types, step.actions);
getRootActions(expr, only_types, step.actions());
step.required_output.push_back(expr->getColumnName());
}
@ -1076,7 +1068,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
if (query_analyzer.appendPrewhere(chain, !first_stage, additional_required_columns_after_prewhere))
{
prewhere_info = std::make_shared<PrewhereInfo>(
chain.steps.front().actions, query.prewhere()->getColumnName());
chain.steps.front()->actions(), query.prewhere()->getColumnName());
if (allowEarlyConstantFolding(*prewhere_info->prewhere_actions, settings))
{
@ -1093,7 +1085,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
chain.addStep();
}
query_analyzer.appendArrayJoin(chain, only_types || !first_stage);
array_join = query_analyzer.appendArrayJoin(chain, before_array_join, only_types || !first_stage);
if (query_analyzer.hasTableJoin())
{
@ -1119,7 +1111,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
{
Block before_where_sample;
if (chain.steps.size() > 1)
before_where_sample = chain.steps[chain.steps.size() - 2].actions->getSampleBlock();
before_where_sample = Block(chain.steps[chain.steps.size() - 2]->getResultColumns());
else
before_where_sample = source_header;
if (sanitizeBlock(before_where_sample))
@ -1200,7 +1192,7 @@ void ExpressionAnalysisResult::finalize(const ExpressionActionsChain & chain, co
{
if (hasPrewhere())
{
const ExpressionActionsChain::Step & step = chain.steps.at(0);
const ExpressionActionsChain::Step & step = *chain.steps.at(0);
prewhere_info->remove_prewhere_column = step.can_remove_required_output.at(0);
Names columns_to_remove;
@ -1225,10 +1217,10 @@ void ExpressionAnalysisResult::finalize(const ExpressionActionsChain & chain, co
else if (hasFilter())
{
/// Can't have prewhere and filter set simultaneously
filter_info->do_remove_column = chain.steps.at(0).can_remove_required_output.at(0);
filter_info->do_remove_column = chain.steps.at(0)->can_remove_required_output.at(0);
}
if (hasWhere())
remove_where_filter = chain.steps.at(where_step_num).can_remove_required_output.at(0);
remove_where_filter = chain.steps.at(where_step_num)->can_remove_required_output.at(0);
}
void ExpressionAnalysisResult::removeExtraColumns() const

16
src/Interpreters/ExpressionAnalyzer.h
View File

@ -34,6 +34,9 @@ struct ASTTablesInSelectQueryElement;
struct StorageInMemoryMetadata;
using StorageMetadataPtr = std::shared_ptr<const StorageInMemoryMetadata>;
class ArrayJoinAction;
using ArrayJoinActionPtr = std::shared_ptr<ArrayJoinAction>;
/// Create columns in block or return false if not possible
bool sanitizeBlock(Block & block, bool throw_if_cannot_create_column = false);
@ -43,9 +46,12 @@ struct ExpressionAnalyzerData
SubqueriesForSets subqueries_for_sets;
PreparedSets prepared_sets;
/// Columns after ARRAY JOIN. It there is no ARRAY JOIN, it's source_columns.
NamesAndTypesList columns_after_array_join;
/// Columns after Columns after ARRAY JOIN and JOIN. If there is no JOIN, it's columns_after_array_join.
NamesAndTypesList columns_after_join;
/// Columns after ARRAY JOIN, JOIN, and/or aggregation.
NamesAndTypesList aggregated_columns;
NamesAndTypesList array_join_columns;
bool has_aggregation = false;
NamesAndTypesList aggregation_keys;
@ -128,12 +134,10 @@ protected:
const TableJoin & analyzedJoin() const { return *syntax->analyzed_join; }
const NamesAndTypesList & sourceColumns() const { return syntax->required_source_columns; }
const std::vector<const ASTFunction *> & aggregates() const { return syntax->aggregates; }
NamesAndTypesList sourceWithJoinedColumns() const;
/// Find global subqueries in the GLOBAL IN/JOIN sections. Fills in external_tables.
void initGlobalSubqueriesAndExternalTables(bool do_global);
void addMultipleArrayJoinAction(ExpressionActionsPtr & actions, bool is_left) const;
ArrayJoinActionPtr addMultipleArrayJoinAction(ExpressionActionsPtr & actions, bool is_left) const;
void addJoinAction(ExpressionActionsPtr & actions, JoinPtr = {}) const;
@ -175,6 +179,8 @@ struct ExpressionAnalysisResult
bool optimize_read_in_order = false;
bool optimize_aggregation_in_order = false;
ExpressionActionsPtr before_array_join;
ArrayJoinActionPtr array_join;
ExpressionActionsPtr before_join;
ExpressionActionsPtr join;
ExpressionActionsPtr before_where;
@ -305,7 +311,7 @@ private:
*/
/// Before aggregation:
bool appendArrayJoin(ExpressionActionsChain & chain, bool only_types);
ArrayJoinActionPtr appendArrayJoin(ExpressionActionsChain & chain, ExpressionActionsPtr & before_array_join, bool only_types);
bool appendJoinLeftKeys(ExpressionActionsChain & chain, bool only_types);
bool appendJoin(ExpressionActionsChain & chain);
/// Add preliminary rows filtration. Actions are created in other expression analyzer to prevent any possible alias injection.

20
src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -34,6 +34,7 @@
#include <Processors/Transforms/ExpressionTransform.h>
#include <Processors/Transforms/InflatingExpressionTransform.h>
#include <Processors/Transforms/AggregatingTransform.h>
#include <Processors/QueryPlan/ArrayJoinStep.h>
#include <Processors/QueryPlan/ReadFromStorageStep.h>
#include <Processors/QueryPlan/ExpressionStep.h>
#include <Processors/QueryPlan/FilterStep.h>
@ -862,6 +863,25 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu
query_plan.addStep(std::move(row_level_security_step));
}
if (expressions.before_array_join)
{
QueryPlanStepPtr before_array_join_step = std::make_unique<ExpressionStep>(
query_plan.getCurrentDataStream(),
expressions.before_array_join);
before_array_join_step->setStepDescription("Before ARRAY JOIN");
query_plan.addStep(std::move(before_array_join_step));
}
if (expressions.array_join)
{
QueryPlanStepPtr array_join_step = std::make_unique<ArrayJoinStep>(
query_plan.getCurrentDataStream(),
expressions.array_join);
array_join_step->setStepDescription("ARRAY JOIN");
query_plan.addStep(std::move(array_join_step));
}
if (expressions.before_join)
{
QueryPlanStepPtr before_join_step = std::make_unique<ExpressionStep>(

6
src/Interpreters/MutationsInterpreter.cpp
View File

@ -623,7 +623,7 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector<Stage> &
actions_chain.finalize();
/// Propagate information about columns needed as input.
for (const auto & column : actions_chain.steps.front().actions->getRequiredColumnsWithTypes())
for (const auto & column : actions_chain.steps.front()->actions()->getRequiredColumnsWithTypes())
prepared_stages[i - 1].output_columns.insert(column.name);
}
@ -667,12 +667,12 @@ BlockInputStreamPtr MutationsInterpreter::addStreamsForLaterStages(const std::ve
if (i < stage.filter_column_names.size())
{
/// Execute DELETEs.
in = std::make_shared<FilterBlockInputStream>(in, step.actions, stage.filter_column_names[i]);
in = std::make_shared<FilterBlockInputStream>(in, step->actions(), stage.filter_column_names[i]);
}
else
{
/// Execute UPDATE or final projection.
in = std::make_shared<ExpressionBlockInputStream>(in, step.actions);
in = std::make_shared<ExpressionBlockInputStream>(in, step->actions());
}
}

10
src/Interpreters/executeQuery.cpp
View File

@ -487,15 +487,15 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
}
/// Common code for finish and exception callbacks
auto status_info_to_query_log = [ast](QueryLogElement &element, const QueryStatusInfo &info) mutable
auto status_info_to_query_log = [](QueryLogElement &element, const QueryStatusInfo &info, const ASTPtr query_ast) mutable
{
DB::UInt64 query_time = info.elapsed_seconds * 1000000;
ProfileEvents::increment(ProfileEvents::QueryTimeMicroseconds, query_time);
if (ast->as<ASTSelectQuery>() || ast->as<ASTSelectWithUnionQuery>())
if (query_ast->as<ASTSelectQuery>() || query_ast->as<ASTSelectWithUnionQuery>())
{
ProfileEvents::increment(ProfileEvents::SelectQueryTimeMicroseconds, query_time);
}
else if (ast->as<ASTInsertQuery>())
else if (query_ast->as<ASTInsertQuery>())
{
ProfileEvents::increment(ProfileEvents::InsertQueryTimeMicroseconds, query_time);
}
@ -535,7 +535,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
elem.event_time = time(nullptr);
status_info_to_query_log(elem, info);
status_info_to_query_log(elem, info, ast);
auto progress_callback = context.getProgressCallback();
@ -608,7 +608,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
if (process_list_elem)
{
QueryStatusInfo info = process_list_elem->getInfo(true, current_settings.log_profile_events, false);
status_info_to_query_log(elem, info);
status_info_to_query_log(elem, info, ast);
}
if (current_settings.calculate_text_stack_trace)

8
src/Parsers/ASTColumnsMatcher.cpp
View File

@ -2,6 +2,7 @@
#include <IO/WriteHelpers.h>
#include <Common/quoteString.h>
#include <re2/re2.h>
#include <Common/SipHash.h>
namespace DB
@ -20,6 +21,13 @@ ASTPtr ASTColumnsMatcher::clone() const
void ASTColumnsMatcher::appendColumnName(WriteBuffer & ostr) const { writeString(original_pattern, ostr); }
void ASTColumnsMatcher::updateTreeHashImpl(SipHash & hash_state) const
{
hash_state.update(original_pattern.size());
hash_state.update(original_pattern);
IAST::updateTreeHashImpl(hash_state);
}
void ASTColumnsMatcher::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const
{
settings.ostr << (settings.hilite ? hilite_keyword : "") << "COLUMNS" << (settings.hilite ? hilite_none : "") << "("

1
src/Parsers/ASTColumnsMatcher.h
View File

@ -33,6 +33,7 @@ public:
void appendColumnName(WriteBuffer & ostr) const override;
void setPattern(String pattern);
bool isColumnMatching(const String & column_name) const;
void updateTreeHashImpl(SipHash & hash_state) const override;
protected:
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;

39
src/Parsers/ASTEnumElement.h
View File

@ -1,39 +0,0 @@
#pragma once
#include "IAST.h"
#include <Core/Field.h>
#include <Common/FieldVisitors.h>
namespace DB
{
class ASTEnumElement : public IAST
{
public:
String name;
Field value;
ASTEnumElement(const String & name, const Field & value)
: name{name}, value {value} {}
String getID(char) const override { return "EnumElement"; }
ASTPtr clone() const override
{
return std::make_shared<ASTEnumElement>(name, value);
}
protected:
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override
{
frame.need_parens = false;
const std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' ');
settings.ostr << settings.nl_or_ws << indent_str << '\'' << name << "' = " << applyVisitor(FieldVisitorToString{}, value);
}
};
}

9
src/Parsers/ASTFunction.cpp
View File

@ -5,6 +5,7 @@
#include <Parsers/ASTSubquery.h>
#include <IO/WriteHelpers.h>
#include <IO/WriteBufferFromString.h>
#include <Common/SipHash.h>
namespace DB
@ -54,6 +55,14 @@ ASTPtr ASTFunction::clone() const
}
void ASTFunction::updateTreeHashImpl(SipHash & hash_state) const
{
hash_state.update(name.size());
hash_state.update(name);
IAST::updateTreeHashImpl(hash_state);
}
/** A special hack. If it's [I]LIKE or NOT [I]LIKE expression and the right hand side is a string literal,
* we will highlight unescaped metacharacters % and _ in string literal for convenience.
* Motivation: most people are unaware that _ is a metacharacter and forgot to properly escape it with two backslashes.

2
src/Parsers/ASTFunction.h
View File

@ -23,6 +23,8 @@ public:
ASTPtr clone() const override;
void updateTreeHashImpl(SipHash & hash_state) const override;
protected:
void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
void appendColumnNameImpl(WriteBuffer & ostr) const override;

20
src/Parsers/ASTFunctionWithKeyValueArguments.cpp
View File

@ -1,6 +1,7 @@
#include <Parsers/ASTFunctionWithKeyValueArguments.h>
#include <Poco/String.h>
#include <Common/SipHash.h>
namespace DB
{
@ -35,6 +36,16 @@ void ASTPair::formatImpl(const FormatSettings & settings, FormatState & state, F
settings.ostr << (settings.hilite ? hilite_none : "");
}
void ASTPair::updateTreeHashImpl(SipHash & hash_state) const
{
hash_state.update(first.size());
hash_state.update(first);
hash_state.update(second_with_brackets);
IAST::updateTreeHashImpl(hash_state);
}
String ASTFunctionWithKeyValueArguments::getID(char delim) const
{
return "FunctionWithKeyValueArguments " + (delim + name);
@ -64,4 +75,13 @@ void ASTFunctionWithKeyValueArguments::formatImpl(const FormatSettings & setting
settings.ostr << (settings.hilite ? hilite_none : "");
}
void ASTFunctionWithKeyValueArguments::updateTreeHashImpl(SipHash & hash_state) const
{
hash_state.update(name.size());
hash_state.update(name);
hash_state.update(has_brackets);
IAST::updateTreeHashImpl(hash_state);
}
}

4
src/Parsers/ASTFunctionWithKeyValueArguments.h
View File

@ -30,6 +30,8 @@ public:
ASTPtr clone() const override;
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
void updateTreeHashImpl(SipHash & hash_state) const override;
};
@ -58,6 +60,8 @@ public:
ASTPtr clone() const override;
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
void updateTreeHashImpl(SipHash & hash_state) const override;
};
}

6
src/Parsers/ASTIdentifier.cpp
View File

@ -111,6 +111,12 @@ void ASTIdentifier::resetTable(const String & database_name, const String & tabl
uuid = ident.uuid;
}
void ASTIdentifier::updateTreeHashImpl(SipHash & hash_state) const
{
hash_state.update(uuid);
IAST::updateTreeHashImpl(hash_state);
}
ASTPtr createTableIdentifier(const String & database_name, const String & table_name)
{
assert(database_name != "_temporary_and_external_tables");

2
src/Parsers/ASTIdentifier.h
View File

@ -53,6 +53,8 @@ public:
void resetTable(const String & database_name, const String & table_name);
void updateTreeHashImpl(SipHash & hash_state) const override;
protected:
void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
void appendColumnNameImpl(WriteBuffer & ostr) const override;

10
src/Parsers/ASTOrderByElement.cpp
View File

@ -1,10 +1,20 @@
#include <Columns/Collator.h>
#include <Parsers/ASTOrderByElement.h>
#include <Common/SipHash.h>
namespace DB
{
void ASTOrderByElement::updateTreeHashImpl(SipHash & hash_state) const
{
hash_state.update(direction);
hash_state.update(nulls_direction);
hash_state.update(nulls_direction_was_explicitly_specified);
hash_state.update(with_fill);
IAST::updateTreeHashImpl(hash_state);
}
void ASTOrderByElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
{
children.front()->formatImpl(settings, state, frame);

2
src/Parsers/ASTOrderByElement.h
View File

@ -46,6 +46,8 @@ public:
return clone;
}
void updateTreeHashImpl(SipHash & hash_state) const override;
protected:
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
};

11
src/Parsers/ASTSelectQuery.cpp
View File

@ -57,6 +57,17 @@ ASTPtr ASTSelectQuery::clone() const
}
void ASTSelectQuery::updateTreeHashImpl(SipHash & hash_state) const
{
hash_state.update(distinct);
hash_state.update(group_by_with_totals);
hash_state.update(group_by_with_rollup);
hash_state.update(group_by_with_cube);
hash_state.update(limit_with_ties);
IAST::updateTreeHashImpl(hash_state);
}
void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const
{
frame.current_select = this;

1
src/Parsers/ASTSelectQuery.h
View File

@ -88,6 +88,7 @@ public:
void replaceDatabaseAndTable(const String & database_name, const String & table_name);
void replaceDatabaseAndTable(const StorageID & table_id);
void addTableFunction(ASTPtr & table_function_ptr);
void updateTreeHashImpl(SipHash & hash_state) const override;
protected:
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;

12
src/Parsers/ASTSetQuery.cpp
View File

@ -1,9 +1,21 @@
#include <Parsers/ASTSetQuery.h>
#include <Parsers/formatSettingName.h>
#include <Common/SipHash.h>
namespace DB
{
void ASTSetQuery::updateTreeHashImpl(SipHash & hash_state) const
{
for (const auto & change : changes)
{
hash_state.update(change.name.size());
hash_state.update(change.name);
applyVisitor(FieldVisitorHash(hash_state), change.value);
}
}
void ASTSetQuery::formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const
{
if (is_standalone)

2
src/Parsers/ASTSetQuery.h
View File

@ -23,6 +23,8 @@ public:
ASTPtr clone() const override { return std::make_shared<ASTSetQuery>(*this); }
void formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const override;
void updateTreeHashImpl(SipHash & hash_state) const override;
};
}

22
src/Parsers/ASTTablesInSelectQuery.cpp
View File

@ -1,6 +1,7 @@
#include <Common/typeid_cast.h>
#include <Parsers/ASTExpressionList.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Common/SipHash.h>
namespace DB
@ -18,6 +19,13 @@ do \
while (false)
void ASTTableExpression::updateTreeHashImpl(SipHash & hash_state) const
{
hash_state.update(final);
IAST::updateTreeHashImpl(hash_state);
}
ASTPtr ASTTableExpression::clone() const
{
auto res = std::make_shared<ASTTableExpression>(*this);
@ -32,6 +40,14 @@ ASTPtr ASTTableExpression::clone() const
return res;
}
void ASTTableJoin::updateTreeHashImpl(SipHash & hash_state) const
{
hash_state.update(locality);
hash_state.update(strictness);
hash_state.update(kind);
IAST::updateTreeHashImpl(hash_state);
}
ASTPtr ASTTableJoin::clone() const
{
auto res = std::make_shared<ASTTableJoin>(*this);
@ -43,6 +59,12 @@ ASTPtr ASTTableJoin::clone() const
return res;
}
void ASTArrayJoin::updateTreeHashImpl(SipHash & hash_state) const
{
hash_state.update(kind);
IAST::updateTreeHashImpl(hash_state);
}
ASTPtr ASTArrayJoin::clone() const
{
auto res = std::make_shared<ASTArrayJoin>(*this);

3
src/Parsers/ASTTablesInSelectQuery.h
View File

@ -56,6 +56,7 @@ struct ASTTableExpression : public IAST
String getID(char) const override { return "TableExpression"; }
ASTPtr clone() const override;
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
void updateTreeHashImpl(SipHash & hash_state) const override;
};
@ -108,6 +109,7 @@ struct ASTTableJoin : public IAST
void formatImplBeforeTable(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const;
void formatImplAfterTable(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const;
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
void updateTreeHashImpl(SipHash & hash_state) const override;
};
inline bool isLeft(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Left; }
@ -139,6 +141,7 @@ struct ASTArrayJoin : public IAST
String getID(char) const override { return "ArrayJoin"; }
ASTPtr clone() const override;
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
void updateTreeHashImpl(SipHash & hash_state) const override;
};

2
src/Parsers/Lexer.cpp
View File

@ -50,7 +50,7 @@ Token quotedString(const char *& pos, const char * const token_begin, const char
Token Lexer::nextToken()
{
Token res = nextTokenImpl();
if (res.type != TokenType::EndOfStream && max_query_size && res.end > begin + max_query_size)
if (max_query_size && res.end > begin + max_query_size)
res.type = TokenType::ErrorMaxQuerySizeExceeded;
if (res.isSignificant())
prev_significant_token_type = res.type;

17
src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp
View File

@ -331,6 +331,23 @@ void TabSeparatedRowInputFormat::tryDeserializeField(const DataTypePtr & type, I
{
if (column_indexes_for_input_fields[file_column])
{
// check null value for type is not nullable. don't cross buffer bound for simplicity, so maybe missing some case
if (!type->isNullable() && !in.eof())
{
if (*in.position() == '\\' && in.available() >= 2)
{
++in.position();
if (*in.position() == 'N')
{
++in.position();
throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected NULL value of not Nullable type {}", type->getName());
}
else
{
--in.position();
}
}
}
const bool is_last_file_column = file_column + 1 == column_indexes_for_input_fields.size();
readField(column, type, is_last_file_column);
}

2
src/Processors/Formats/RowInputFormatWithDiagnosticInfo.cpp
View File

@ -140,6 +140,8 @@ bool RowInputFormatWithDiagnosticInfo::deserializeFieldAndPrintDiagnosticInfo(co
out << "ERROR: Date must be in YYYY-MM-DD format.\n";
else
out << "ERROR\n";
// Print exception message
out << getExceptionMessage(exception, false) << '\n';
return false;
}

83
src/Processors/QueryPlan/ArrayJoinStep.cpp Normal file
View File

@ -0,0 +1,83 @@
#include <Processors/QueryPlan/ArrayJoinStep.h>
#include <Processors/Transforms/ArrayJoinTransform.h>
#include <Processors/Transforms/ConvertingTransform.h>
#include <Processors/QueryPipeline.h>
#include <Interpreters/ArrayJoinAction.h>
#include <IO/Operators.h>
namespace DB
{
static ITransformingStep::Traits getTraits()
{
return ITransformingStep::Traits
{
{
.preserves_distinct_columns = false,
.returns_single_stream = false,
.preserves_number_of_streams = true,
.preserves_sorting = false,
},
{
.preserves_number_of_rows = false,
}
};
}
ArrayJoinStep::ArrayJoinStep(const DataStream & input_stream_, ArrayJoinActionPtr array_join_)
: ITransformingStep(
input_stream_,
ArrayJoinTransform::transformHeader(input_stream_.header, array_join_),
getTraits())
, array_join(std::move(array_join_))
{
}
void ArrayJoinStep::updateInputStream(DataStream input_stream, Block result_header)
{
output_stream = createOutputStream(
input_stream,
ArrayJoinTransform::transformHeader(input_stream.header, array_join),
getDataStreamTraits());
input_streams.clear();
input_streams.emplace_back(std::move(input_stream));
res_header = std::move(result_header);
}
void ArrayJoinStep::transformPipeline(QueryPipeline & pipeline)
{
pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type)
{
bool on_totals = stream_type == QueryPipeline::StreamType::Totals;
return std::make_shared<ArrayJoinTransform>(header, array_join, on_totals);
});
if (res_header && !blocksHaveEqualStructure(res_header, output_stream->header))
{
pipeline.addSimpleTransform([&](const Block & header)
{
return std::make_shared<ConvertingTransform>(header, res_header, ConvertingTransform::MatchColumnsMode::Name);
});
}
}
void ArrayJoinStep::describeActions(FormatSettings & settings) const
{
String prefix(settings.offset, ' ');
bool first = true;
settings.out << prefix << (array_join->is_left ? "LEFT " : "") << "ARRAY JOIN ";
for (const auto & column : array_join->columns)
{
if (!first)
settings.out << ", ";
first = false;
settings.out << column;
}
settings.out << '\n';
}
}

29
src/Processors/QueryPlan/ArrayJoinStep.h Normal file
View File

@ -0,0 +1,29 @@
#pragma once
#include <Processors/QueryPlan/ITransformingStep.h>
namespace DB
{
class ArrayJoinAction;
using ArrayJoinActionPtr = std::shared_ptr<ArrayJoinAction>;
class ArrayJoinStep : public ITransformingStep
{
public:
explicit ArrayJoinStep(const DataStream & input_stream_, ArrayJoinActionPtr array_join_);
String getName() const override { return "ArrayJoin"; }
void transformPipeline(QueryPipeline & pipeline) override;
void describeActions(FormatSettings & settings) const override;
void updateInputStream(DataStream input_stream, Block result_header);
const ArrayJoinActionPtr & arrayJoin() const { return array_join; }
private:
ArrayJoinActionPtr array_join;
Block res_header;
};
}

23
src/Processors/QueryPlan/ExpressionStep.cpp
View File

@ -1,6 +1,7 @@
#include <Processors/QueryPlan/ExpressionStep.h>
#include <Processors/Transforms/ExpressionTransform.h>
#include <Processors/QueryPipeline.h>
#include <Processors/Transforms/ConvertingTransform.h>
#include <Processors/Transforms/InflatingExpressionTransform.h>
#include <Interpreters/ExpressionActions.h>
#include <IO/Operators.h>
@ -35,6 +36,19 @@ ExpressionStep::ExpressionStep(const DataStream & input_stream_, ExpressionActio
updateDistinctColumns(output_stream->header, output_stream->distinct_columns);
}
void ExpressionStep::updateInputStream(DataStream input_stream, bool keep_header)
{
Block out_header = keep_header ? std::move(output_stream->header)
: Transform::transformHeader(input_stream.header, expression);
output_stream = createOutputStream(
input_stream,
std::move(out_header),
getDataStreamTraits());
input_streams.clear();
input_streams.emplace_back(std::move(input_stream));
}
void ExpressionStep::transformPipeline(QueryPipeline & pipeline)
{
pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type)
@ -42,6 +56,15 @@ void ExpressionStep::transformPipeline(QueryPipeline & pipeline)
bool on_totals = stream_type == QueryPipeline::StreamType::Totals;
return std::make_shared<Transform>(header, expression, on_totals);
});
if (!blocksHaveEqualStructure(pipeline.getHeader(), output_stream->header))
{
pipeline.addSimpleTransform([&](const Block & header)
{
return std::make_shared<ConvertingTransform>(header, output_stream->header,
ConvertingTransform::MatchColumnsMode::Name);
});
}
}
static void doDescribeActions(const ExpressionActionsPtr & expression, IQueryPlanStep::FormatSettings & settings)

4
src/Processors/QueryPlan/ExpressionStep.h
View File

@ -21,8 +21,12 @@ public:
void transformPipeline(QueryPipeline & pipeline) override;
void updateInputStream(DataStream input_stream, bool keep_header);
void describeActions(FormatSettings & settings) const override;
const ExpressionActionsPtr & getExpression() const { return expression; }
private:
ExpressionActionsPtr expression;
};

24
src/Processors/QueryPlan/FilterStep.cpp
View File

@ -1,6 +1,7 @@
#include <Processors/QueryPlan/FilterStep.h>
#include <Processors/Transforms/FilterTransform.h>
#include <Processors/QueryPipeline.h>
#include <Processors/Transforms/ConvertingTransform.h>
#include <Interpreters/ExpressionActions.h>
#include <IO/Operators.h>
@ -40,6 +41,21 @@ FilterStep::FilterStep(
updateDistinctColumns(output_stream->header, output_stream->distinct_columns);
}
void FilterStep::updateInputStream(DataStream input_stream, bool keep_header)
{
Block out_header = std::move(output_stream->header);
if (keep_header)
out_header = FilterTransform::transformHeader(input_stream.header, expression, filter_column_name, remove_filter_column);
output_stream = createOutputStream(
input_stream,
std::move(out_header),
getDataStreamTraits());
input_streams.clear();
input_streams.emplace_back(std::move(input_stream));
}
void FilterStep::transformPipeline(QueryPipeline & pipeline)
{
pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type)
@ -47,6 +63,14 @@ void FilterStep::transformPipeline(QueryPipeline & pipeline)
bool on_totals = stream_type == QueryPipeline::StreamType::Totals;
return std::make_shared<FilterTransform>(header, expression, filter_column_name, remove_filter_column, on_totals);
});
if (!blocksHaveEqualStructure(pipeline.getHeader(), output_stream->header))
{
pipeline.addSimpleTransform([&](const Block & header)
{
return std::make_shared<ConvertingTransform>(header, output_stream->header, ConvertingTransform::MatchColumnsMode::Name);
});
}
}
void FilterStep::describeActions(FormatSettings & settings) const

6
src/Processors/QueryPlan/FilterStep.h
View File

@ -20,8 +20,14 @@ public:
String getName() const override { return "Filter"; }
void transformPipeline(QueryPipeline & pipeline) override;
void updateInputStream(DataStream input_stream, bool keep_header);
void describeActions(FormatSettings & settings) const override;
const ExpressionActionsPtr & getExpression() const { return expression; }
const String & getFilterColumnName() const { return filter_column_name; }
bool removesFilterColumn() const { return remove_filter_column; }
private:
ExpressionActionsPtr expression;
String filter_column_name;

69
src/Processors/QueryPlan/QueryPlan.cpp
View File

@ -3,6 +3,8 @@
#include <Processors/QueryPipeline.h>
#include <IO/WriteBuffer.h>
#include <IO/Operators.h>
#include <Interpreters/ExpressionActions.h>
#include <Interpreters/ArrayJoinAction.h>
#include <stack>
#include <Processors/QueryPlan/LimitStep.h>
#include "MergingSortedStep.h"
@ -10,6 +12,9 @@
#include "MergeSortingStep.h"
#include "PartialSortingStep.h"
#include "TotalsHavingStep.h"
#include "ExpressionStep.h"
#include "ArrayJoinStep.h"
#include "FilterStep.h"
namespace DB
{
@ -408,6 +413,64 @@ static void tryPushDownLimit(QueryPlanStepPtr & parent, QueryPlan::Node * child_
parent.swap(child);
}
/// Move ARRAY JOIN up if possible.
static void tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Node * child_node, QueryPlan::Nodes & nodes)
{
auto & parent = parent_node->step;
auto & child = child_node->step;
auto * expression_step = typeid_cast<ExpressionStep *>(parent.get());
auto * filter_step = typeid_cast<FilterStep *>(parent.get());
auto * array_join_step = typeid_cast<ArrayJoinStep *>(child.get());
if (!(expression_step || filter_step) || !array_join_step)
return;
const auto & array_join = array_join_step->arrayJoin();
const auto & expression = expression_step ? expression_step->getExpression()
: filter_step->getExpression();
auto split_actions = expression->splitActionsBeforeArrayJoin(array_join->columns);
/// No actions can be moved before ARRAY JOIN.
if (!split_actions)
return;
/// All actions was moved before ARRAY JOIN. Swap Expression and ArrayJoin.
if (expression->getActions().empty())
{
auto expected_header = parent->getOutputStream().header;
/// Expression/Filter -> ArrayJoin
std::swap(parent, child);
/// ArrayJoin -> Expression/Filter
if (expression_step)
child = std::make_unique<ExpressionStep>(child_node->children.at(0)->step->getOutputStream(),
std::move(split_actions));
else
child = std::make_unique<FilterStep>(child_node->children.at(0)->step->getOutputStream(),
std::move(split_actions),
filter_step->getFilterColumnName(),
filter_step->removesFilterColumn());
array_join_step->updateInputStream(child->getOutputStream(), expected_header);
return;
}
/// Add new expression step before ARRAY JOIN.
/// Expression/Filter -> ArrayJoin -> Something
auto & node = nodes.emplace_back();
node.children.swap(child_node->children);
child_node->children.emplace_back(&node);
/// Expression/Filter -> ArrayJoin -> node -> Something
node.step = std::make_unique<ExpressionStep>(node.children.at(0)->step->getOutputStream(),
std::move(split_actions));
array_join_step->updateInputStream(node.step->getOutputStream(), {});
expression_step ? expression_step->updateInputStream(array_join_step->getOutputStream(), true)
: filter_step->updateInputStream(array_join_step->getOutputStream(), true);
}
void QueryPlan::optimize()
{
struct Frame
@ -436,7 +499,13 @@ void QueryPlan::optimize()
++frame.next_child;
}
else
{
/// Last entrance, try lift up.
if (frame.node->children.size() == 1)
tryLiftUpArrayJoin(frame.node, frame.node->children.front(), nodes);
stack.pop();
}
}
}

5
src/Processors/QueryPlan/QueryPlan.h
View File

@ -69,11 +69,10 @@ public:
std::vector<Node *> children = {};
};
private:
using Nodes = std::list<Node>;
Nodes nodes;
private:
Nodes nodes;
Node * root = nullptr;
void checkInitialized() const;

37
src/Processors/Transforms/ArrayJoinTransform.cpp Normal file
View File

@ -0,0 +1,37 @@
#include <Processors/Transforms/ArrayJoinTransform.h>
#include <Interpreters/ArrayJoinAction.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
Block ArrayJoinTransform::transformHeader(Block header, const ArrayJoinActionPtr & array_join)
{
array_join->execute(header);
return header;
}
ArrayJoinTransform::ArrayJoinTransform(
const Block & header_,
ArrayJoinActionPtr array_join_,
bool /*on_totals_*/)
: ISimpleTransform(header_, transformHeader(header_, array_join_), false)
, array_join(std::move(array_join_))
{
/// TODO
// if (on_totals_)
// throw Exception("ARRAY JOIN is not supported for totals", ErrorCodes::LOGICAL_ERROR);
}
void ArrayJoinTransform::transform(Chunk & chunk)
{
auto block = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns());
array_join->execute(block);
chunk.setColumns(block.getColumns(), block.rows());
}
}

30
src/Processors/Transforms/ArrayJoinTransform.h Normal file
View File

@ -0,0 +1,30 @@
#pragma once
#include <Processors/ISimpleTransform.h>
namespace DB
{
class ArrayJoinAction;
using ArrayJoinActionPtr = std::shared_ptr<ArrayJoinAction>;
/// Execute ARRAY JOIN
class ArrayJoinTransform : public ISimpleTransform
{
public:
ArrayJoinTransform(
const Block & header_,
ArrayJoinActionPtr array_join_,
bool on_totals_ = false);
String getName() const override { return "ArrayJoinTransform"; }
static Block transformHeader(Block header, const ArrayJoinActionPtr & array_join);
protected:
void transform(Chunk & chunk) override;
private:
ArrayJoinActionPtr array_join;
};
}

Some files were not shown because too many files have changed in this diff Show More