diff --git a/.gitmodules b/.gitmodules index fb8f6796ebc..f7a16b84d37 100644 --- a/.gitmodules +++ b/.gitmodules @@ -154,3 +154,6 @@ [submodule "contrib/libcpuid"] path = contrib/libcpuid url = https://github.com/ClickHouse-Extras/libcpuid.git +[submodule "contrib/openldap"] + path = contrib/openldap + url = https://github.com/openldap/openldap.git diff --git a/CMakeLists.txt b/CMakeLists.txt index dd21ff123da..fb36aff6603 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -331,6 +331,7 @@ include (cmake/lib_name.cmake) find_contrib_lib(double-conversion) # Must be before parquet include (cmake/find/ssl.cmake) +include (cmake/find/ldap.cmake) # after ssl include (cmake/find/icu.cmake) include (cmake/find/boost.cmake) include (cmake/find/zlib.cmake) diff --git a/base/ext/range.h b/base/ext/range.h index c379d453f7b..266016f5779 100644 --- a/base/ext/range.h +++ b/base/ext/range.h @@ -1,42 +1,62 @@ #pragma once -#include #include #include +#include namespace ext { - /// For loop adaptor which is used to iterate through a half-closed interval [begin, end). - template - inline auto range(BeginType begin, EndType end) +namespace internal +{ + template + auto rangeImpl(BeginType begin, EndType end) { - using CommonType = typename std::common_type::type; - return boost::counting_range(begin, end); - } - - template - inline auto range(Type end) - { - return range(static_cast(0), end); - } - - /// The same as range(), but every value is casted statically to a specified `ValueType`. - /// This is useful to iterate through all constants of a enum. - template - inline auto range_with_static_cast(BeginType begin, EndType end) - { - using CommonType = typename std::common_type::type; - if constexpr (std::is_same_v) - return boost::counting_range(begin, end); + if constexpr (std::is_same_v) + return boost::counting_range(static_cast(begin), static_cast(end)); else - return boost::counting_range(begin, end) - | boost::adaptors::transformed([](CommonType x) -> ValueType { return static_cast(x); }); - } - - template - inline auto range_with_static_cast(EndType end) - { - return range_with_static_cast(static_cast(0), end); + return boost::counting_range(static_cast(begin), static_cast(end)) + | boost::adaptors::transformed([](CountingType x) { return static_cast(x); }); } } + + +/// For loop adaptor which is used to iterate through a half-closed interval [begin, end). +/// The parameters `begin` and `end` can have any integral or enum types. +template || std::is_enum_v) && + (std::is_integral_v || std::is_enum_v) && + (!std::is_enum_v || !std::is_enum_v || std::is_same_v), void>> +inline auto range(BeginType begin, EndType end) +{ + if constexpr (std::is_integral_v && std::is_integral_v) + { + using CommonType = std::common_type_t; + return internal::rangeImpl(begin, end); + } + else if constexpr (std::is_enum_v) + { + return internal::rangeImpl>(begin, end); + } + else + { + return internal::rangeImpl>(begin, end); + } +} + + +/// For loop adaptor which is used to iterate through a half-closed interval [0, end). +/// The parameter `end` can have any integral or enum type. +/// The same as range(0, end). +template || std::is_enum_v, void>> +inline auto range(Type end) +{ + if constexpr (std::is_integral_v) + return internal::rangeImpl(0, end); + else + return internal::rangeImpl>(0, end); +} +} diff --git a/cmake/Modules/FindOpenLDAP.cmake b/cmake/Modules/FindOpenLDAP.cmake new file mode 100644 index 00000000000..c33eafdcb2e --- /dev/null +++ b/cmake/Modules/FindOpenLDAP.cmake @@ -0,0 +1,55 @@ +# Find OpenLDAP libraries. +# +# Can be configured with: +# OPENLDAP_ROOT_DIR - path to the OpenLDAP installation prefix +# OPENLDAP_USE_STATIC_LIBS - look for static version of the libraries +# OPENLDAP_USE_REENTRANT_LIBS - look for thread-safe version of the libraries +# +# Sets values of: +# OPENLDAP_FOUND - TRUE if found +# OPENLDAP_INCLUDE_DIR - path to the include directory +# OPENLDAP_LIBRARIES - paths to the libldap and liblber libraries +# OPENLDAP_LDAP_LIBRARY - paths to the libldap library +# OPENLDAP_LBER_LIBRARY - paths to the liblber library +# + +if(OPENLDAP_USE_STATIC_LIBS) + set(_orig_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) + if(WIN32) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".a" ${CMAKE_FIND_LIBRARY_SUFFIXES}) + else() + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") + endif() +endif() + +set(_r_suffix) +if(OPENLDAP_USE_REENTRANT_LIBS) + set(_r_suffix "_r") +endif() + +if(OPENLDAP_ROOT_DIR) + find_path(OPENLDAP_INCLUDE_DIR NAMES "ldap.h" "lber.h" PATHS "${OPENLDAP_ROOT_DIR}" PATH_SUFFIXES "include" NO_DEFAULT_PATH) + find_library(OPENLDAP_LDAP_LIBRARY NAMES "ldap${_r_suffix}" PATHS "${OPENLDAP_ROOT_DIR}" PATH_SUFFIXES "lib" NO_DEFAULT_PATH) + find_library(OPENLDAP_LBER_LIBRARY NAMES "lber" PATHS "${OPENLDAP_ROOT_DIR}" PATH_SUFFIXES "lib" NO_DEFAULT_PATH) +else() + find_path(OPENLDAP_INCLUDE_DIR NAMES "ldap.h" "lber.h") + find_library(OPENLDAP_LDAP_LIBRARY NAMES "ldap${_r_suffix}") + find_library(OPENLDAP_LBER_LIBRARY NAMES "lber") +endif() + +unset(_r_suffix) + +set(OPENLDAP_LIBRARIES ${OPENLDAP_LDAP_LIBRARY} ${OPENLDAP_LBER_LIBRARY}) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + OpenLDAP DEFAULT_MSG + OPENLDAP_INCLUDE_DIR OPENLDAP_LDAP_LIBRARY OPENLDAP_LBER_LIBRARY +) + +mark_as_advanced(OPENLDAP_INCLUDE_DIR OPENLDAP_LIBRARIES OPENLDAP_LDAP_LIBRARY OPENLDAP_LBER_LIBRARY) + +if(OPENLDAP_USE_STATIC_LIBS) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${_orig_CMAKE_FIND_LIBRARY_SUFFIXES}) + unset(_orig_CMAKE_FIND_LIBRARY_SUFFIXES) +endif() diff --git a/cmake/find/ldap.cmake b/cmake/find/ldap.cmake new file mode 100644 index 00000000000..230727819e4 --- /dev/null +++ b/cmake/find/ldap.cmake @@ -0,0 +1,76 @@ +option (ENABLE_LDAP "Enable LDAP" ${ENABLE_LIBRARIES}) + +if (ENABLE_LDAP) + option (USE_INTERNAL_LDAP_LIBRARY "Set to FALSE to use system *LDAP library instead of bundled" ${NOT_UNBUNDLED}) + + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/openldap/README") + if (USE_INTERNAL_LDAP_LIBRARY) + message (WARNING "Submodule contrib/openldap is missing. To fix try running:\n git submodule update --init --recursive") + endif () + + set (USE_INTERNAL_LDAP_LIBRARY 0) + set (MISSING_INTERNAL_LDAP_LIBRARY 1) + endif () + + set (OPENLDAP_USE_STATIC_LIBS ${USE_STATIC_LIBRARIES}) + set (OPENLDAP_USE_REENTRANT_LIBS 1) + + if (NOT USE_INTERNAL_LDAP_LIBRARY) + if (APPLE AND NOT OPENLDAP_ROOT_DIR) + set (OPENLDAP_ROOT_DIR "/usr/local/opt/openldap") + endif () + + find_package (OpenLDAP) + endif () + + if (NOT OPENLDAP_FOUND AND NOT MISSING_INTERNAL_LDAP_LIBRARY) + string (TOLOWER "${CMAKE_SYSTEM_NAME}" _system_name) + string (TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _system_processor) + + if ( + "${_system_processor}" STREQUAL "amd64" OR + "${_system_processor}" STREQUAL "x64" + ) + set (_system_processor "x86_64") + elseif ( + "${_system_processor}" STREQUAL "arm64" + ) + set (_system_processor "aarch64") + endif () + + if ( + ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "x86_64" ) OR + ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "aarch64" ) OR + ( "${_system_name}" STREQUAL "freebsd" AND "${_system_processor}" STREQUAL "x86_64" ) OR + ( "${_system_name}" STREQUAL "darwin" AND "${_system_processor}" STREQUAL "x86_64" ) + ) + set (_ldap_supported_platform TRUE) + endif () + + if (NOT _ldap_supported_platform) + message (WARNING "LDAP support using the bundled library is not implemented for ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR} platform.") + elseif (NOT USE_SSL) + message (WARNING "LDAP support using the bundled library is not possible if SSL is not used.") + else () + set (USE_INTERNAL_LDAP_LIBRARY 1) + set (OPENLDAP_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/openldap") + set (OPENLDAP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/openldap/include") + # Below, 'ldap'/'ldap_r' and 'lber' will be resolved to + # the targets defined in contrib/openldap-cmake/CMakeLists.txt + if (OPENLDAP_USE_REENTRANT_LIBS) + set (OPENLDAP_LDAP_LIBRARY "ldap_r") + else () + set (OPENLDAP_LDAP_LIBRARY "ldap") + endif() + set (OPENLDAP_LBER_LIBRARY "lber") + set (OPENLDAP_LIBRARIES ${OPENLDAP_LDAP_LIBRARY} ${OPENLDAP_LBER_LIBRARY}) + set (OPENLDAP_FOUND 1) + endif () + endif () + + if (OPENLDAP_FOUND) + set (USE_LDAP 1) + endif () +endif () + +message (STATUS "Using ldap=${USE_LDAP}: ${OPENLDAP_INCLUDE_DIR} : ${OPENLDAP_LIBRARIES}") diff --git a/cmake/freebsd/toolchain-x86_64.cmake b/cmake/freebsd/toolchain-x86_64.cmake index 0961250ef8e..d9839ec74ee 100644 --- a/cmake/freebsd/toolchain-x86_64.cmake +++ b/cmake/freebsd/toolchain-x86_64.cmake @@ -1,8 +1,8 @@ set (CMAKE_SYSTEM_NAME "FreeBSD") set (CMAKE_SYSTEM_PROCESSOR "x86_64") -set (CMAKE_C_COMPILER_TARGET "x86_64-pc-freebsd12.1") -set (CMAKE_CXX_COMPILER_TARGET "x86_64-pc-freebsd12.1") -set (CMAKE_ASM_COMPILER_TARGET "x86_64-pc-freebsd12.1") +set (CMAKE_C_COMPILER_TARGET "x86_64-pc-freebsd11") +set (CMAKE_CXX_COMPILER_TARGET "x86_64-pc-freebsd11") +set (CMAKE_ASM_COMPILER_TARGET "x86_64-pc-freebsd11") set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/freebsd-x86_64") set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake index d18a996e2c9..0ecdfd2a3ad 100644 --- a/cmake/linux/default_libs.cmake +++ b/cmake/linux/default_libs.cmake @@ -21,7 +21,7 @@ set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS}) # glibc-compatibility library relies to fixed version of libc headers # (because minor changes in function attributes between different glibc versions will introduce incompatibilities) # This is for x86_64. For other architectures we have separate toolchains. -if (ARCH_AMD64) +if (ARCH_AMD64 AND NOT_UNBUNDLED) set(CMAKE_C_STANDARD_INCLUDE_DIRECTORIES ${ClickHouse_SOURCE_DIR}/contrib/libc-headers/x86_64-linux-gnu ${ClickHouse_SOURCE_DIR}/contrib/libc-headers) set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES ${ClickHouse_SOURCE_DIR}/contrib/libc-headers/x86_64-linux-gnu ${ClickHouse_SOURCE_DIR}/contrib/libc-headers) endif () diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index d98a854ace1..1031285eac7 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -110,6 +110,10 @@ if (USE_INTERNAL_SSL_LIBRARY) add_library(OpenSSL::SSL ALIAS ${OPENSSL_SSL_LIBRARY}) endif () +if (ENABLE_LDAP AND USE_INTERNAL_LDAP_LIBRARY) + add_subdirectory (openldap-cmake) +endif () + function(mysql_support) set(CLIENT_PLUGIN_CACHING_SHA2_PASSWORD STATIC) set(CLIENT_PLUGIN_SHA256_PASSWORD STATIC) diff --git a/contrib/openldap b/contrib/openldap new file mode 160000 index 00000000000..34b9ba94b30 --- /dev/null +++ b/contrib/openldap @@ -0,0 +1 @@ +Subproject commit 34b9ba94b30319ed6389a4e001d057f7983fe363 diff --git a/contrib/openldap-cmake/CMakeLists.txt b/contrib/openldap-cmake/CMakeLists.txt new file mode 100644 index 00000000000..b0a5f4048ff --- /dev/null +++ b/contrib/openldap-cmake/CMakeLists.txt @@ -0,0 +1,202 @@ +set(OPENLDAP_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/openldap) + +# How these lists were generated? +# I compiled the original OpenLDAP with it's original build system and copied the list of source files from build commands. + +set(_libs_type SHARED) +if(OPENLDAP_USE_STATIC_LIBS) + set(_libs_type STATIC) +endif() + +set(OPENLDAP_VERSION_STRING "2.5.X") + +macro(mkversion _lib_name) + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${_lib_name}-version.c + COMMAND ${CMAKE_COMMAND} -E env bash -c "${OPENLDAP_SOURCE_DIR}/build/mkversion -v '${OPENLDAP_VERSION_STRING}' liblber.la > ${CMAKE_CURRENT_BINARY_DIR}/${_lib_name}-version.c" + MAIN_DEPENDENCY ${OPENLDAP_SOURCE_DIR}/build/mkversion + WORKING_DIRECTORY ${OPENLDAP_SOURCE_DIR} + VERBATIM + ) +endmacro() + +string(TOLOWER "${CMAKE_SYSTEM_NAME}" _system_name) +string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _system_processor) + +if( + "${_system_processor}" STREQUAL "amd64" OR + "${_system_processor}" STREQUAL "x64" +) + set(_system_processor "x86_64") +elseif( + "${_system_processor}" STREQUAL "arm64" +) + set (_system_processor "aarch64") +endif() + +set(_extra_build_dir "${CMAKE_CURRENT_SOURCE_DIR}/${_system_name}_${_system_processor}") + +set(_lber_srcs + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/assert.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/decode.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/encode.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/io.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/bprint.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/debug.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/memory.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/options.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/sockbuf.c + ${OPENLDAP_SOURCE_DIR}/libraries/liblber/stdio.c +) + +mkversion(lber) + +add_library(lber ${_libs_type} + ${_lber_srcs} + ${CMAKE_CURRENT_BINARY_DIR}/lber-version.c +) + +target_link_libraries(lber + PRIVATE ${OPENSSL_LIBRARIES} +) + +target_include_directories(lber + PRIVATE ${_extra_build_dir}/include + PRIVATE ${OPENLDAP_SOURCE_DIR}/include + PRIVATE ${OPENLDAP_SOURCE_DIR}/libraries/liblber + PRIVATE ${OPENSSL_INCLUDE_DIR} +) + +target_compile_definitions(lber + PRIVATE LBER_LIBRARY +) + +set(_ldap_srcs + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/bind.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/open.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/result.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/error.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/compare.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/search.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/controls.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/messages.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/references.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/extended.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/cyrus.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/modify.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/add.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/modrdn.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/delete.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/abandon.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/sasl.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/sbind.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/unbind.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/cancel.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/filter.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/free.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/sort.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/passwd.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/whoami.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/vc.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/getdn.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/getentry.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/getattr.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/getvalues.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/addentry.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/request.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/os-ip.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/url.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/pagectrl.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/sortctrl.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/vlvctrl.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/init.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/options.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/print.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/string.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/util-int.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/schema.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/charray.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/os-local.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/dnssrv.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/utf-8.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/utf-8-conv.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/tls2.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/tls_o.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/tls_g.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/turn.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/ppolicy.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/dds.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/txn.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/ldap_sync.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/stctrl.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/assertion.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/deref.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/ldifutil.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/ldif.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/fetch.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/lbase64.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/msctrl.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap/psearchctrl.c +) + +mkversion(ldap) + +add_library(ldap ${_libs_type} + ${_ldap_srcs} + ${CMAKE_CURRENT_BINARY_DIR}/ldap-version.c +) + +target_link_libraries(ldap + PRIVATE lber + PRIVATE ${OPENSSL_LIBRARIES} +) + +target_include_directories(ldap + PRIVATE ${_extra_build_dir}/include + PRIVATE ${OPENLDAP_SOURCE_DIR}/include + PRIVATE ${OPENLDAP_SOURCE_DIR}/libraries/libldap + PRIVATE ${OPENSSL_INCLUDE_DIR} +) + +target_compile_definitions(ldap + PRIVATE LDAP_LIBRARY +) + +set(_ldap_r_specific_srcs + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/threads.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/rdwr.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/tpool.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/rq.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/thr_posix.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/thr_thr.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/thr_nt.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/thr_pth.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/thr_stub.c + ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r/thr_debug.c +) + +mkversion(ldap_r) + +add_library(ldap_r ${_libs_type} + ${_ldap_r_specific_srcs} + ${_ldap_srcs} + ${CMAKE_CURRENT_BINARY_DIR}/ldap_r-version.c +) + +target_link_libraries(ldap_r + PRIVATE lber + PRIVATE ${OPENSSL_LIBRARIES} +) + +target_include_directories(ldap_r + PRIVATE ${_extra_build_dir}/include + PRIVATE ${OPENLDAP_SOURCE_DIR}/include + PRIVATE ${OPENLDAP_SOURCE_DIR}/libraries/libldap_r + PRIVATE ${OPENLDAP_SOURCE_DIR}/libraries/libldap + PRIVATE ${OPENSSL_INCLUDE_DIR} +) + +target_compile_definitions(ldap_r + PRIVATE LDAP_R_COMPILE + PRIVATE LDAP_LIBRARY +) diff --git a/contrib/openldap-cmake/darwin_x86_64/include/lber_types.h b/contrib/openldap-cmake/darwin_x86_64/include/lber_types.h new file mode 100644 index 00000000000..dbd59430527 --- /dev/null +++ b/contrib/openldap-cmake/darwin_x86_64/include/lber_types.h @@ -0,0 +1,63 @@ +/* include/lber_types.h. Generated from lber_types.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LBER types + */ + +#ifndef _LBER_TYPES_H +#define _LBER_TYPES_H + +#include + +LDAP_BEGIN_DECL + +/* LBER boolean, enum, integers (32 bits or larger) */ +#define LBER_INT_T int + +/* LBER tags (32 bits or larger) */ +#define LBER_TAG_T long + +/* LBER socket descriptor */ +#define LBER_SOCKET_T int + +/* LBER lengths (32 bits or larger) */ +#define LBER_LEN_T long + +/* ------------------------------------------------------------ */ + +/* booleans, enumerations, and integers */ +typedef LBER_INT_T ber_int_t; + +/* signed and unsigned versions */ +typedef signed LBER_INT_T ber_sint_t; +typedef unsigned LBER_INT_T ber_uint_t; + +/* tags */ +typedef unsigned LBER_TAG_T ber_tag_t; + +/* "socket" descriptors */ +typedef LBER_SOCKET_T ber_socket_t; + +/* lengths */ +typedef unsigned LBER_LEN_T ber_len_t; + +/* signed lengths */ +typedef signed LBER_LEN_T ber_slen_t; + +LDAP_END_DECL + +#endif /* _LBER_TYPES_H */ diff --git a/contrib/openldap-cmake/darwin_x86_64/include/ldap_config.h b/contrib/openldap-cmake/darwin_x86_64/include/ldap_config.h new file mode 100644 index 00000000000..89f7b40b884 --- /dev/null +++ b/contrib/openldap-cmake/darwin_x86_64/include/ldap_config.h @@ -0,0 +1,74 @@ +/* include/ldap_config.h. Generated from ldap_config.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * This file works in conjunction with OpenLDAP configure system. + * If you do no like the values below, adjust your configure options. + */ + +#ifndef _LDAP_CONFIG_H +#define _LDAP_CONFIG_H + +/* directory separator */ +#ifndef LDAP_DIRSEP +#ifndef _WIN32 +#define LDAP_DIRSEP "/" +#else +#define LDAP_DIRSEP "\\" +#endif +#endif + +/* directory for temporary files */ +#if defined(_WIN32) +# define LDAP_TMPDIR "C:\\." /* we don't have much of a choice */ +#elif defined( _P_tmpdir ) +# define LDAP_TMPDIR _P_tmpdir +#elif defined( P_tmpdir ) +# define LDAP_TMPDIR P_tmpdir +#elif defined( _PATH_TMPDIR ) +# define LDAP_TMPDIR _PATH_TMPDIR +#else +# define LDAP_TMPDIR LDAP_DIRSEP "tmp" +#endif + +/* directories */ +#ifndef LDAP_BINDIR +#define LDAP_BINDIR "/tmp/ldap-prefix/bin" +#endif +#ifndef LDAP_SBINDIR +#define LDAP_SBINDIR "/tmp/ldap-prefix/sbin" +#endif +#ifndef LDAP_DATADIR +#define LDAP_DATADIR "/tmp/ldap-prefix/share/openldap" +#endif +#ifndef LDAP_SYSCONFDIR +#define LDAP_SYSCONFDIR "/tmp/ldap-prefix/etc/openldap" +#endif +#ifndef LDAP_LIBEXECDIR +#define LDAP_LIBEXECDIR "/tmp/ldap-prefix/libexec" +#endif +#ifndef LDAP_MODULEDIR +#define LDAP_MODULEDIR "/tmp/ldap-prefix/libexec/openldap" +#endif +#ifndef LDAP_RUNDIR +#define LDAP_RUNDIR "/tmp/ldap-prefix/var" +#endif +#ifndef LDAP_LOCALEDIR +#define LDAP_LOCALEDIR "" +#endif + + +#endif /* _LDAP_CONFIG_H */ diff --git a/contrib/openldap-cmake/darwin_x86_64/include/ldap_features.h b/contrib/openldap-cmake/darwin_x86_64/include/ldap_features.h new file mode 100644 index 00000000000..f0cc7c3626f --- /dev/null +++ b/contrib/openldap-cmake/darwin_x86_64/include/ldap_features.h @@ -0,0 +1,61 @@ +/* include/ldap_features.h. Generated from ldap_features.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LDAP Features + */ + +#ifndef _LDAP_FEATURES_H +#define _LDAP_FEATURES_H 1 + +/* OpenLDAP API version macros */ +#define LDAP_VENDOR_VERSION 20501 +#define LDAP_VENDOR_VERSION_MAJOR 2 +#define LDAP_VENDOR_VERSION_MINOR 5 +#define LDAP_VENDOR_VERSION_PATCH X + +/* +** WORK IN PROGRESS! +** +** OpenLDAP reentrancy/thread-safeness should be dynamically +** checked using ldap_get_option(). +** +** The -lldap implementation is not thread-safe. +** +** The -lldap_r implementation is: +** LDAP_API_FEATURE_THREAD_SAFE (basic thread safety) +** but also be: +** LDAP_API_FEATURE_SESSION_THREAD_SAFE +** LDAP_API_FEATURE_OPERATION_THREAD_SAFE +** +** The preprocessor flag LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE +** can be used to determine if -lldap_r is available at compile +** time. You must define LDAP_THREAD_SAFE if and only if you +** link with -lldap_r. +** +** If you fail to define LDAP_THREAD_SAFE when linking with +** -lldap_r or define LDAP_THREAD_SAFE when linking with -lldap, +** provided header definitions and declarations may be incorrect. +** +*/ + +/* is -lldap_r available or not */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* LDAP v2 Referrals */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +#endif /* LDAP_FEATURES */ diff --git a/contrib/openldap-cmake/darwin_x86_64/include/portable.h b/contrib/openldap-cmake/darwin_x86_64/include/portable.h new file mode 100644 index 00000000000..fdf4e89017e --- /dev/null +++ b/contrib/openldap-cmake/darwin_x86_64/include/portable.h @@ -0,0 +1,1169 @@ +/* include/portable.h. Generated from portable.hin by configure. */ +/* include/portable.hin. Generated from configure.in by autoheader. */ + + +/* begin of portable.h.pre */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#ifndef _LDAP_PORTABLE_H +#define _LDAP_PORTABLE_H + +/* define this if needed to get reentrant functions */ +#ifndef REENTRANT +#define REENTRANT 1 +#endif +#ifndef _REENTRANT +#define _REENTRANT 1 +#endif + +/* define this if needed to get threadsafe functions */ +#ifndef THREADSAFE +#define THREADSAFE 1 +#endif +#ifndef _THREADSAFE +#define _THREADSAFE 1 +#endif +#ifndef THREAD_SAFE +#define THREAD_SAFE 1 +#endif +#ifndef _THREAD_SAFE +#define _THREAD_SAFE 1 +#endif + +#ifndef _SGI_MP_SOURCE +#define _SGI_MP_SOURCE 1 +#endif + +/* end of portable.h.pre */ + + +/* Define if building universal (internal helper macro) */ +/* #undef AC_APPLE_UNIVERSAL_BUILD */ + +/* define to use both and */ +/* #undef BOTH_STRINGS_H */ + +/* define if cross compiling */ +/* #undef CROSS_COMPILING */ + +/* set to the number of arguments ctime_r() expects */ +#define CTIME_R_NARGS 2 + +/* define if toupper() requires islower() */ +/* #undef C_UPPER_LOWER */ + +/* define if sys_errlist is not declared in stdio.h or errno.h */ +/* #undef DECL_SYS_ERRLIST */ + +/* define to enable slapi library */ +/* #undef ENABLE_SLAPI */ + +/* defined to be the EXE extension */ +#define EXEEXT "" + +/* set to the number of arguments gethostbyaddr_r() expects */ +/* #undef GETHOSTBYADDR_R_NARGS */ + +/* set to the number of arguments gethostbyname_r() expects */ +/* #undef GETHOSTBYNAME_R_NARGS */ + +/* Define to 1 if `TIOCGWINSZ' requires . */ +/* #undef GWINSZ_IN_SYS_IOCTL */ + +/* define if you have AIX security lib */ +/* #undef HAVE_AIX_SECURITY */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H 1 + +/* Define to 1 if you have the `bcopy' function. */ +#define HAVE_BCOPY 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_BITS_TYPES_H */ + +/* Define to 1 if you have the `chroot' function. */ +#define HAVE_CHROOT 1 + +/* Define to 1 if you have the `closesocket' function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CONIO_H */ + +/* define if crypt(3) is available */ +/* #undef HAVE_CRYPT */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CRYPT_H */ + +/* define if crypt_r() is also available */ +/* #undef HAVE_CRYPT_R */ + +/* Define to 1 if you have the `ctime_r' function. */ +#define HAVE_CTIME_R 1 + +/* define if you have Cyrus SASL */ +/* #undef HAVE_CYRUS_SASL */ + +/* define if your system supports /dev/poll */ +/* #undef HAVE_DEVPOLL */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DIRECT_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#define HAVE_DIRENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ +/* #undef HAVE_DOPRNT */ + +/* define if system uses EBCDIC instead of ASCII */ +/* #undef HAVE_EBCDIC */ + +/* Define to 1 if you have the `endgrent' function. */ +#define HAVE_ENDGRENT 1 + +/* Define to 1 if you have the `endpwent' function. */ +#define HAVE_ENDPWENT 1 + +/* define if your system supports epoll */ +/* #undef HAVE_EPOLL */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H 1 + +/* Define to 1 if you have the `fcntl' function. */ +#define HAVE_FCNTL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H 1 + +/* define if you actually have FreeBSD fetch(3) */ +/* #undef HAVE_FETCH */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_FILIO_H */ + +/* Define to 1 if you have the `flock' function. */ +#define HAVE_FLOCK 1 + +/* Define to 1 if you have the `fstat' function. */ +#define HAVE_FSTAT 1 + +/* Define to 1 if you have the `gai_strerror' function. */ +#define HAVE_GAI_STRERROR 1 + +/* Define to 1 if you have the `getaddrinfo' function. */ +#define HAVE_GETADDRINFO 1 + +/* Define to 1 if you have the `getdtablesize' function. */ +#define HAVE_GETDTABLESIZE 1 + +/* Define to 1 if you have the `geteuid' function. */ +#define HAVE_GETEUID 1 + +/* Define to 1 if you have the `getgrgid' function. */ +#define HAVE_GETGRGID 1 + +/* Define to 1 if you have the `gethostbyaddr_r' function. */ +/* #undef HAVE_GETHOSTBYADDR_R */ + +/* Define to 1 if you have the `gethostbyname_r' function. */ +/* #undef HAVE_GETHOSTBYNAME_R */ + +/* Define to 1 if you have the `gethostname' function. */ +#define HAVE_GETHOSTNAME 1 + +/* Define to 1 if you have the `getnameinfo' function. */ +#define HAVE_GETNAMEINFO 1 + +/* Define to 1 if you have the `getopt' function. */ +#define HAVE_GETOPT 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_GETOPT_H 1 + +/* Define to 1 if you have the `getpassphrase' function. */ +/* #undef HAVE_GETPASSPHRASE */ + +/* Define to 1 if you have the `getpeereid' function. */ +#define HAVE_GETPEEREID 1 + +/* Define to 1 if you have the `getpeerucred' function. */ +/* #undef HAVE_GETPEERUCRED */ + +/* Define to 1 if you have the `getpwnam' function. */ +#define HAVE_GETPWNAM 1 + +/* Define to 1 if you have the `getpwuid' function. */ +#define HAVE_GETPWUID 1 + +/* Define to 1 if you have the `getspnam' function. */ +/* #undef HAVE_GETSPNAM */ + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GMP_H */ + +/* Define to 1 if you have the `gmtime_r' function. */ +#define HAVE_GMTIME_R 1 + +/* define if you have GNUtls */ +/* #undef HAVE_GNUTLS */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GNUTLS_GNUTLS_H */ + +/* if you have GNU Pth */ +/* #undef HAVE_GNU_PTH */ + +/* Define to 1 if you have the header file. */ +#define HAVE_GRP_H 1 + +/* Define to 1 if you have the `hstrerror' function. */ +#define HAVE_HSTRERROR 1 + +/* define to you inet_aton(3) is available */ +#define HAVE_INET_ATON 1 + +/* Define to 1 if you have the `inet_ntoa_b' function. */ +/* #undef HAVE_INET_NTOA_B */ + +/* Define to 1 if you have the `inet_ntop' function. */ +#define HAVE_INET_NTOP 1 + +/* Define to 1 if you have the `initgroups' function. */ +#define HAVE_INITGROUPS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the `ioctl' function. */ +#define HAVE_IOCTL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_IO_H */ + +/* define if your system supports kqueue */ +#define HAVE_KQUEUE 1 + +/* Define to 1 if you have the `gen' library (-lgen). */ +/* #undef HAVE_LIBGEN */ + +/* Define to 1 if you have the `gmp' library (-lgmp). */ +/* #undef HAVE_LIBGMP */ + +/* Define to 1 if you have the `inet' library (-linet). */ +/* #undef HAVE_LIBINET */ + +/* define if you have libtool -ltdl */ +/* #undef HAVE_LIBLTDL */ + +/* Define to 1 if you have the `net' library (-lnet). */ +/* #undef HAVE_LIBNET */ + +/* Define to 1 if you have the `nsl' library (-lnsl). */ +/* #undef HAVE_LIBNSL */ + +/* Define to 1 if you have the `nsl_s' library (-lnsl_s). */ +/* #undef HAVE_LIBNSL_S */ + +/* Define to 1 if you have the `socket' library (-lsocket). */ +/* #undef HAVE_LIBSOCKET */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LIBUTIL_H */ + +/* Define to 1 if you have the `V3' library (-lV3). */ +/* #undef HAVE_LIBV3 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* if you have LinuxThreads */ +/* #undef HAVE_LINUX_THREADS */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LOCALE_H 1 + +/* Define to 1 if you have the `localtime_r' function. */ +#define HAVE_LOCALTIME_R 1 + +/* Define to 1 if you have the `lockf' function. */ +#define HAVE_LOCKF 1 + +/* Define to 1 if the system has the type `long long'. */ +#define HAVE_LONG_LONG 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LTDL_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_MALLOC_H */ + +/* Define to 1 if you have the `memcpy' function. */ +#define HAVE_MEMCPY 1 + +/* Define to 1 if you have the `memmove' function. */ +#define HAVE_MEMMOVE 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `memrchr' function. */ +/* #undef HAVE_MEMRCHR */ + +/* Define to 1 if you have the `mkstemp' function. */ +#define HAVE_MKSTEMP 1 + +/* Define to 1 if you have the `mktemp' function. */ +#define HAVE_MKTEMP 1 + +/* define this if you have mkversion */ +#define HAVE_MKVERSION 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. */ +/* #undef HAVE_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H 1 + +/* define if strerror_r returns char* instead of int */ +/* #undef HAVE_NONPOSIX_STRERROR_R */ + +/* if you have NT Event Log */ +/* #undef HAVE_NT_EVENT_LOG */ + +/* if you have NT Service Manager */ +/* #undef HAVE_NT_SERVICE_MANAGER */ + +/* if you have NT Threads */ +/* #undef HAVE_NT_THREADS */ + +/* define if you have OpenSSL */ +#define HAVE_OPENSSL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_BN_H 1 + +/* define if you have OpenSSL with CRL checking capability */ +#define HAVE_OPENSSL_CRL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_CRYPTO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_SSL_H 1 + +/* Define to 1 if you have the `pipe' function. */ +#define HAVE_PIPE 1 + +/* Define to 1 if you have the `poll' function. */ +#define HAVE_POLL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PROCESS_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PSAP_H */ + +/* define to pthreads API spec revision */ +#define HAVE_PTHREADS 10 + +/* define if you have pthread_detach function */ +#define HAVE_PTHREAD_DETACH 1 + +/* Define to 1 if you have the `pthread_getconcurrency' function. */ +#define HAVE_PTHREAD_GETCONCURRENCY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PTHREAD_H 1 + +/* Define to 1 if you have the `pthread_kill' function. */ +#define HAVE_PTHREAD_KILL 1 + +/* Define to 1 if you have the `pthread_kill_other_threads_np' function. */ +/* #undef HAVE_PTHREAD_KILL_OTHER_THREADS_NP */ + +/* define if you have pthread_rwlock_destroy function */ +#define HAVE_PTHREAD_RWLOCK_DESTROY 1 + +/* Define to 1 if you have the `pthread_setconcurrency' function. */ +#define HAVE_PTHREAD_SETCONCURRENCY 1 + +/* Define to 1 if you have the `pthread_yield' function. */ +/* #undef HAVE_PTHREAD_YIELD */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PTH_H */ + +/* Define to 1 if the system has the type `ptrdiff_t'. */ +#define HAVE_PTRDIFF_T 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PWD_H 1 + +/* Define to 1 if you have the `read' function. */ +#define HAVE_READ 1 + +/* Define to 1 if you have the `recv' function. */ +#define HAVE_RECV 1 + +/* Define to 1 if you have the `recvfrom' function. */ +#define HAVE_RECVFROM 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_REGEX_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_RESOLV_H */ + +/* define if you have res_query() */ +/* #undef HAVE_RES_QUERY */ + +/* define if OpenSSL needs RSAref */ +/* #undef HAVE_RSAREF */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_SASL_H */ + +/* define if your SASL library has sasl_version() */ +/* #undef HAVE_SASL_VERSION */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SCHED_H 1 + +/* Define to 1 if you have the `sched_yield' function. */ +#define HAVE_SCHED_YIELD 1 + +/* Define to 1 if you have the `send' function. */ +#define HAVE_SEND 1 + +/* Define to 1 if you have the `sendmsg' function. */ +#define HAVE_SENDMSG 1 + +/* Define to 1 if you have the `sendto' function. */ +#define HAVE_SENDTO 1 + +/* Define to 1 if you have the `setegid' function. */ +#define HAVE_SETEGID 1 + +/* Define to 1 if you have the `seteuid' function. */ +#define HAVE_SETEUID 1 + +/* Define to 1 if you have the `setgid' function. */ +#define HAVE_SETGID 1 + +/* Define to 1 if you have the `setpwfile' function. */ +/* #undef HAVE_SETPWFILE */ + +/* Define to 1 if you have the `setsid' function. */ +#define HAVE_SETSID 1 + +/* Define to 1 if you have the `setuid' function. */ +#define HAVE_SETUID 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SGTTY_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SHADOW_H */ + +/* Define to 1 if you have the `sigaction' function. */ +#define HAVE_SIGACTION 1 + +/* Define to 1 if you have the `signal' function. */ +#define HAVE_SIGNAL 1 + +/* Define to 1 if you have the `sigset' function. */ +#define HAVE_SIGSET 1 + +/* define if you have -lslp */ +/* #undef HAVE_SLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SLP_H */ + +/* Define to 1 if you have the `snprintf' function. */ +#define HAVE_SNPRINTF 1 + +/* if you have spawnlp() */ +/* #undef HAVE_SPAWNLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQLEXT_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQL_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDDEF_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strdup' function. */ +#define HAVE_STRDUP 1 + +/* Define to 1 if you have the `strerror' function. */ +#define HAVE_STRERROR 1 + +/* Define to 1 if you have the `strerror_r' function. */ +#define HAVE_STRERROR_R 1 + +/* Define to 1 if you have the `strftime' function. */ +#define HAVE_STRFTIME 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strpbrk' function. */ +#define HAVE_STRPBRK 1 + +/* Define to 1 if you have the `strrchr' function. */ +#define HAVE_STRRCHR 1 + +/* Define to 1 if you have the `strsep' function. */ +#define HAVE_STRSEP 1 + +/* Define to 1 if you have the `strspn' function. */ +#define HAVE_STRSPN 1 + +/* Define to 1 if you have the `strstr' function. */ +#define HAVE_STRSTR 1 + +/* Define to 1 if you have the `strtol' function. */ +#define HAVE_STRTOL 1 + +/* Define to 1 if you have the `strtoll' function. */ +#define HAVE_STRTOLL 1 + +/* Define to 1 if you have the `strtoq' function. */ +#define HAVE_STRTOQ 1 + +/* Define to 1 if you have the `strtoul' function. */ +#define HAVE_STRTOUL 1 + +/* Define to 1 if you have the `strtoull' function. */ +#define HAVE_STRTOULL 1 + +/* Define to 1 if you have the `strtouq' function. */ +#define HAVE_STRTOUQ 1 + +/* Define to 1 if `msg_accrightslen' is a member of `struct msghdr'. */ +/* #undef HAVE_STRUCT_MSGHDR_MSG_ACCRIGHTSLEN */ + +/* Define to 1 if `msg_control' is a member of `struct msghdr'. */ +/* #undef HAVE_STRUCT_MSGHDR_MSG_CONTROL */ + +/* Define to 1 if `pw_gecos' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_GECOS 1 + +/* Define to 1 if `pw_passwd' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_PASSWD 1 + +/* Define to 1 if `st_blksize' is a member of `struct stat'. */ +#define HAVE_STRUCT_STAT_ST_BLKSIZE 1 + +/* Define to 1 if `st_fstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE */ + +/* define to 1 if st_fstype is char * */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_CHAR */ + +/* define to 1 if st_fstype is int */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_INT */ + +/* Define to 1 if `st_vfstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_VFSTYPE */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYNCH_H */ + +/* Define to 1 if you have the `sysconf' function. */ +#define HAVE_SYSCONF 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSEXITS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_DEVPOLL_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_DIR_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_EPOLL_H */ + +/* define if you actually have sys_errlist in your libs */ +#define HAVE_SYS_ERRLIST 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_ERRNO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_EVENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_FILE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_FILIO_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_FSTYP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_PRIVGRP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_RESOURCE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UCRED_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UN_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_UUID_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_VMOUNT_H */ + +/* Define to 1 if you have that is POSIX.1 compatible. */ +#define HAVE_SYS_WAIT_H 1 + +/* define if you have -lwrap */ +/* #undef HAVE_TCPD */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_TCPD_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_TERMIOS_H 1 + +/* if you have Solaris LWP (thr) package */ +/* #undef HAVE_THR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_THREAD_H */ + +/* Define to 1 if you have the `thr_getconcurrency' function. */ +/* #undef HAVE_THR_GETCONCURRENCY */ + +/* Define to 1 if you have the `thr_setconcurrency' function. */ +/* #undef HAVE_THR_SETCONCURRENCY */ + +/* Define to 1 if you have the `thr_yield' function. */ +/* #undef HAVE_THR_YIELD */ + +/* define if you have TLS */ +#define HAVE_TLS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UTIME_H 1 + +/* define if you have uuid_generate() */ +/* #undef HAVE_UUID_GENERATE */ + +/* define if you have uuid_to_str() */ +/* #undef HAVE_UUID_TO_STR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_UUID_UUID_H */ + +/* Define to 1 if you have the `vprintf' function. */ +#define HAVE_VPRINTF 1 + +/* Define to 1 if you have the `vsnprintf' function. */ +#define HAVE_VSNPRINTF 1 + +/* Define to 1 if you have the `wait4' function. */ +#define HAVE_WAIT4 1 + +/* Define to 1 if you have the `waitpid' function. */ +#define HAVE_WAITPID 1 + +/* define if you have winsock */ +/* #undef HAVE_WINSOCK */ + +/* define if you have winsock2 */ +/* #undef HAVE_WINSOCK2 */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WIREDTIGER_H */ + +/* Define to 1 if you have the `write' function. */ +#define HAVE_WRITE 1 + +/* define if select implicitly yields */ +#define HAVE_YIELDING_SELECT 1 + +/* Define to 1 if you have the `_vsnprintf' function. */ +/* #undef HAVE__VSNPRINTF */ + +/* define to 32-bit or greater integer type */ +#define LBER_INT_T int + +/* define to large integer type */ +#define LBER_LEN_T long + +/* define to socket descriptor type */ +#define LBER_SOCKET_T int + +/* define to large integer type */ +#define LBER_TAG_T long + +/* define to 1 if library is thread safe */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* define to LDAP VENDOR VERSION */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +/* define this to add debugging code */ +/* #undef LDAP_DEBUG */ + +/* define if LDAP libs are dynamic */ +/* #undef LDAP_LIBS_DYNAMIC */ + +/* define to support PF_INET6 */ +#define LDAP_PF_INET6 1 + +/* define to support PF_LOCAL */ +#define LDAP_PF_LOCAL 1 + +/* define this to add SLAPI code */ +/* #undef LDAP_SLAPI */ + +/* define this to add syslog code */ +/* #undef LDAP_SYSLOG */ + +/* Version */ +#define LDAP_VENDOR_VERSION 20501 + +/* Major */ +#define LDAP_VENDOR_VERSION_MAJOR 2 + +/* Minor */ +#define LDAP_VENDOR_VERSION_MINOR 5 + +/* Patch */ +#define LDAP_VENDOR_VERSION_PATCH X + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* define if memcmp is not 8-bit clean or is otherwise broken */ +/* #undef NEED_MEMCMP_REPLACEMENT */ + +/* define if you have (or want) no threads */ +/* #undef NO_THREADS */ + +/* define to use the original debug style */ +/* #undef OLD_DEBUG */ + +/* Package */ +#define OPENLDAP_PACKAGE "OpenLDAP" + +/* Version */ +#define OPENLDAP_VERSION "2.5.X" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "" + +/* define if sched_yield yields the entire process */ +/* #undef REPLACE_BROKEN_YIELD */ + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE void + +/* Define to the type of arg 1 for `select'. */ +#define SELECT_TYPE_ARG1 int + +/* Define to the type of args 2, 3 and 4 for `select'. */ +#define SELECT_TYPE_ARG234 (fd_set *) + +/* Define to the type of arg 5 for `select'. */ +#define SELECT_TYPE_ARG5 (struct timeval *) + +/* The size of `int', as computed by sizeof. */ +#define SIZEOF_INT 4 + +/* The size of `long', as computed by sizeof. */ +#define SIZEOF_LONG 8 + +/* The size of `long long', as computed by sizeof. */ +#define SIZEOF_LONG_LONG 8 + +/* The size of `short', as computed by sizeof. */ +#define SIZEOF_SHORT 2 + +/* The size of `wchar_t', as computed by sizeof. */ +#define SIZEOF_WCHAR_T 4 + +/* define to support per-object ACIs */ +/* #undef SLAPD_ACI_ENABLED */ + +/* define to support LDAP Async Metadirectory backend */ +/* #undef SLAPD_ASYNCMETA */ + +/* define to support cleartext passwords */ +/* #undef SLAPD_CLEARTEXT */ + +/* define to support crypt(3) passwords */ +/* #undef SLAPD_CRYPT */ + +/* define to support DNS SRV backend */ +/* #undef SLAPD_DNSSRV */ + +/* define to support LDAP backend */ +/* #undef SLAPD_LDAP */ + +/* define to support MDB backend */ +/* #undef SLAPD_MDB */ + +/* define to support LDAP Metadirectory backend */ +/* #undef SLAPD_META */ + +/* define to support modules */ +/* #undef SLAPD_MODULES */ + +/* dynamically linked module */ +#define SLAPD_MOD_DYNAMIC 2 + +/* statically linked module */ +#define SLAPD_MOD_STATIC 1 + +/* define to support cn=Monitor backend */ +/* #undef SLAPD_MONITOR */ + +/* define to support NDB backend */ +/* #undef SLAPD_NDB */ + +/* define to support NULL backend */ +/* #undef SLAPD_NULL */ + +/* define for In-Directory Access Logging overlay */ +/* #undef SLAPD_OVER_ACCESSLOG */ + +/* define for Audit Logging overlay */ +/* #undef SLAPD_OVER_AUDITLOG */ + +/* define for Automatic Certificate Authority overlay */ +/* #undef SLAPD_OVER_AUTOCA */ + +/* define for Collect overlay */ +/* #undef SLAPD_OVER_COLLECT */ + +/* define for Attribute Constraint overlay */ +/* #undef SLAPD_OVER_CONSTRAINT */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DDS */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DEREF */ + +/* define for Dynamic Group overlay */ +/* #undef SLAPD_OVER_DYNGROUP */ + +/* define for Dynamic List overlay */ +/* #undef SLAPD_OVER_DYNLIST */ + +/* define for Reverse Group Membership overlay */ +/* #undef SLAPD_OVER_MEMBEROF */ + +/* define for Password Policy overlay */ +/* #undef SLAPD_OVER_PPOLICY */ + +/* define for Proxy Cache overlay */ +/* #undef SLAPD_OVER_PROXYCACHE */ + +/* define for Referential Integrity overlay */ +/* #undef SLAPD_OVER_REFINT */ + +/* define for Return Code overlay */ +/* #undef SLAPD_OVER_RETCODE */ + +/* define for Rewrite/Remap overlay */ +/* #undef SLAPD_OVER_RWM */ + +/* define for Sequential Modify overlay */ +/* #undef SLAPD_OVER_SEQMOD */ + +/* define for ServerSideSort/VLV overlay */ +/* #undef SLAPD_OVER_SSSVLV */ + +/* define for Syncrepl Provider overlay */ +/* #undef SLAPD_OVER_SYNCPROV */ + +/* define for Translucent Proxy overlay */ +/* #undef SLAPD_OVER_TRANSLUCENT */ + +/* define for Attribute Uniqueness overlay */ +/* #undef SLAPD_OVER_UNIQUE */ + +/* define for Value Sorting overlay */ +/* #undef SLAPD_OVER_VALSORT */ + +/* define to support PASSWD backend */ +/* #undef SLAPD_PASSWD */ + +/* define to support PERL backend */ +/* #undef SLAPD_PERL */ + +/* define to support relay backend */ +/* #undef SLAPD_RELAY */ + +/* define to support reverse lookups */ +/* #undef SLAPD_RLOOKUPS */ + +/* define to support SHELL backend */ +/* #undef SLAPD_SHELL */ + +/* define to support SOCK backend */ +/* #undef SLAPD_SOCK */ + +/* define to support SASL passwords */ +/* #undef SLAPD_SPASSWD */ + +/* define to support SQL backend */ +/* #undef SLAPD_SQL */ + +/* define to support WiredTiger backend */ +/* #undef SLAPD_WT */ + +/* define to support run-time loadable ACL */ +/* #undef SLAP_DYNACL */ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME 1 + +/* Define to 1 if your declares `struct tm'. */ +/* #undef TM_IN_SYS_TIME */ + +/* set to urandom device */ +#define URANDOM_DEVICE "/dev/urandom" + +/* define to use OpenSSL BIGNUM for MP */ +/* #undef USE_MP_BIGNUM */ + +/* define to use GMP for MP */ +/* #undef USE_MP_GMP */ + +/* define to use 'long' for MP */ +/* #undef USE_MP_LONG */ + +/* define to use 'long long' for MP */ +/* #undef USE_MP_LONG_LONG */ + +/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most + significant byte first (like Motorola and SPARC, unlike Intel). */ +#if defined AC_APPLE_UNIVERSAL_BUILD +# if defined __BIG_ENDIAN__ +# define WORDS_BIGENDIAN 1 +# endif +#else +# ifndef WORDS_BIGENDIAN +/* # undef WORDS_BIGENDIAN */ +# endif +#endif + +/* Define to the type of arg 3 for `accept'. */ +#define ber_socklen_t socklen_t + +/* Define to `char *' if does not define. */ +/* #undef caddr_t */ + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to `int' if doesn't define. */ +/* #undef gid_t */ + +/* Define to `int' if does not define. */ +/* #undef mode_t */ + +/* Define to `long' if does not define. */ +/* #undef off_t */ + +/* Define to `int' if does not define. */ +/* #undef pid_t */ + +/* Define to `int' if does not define. */ +/* #undef sig_atomic_t */ + +/* Define to `unsigned' if does not define. */ +/* #undef size_t */ + +/* define to snprintf routine */ +/* #undef snprintf */ + +/* Define like ber_socklen_t if does not define. */ +/* #undef socklen_t */ + +/* Define to `signed int' if does not define. */ +/* #undef ssize_t */ + +/* Define to `int' if doesn't define. */ +/* #undef uid_t */ + +/* define as empty if volatile is not supported */ +/* #undef volatile */ + +/* define to snprintf routine */ +/* #undef vsnprintf */ + + +/* begin of portable.h.post */ + +#ifdef _WIN32 +/* don't suck in all of the win32 api */ +# define WIN32_LEAN_AND_MEAN 1 +#endif + +#ifndef LDAP_NEEDS_PROTOTYPES +/* force LDAP_P to always include prototypes */ +#define LDAP_NEEDS_PROTOTYPES 1 +#endif + +#ifndef LDAP_REL_ENG +#if (LDAP_VENDOR_VERSION == 000000) && !defined(LDAP_DEVEL) +#define LDAP_DEVEL +#endif +#if defined(LDAP_DEVEL) && !defined(LDAP_TEST) +#define LDAP_TEST +#endif +#endif + +#ifdef HAVE_STDDEF_H +# include +#endif + +#ifdef HAVE_EBCDIC +/* ASCII/EBCDIC converting replacements for stdio funcs + * vsnprintf and snprintf are used too, but they are already + * checked by the configure script + */ +#define fputs ber_pvt_fputs +#define fgets ber_pvt_fgets +#define printf ber_pvt_printf +#define fprintf ber_pvt_fprintf +#define vfprintf ber_pvt_vfprintf +#define vsprintf ber_pvt_vsprintf +#endif + +#include "ac/fdset.h" + +#include "ldap_cdefs.h" +#include "ldap_features.h" + +#include "ac/assert.h" +#include "ac/localize.h" + +#endif /* _LDAP_PORTABLE_H */ +/* end of portable.h.post */ + diff --git a/contrib/openldap-cmake/freebsd_x86_64/include/lber_types.h b/contrib/openldap-cmake/freebsd_x86_64/include/lber_types.h new file mode 100644 index 00000000000..dbd59430527 --- /dev/null +++ b/contrib/openldap-cmake/freebsd_x86_64/include/lber_types.h @@ -0,0 +1,63 @@ +/* include/lber_types.h. Generated from lber_types.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LBER types + */ + +#ifndef _LBER_TYPES_H +#define _LBER_TYPES_H + +#include + +LDAP_BEGIN_DECL + +/* LBER boolean, enum, integers (32 bits or larger) */ +#define LBER_INT_T int + +/* LBER tags (32 bits or larger) */ +#define LBER_TAG_T long + +/* LBER socket descriptor */ +#define LBER_SOCKET_T int + +/* LBER lengths (32 bits or larger) */ +#define LBER_LEN_T long + +/* ------------------------------------------------------------ */ + +/* booleans, enumerations, and integers */ +typedef LBER_INT_T ber_int_t; + +/* signed and unsigned versions */ +typedef signed LBER_INT_T ber_sint_t; +typedef unsigned LBER_INT_T ber_uint_t; + +/* tags */ +typedef unsigned LBER_TAG_T ber_tag_t; + +/* "socket" descriptors */ +typedef LBER_SOCKET_T ber_socket_t; + +/* lengths */ +typedef unsigned LBER_LEN_T ber_len_t; + +/* signed lengths */ +typedef signed LBER_LEN_T ber_slen_t; + +LDAP_END_DECL + +#endif /* _LBER_TYPES_H */ diff --git a/contrib/openldap-cmake/freebsd_x86_64/include/ldap_config.h b/contrib/openldap-cmake/freebsd_x86_64/include/ldap_config.h new file mode 100644 index 00000000000..89f7b40b884 --- /dev/null +++ b/contrib/openldap-cmake/freebsd_x86_64/include/ldap_config.h @@ -0,0 +1,74 @@ +/* include/ldap_config.h. Generated from ldap_config.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * This file works in conjunction with OpenLDAP configure system. + * If you do no like the values below, adjust your configure options. + */ + +#ifndef _LDAP_CONFIG_H +#define _LDAP_CONFIG_H + +/* directory separator */ +#ifndef LDAP_DIRSEP +#ifndef _WIN32 +#define LDAP_DIRSEP "/" +#else +#define LDAP_DIRSEP "\\" +#endif +#endif + +/* directory for temporary files */ +#if defined(_WIN32) +# define LDAP_TMPDIR "C:\\." /* we don't have much of a choice */ +#elif defined( _P_tmpdir ) +# define LDAP_TMPDIR _P_tmpdir +#elif defined( P_tmpdir ) +# define LDAP_TMPDIR P_tmpdir +#elif defined( _PATH_TMPDIR ) +# define LDAP_TMPDIR _PATH_TMPDIR +#else +# define LDAP_TMPDIR LDAP_DIRSEP "tmp" +#endif + +/* directories */ +#ifndef LDAP_BINDIR +#define LDAP_BINDIR "/tmp/ldap-prefix/bin" +#endif +#ifndef LDAP_SBINDIR +#define LDAP_SBINDIR "/tmp/ldap-prefix/sbin" +#endif +#ifndef LDAP_DATADIR +#define LDAP_DATADIR "/tmp/ldap-prefix/share/openldap" +#endif +#ifndef LDAP_SYSCONFDIR +#define LDAP_SYSCONFDIR "/tmp/ldap-prefix/etc/openldap" +#endif +#ifndef LDAP_LIBEXECDIR +#define LDAP_LIBEXECDIR "/tmp/ldap-prefix/libexec" +#endif +#ifndef LDAP_MODULEDIR +#define LDAP_MODULEDIR "/tmp/ldap-prefix/libexec/openldap" +#endif +#ifndef LDAP_RUNDIR +#define LDAP_RUNDIR "/tmp/ldap-prefix/var" +#endif +#ifndef LDAP_LOCALEDIR +#define LDAP_LOCALEDIR "" +#endif + + +#endif /* _LDAP_CONFIG_H */ diff --git a/contrib/openldap-cmake/freebsd_x86_64/include/ldap_features.h b/contrib/openldap-cmake/freebsd_x86_64/include/ldap_features.h new file mode 100644 index 00000000000..f0cc7c3626f --- /dev/null +++ b/contrib/openldap-cmake/freebsd_x86_64/include/ldap_features.h @@ -0,0 +1,61 @@ +/* include/ldap_features.h. Generated from ldap_features.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LDAP Features + */ + +#ifndef _LDAP_FEATURES_H +#define _LDAP_FEATURES_H 1 + +/* OpenLDAP API version macros */ +#define LDAP_VENDOR_VERSION 20501 +#define LDAP_VENDOR_VERSION_MAJOR 2 +#define LDAP_VENDOR_VERSION_MINOR 5 +#define LDAP_VENDOR_VERSION_PATCH X + +/* +** WORK IN PROGRESS! +** +** OpenLDAP reentrancy/thread-safeness should be dynamically +** checked using ldap_get_option(). +** +** The -lldap implementation is not thread-safe. +** +** The -lldap_r implementation is: +** LDAP_API_FEATURE_THREAD_SAFE (basic thread safety) +** but also be: +** LDAP_API_FEATURE_SESSION_THREAD_SAFE +** LDAP_API_FEATURE_OPERATION_THREAD_SAFE +** +** The preprocessor flag LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE +** can be used to determine if -lldap_r is available at compile +** time. You must define LDAP_THREAD_SAFE if and only if you +** link with -lldap_r. +** +** If you fail to define LDAP_THREAD_SAFE when linking with +** -lldap_r or define LDAP_THREAD_SAFE when linking with -lldap, +** provided header definitions and declarations may be incorrect. +** +*/ + +/* is -lldap_r available or not */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* LDAP v2 Referrals */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +#endif /* LDAP_FEATURES */ diff --git a/contrib/openldap-cmake/freebsd_x86_64/include/portable.h b/contrib/openldap-cmake/freebsd_x86_64/include/portable.h new file mode 100644 index 00000000000..10a15fe3ca1 --- /dev/null +++ b/contrib/openldap-cmake/freebsd_x86_64/include/portable.h @@ -0,0 +1,1169 @@ +/* include/portable.h. Generated from portable.hin by configure. */ +/* include/portable.hin. Generated from configure.in by autoheader. */ + + +/* begin of portable.h.pre */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#ifndef _LDAP_PORTABLE_H +#define _LDAP_PORTABLE_H + +/* define this if needed to get reentrant functions */ +#ifndef REENTRANT +#define REENTRANT 1 +#endif +#ifndef _REENTRANT +#define _REENTRANT 1 +#endif + +/* define this if needed to get threadsafe functions */ +#ifndef THREADSAFE +#define THREADSAFE 1 +#endif +#ifndef _THREADSAFE +#define _THREADSAFE 1 +#endif +#ifndef THREAD_SAFE +#define THREAD_SAFE 1 +#endif +#ifndef _THREAD_SAFE +#define _THREAD_SAFE 1 +#endif + +#ifndef _SGI_MP_SOURCE +#define _SGI_MP_SOURCE 1 +#endif + +/* end of portable.h.pre */ + + +/* Define if building universal (internal helper macro) */ +/* #undef AC_APPLE_UNIVERSAL_BUILD */ + +/* define to use both and */ +/* #undef BOTH_STRINGS_H */ + +/* define if cross compiling */ +/* #undef CROSS_COMPILING */ + +/* set to the number of arguments ctime_r() expects */ +#define CTIME_R_NARGS 2 + +/* define if toupper() requires islower() */ +/* #undef C_UPPER_LOWER */ + +/* define if sys_errlist is not declared in stdio.h or errno.h */ +/* #undef DECL_SYS_ERRLIST */ + +/* define to enable slapi library */ +/* #undef ENABLE_SLAPI */ + +/* defined to be the EXE extension */ +#define EXEEXT "" + +/* set to the number of arguments gethostbyaddr_r() expects */ +#define GETHOSTBYADDR_R_NARGS 8 + +/* set to the number of arguments gethostbyname_r() expects */ +#define GETHOSTBYNAME_R_NARGS 6 + +/* Define to 1 if `TIOCGWINSZ' requires . */ +/* #undef GWINSZ_IN_SYS_IOCTL */ + +/* define if you have AIX security lib */ +/* #undef HAVE_AIX_SECURITY */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H 1 + +/* Define to 1 if you have the `bcopy' function. */ +#define HAVE_BCOPY 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_BITS_TYPES_H */ + +/* Define to 1 if you have the `chroot' function. */ +#define HAVE_CHROOT 1 + +/* Define to 1 if you have the `closesocket' function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CONIO_H */ + +/* define if crypt(3) is available */ +/* #undef HAVE_CRYPT */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CRYPT_H */ + +/* define if crypt_r() is also available */ +/* #undef HAVE_CRYPT_R */ + +/* Define to 1 if you have the `ctime_r' function. */ +#define HAVE_CTIME_R 1 + +/* define if you have Cyrus SASL */ +/* #undef HAVE_CYRUS_SASL */ + +/* define if your system supports /dev/poll */ +/* #undef HAVE_DEVPOLL */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DIRECT_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#define HAVE_DIRENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ +/* #undef HAVE_DOPRNT */ + +/* define if system uses EBCDIC instead of ASCII */ +/* #undef HAVE_EBCDIC */ + +/* Define to 1 if you have the `endgrent' function. */ +#define HAVE_ENDGRENT 1 + +/* Define to 1 if you have the `endpwent' function. */ +#define HAVE_ENDPWENT 1 + +/* define if your system supports epoll */ +/* #undef HAVE_EPOLL */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H 1 + +/* Define to 1 if you have the `fcntl' function. */ +#define HAVE_FCNTL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H 1 + +/* define if you actually have FreeBSD fetch(3) */ +/* #undef HAVE_FETCH */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_FILIO_H */ + +/* Define to 1 if you have the `flock' function. */ +#define HAVE_FLOCK 1 + +/* Define to 1 if you have the `fstat' function. */ +#define HAVE_FSTAT 1 + +/* Define to 1 if you have the `gai_strerror' function. */ +#define HAVE_GAI_STRERROR 1 + +/* Define to 1 if you have the `getaddrinfo' function. */ +#define HAVE_GETADDRINFO 1 + +/* Define to 1 if you have the `getdtablesize' function. */ +#define HAVE_GETDTABLESIZE 1 + +/* Define to 1 if you have the `geteuid' function. */ +#define HAVE_GETEUID 1 + +/* Define to 1 if you have the `getgrgid' function. */ +#define HAVE_GETGRGID 1 + +/* Define to 1 if you have the `gethostbyaddr_r' function. */ +#define HAVE_GETHOSTBYADDR_R 1 + +/* Define to 1 if you have the `gethostbyname_r' function. */ +#define HAVE_GETHOSTBYNAME_R 1 + +/* Define to 1 if you have the `gethostname' function. */ +#define HAVE_GETHOSTNAME 1 + +/* Define to 1 if you have the `getnameinfo' function. */ +#define HAVE_GETNAMEINFO 1 + +/* Define to 1 if you have the `getopt' function. */ +#define HAVE_GETOPT 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_GETOPT_H 1 + +/* Define to 1 if you have the `getpassphrase' function. */ +/* #undef HAVE_GETPASSPHRASE */ + +/* Define to 1 if you have the `getpeereid' function. */ +#define HAVE_GETPEEREID 1 + +/* Define to 1 if you have the `getpeerucred' function. */ +/* #undef HAVE_GETPEERUCRED */ + +/* Define to 1 if you have the `getpwnam' function. */ +#define HAVE_GETPWNAM 1 + +/* Define to 1 if you have the `getpwuid' function. */ +#define HAVE_GETPWUID 1 + +/* Define to 1 if you have the `getspnam' function. */ +/* #undef HAVE_GETSPNAM */ + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GMP_H */ + +/* Define to 1 if you have the `gmtime_r' function. */ +#define HAVE_GMTIME_R 1 + +/* define if you have GNUtls */ +/* #undef HAVE_GNUTLS */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GNUTLS_GNUTLS_H */ + +/* if you have GNU Pth */ +/* #undef HAVE_GNU_PTH */ + +/* Define to 1 if you have the header file. */ +#define HAVE_GRP_H 1 + +/* Define to 1 if you have the `hstrerror' function. */ +#define HAVE_HSTRERROR 1 + +/* define to you inet_aton(3) is available */ +#define HAVE_INET_ATON 1 + +/* Define to 1 if you have the `inet_ntoa_b' function. */ +/* #undef HAVE_INET_NTOA_B */ + +/* Define to 1 if you have the `inet_ntop' function. */ +#define HAVE_INET_NTOP 1 + +/* Define to 1 if you have the `initgroups' function. */ +#define HAVE_INITGROUPS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the `ioctl' function. */ +#define HAVE_IOCTL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_IO_H */ + +/* define if your system supports kqueue */ +#define HAVE_KQUEUE 1 + +/* Define to 1 if you have the `gen' library (-lgen). */ +/* #undef HAVE_LIBGEN */ + +/* Define to 1 if you have the `gmp' library (-lgmp). */ +/* #undef HAVE_LIBGMP */ + +/* Define to 1 if you have the `inet' library (-linet). */ +/* #undef HAVE_LIBINET */ + +/* define if you have libtool -ltdl */ +/* #undef HAVE_LIBLTDL */ + +/* Define to 1 if you have the `net' library (-lnet). */ +/* #undef HAVE_LIBNET */ + +/* Define to 1 if you have the `nsl' library (-lnsl). */ +/* #undef HAVE_LIBNSL */ + +/* Define to 1 if you have the `nsl_s' library (-lnsl_s). */ +/* #undef HAVE_LIBNSL_S */ + +/* Define to 1 if you have the `socket' library (-lsocket). */ +/* #undef HAVE_LIBSOCKET */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIBUTIL_H 1 + +/* Define to 1 if you have the `V3' library (-lV3). */ +/* #undef HAVE_LIBV3 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* if you have LinuxThreads */ +/* #undef HAVE_LINUX_THREADS */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LOCALE_H 1 + +/* Define to 1 if you have the `localtime_r' function. */ +#define HAVE_LOCALTIME_R 1 + +/* Define to 1 if you have the `lockf' function. */ +#define HAVE_LOCKF 1 + +/* Define to 1 if the system has the type `long long'. */ +#define HAVE_LONG_LONG 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LTDL_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_MALLOC_H */ + +/* Define to 1 if you have the `memcpy' function. */ +#define HAVE_MEMCPY 1 + +/* Define to 1 if you have the `memmove' function. */ +#define HAVE_MEMMOVE 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `memrchr' function. */ +#define HAVE_MEMRCHR 1 + +/* Define to 1 if you have the `mkstemp' function. */ +#define HAVE_MKSTEMP 1 + +/* Define to 1 if you have the `mktemp' function. */ +#define HAVE_MKTEMP 1 + +/* define this if you have mkversion */ +#define HAVE_MKVERSION 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. */ +/* #undef HAVE_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H 1 + +/* define if strerror_r returns char* instead of int */ +/* #undef HAVE_NONPOSIX_STRERROR_R */ + +/* if you have NT Event Log */ +/* #undef HAVE_NT_EVENT_LOG */ + +/* if you have NT Service Manager */ +/* #undef HAVE_NT_SERVICE_MANAGER */ + +/* if you have NT Threads */ +/* #undef HAVE_NT_THREADS */ + +/* define if you have OpenSSL */ +#define HAVE_OPENSSL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_BN_H 1 + +/* define if you have OpenSSL with CRL checking capability */ +#define HAVE_OPENSSL_CRL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_CRYPTO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_SSL_H 1 + +/* Define to 1 if you have the `pipe' function. */ +#define HAVE_PIPE 1 + +/* Define to 1 if you have the `poll' function. */ +#define HAVE_POLL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PROCESS_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PSAP_H */ + +/* define to pthreads API spec revision */ +#define HAVE_PTHREADS 10 + +/* define if you have pthread_detach function */ +#define HAVE_PTHREAD_DETACH 1 + +/* Define to 1 if you have the `pthread_getconcurrency' function. */ +#define HAVE_PTHREAD_GETCONCURRENCY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PTHREAD_H 1 + +/* Define to 1 if you have the `pthread_kill' function. */ +#define HAVE_PTHREAD_KILL 1 + +/* Define to 1 if you have the `pthread_kill_other_threads_np' function. */ +/* #undef HAVE_PTHREAD_KILL_OTHER_THREADS_NP */ + +/* define if you have pthread_rwlock_destroy function */ +#define HAVE_PTHREAD_RWLOCK_DESTROY 1 + +/* Define to 1 if you have the `pthread_setconcurrency' function. */ +#define HAVE_PTHREAD_SETCONCURRENCY 1 + +/* Define to 1 if you have the `pthread_yield' function. */ +#define HAVE_PTHREAD_YIELD 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PTH_H */ + +/* Define to 1 if the system has the type `ptrdiff_t'. */ +#define HAVE_PTRDIFF_T 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PWD_H 1 + +/* Define to 1 if you have the `read' function. */ +#define HAVE_READ 1 + +/* Define to 1 if you have the `recv' function. */ +#define HAVE_RECV 1 + +/* Define to 1 if you have the `recvfrom' function. */ +#define HAVE_RECVFROM 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_REGEX_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_RESOLV_H */ + +/* define if you have res_query() */ +/* #undef HAVE_RES_QUERY */ + +/* define if OpenSSL needs RSAref */ +/* #undef HAVE_RSAREF */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_SASL_H */ + +/* define if your SASL library has sasl_version() */ +/* #undef HAVE_SASL_VERSION */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SCHED_H 1 + +/* Define to 1 if you have the `sched_yield' function. */ +#define HAVE_SCHED_YIELD 1 + +/* Define to 1 if you have the `send' function. */ +#define HAVE_SEND 1 + +/* Define to 1 if you have the `sendmsg' function. */ +#define HAVE_SENDMSG 1 + +/* Define to 1 if you have the `sendto' function. */ +#define HAVE_SENDTO 1 + +/* Define to 1 if you have the `setegid' function. */ +#define HAVE_SETEGID 1 + +/* Define to 1 if you have the `seteuid' function. */ +#define HAVE_SETEUID 1 + +/* Define to 1 if you have the `setgid' function. */ +#define HAVE_SETGID 1 + +/* Define to 1 if you have the `setpwfile' function. */ +/* #undef HAVE_SETPWFILE */ + +/* Define to 1 if you have the `setsid' function. */ +#define HAVE_SETSID 1 + +/* Define to 1 if you have the `setuid' function. */ +#define HAVE_SETUID 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SGTTY_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SHADOW_H */ + +/* Define to 1 if you have the `sigaction' function. */ +#define HAVE_SIGACTION 1 + +/* Define to 1 if you have the `signal' function. */ +#define HAVE_SIGNAL 1 + +/* Define to 1 if you have the `sigset' function. */ +#define HAVE_SIGSET 1 + +/* define if you have -lslp */ +/* #undef HAVE_SLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SLP_H */ + +/* Define to 1 if you have the `snprintf' function. */ +#define HAVE_SNPRINTF 1 + +/* if you have spawnlp() */ +/* #undef HAVE_SPAWNLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQLEXT_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQL_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDDEF_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strdup' function. */ +#define HAVE_STRDUP 1 + +/* Define to 1 if you have the `strerror' function. */ +#define HAVE_STRERROR 1 + +/* Define to 1 if you have the `strerror_r' function. */ +#define HAVE_STRERROR_R 1 + +/* Define to 1 if you have the `strftime' function. */ +#define HAVE_STRFTIME 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strpbrk' function. */ +#define HAVE_STRPBRK 1 + +/* Define to 1 if you have the `strrchr' function. */ +#define HAVE_STRRCHR 1 + +/* Define to 1 if you have the `strsep' function. */ +#define HAVE_STRSEP 1 + +/* Define to 1 if you have the `strspn' function. */ +#define HAVE_STRSPN 1 + +/* Define to 1 if you have the `strstr' function. */ +#define HAVE_STRSTR 1 + +/* Define to 1 if you have the `strtol' function. */ +#define HAVE_STRTOL 1 + +/* Define to 1 if you have the `strtoll' function. */ +#define HAVE_STRTOLL 1 + +/* Define to 1 if you have the `strtoq' function. */ +#define HAVE_STRTOQ 1 + +/* Define to 1 if you have the `strtoul' function. */ +#define HAVE_STRTOUL 1 + +/* Define to 1 if you have the `strtoull' function. */ +#define HAVE_STRTOULL 1 + +/* Define to 1 if you have the `strtouq' function. */ +#define HAVE_STRTOUQ 1 + +/* Define to 1 if `msg_accrightslen' is a member of `struct msghdr'. */ +/* #undef HAVE_STRUCT_MSGHDR_MSG_ACCRIGHTSLEN */ + +/* Define to 1 if `msg_control' is a member of `struct msghdr'. */ +/* #undef HAVE_STRUCT_MSGHDR_MSG_CONTROL */ + +/* Define to 1 if `pw_gecos' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_GECOS 1 + +/* Define to 1 if `pw_passwd' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_PASSWD 1 + +/* Define to 1 if `st_blksize' is a member of `struct stat'. */ +#define HAVE_STRUCT_STAT_ST_BLKSIZE 1 + +/* Define to 1 if `st_fstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE */ + +/* define to 1 if st_fstype is char * */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_CHAR */ + +/* define to 1 if st_fstype is int */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_INT */ + +/* Define to 1 if `st_vfstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_VFSTYPE */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYNCH_H */ + +/* Define to 1 if you have the `sysconf' function. */ +#define HAVE_SYSCONF 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSEXITS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_DEVPOLL_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_DIR_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_EPOLL_H */ + +/* define if you actually have sys_errlist in your libs */ +#define HAVE_SYS_ERRLIST 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_ERRNO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_EVENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_FILE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_FILIO_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_FSTYP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_PRIVGRP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_RESOURCE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UCRED_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UN_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_UUID_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_VMOUNT_H */ + +/* Define to 1 if you have that is POSIX.1 compatible. */ +#define HAVE_SYS_WAIT_H 1 + +/* define if you have -lwrap */ +/* #undef HAVE_TCPD */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_TCPD_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_TERMIOS_H 1 + +/* if you have Solaris LWP (thr) package */ +/* #undef HAVE_THR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_THREAD_H */ + +/* Define to 1 if you have the `thr_getconcurrency' function. */ +/* #undef HAVE_THR_GETCONCURRENCY */ + +/* Define to 1 if you have the `thr_setconcurrency' function. */ +/* #undef HAVE_THR_SETCONCURRENCY */ + +/* Define to 1 if you have the `thr_yield' function. */ +/* #undef HAVE_THR_YIELD */ + +/* define if you have TLS */ +#define HAVE_TLS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UTIME_H 1 + +/* define if you have uuid_generate() */ +/* #undef HAVE_UUID_GENERATE */ + +/* define if you have uuid_to_str() */ +/* #undef HAVE_UUID_TO_STR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_UUID_UUID_H */ + +/* Define to 1 if you have the `vprintf' function. */ +#define HAVE_VPRINTF 1 + +/* Define to 1 if you have the `vsnprintf' function. */ +#define HAVE_VSNPRINTF 1 + +/* Define to 1 if you have the `wait4' function. */ +#define HAVE_WAIT4 1 + +/* Define to 1 if you have the `waitpid' function. */ +#define HAVE_WAITPID 1 + +/* define if you have winsock */ +/* #undef HAVE_WINSOCK */ + +/* define if you have winsock2 */ +/* #undef HAVE_WINSOCK2 */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WIREDTIGER_H */ + +/* Define to 1 if you have the `write' function. */ +#define HAVE_WRITE 1 + +/* define if select implicitly yields */ +#define HAVE_YIELDING_SELECT 1 + +/* Define to 1 if you have the `_vsnprintf' function. */ +/* #undef HAVE__VSNPRINTF */ + +/* define to 32-bit or greater integer type */ +#define LBER_INT_T int + +/* define to large integer type */ +#define LBER_LEN_T long + +/* define to socket descriptor type */ +#define LBER_SOCKET_T int + +/* define to large integer type */ +#define LBER_TAG_T long + +/* define to 1 if library is thread safe */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* define to LDAP VENDOR VERSION */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +/* define this to add debugging code */ +/* #undef LDAP_DEBUG */ + +/* define if LDAP libs are dynamic */ +/* #undef LDAP_LIBS_DYNAMIC */ + +/* define to support PF_INET6 */ +#define LDAP_PF_INET6 1 + +/* define to support PF_LOCAL */ +#define LDAP_PF_LOCAL 1 + +/* define this to add SLAPI code */ +/* #undef LDAP_SLAPI */ + +/* define this to add syslog code */ +/* #undef LDAP_SYSLOG */ + +/* Version */ +#define LDAP_VENDOR_VERSION 20501 + +/* Major */ +#define LDAP_VENDOR_VERSION_MAJOR 2 + +/* Minor */ +#define LDAP_VENDOR_VERSION_MINOR 5 + +/* Patch */ +#define LDAP_VENDOR_VERSION_PATCH X + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* define if memcmp is not 8-bit clean or is otherwise broken */ +/* #undef NEED_MEMCMP_REPLACEMENT */ + +/* define if you have (or want) no threads */ +/* #undef NO_THREADS */ + +/* define to use the original debug style */ +/* #undef OLD_DEBUG */ + +/* Package */ +#define OPENLDAP_PACKAGE "OpenLDAP" + +/* Version */ +#define OPENLDAP_VERSION "2.5.X" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "" + +/* define if sched_yield yields the entire process */ +/* #undef REPLACE_BROKEN_YIELD */ + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE void + +/* Define to the type of arg 1 for `select'. */ +#define SELECT_TYPE_ARG1 int + +/* Define to the type of args 2, 3 and 4 for `select'. */ +#define SELECT_TYPE_ARG234 (fd_set *) + +/* Define to the type of arg 5 for `select'. */ +#define SELECT_TYPE_ARG5 (struct timeval *) + +/* The size of `int', as computed by sizeof. */ +#define SIZEOF_INT 4 + +/* The size of `long', as computed by sizeof. */ +#define SIZEOF_LONG 8 + +/* The size of `long long', as computed by sizeof. */ +#define SIZEOF_LONG_LONG 8 + +/* The size of `short', as computed by sizeof. */ +#define SIZEOF_SHORT 2 + +/* The size of `wchar_t', as computed by sizeof. */ +#define SIZEOF_WCHAR_T 4 + +/* define to support per-object ACIs */ +/* #undef SLAPD_ACI_ENABLED */ + +/* define to support LDAP Async Metadirectory backend */ +/* #undef SLAPD_ASYNCMETA */ + +/* define to support cleartext passwords */ +/* #undef SLAPD_CLEARTEXT */ + +/* define to support crypt(3) passwords */ +/* #undef SLAPD_CRYPT */ + +/* define to support DNS SRV backend */ +/* #undef SLAPD_DNSSRV */ + +/* define to support LDAP backend */ +/* #undef SLAPD_LDAP */ + +/* define to support MDB backend */ +/* #undef SLAPD_MDB */ + +/* define to support LDAP Metadirectory backend */ +/* #undef SLAPD_META */ + +/* define to support modules */ +/* #undef SLAPD_MODULES */ + +/* dynamically linked module */ +#define SLAPD_MOD_DYNAMIC 2 + +/* statically linked module */ +#define SLAPD_MOD_STATIC 1 + +/* define to support cn=Monitor backend */ +/* #undef SLAPD_MONITOR */ + +/* define to support NDB backend */ +/* #undef SLAPD_NDB */ + +/* define to support NULL backend */ +/* #undef SLAPD_NULL */ + +/* define for In-Directory Access Logging overlay */ +/* #undef SLAPD_OVER_ACCESSLOG */ + +/* define for Audit Logging overlay */ +/* #undef SLAPD_OVER_AUDITLOG */ + +/* define for Automatic Certificate Authority overlay */ +/* #undef SLAPD_OVER_AUTOCA */ + +/* define for Collect overlay */ +/* #undef SLAPD_OVER_COLLECT */ + +/* define for Attribute Constraint overlay */ +/* #undef SLAPD_OVER_CONSTRAINT */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DDS */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DEREF */ + +/* define for Dynamic Group overlay */ +/* #undef SLAPD_OVER_DYNGROUP */ + +/* define for Dynamic List overlay */ +/* #undef SLAPD_OVER_DYNLIST */ + +/* define for Reverse Group Membership overlay */ +/* #undef SLAPD_OVER_MEMBEROF */ + +/* define for Password Policy overlay */ +/* #undef SLAPD_OVER_PPOLICY */ + +/* define for Proxy Cache overlay */ +/* #undef SLAPD_OVER_PROXYCACHE */ + +/* define for Referential Integrity overlay */ +/* #undef SLAPD_OVER_REFINT */ + +/* define for Return Code overlay */ +/* #undef SLAPD_OVER_RETCODE */ + +/* define for Rewrite/Remap overlay */ +/* #undef SLAPD_OVER_RWM */ + +/* define for Sequential Modify overlay */ +/* #undef SLAPD_OVER_SEQMOD */ + +/* define for ServerSideSort/VLV overlay */ +/* #undef SLAPD_OVER_SSSVLV */ + +/* define for Syncrepl Provider overlay */ +/* #undef SLAPD_OVER_SYNCPROV */ + +/* define for Translucent Proxy overlay */ +/* #undef SLAPD_OVER_TRANSLUCENT */ + +/* define for Attribute Uniqueness overlay */ +/* #undef SLAPD_OVER_UNIQUE */ + +/* define for Value Sorting overlay */ +/* #undef SLAPD_OVER_VALSORT */ + +/* define to support PASSWD backend */ +/* #undef SLAPD_PASSWD */ + +/* define to support PERL backend */ +/* #undef SLAPD_PERL */ + +/* define to support relay backend */ +/* #undef SLAPD_RELAY */ + +/* define to support reverse lookups */ +/* #undef SLAPD_RLOOKUPS */ + +/* define to support SHELL backend */ +/* #undef SLAPD_SHELL */ + +/* define to support SOCK backend */ +/* #undef SLAPD_SOCK */ + +/* define to support SASL passwords */ +/* #undef SLAPD_SPASSWD */ + +/* define to support SQL backend */ +/* #undef SLAPD_SQL */ + +/* define to support WiredTiger backend */ +/* #undef SLAPD_WT */ + +/* define to support run-time loadable ACL */ +/* #undef SLAP_DYNACL */ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME 1 + +/* Define to 1 if your declares `struct tm'. */ +/* #undef TM_IN_SYS_TIME */ + +/* set to urandom device */ +#define URANDOM_DEVICE "/dev/urandom" + +/* define to use OpenSSL BIGNUM for MP */ +/* #undef USE_MP_BIGNUM */ + +/* define to use GMP for MP */ +/* #undef USE_MP_GMP */ + +/* define to use 'long' for MP */ +/* #undef USE_MP_LONG */ + +/* define to use 'long long' for MP */ +/* #undef USE_MP_LONG_LONG */ + +/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most + significant byte first (like Motorola and SPARC, unlike Intel). */ +#if defined AC_APPLE_UNIVERSAL_BUILD +# if defined __BIG_ENDIAN__ +# define WORDS_BIGENDIAN 1 +# endif +#else +# ifndef WORDS_BIGENDIAN +/* # undef WORDS_BIGENDIAN */ +# endif +#endif + +/* Define to the type of arg 3 for `accept'. */ +#define ber_socklen_t socklen_t + +/* Define to `char *' if does not define. */ +/* #undef caddr_t */ + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to `int' if doesn't define. */ +/* #undef gid_t */ + +/* Define to `int' if does not define. */ +/* #undef mode_t */ + +/* Define to `long' if does not define. */ +/* #undef off_t */ + +/* Define to `int' if does not define. */ +/* #undef pid_t */ + +/* Define to `int' if does not define. */ +/* #undef sig_atomic_t */ + +/* Define to `unsigned' if does not define. */ +/* #undef size_t */ + +/* define to snprintf routine */ +/* #undef snprintf */ + +/* Define like ber_socklen_t if does not define. */ +/* #undef socklen_t */ + +/* Define to `signed int' if does not define. */ +/* #undef ssize_t */ + +/* Define to `int' if doesn't define. */ +/* #undef uid_t */ + +/* define as empty if volatile is not supported */ +/* #undef volatile */ + +/* define to snprintf routine */ +/* #undef vsnprintf */ + + +/* begin of portable.h.post */ + +#ifdef _WIN32 +/* don't suck in all of the win32 api */ +# define WIN32_LEAN_AND_MEAN 1 +#endif + +#ifndef LDAP_NEEDS_PROTOTYPES +/* force LDAP_P to always include prototypes */ +#define LDAP_NEEDS_PROTOTYPES 1 +#endif + +#ifndef LDAP_REL_ENG +#if (LDAP_VENDOR_VERSION == 000000) && !defined(LDAP_DEVEL) +#define LDAP_DEVEL +#endif +#if defined(LDAP_DEVEL) && !defined(LDAP_TEST) +#define LDAP_TEST +#endif +#endif + +#ifdef HAVE_STDDEF_H +# include +#endif + +#ifdef HAVE_EBCDIC +/* ASCII/EBCDIC converting replacements for stdio funcs + * vsnprintf and snprintf are used too, but they are already + * checked by the configure script + */ +#define fputs ber_pvt_fputs +#define fgets ber_pvt_fgets +#define printf ber_pvt_printf +#define fprintf ber_pvt_fprintf +#define vfprintf ber_pvt_vfprintf +#define vsprintf ber_pvt_vsprintf +#endif + +#include "ac/fdset.h" + +#include "ldap_cdefs.h" +#include "ldap_features.h" + +#include "ac/assert.h" +#include "ac/localize.h" + +#endif /* _LDAP_PORTABLE_H */ +/* end of portable.h.post */ + diff --git a/contrib/openldap-cmake/linux_aarch64/include/lber_types.h b/contrib/openldap-cmake/linux_aarch64/include/lber_types.h new file mode 100644 index 00000000000..dbd59430527 --- /dev/null +++ b/contrib/openldap-cmake/linux_aarch64/include/lber_types.h @@ -0,0 +1,63 @@ +/* include/lber_types.h. Generated from lber_types.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LBER types + */ + +#ifndef _LBER_TYPES_H +#define _LBER_TYPES_H + +#include + +LDAP_BEGIN_DECL + +/* LBER boolean, enum, integers (32 bits or larger) */ +#define LBER_INT_T int + +/* LBER tags (32 bits or larger) */ +#define LBER_TAG_T long + +/* LBER socket descriptor */ +#define LBER_SOCKET_T int + +/* LBER lengths (32 bits or larger) */ +#define LBER_LEN_T long + +/* ------------------------------------------------------------ */ + +/* booleans, enumerations, and integers */ +typedef LBER_INT_T ber_int_t; + +/* signed and unsigned versions */ +typedef signed LBER_INT_T ber_sint_t; +typedef unsigned LBER_INT_T ber_uint_t; + +/* tags */ +typedef unsigned LBER_TAG_T ber_tag_t; + +/* "socket" descriptors */ +typedef LBER_SOCKET_T ber_socket_t; + +/* lengths */ +typedef unsigned LBER_LEN_T ber_len_t; + +/* signed lengths */ +typedef signed LBER_LEN_T ber_slen_t; + +LDAP_END_DECL + +#endif /* _LBER_TYPES_H */ diff --git a/contrib/openldap-cmake/linux_aarch64/include/ldap_config.h b/contrib/openldap-cmake/linux_aarch64/include/ldap_config.h new file mode 100644 index 00000000000..89f7b40b884 --- /dev/null +++ b/contrib/openldap-cmake/linux_aarch64/include/ldap_config.h @@ -0,0 +1,74 @@ +/* include/ldap_config.h. Generated from ldap_config.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * This file works in conjunction with OpenLDAP configure system. + * If you do no like the values below, adjust your configure options. + */ + +#ifndef _LDAP_CONFIG_H +#define _LDAP_CONFIG_H + +/* directory separator */ +#ifndef LDAP_DIRSEP +#ifndef _WIN32 +#define LDAP_DIRSEP "/" +#else +#define LDAP_DIRSEP "\\" +#endif +#endif + +/* directory for temporary files */ +#if defined(_WIN32) +# define LDAP_TMPDIR "C:\\." /* we don't have much of a choice */ +#elif defined( _P_tmpdir ) +# define LDAP_TMPDIR _P_tmpdir +#elif defined( P_tmpdir ) +# define LDAP_TMPDIR P_tmpdir +#elif defined( _PATH_TMPDIR ) +# define LDAP_TMPDIR _PATH_TMPDIR +#else +# define LDAP_TMPDIR LDAP_DIRSEP "tmp" +#endif + +/* directories */ +#ifndef LDAP_BINDIR +#define LDAP_BINDIR "/tmp/ldap-prefix/bin" +#endif +#ifndef LDAP_SBINDIR +#define LDAP_SBINDIR "/tmp/ldap-prefix/sbin" +#endif +#ifndef LDAP_DATADIR +#define LDAP_DATADIR "/tmp/ldap-prefix/share/openldap" +#endif +#ifndef LDAP_SYSCONFDIR +#define LDAP_SYSCONFDIR "/tmp/ldap-prefix/etc/openldap" +#endif +#ifndef LDAP_LIBEXECDIR +#define LDAP_LIBEXECDIR "/tmp/ldap-prefix/libexec" +#endif +#ifndef LDAP_MODULEDIR +#define LDAP_MODULEDIR "/tmp/ldap-prefix/libexec/openldap" +#endif +#ifndef LDAP_RUNDIR +#define LDAP_RUNDIR "/tmp/ldap-prefix/var" +#endif +#ifndef LDAP_LOCALEDIR +#define LDAP_LOCALEDIR "" +#endif + + +#endif /* _LDAP_CONFIG_H */ diff --git a/contrib/openldap-cmake/linux_aarch64/include/ldap_features.h b/contrib/openldap-cmake/linux_aarch64/include/ldap_features.h new file mode 100644 index 00000000000..f0cc7c3626f --- /dev/null +++ b/contrib/openldap-cmake/linux_aarch64/include/ldap_features.h @@ -0,0 +1,61 @@ +/* include/ldap_features.h. Generated from ldap_features.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LDAP Features + */ + +#ifndef _LDAP_FEATURES_H +#define _LDAP_FEATURES_H 1 + +/* OpenLDAP API version macros */ +#define LDAP_VENDOR_VERSION 20501 +#define LDAP_VENDOR_VERSION_MAJOR 2 +#define LDAP_VENDOR_VERSION_MINOR 5 +#define LDAP_VENDOR_VERSION_PATCH X + +/* +** WORK IN PROGRESS! +** +** OpenLDAP reentrancy/thread-safeness should be dynamically +** checked using ldap_get_option(). +** +** The -lldap implementation is not thread-safe. +** +** The -lldap_r implementation is: +** LDAP_API_FEATURE_THREAD_SAFE (basic thread safety) +** but also be: +** LDAP_API_FEATURE_SESSION_THREAD_SAFE +** LDAP_API_FEATURE_OPERATION_THREAD_SAFE +** +** The preprocessor flag LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE +** can be used to determine if -lldap_r is available at compile +** time. You must define LDAP_THREAD_SAFE if and only if you +** link with -lldap_r. +** +** If you fail to define LDAP_THREAD_SAFE when linking with +** -lldap_r or define LDAP_THREAD_SAFE when linking with -lldap, +** provided header definitions and declarations may be incorrect. +** +*/ + +/* is -lldap_r available or not */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* LDAP v2 Referrals */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +#endif /* LDAP_FEATURES */ diff --git a/contrib/openldap-cmake/linux_aarch64/include/portable.h b/contrib/openldap-cmake/linux_aarch64/include/portable.h new file mode 100644 index 00000000000..2924b6713a4 --- /dev/null +++ b/contrib/openldap-cmake/linux_aarch64/include/portable.h @@ -0,0 +1,1169 @@ +/* include/portable.h. Generated from portable.hin by configure. */ +/* include/portable.hin. Generated from configure.in by autoheader. */ + + +/* begin of portable.h.pre */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#ifndef _LDAP_PORTABLE_H +#define _LDAP_PORTABLE_H + +/* define this if needed to get reentrant functions */ +#ifndef REENTRANT +#define REENTRANT 1 +#endif +#ifndef _REENTRANT +#define _REENTRANT 1 +#endif + +/* define this if needed to get threadsafe functions */ +#ifndef THREADSAFE +#define THREADSAFE 1 +#endif +#ifndef _THREADSAFE +#define _THREADSAFE 1 +#endif +#ifndef THREAD_SAFE +#define THREAD_SAFE 1 +#endif +#ifndef _THREAD_SAFE +#define _THREAD_SAFE 1 +#endif + +#ifndef _SGI_MP_SOURCE +#define _SGI_MP_SOURCE 1 +#endif + +/* end of portable.h.pre */ + + +/* Define if building universal (internal helper macro) */ +/* #undef AC_APPLE_UNIVERSAL_BUILD */ + +/* define to use both and */ +/* #undef BOTH_STRINGS_H */ + +/* define if cross compiling */ +/* #undef CROSS_COMPILING */ + +/* set to the number of arguments ctime_r() expects */ +#define CTIME_R_NARGS 2 + +/* define if toupper() requires islower() */ +/* #undef C_UPPER_LOWER */ + +/* define if sys_errlist is not declared in stdio.h or errno.h */ +/* #undef DECL_SYS_ERRLIST */ + +/* define to enable slapi library */ +/* #undef ENABLE_SLAPI */ + +/* defined to be the EXE extension */ +#define EXEEXT "" + +/* set to the number of arguments gethostbyaddr_r() expects */ +#define GETHOSTBYADDR_R_NARGS 8 + +/* set to the number of arguments gethostbyname_r() expects */ +#define GETHOSTBYNAME_R_NARGS 6 + +/* Define to 1 if `TIOCGWINSZ' requires . */ +#define GWINSZ_IN_SYS_IOCTL 1 + +/* define if you have AIX security lib */ +/* #undef HAVE_AIX_SECURITY */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H 1 + +/* Define to 1 if you have the `bcopy' function. */ +#define HAVE_BCOPY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_BITS_TYPES_H 1 + +/* Define to 1 if you have the `chroot' function. */ +#define HAVE_CHROOT 1 + +/* Define to 1 if you have the `closesocket' function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CONIO_H */ + +/* define if crypt(3) is available */ +/* #undef HAVE_CRYPT */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CRYPT_H */ + +/* define if crypt_r() is also available */ +/* #undef HAVE_CRYPT_R */ + +/* Define to 1 if you have the `ctime_r' function. */ +#define HAVE_CTIME_R 1 + +/* define if you have Cyrus SASL */ +/* #undef HAVE_CYRUS_SASL */ + +/* define if your system supports /dev/poll */ +/* #undef HAVE_DEVPOLL */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DIRECT_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#define HAVE_DIRENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ +/* #undef HAVE_DOPRNT */ + +/* define if system uses EBCDIC instead of ASCII */ +/* #undef HAVE_EBCDIC */ + +/* Define to 1 if you have the `endgrent' function. */ +#define HAVE_ENDGRENT 1 + +/* Define to 1 if you have the `endpwent' function. */ +#define HAVE_ENDPWENT 1 + +/* define if your system supports epoll */ +#define HAVE_EPOLL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H 1 + +/* Define to 1 if you have the `fcntl' function. */ +#define HAVE_FCNTL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H 1 + +/* define if you actually have FreeBSD fetch(3) */ +/* #undef HAVE_FETCH */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_FILIO_H */ + +/* Define to 1 if you have the `flock' function. */ +#define HAVE_FLOCK 1 + +/* Define to 1 if you have the `fstat' function. */ +#define HAVE_FSTAT 1 + +/* Define to 1 if you have the `gai_strerror' function. */ +#define HAVE_GAI_STRERROR 1 + +/* Define to 1 if you have the `getaddrinfo' function. */ +#define HAVE_GETADDRINFO 1 + +/* Define to 1 if you have the `getdtablesize' function. */ +#define HAVE_GETDTABLESIZE 1 + +/* Define to 1 if you have the `geteuid' function. */ +#define HAVE_GETEUID 1 + +/* Define to 1 if you have the `getgrgid' function. */ +#define HAVE_GETGRGID 1 + +/* Define to 1 if you have the `gethostbyaddr_r' function. */ +#define HAVE_GETHOSTBYADDR_R 1 + +/* Define to 1 if you have the `gethostbyname_r' function. */ +#define HAVE_GETHOSTBYNAME_R 1 + +/* Define to 1 if you have the `gethostname' function. */ +#define HAVE_GETHOSTNAME 1 + +/* Define to 1 if you have the `getnameinfo' function. */ +#define HAVE_GETNAMEINFO 1 + +/* Define to 1 if you have the `getopt' function. */ +#define HAVE_GETOPT 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_GETOPT_H 1 + +/* Define to 1 if you have the `getpassphrase' function. */ +/* #undef HAVE_GETPASSPHRASE */ + +/* Define to 1 if you have the `getpeereid' function. */ +/* #undef HAVE_GETPEEREID */ + +/* Define to 1 if you have the `getpeerucred' function. */ +/* #undef HAVE_GETPEERUCRED */ + +/* Define to 1 if you have the `getpwnam' function. */ +#define HAVE_GETPWNAM 1 + +/* Define to 1 if you have the `getpwuid' function. */ +#define HAVE_GETPWUID 1 + +/* Define to 1 if you have the `getspnam' function. */ +#define HAVE_GETSPNAM 1 + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GMP_H */ + +/* Define to 1 if you have the `gmtime_r' function. */ +#define HAVE_GMTIME_R 1 + +/* define if you have GNUtls */ +/* #undef HAVE_GNUTLS */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GNUTLS_GNUTLS_H */ + +/* if you have GNU Pth */ +/* #undef HAVE_GNU_PTH */ + +/* Define to 1 if you have the header file. */ +#define HAVE_GRP_H 1 + +/* Define to 1 if you have the `hstrerror' function. */ +#define HAVE_HSTRERROR 1 + +/* define to you inet_aton(3) is available */ +#define HAVE_INET_ATON 1 + +/* Define to 1 if you have the `inet_ntoa_b' function. */ +/* #undef HAVE_INET_NTOA_B */ + +/* Define to 1 if you have the `inet_ntop' function. */ +#define HAVE_INET_NTOP 1 + +/* Define to 1 if you have the `initgroups' function. */ +#define HAVE_INITGROUPS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the `ioctl' function. */ +#define HAVE_IOCTL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_IO_H */ + +/* define if your system supports kqueue */ +/* #undef HAVE_KQUEUE */ + +/* Define to 1 if you have the `gen' library (-lgen). */ +/* #undef HAVE_LIBGEN */ + +/* Define to 1 if you have the `gmp' library (-lgmp). */ +/* #undef HAVE_LIBGMP */ + +/* Define to 1 if you have the `inet' library (-linet). */ +/* #undef HAVE_LIBINET */ + +/* define if you have libtool -ltdl */ +/* #undef HAVE_LIBLTDL */ + +/* Define to 1 if you have the `net' library (-lnet). */ +/* #undef HAVE_LIBNET */ + +/* Define to 1 if you have the `nsl' library (-lnsl). */ +/* #undef HAVE_LIBNSL */ + +/* Define to 1 if you have the `nsl_s' library (-lnsl_s). */ +/* #undef HAVE_LIBNSL_S */ + +/* Define to 1 if you have the `socket' library (-lsocket). */ +/* #undef HAVE_LIBSOCKET */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LIBUTIL_H */ + +/* Define to 1 if you have the `V3' library (-lV3). */ +/* #undef HAVE_LIBV3 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* if you have LinuxThreads */ +/* #undef HAVE_LINUX_THREADS */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LOCALE_H 1 + +/* Define to 1 if you have the `localtime_r' function. */ +#define HAVE_LOCALTIME_R 1 + +/* Define to 1 if you have the `lockf' function. */ +#define HAVE_LOCKF 1 + +/* Define to 1 if the system has the type `long long'. */ +#define HAVE_LONG_LONG 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LTDL_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_MALLOC_H 1 + +/* Define to 1 if you have the `memcpy' function. */ +#define HAVE_MEMCPY 1 + +/* Define to 1 if you have the `memmove' function. */ +#define HAVE_MEMMOVE 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `memrchr' function. */ +#define HAVE_MEMRCHR 1 + +/* Define to 1 if you have the `mkstemp' function. */ +#define HAVE_MKSTEMP 1 + +/* Define to 1 if you have the `mktemp' function. */ +#define HAVE_MKTEMP 1 + +/* define this if you have mkversion */ +#define HAVE_MKVERSION 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. */ +/* #undef HAVE_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H 1 + +/* define if strerror_r returns char* instead of int */ +/* #undef HAVE_NONPOSIX_STRERROR_R */ + +/* if you have NT Event Log */ +/* #undef HAVE_NT_EVENT_LOG */ + +/* if you have NT Service Manager */ +/* #undef HAVE_NT_SERVICE_MANAGER */ + +/* if you have NT Threads */ +/* #undef HAVE_NT_THREADS */ + +/* define if you have OpenSSL */ +#define HAVE_OPENSSL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_BN_H 1 + +/* define if you have OpenSSL with CRL checking capability */ +#define HAVE_OPENSSL_CRL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_CRYPTO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_SSL_H 1 + +/* Define to 1 if you have the `pipe' function. */ +#define HAVE_PIPE 1 + +/* Define to 1 if you have the `poll' function. */ +#define HAVE_POLL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PROCESS_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PSAP_H */ + +/* define to pthreads API spec revision */ +#define HAVE_PTHREADS 10 + +/* define if you have pthread_detach function */ +#define HAVE_PTHREAD_DETACH 1 + +/* Define to 1 if you have the `pthread_getconcurrency' function. */ +#define HAVE_PTHREAD_GETCONCURRENCY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PTHREAD_H 1 + +/* Define to 1 if you have the `pthread_kill' function. */ +#define HAVE_PTHREAD_KILL 1 + +/* Define to 1 if you have the `pthread_kill_other_threads_np' function. */ +/* #undef HAVE_PTHREAD_KILL_OTHER_THREADS_NP */ + +/* define if you have pthread_rwlock_destroy function */ +#define HAVE_PTHREAD_RWLOCK_DESTROY 1 + +/* Define to 1 if you have the `pthread_setconcurrency' function. */ +#define HAVE_PTHREAD_SETCONCURRENCY 1 + +/* Define to 1 if you have the `pthread_yield' function. */ +#define HAVE_PTHREAD_YIELD 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PTH_H */ + +/* Define to 1 if the system has the type `ptrdiff_t'. */ +#define HAVE_PTRDIFF_T 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PWD_H 1 + +/* Define to 1 if you have the `read' function. */ +#define HAVE_READ 1 + +/* Define to 1 if you have the `recv' function. */ +#define HAVE_RECV 1 + +/* Define to 1 if you have the `recvfrom' function. */ +#define HAVE_RECVFROM 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_REGEX_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_RESOLV_H */ + +/* define if you have res_query() */ +/* #undef HAVE_RES_QUERY */ + +/* define if OpenSSL needs RSAref */ +/* #undef HAVE_RSAREF */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_SASL_H */ + +/* define if your SASL library has sasl_version() */ +/* #undef HAVE_SASL_VERSION */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SCHED_H 1 + +/* Define to 1 if you have the `sched_yield' function. */ +#define HAVE_SCHED_YIELD 1 + +/* Define to 1 if you have the `send' function. */ +#define HAVE_SEND 1 + +/* Define to 1 if you have the `sendmsg' function. */ +#define HAVE_SENDMSG 1 + +/* Define to 1 if you have the `sendto' function. */ +#define HAVE_SENDTO 1 + +/* Define to 1 if you have the `setegid' function. */ +#define HAVE_SETEGID 1 + +/* Define to 1 if you have the `seteuid' function. */ +#define HAVE_SETEUID 1 + +/* Define to 1 if you have the `setgid' function. */ +#define HAVE_SETGID 1 + +/* Define to 1 if you have the `setpwfile' function. */ +/* #undef HAVE_SETPWFILE */ + +/* Define to 1 if you have the `setsid' function. */ +#define HAVE_SETSID 1 + +/* Define to 1 if you have the `setuid' function. */ +#define HAVE_SETUID 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SGTTY_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SHADOW_H */ + +/* Define to 1 if you have the `sigaction' function. */ +#define HAVE_SIGACTION 1 + +/* Define to 1 if you have the `signal' function. */ +#define HAVE_SIGNAL 1 + +/* Define to 1 if you have the `sigset' function. */ +#define HAVE_SIGSET 1 + +/* define if you have -lslp */ +/* #undef HAVE_SLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SLP_H */ + +/* Define to 1 if you have the `snprintf' function. */ +#define HAVE_SNPRINTF 1 + +/* if you have spawnlp() */ +/* #undef HAVE_SPAWNLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQLEXT_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQL_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDDEF_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strdup' function. */ +#define HAVE_STRDUP 1 + +/* Define to 1 if you have the `strerror' function. */ +#define HAVE_STRERROR 1 + +/* Define to 1 if you have the `strerror_r' function. */ +#define HAVE_STRERROR_R 1 + +/* Define to 1 if you have the `strftime' function. */ +#define HAVE_STRFTIME 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strpbrk' function. */ +#define HAVE_STRPBRK 1 + +/* Define to 1 if you have the `strrchr' function. */ +#define HAVE_STRRCHR 1 + +/* Define to 1 if you have the `strsep' function. */ +#define HAVE_STRSEP 1 + +/* Define to 1 if you have the `strspn' function. */ +#define HAVE_STRSPN 1 + +/* Define to 1 if you have the `strstr' function. */ +#define HAVE_STRSTR 1 + +/* Define to 1 if you have the `strtol' function. */ +#define HAVE_STRTOL 1 + +/* Define to 1 if you have the `strtoll' function. */ +#define HAVE_STRTOLL 1 + +/* Define to 1 if you have the `strtoq' function. */ +#define HAVE_STRTOQ 1 + +/* Define to 1 if you have the `strtoul' function. */ +#define HAVE_STRTOUL 1 + +/* Define to 1 if you have the `strtoull' function. */ +#define HAVE_STRTOULL 1 + +/* Define to 1 if you have the `strtouq' function. */ +#define HAVE_STRTOUQ 1 + +/* Define to 1 if `msg_accrightslen' is a member of `struct msghdr'. */ +/* #undef HAVE_STRUCT_MSGHDR_MSG_ACCRIGHTSLEN */ + +/* Define to 1 if `msg_control' is a member of `struct msghdr'. */ +#define HAVE_STRUCT_MSGHDR_MSG_CONTROL 1 + +/* Define to 1 if `pw_gecos' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_GECOS 1 + +/* Define to 1 if `pw_passwd' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_PASSWD 1 + +/* Define to 1 if `st_blksize' is a member of `struct stat'. */ +#define HAVE_STRUCT_STAT_ST_BLKSIZE 1 + +/* Define to 1 if `st_fstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE */ + +/* define to 1 if st_fstype is char * */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_CHAR */ + +/* define to 1 if st_fstype is int */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_INT */ + +/* Define to 1 if `st_vfstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_VFSTYPE */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYNCH_H */ + +/* Define to 1 if you have the `sysconf' function. */ +#define HAVE_SYSCONF 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSEXITS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_DEVPOLL_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_DIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_EPOLL_H 1 + +/* define if you actually have sys_errlist in your libs */ +#define HAVE_SYS_ERRLIST 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_ERRNO_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_EVENT_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_FILE_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_FILIO_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_FSTYP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_PRIVGRP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_RESOURCE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_UCRED_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UN_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_UUID_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_VMOUNT_H */ + +/* Define to 1 if you have that is POSIX.1 compatible. */ +#define HAVE_SYS_WAIT_H 1 + +/* define if you have -lwrap */ +/* #undef HAVE_TCPD */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_TCPD_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_TERMIOS_H 1 + +/* if you have Solaris LWP (thr) package */ +/* #undef HAVE_THR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_THREAD_H */ + +/* Define to 1 if you have the `thr_getconcurrency' function. */ +/* #undef HAVE_THR_GETCONCURRENCY */ + +/* Define to 1 if you have the `thr_setconcurrency' function. */ +/* #undef HAVE_THR_SETCONCURRENCY */ + +/* Define to 1 if you have the `thr_yield' function. */ +/* #undef HAVE_THR_YIELD */ + +/* define if you have TLS */ +#define HAVE_TLS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UTIME_H 1 + +/* define if you have uuid_generate() */ +/* #undef HAVE_UUID_GENERATE */ + +/* define if you have uuid_to_str() */ +/* #undef HAVE_UUID_TO_STR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_UUID_UUID_H */ + +/* Define to 1 if you have the `vprintf' function. */ +#define HAVE_VPRINTF 1 + +/* Define to 1 if you have the `vsnprintf' function. */ +#define HAVE_VSNPRINTF 1 + +/* Define to 1 if you have the `wait4' function. */ +#define HAVE_WAIT4 1 + +/* Define to 1 if you have the `waitpid' function. */ +#define HAVE_WAITPID 1 + +/* define if you have winsock */ +/* #undef HAVE_WINSOCK */ + +/* define if you have winsock2 */ +/* #undef HAVE_WINSOCK2 */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WIREDTIGER_H */ + +/* Define to 1 if you have the `write' function. */ +#define HAVE_WRITE 1 + +/* define if select implicitly yields */ +#define HAVE_YIELDING_SELECT 1 + +/* Define to 1 if you have the `_vsnprintf' function. */ +/* #undef HAVE__VSNPRINTF */ + +/* define to 32-bit or greater integer type */ +#define LBER_INT_T int + +/* define to large integer type */ +#define LBER_LEN_T long + +/* define to socket descriptor type */ +#define LBER_SOCKET_T int + +/* define to large integer type */ +#define LBER_TAG_T long + +/* define to 1 if library is thread safe */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* define to LDAP VENDOR VERSION */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +/* define this to add debugging code */ +/* #undef LDAP_DEBUG */ + +/* define if LDAP libs are dynamic */ +/* #undef LDAP_LIBS_DYNAMIC */ + +/* define to support PF_INET6 */ +#define LDAP_PF_INET6 1 + +/* define to support PF_LOCAL */ +#define LDAP_PF_LOCAL 1 + +/* define this to add SLAPI code */ +/* #undef LDAP_SLAPI */ + +/* define this to add syslog code */ +/* #undef LDAP_SYSLOG */ + +/* Version */ +#define LDAP_VENDOR_VERSION 20501 + +/* Major */ +#define LDAP_VENDOR_VERSION_MAJOR 2 + +/* Minor */ +#define LDAP_VENDOR_VERSION_MINOR 5 + +/* Patch */ +#define LDAP_VENDOR_VERSION_PATCH X + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* define if memcmp is not 8-bit clean or is otherwise broken */ +/* #undef NEED_MEMCMP_REPLACEMENT */ + +/* define if you have (or want) no threads */ +/* #undef NO_THREADS */ + +/* define to use the original debug style */ +/* #undef OLD_DEBUG */ + +/* Package */ +#define OPENLDAP_PACKAGE "OpenLDAP" + +/* Version */ +#define OPENLDAP_VERSION "2.5.X" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "" + +/* define if sched_yield yields the entire process */ +/* #undef REPLACE_BROKEN_YIELD */ + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE void + +/* Define to the type of arg 1 for `select'. */ +#define SELECT_TYPE_ARG1 int + +/* Define to the type of args 2, 3 and 4 for `select'. */ +#define SELECT_TYPE_ARG234 (fd_set *) + +/* Define to the type of arg 5 for `select'. */ +#define SELECT_TYPE_ARG5 (struct timeval *) + +/* The size of `int', as computed by sizeof. */ +#define SIZEOF_INT 4 + +/* The size of `long', as computed by sizeof. */ +#define SIZEOF_LONG 8 + +/* The size of `long long', as computed by sizeof. */ +#define SIZEOF_LONG_LONG 8 + +/* The size of `short', as computed by sizeof. */ +#define SIZEOF_SHORT 2 + +/* The size of `wchar_t', as computed by sizeof. */ +#define SIZEOF_WCHAR_T 4 + +/* define to support per-object ACIs */ +/* #undef SLAPD_ACI_ENABLED */ + +/* define to support LDAP Async Metadirectory backend */ +/* #undef SLAPD_ASYNCMETA */ + +/* define to support cleartext passwords */ +/* #undef SLAPD_CLEARTEXT */ + +/* define to support crypt(3) passwords */ +/* #undef SLAPD_CRYPT */ + +/* define to support DNS SRV backend */ +/* #undef SLAPD_DNSSRV */ + +/* define to support LDAP backend */ +/* #undef SLAPD_LDAP */ + +/* define to support MDB backend */ +/* #undef SLAPD_MDB */ + +/* define to support LDAP Metadirectory backend */ +/* #undef SLAPD_META */ + +/* define to support modules */ +/* #undef SLAPD_MODULES */ + +/* dynamically linked module */ +#define SLAPD_MOD_DYNAMIC 2 + +/* statically linked module */ +#define SLAPD_MOD_STATIC 1 + +/* define to support cn=Monitor backend */ +/* #undef SLAPD_MONITOR */ + +/* define to support NDB backend */ +/* #undef SLAPD_NDB */ + +/* define to support NULL backend */ +/* #undef SLAPD_NULL */ + +/* define for In-Directory Access Logging overlay */ +/* #undef SLAPD_OVER_ACCESSLOG */ + +/* define for Audit Logging overlay */ +/* #undef SLAPD_OVER_AUDITLOG */ + +/* define for Automatic Certificate Authority overlay */ +/* #undef SLAPD_OVER_AUTOCA */ + +/* define for Collect overlay */ +/* #undef SLAPD_OVER_COLLECT */ + +/* define for Attribute Constraint overlay */ +/* #undef SLAPD_OVER_CONSTRAINT */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DDS */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DEREF */ + +/* define for Dynamic Group overlay */ +/* #undef SLAPD_OVER_DYNGROUP */ + +/* define for Dynamic List overlay */ +/* #undef SLAPD_OVER_DYNLIST */ + +/* define for Reverse Group Membership overlay */ +/* #undef SLAPD_OVER_MEMBEROF */ + +/* define for Password Policy overlay */ +/* #undef SLAPD_OVER_PPOLICY */ + +/* define for Proxy Cache overlay */ +/* #undef SLAPD_OVER_PROXYCACHE */ + +/* define for Referential Integrity overlay */ +/* #undef SLAPD_OVER_REFINT */ + +/* define for Return Code overlay */ +/* #undef SLAPD_OVER_RETCODE */ + +/* define for Rewrite/Remap overlay */ +/* #undef SLAPD_OVER_RWM */ + +/* define for Sequential Modify overlay */ +/* #undef SLAPD_OVER_SEQMOD */ + +/* define for ServerSideSort/VLV overlay */ +/* #undef SLAPD_OVER_SSSVLV */ + +/* define for Syncrepl Provider overlay */ +/* #undef SLAPD_OVER_SYNCPROV */ + +/* define for Translucent Proxy overlay */ +/* #undef SLAPD_OVER_TRANSLUCENT */ + +/* define for Attribute Uniqueness overlay */ +/* #undef SLAPD_OVER_UNIQUE */ + +/* define for Value Sorting overlay */ +/* #undef SLAPD_OVER_VALSORT */ + +/* define to support PASSWD backend */ +/* #undef SLAPD_PASSWD */ + +/* define to support PERL backend */ +/* #undef SLAPD_PERL */ + +/* define to support relay backend */ +/* #undef SLAPD_RELAY */ + +/* define to support reverse lookups */ +/* #undef SLAPD_RLOOKUPS */ + +/* define to support SHELL backend */ +/* #undef SLAPD_SHELL */ + +/* define to support SOCK backend */ +/* #undef SLAPD_SOCK */ + +/* define to support SASL passwords */ +/* #undef SLAPD_SPASSWD */ + +/* define to support SQL backend */ +/* #undef SLAPD_SQL */ + +/* define to support WiredTiger backend */ +/* #undef SLAPD_WT */ + +/* define to support run-time loadable ACL */ +/* #undef SLAP_DYNACL */ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME 1 + +/* Define to 1 if your declares `struct tm'. */ +/* #undef TM_IN_SYS_TIME */ + +/* set to urandom device */ +#define URANDOM_DEVICE "/dev/urandom" + +/* define to use OpenSSL BIGNUM for MP */ +/* #undef USE_MP_BIGNUM */ + +/* define to use GMP for MP */ +/* #undef USE_MP_GMP */ + +/* define to use 'long' for MP */ +/* #undef USE_MP_LONG */ + +/* define to use 'long long' for MP */ +/* #undef USE_MP_LONG_LONG */ + +/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most + significant byte first (like Motorola and SPARC, unlike Intel). */ +#if defined AC_APPLE_UNIVERSAL_BUILD +# if defined __BIG_ENDIAN__ +# define WORDS_BIGENDIAN 1 +# endif +#else +# ifndef WORDS_BIGENDIAN +/* # undef WORDS_BIGENDIAN */ +# endif +#endif + +/* Define to the type of arg 3 for `accept'. */ +#define ber_socklen_t socklen_t + +/* Define to `char *' if does not define. */ +/* #undef caddr_t */ + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to `int' if doesn't define. */ +/* #undef gid_t */ + +/* Define to `int' if does not define. */ +/* #undef mode_t */ + +/* Define to `long' if does not define. */ +/* #undef off_t */ + +/* Define to `int' if does not define. */ +/* #undef pid_t */ + +/* Define to `int' if does not define. */ +/* #undef sig_atomic_t */ + +/* Define to `unsigned' if does not define. */ +/* #undef size_t */ + +/* define to snprintf routine */ +/* #undef snprintf */ + +/* Define like ber_socklen_t if does not define. */ +/* #undef socklen_t */ + +/* Define to `signed int' if does not define. */ +/* #undef ssize_t */ + +/* Define to `int' if doesn't define. */ +/* #undef uid_t */ + +/* define as empty if volatile is not supported */ +/* #undef volatile */ + +/* define to snprintf routine */ +/* #undef vsnprintf */ + + +/* begin of portable.h.post */ + +#ifdef _WIN32 +/* don't suck in all of the win32 api */ +# define WIN32_LEAN_AND_MEAN 1 +#endif + +#ifndef LDAP_NEEDS_PROTOTYPES +/* force LDAP_P to always include prototypes */ +#define LDAP_NEEDS_PROTOTYPES 1 +#endif + +#ifndef LDAP_REL_ENG +#if (LDAP_VENDOR_VERSION == 000000) && !defined(LDAP_DEVEL) +#define LDAP_DEVEL +#endif +#if defined(LDAP_DEVEL) && !defined(LDAP_TEST) +#define LDAP_TEST +#endif +#endif + +#ifdef HAVE_STDDEF_H +# include +#endif + +#ifdef HAVE_EBCDIC +/* ASCII/EBCDIC converting replacements for stdio funcs + * vsnprintf and snprintf are used too, but they are already + * checked by the configure script + */ +#define fputs ber_pvt_fputs +#define fgets ber_pvt_fgets +#define printf ber_pvt_printf +#define fprintf ber_pvt_fprintf +#define vfprintf ber_pvt_vfprintf +#define vsprintf ber_pvt_vsprintf +#endif + +#include "ac/fdset.h" + +#include "ldap_cdefs.h" +#include "ldap_features.h" + +#include "ac/assert.h" +#include "ac/localize.h" + +#endif /* _LDAP_PORTABLE_H */ +/* end of portable.h.post */ + diff --git a/contrib/openldap-cmake/linux_x86_64/include/lber_types.h b/contrib/openldap-cmake/linux_x86_64/include/lber_types.h new file mode 100644 index 00000000000..dbd59430527 --- /dev/null +++ b/contrib/openldap-cmake/linux_x86_64/include/lber_types.h @@ -0,0 +1,63 @@ +/* include/lber_types.h. Generated from lber_types.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LBER types + */ + +#ifndef _LBER_TYPES_H +#define _LBER_TYPES_H + +#include + +LDAP_BEGIN_DECL + +/* LBER boolean, enum, integers (32 bits or larger) */ +#define LBER_INT_T int + +/* LBER tags (32 bits or larger) */ +#define LBER_TAG_T long + +/* LBER socket descriptor */ +#define LBER_SOCKET_T int + +/* LBER lengths (32 bits or larger) */ +#define LBER_LEN_T long + +/* ------------------------------------------------------------ */ + +/* booleans, enumerations, and integers */ +typedef LBER_INT_T ber_int_t; + +/* signed and unsigned versions */ +typedef signed LBER_INT_T ber_sint_t; +typedef unsigned LBER_INT_T ber_uint_t; + +/* tags */ +typedef unsigned LBER_TAG_T ber_tag_t; + +/* "socket" descriptors */ +typedef LBER_SOCKET_T ber_socket_t; + +/* lengths */ +typedef unsigned LBER_LEN_T ber_len_t; + +/* signed lengths */ +typedef signed LBER_LEN_T ber_slen_t; + +LDAP_END_DECL + +#endif /* _LBER_TYPES_H */ diff --git a/contrib/openldap-cmake/linux_x86_64/include/ldap_config.h b/contrib/openldap-cmake/linux_x86_64/include/ldap_config.h new file mode 100644 index 00000000000..89f7b40b884 --- /dev/null +++ b/contrib/openldap-cmake/linux_x86_64/include/ldap_config.h @@ -0,0 +1,74 @@ +/* include/ldap_config.h. Generated from ldap_config.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * This file works in conjunction with OpenLDAP configure system. + * If you do no like the values below, adjust your configure options. + */ + +#ifndef _LDAP_CONFIG_H +#define _LDAP_CONFIG_H + +/* directory separator */ +#ifndef LDAP_DIRSEP +#ifndef _WIN32 +#define LDAP_DIRSEP "/" +#else +#define LDAP_DIRSEP "\\" +#endif +#endif + +/* directory for temporary files */ +#if defined(_WIN32) +# define LDAP_TMPDIR "C:\\." /* we don't have much of a choice */ +#elif defined( _P_tmpdir ) +# define LDAP_TMPDIR _P_tmpdir +#elif defined( P_tmpdir ) +# define LDAP_TMPDIR P_tmpdir +#elif defined( _PATH_TMPDIR ) +# define LDAP_TMPDIR _PATH_TMPDIR +#else +# define LDAP_TMPDIR LDAP_DIRSEP "tmp" +#endif + +/* directories */ +#ifndef LDAP_BINDIR +#define LDAP_BINDIR "/tmp/ldap-prefix/bin" +#endif +#ifndef LDAP_SBINDIR +#define LDAP_SBINDIR "/tmp/ldap-prefix/sbin" +#endif +#ifndef LDAP_DATADIR +#define LDAP_DATADIR "/tmp/ldap-prefix/share/openldap" +#endif +#ifndef LDAP_SYSCONFDIR +#define LDAP_SYSCONFDIR "/tmp/ldap-prefix/etc/openldap" +#endif +#ifndef LDAP_LIBEXECDIR +#define LDAP_LIBEXECDIR "/tmp/ldap-prefix/libexec" +#endif +#ifndef LDAP_MODULEDIR +#define LDAP_MODULEDIR "/tmp/ldap-prefix/libexec/openldap" +#endif +#ifndef LDAP_RUNDIR +#define LDAP_RUNDIR "/tmp/ldap-prefix/var" +#endif +#ifndef LDAP_LOCALEDIR +#define LDAP_LOCALEDIR "" +#endif + + +#endif /* _LDAP_CONFIG_H */ diff --git a/contrib/openldap-cmake/linux_x86_64/include/ldap_features.h b/contrib/openldap-cmake/linux_x86_64/include/ldap_features.h new file mode 100644 index 00000000000..f0cc7c3626f --- /dev/null +++ b/contrib/openldap-cmake/linux_x86_64/include/ldap_features.h @@ -0,0 +1,61 @@ +/* include/ldap_features.h. Generated from ldap_features.hin by configure. */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +/* + * LDAP Features + */ + +#ifndef _LDAP_FEATURES_H +#define _LDAP_FEATURES_H 1 + +/* OpenLDAP API version macros */ +#define LDAP_VENDOR_VERSION 20501 +#define LDAP_VENDOR_VERSION_MAJOR 2 +#define LDAP_VENDOR_VERSION_MINOR 5 +#define LDAP_VENDOR_VERSION_PATCH X + +/* +** WORK IN PROGRESS! +** +** OpenLDAP reentrancy/thread-safeness should be dynamically +** checked using ldap_get_option(). +** +** The -lldap implementation is not thread-safe. +** +** The -lldap_r implementation is: +** LDAP_API_FEATURE_THREAD_SAFE (basic thread safety) +** but also be: +** LDAP_API_FEATURE_SESSION_THREAD_SAFE +** LDAP_API_FEATURE_OPERATION_THREAD_SAFE +** +** The preprocessor flag LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE +** can be used to determine if -lldap_r is available at compile +** time. You must define LDAP_THREAD_SAFE if and only if you +** link with -lldap_r. +** +** If you fail to define LDAP_THREAD_SAFE when linking with +** -lldap_r or define LDAP_THREAD_SAFE when linking with -lldap, +** provided header definitions and declarations may be incorrect. +** +*/ + +/* is -lldap_r available or not */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* LDAP v2 Referrals */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +#endif /* LDAP_FEATURES */ diff --git a/contrib/openldap-cmake/linux_x86_64/include/portable.h b/contrib/openldap-cmake/linux_x86_64/include/portable.h new file mode 100644 index 00000000000..2924b6713a4 --- /dev/null +++ b/contrib/openldap-cmake/linux_x86_64/include/portable.h @@ -0,0 +1,1169 @@ +/* include/portable.h. Generated from portable.hin by configure. */ +/* include/portable.hin. Generated from configure.in by autoheader. */ + + +/* begin of portable.h.pre */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2020 The OpenLDAP Foundation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#ifndef _LDAP_PORTABLE_H +#define _LDAP_PORTABLE_H + +/* define this if needed to get reentrant functions */ +#ifndef REENTRANT +#define REENTRANT 1 +#endif +#ifndef _REENTRANT +#define _REENTRANT 1 +#endif + +/* define this if needed to get threadsafe functions */ +#ifndef THREADSAFE +#define THREADSAFE 1 +#endif +#ifndef _THREADSAFE +#define _THREADSAFE 1 +#endif +#ifndef THREAD_SAFE +#define THREAD_SAFE 1 +#endif +#ifndef _THREAD_SAFE +#define _THREAD_SAFE 1 +#endif + +#ifndef _SGI_MP_SOURCE +#define _SGI_MP_SOURCE 1 +#endif + +/* end of portable.h.pre */ + + +/* Define if building universal (internal helper macro) */ +/* #undef AC_APPLE_UNIVERSAL_BUILD */ + +/* define to use both and */ +/* #undef BOTH_STRINGS_H */ + +/* define if cross compiling */ +/* #undef CROSS_COMPILING */ + +/* set to the number of arguments ctime_r() expects */ +#define CTIME_R_NARGS 2 + +/* define if toupper() requires islower() */ +/* #undef C_UPPER_LOWER */ + +/* define if sys_errlist is not declared in stdio.h or errno.h */ +/* #undef DECL_SYS_ERRLIST */ + +/* define to enable slapi library */ +/* #undef ENABLE_SLAPI */ + +/* defined to be the EXE extension */ +#define EXEEXT "" + +/* set to the number of arguments gethostbyaddr_r() expects */ +#define GETHOSTBYADDR_R_NARGS 8 + +/* set to the number of arguments gethostbyname_r() expects */ +#define GETHOSTBYNAME_R_NARGS 6 + +/* Define to 1 if `TIOCGWINSZ' requires . */ +#define GWINSZ_IN_SYS_IOCTL 1 + +/* define if you have AIX security lib */ +/* #undef HAVE_AIX_SECURITY */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ASSERT_H 1 + +/* Define to 1 if you have the `bcopy' function. */ +#define HAVE_BCOPY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_BITS_TYPES_H 1 + +/* Define to 1 if you have the `chroot' function. */ +#define HAVE_CHROOT 1 + +/* Define to 1 if you have the `closesocket' function. */ +/* #undef HAVE_CLOSESOCKET */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CONIO_H */ + +/* define if crypt(3) is available */ +/* #undef HAVE_CRYPT */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CRYPT_H */ + +/* define if crypt_r() is also available */ +/* #undef HAVE_CRYPT_R */ + +/* Define to 1 if you have the `ctime_r' function. */ +#define HAVE_CTIME_R 1 + +/* define if you have Cyrus SASL */ +/* #undef HAVE_CYRUS_SASL */ + +/* define if your system supports /dev/poll */ +/* #undef HAVE_DEVPOLL */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DIRECT_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#define HAVE_DIRENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ +/* #undef HAVE_DOPRNT */ + +/* define if system uses EBCDIC instead of ASCII */ +/* #undef HAVE_EBCDIC */ + +/* Define to 1 if you have the `endgrent' function. */ +#define HAVE_ENDGRENT 1 + +/* Define to 1 if you have the `endpwent' function. */ +#define HAVE_ENDPWENT 1 + +/* define if your system supports epoll */ +#define HAVE_EPOLL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H 1 + +/* Define to 1 if you have the `fcntl' function. */ +#define HAVE_FCNTL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H 1 + +/* define if you actually have FreeBSD fetch(3) */ +/* #undef HAVE_FETCH */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_FILIO_H */ + +/* Define to 1 if you have the `flock' function. */ +#define HAVE_FLOCK 1 + +/* Define to 1 if you have the `fstat' function. */ +#define HAVE_FSTAT 1 + +/* Define to 1 if you have the `gai_strerror' function. */ +#define HAVE_GAI_STRERROR 1 + +/* Define to 1 if you have the `getaddrinfo' function. */ +#define HAVE_GETADDRINFO 1 + +/* Define to 1 if you have the `getdtablesize' function. */ +#define HAVE_GETDTABLESIZE 1 + +/* Define to 1 if you have the `geteuid' function. */ +#define HAVE_GETEUID 1 + +/* Define to 1 if you have the `getgrgid' function. */ +#define HAVE_GETGRGID 1 + +/* Define to 1 if you have the `gethostbyaddr_r' function. */ +#define HAVE_GETHOSTBYADDR_R 1 + +/* Define to 1 if you have the `gethostbyname_r' function. */ +#define HAVE_GETHOSTBYNAME_R 1 + +/* Define to 1 if you have the `gethostname' function. */ +#define HAVE_GETHOSTNAME 1 + +/* Define to 1 if you have the `getnameinfo' function. */ +#define HAVE_GETNAMEINFO 1 + +/* Define to 1 if you have the `getopt' function. */ +#define HAVE_GETOPT 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_GETOPT_H 1 + +/* Define to 1 if you have the `getpassphrase' function. */ +/* #undef HAVE_GETPASSPHRASE */ + +/* Define to 1 if you have the `getpeereid' function. */ +/* #undef HAVE_GETPEEREID */ + +/* Define to 1 if you have the `getpeerucred' function. */ +/* #undef HAVE_GETPEERUCRED */ + +/* Define to 1 if you have the `getpwnam' function. */ +#define HAVE_GETPWNAM 1 + +/* Define to 1 if you have the `getpwuid' function. */ +#define HAVE_GETPWUID 1 + +/* Define to 1 if you have the `getspnam' function. */ +#define HAVE_GETSPNAM 1 + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GMP_H */ + +/* Define to 1 if you have the `gmtime_r' function. */ +#define HAVE_GMTIME_R 1 + +/* define if you have GNUtls */ +/* #undef HAVE_GNUTLS */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_GNUTLS_GNUTLS_H */ + +/* if you have GNU Pth */ +/* #undef HAVE_GNU_PTH */ + +/* Define to 1 if you have the header file. */ +#define HAVE_GRP_H 1 + +/* Define to 1 if you have the `hstrerror' function. */ +#define HAVE_HSTRERROR 1 + +/* define to you inet_aton(3) is available */ +#define HAVE_INET_ATON 1 + +/* Define to 1 if you have the `inet_ntoa_b' function. */ +/* #undef HAVE_INET_NTOA_B */ + +/* Define to 1 if you have the `inet_ntop' function. */ +#define HAVE_INET_NTOP 1 + +/* Define to 1 if you have the `initgroups' function. */ +#define HAVE_INITGROUPS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the `ioctl' function. */ +#define HAVE_IOCTL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_IO_H */ + +/* define if your system supports kqueue */ +/* #undef HAVE_KQUEUE */ + +/* Define to 1 if you have the `gen' library (-lgen). */ +/* #undef HAVE_LIBGEN */ + +/* Define to 1 if you have the `gmp' library (-lgmp). */ +/* #undef HAVE_LIBGMP */ + +/* Define to 1 if you have the `inet' library (-linet). */ +/* #undef HAVE_LIBINET */ + +/* define if you have libtool -ltdl */ +/* #undef HAVE_LIBLTDL */ + +/* Define to 1 if you have the `net' library (-lnet). */ +/* #undef HAVE_LIBNET */ + +/* Define to 1 if you have the `nsl' library (-lnsl). */ +/* #undef HAVE_LIBNSL */ + +/* Define to 1 if you have the `nsl_s' library (-lnsl_s). */ +/* #undef HAVE_LIBNSL_S */ + +/* Define to 1 if you have the `socket' library (-lsocket). */ +/* #undef HAVE_LIBSOCKET */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LIBUTIL_H */ + +/* Define to 1 if you have the `V3' library (-lV3). */ +/* #undef HAVE_LIBV3 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* if you have LinuxThreads */ +/* #undef HAVE_LINUX_THREADS */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LOCALE_H 1 + +/* Define to 1 if you have the `localtime_r' function. */ +#define HAVE_LOCALTIME_R 1 + +/* Define to 1 if you have the `lockf' function. */ +#define HAVE_LOCKF 1 + +/* Define to 1 if the system has the type `long long'. */ +#define HAVE_LONG_LONG 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LTDL_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_MALLOC_H 1 + +/* Define to 1 if you have the `memcpy' function. */ +#define HAVE_MEMCPY 1 + +/* Define to 1 if you have the `memmove' function. */ +#define HAVE_MEMMOVE 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `memrchr' function. */ +#define HAVE_MEMRCHR 1 + +/* Define to 1 if you have the `mkstemp' function. */ +#define HAVE_MKSTEMP 1 + +/* Define to 1 if you have the `mktemp' function. */ +#define HAVE_MKTEMP 1 + +/* define this if you have mkversion */ +#define HAVE_MKVERSION 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. */ +/* #undef HAVE_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_TCP_H 1 + +/* define if strerror_r returns char* instead of int */ +/* #undef HAVE_NONPOSIX_STRERROR_R */ + +/* if you have NT Event Log */ +/* #undef HAVE_NT_EVENT_LOG */ + +/* if you have NT Service Manager */ +/* #undef HAVE_NT_SERVICE_MANAGER */ + +/* if you have NT Threads */ +/* #undef HAVE_NT_THREADS */ + +/* define if you have OpenSSL */ +#define HAVE_OPENSSL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_BN_H 1 + +/* define if you have OpenSSL with CRL checking capability */ +#define HAVE_OPENSSL_CRL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_CRYPTO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_OPENSSL_SSL_H 1 + +/* Define to 1 if you have the `pipe' function. */ +#define HAVE_PIPE 1 + +/* Define to 1 if you have the `poll' function. */ +#define HAVE_POLL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PROCESS_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PSAP_H */ + +/* define to pthreads API spec revision */ +#define HAVE_PTHREADS 10 + +/* define if you have pthread_detach function */ +#define HAVE_PTHREAD_DETACH 1 + +/* Define to 1 if you have the `pthread_getconcurrency' function. */ +#define HAVE_PTHREAD_GETCONCURRENCY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PTHREAD_H 1 + +/* Define to 1 if you have the `pthread_kill' function. */ +#define HAVE_PTHREAD_KILL 1 + +/* Define to 1 if you have the `pthread_kill_other_threads_np' function. */ +/* #undef HAVE_PTHREAD_KILL_OTHER_THREADS_NP */ + +/* define if you have pthread_rwlock_destroy function */ +#define HAVE_PTHREAD_RWLOCK_DESTROY 1 + +/* Define to 1 if you have the `pthread_setconcurrency' function. */ +#define HAVE_PTHREAD_SETCONCURRENCY 1 + +/* Define to 1 if you have the `pthread_yield' function. */ +#define HAVE_PTHREAD_YIELD 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_PTH_H */ + +/* Define to 1 if the system has the type `ptrdiff_t'. */ +#define HAVE_PTRDIFF_T 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PWD_H 1 + +/* Define to 1 if you have the `read' function. */ +#define HAVE_READ 1 + +/* Define to 1 if you have the `recv' function. */ +#define HAVE_RECV 1 + +/* Define to 1 if you have the `recvfrom' function. */ +#define HAVE_RECVFROM 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_REGEX_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_RESOLV_H */ + +/* define if you have res_query() */ +/* #undef HAVE_RES_QUERY */ + +/* define if OpenSSL needs RSAref */ +/* #undef HAVE_RSAREF */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_SASL_H */ + +/* define if your SASL library has sasl_version() */ +/* #undef HAVE_SASL_VERSION */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SCHED_H 1 + +/* Define to 1 if you have the `sched_yield' function. */ +#define HAVE_SCHED_YIELD 1 + +/* Define to 1 if you have the `send' function. */ +#define HAVE_SEND 1 + +/* Define to 1 if you have the `sendmsg' function. */ +#define HAVE_SENDMSG 1 + +/* Define to 1 if you have the `sendto' function. */ +#define HAVE_SENDTO 1 + +/* Define to 1 if you have the `setegid' function. */ +#define HAVE_SETEGID 1 + +/* Define to 1 if you have the `seteuid' function. */ +#define HAVE_SETEUID 1 + +/* Define to 1 if you have the `setgid' function. */ +#define HAVE_SETGID 1 + +/* Define to 1 if you have the `setpwfile' function. */ +/* #undef HAVE_SETPWFILE */ + +/* Define to 1 if you have the `setsid' function. */ +#define HAVE_SETSID 1 + +/* Define to 1 if you have the `setuid' function. */ +#define HAVE_SETUID 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SGTTY_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SHADOW_H */ + +/* Define to 1 if you have the `sigaction' function. */ +#define HAVE_SIGACTION 1 + +/* Define to 1 if you have the `signal' function. */ +#define HAVE_SIGNAL 1 + +/* Define to 1 if you have the `sigset' function. */ +#define HAVE_SIGSET 1 + +/* define if you have -lslp */ +/* #undef HAVE_SLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SLP_H */ + +/* Define to 1 if you have the `snprintf' function. */ +#define HAVE_SNPRINTF 1 + +/* if you have spawnlp() */ +/* #undef HAVE_SPAWNLP */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQLEXT_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SQL_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDDEF_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strdup' function. */ +#define HAVE_STRDUP 1 + +/* Define to 1 if you have the `strerror' function. */ +#define HAVE_STRERROR 1 + +/* Define to 1 if you have the `strerror_r' function. */ +#define HAVE_STRERROR_R 1 + +/* Define to 1 if you have the `strftime' function. */ +#define HAVE_STRFTIME 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strpbrk' function. */ +#define HAVE_STRPBRK 1 + +/* Define to 1 if you have the `strrchr' function. */ +#define HAVE_STRRCHR 1 + +/* Define to 1 if you have the `strsep' function. */ +#define HAVE_STRSEP 1 + +/* Define to 1 if you have the `strspn' function. */ +#define HAVE_STRSPN 1 + +/* Define to 1 if you have the `strstr' function. */ +#define HAVE_STRSTR 1 + +/* Define to 1 if you have the `strtol' function. */ +#define HAVE_STRTOL 1 + +/* Define to 1 if you have the `strtoll' function. */ +#define HAVE_STRTOLL 1 + +/* Define to 1 if you have the `strtoq' function. */ +#define HAVE_STRTOQ 1 + +/* Define to 1 if you have the `strtoul' function. */ +#define HAVE_STRTOUL 1 + +/* Define to 1 if you have the `strtoull' function. */ +#define HAVE_STRTOULL 1 + +/* Define to 1 if you have the `strtouq' function. */ +#define HAVE_STRTOUQ 1 + +/* Define to 1 if `msg_accrightslen' is a member of `struct msghdr'. */ +/* #undef HAVE_STRUCT_MSGHDR_MSG_ACCRIGHTSLEN */ + +/* Define to 1 if `msg_control' is a member of `struct msghdr'. */ +#define HAVE_STRUCT_MSGHDR_MSG_CONTROL 1 + +/* Define to 1 if `pw_gecos' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_GECOS 1 + +/* Define to 1 if `pw_passwd' is a member of `struct passwd'. */ +#define HAVE_STRUCT_PASSWD_PW_PASSWD 1 + +/* Define to 1 if `st_blksize' is a member of `struct stat'. */ +#define HAVE_STRUCT_STAT_ST_BLKSIZE 1 + +/* Define to 1 if `st_fstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE */ + +/* define to 1 if st_fstype is char * */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_CHAR */ + +/* define to 1 if st_fstype is int */ +/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_INT */ + +/* Define to 1 if `st_vfstype' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_VFSTYPE */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYNCH_H */ + +/* Define to 1 if you have the `sysconf' function. */ +#define HAVE_SYSCONF 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSEXITS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_DEVPOLL_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_DIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_EPOLL_H 1 + +/* define if you actually have sys_errlist in your libs */ +#define HAVE_SYS_ERRLIST 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_ERRNO_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_EVENT_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_FILE_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_FILIO_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_FSTYP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_POLL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_PRIVGRP_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_RESOURCE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SYSLOG_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_UCRED_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UN_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_UUID_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_VMOUNT_H */ + +/* Define to 1 if you have that is POSIX.1 compatible. */ +#define HAVE_SYS_WAIT_H 1 + +/* define if you have -lwrap */ +/* #undef HAVE_TCPD */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_TCPD_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_TERMIOS_H 1 + +/* if you have Solaris LWP (thr) package */ +/* #undef HAVE_THR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_THREAD_H */ + +/* Define to 1 if you have the `thr_getconcurrency' function. */ +/* #undef HAVE_THR_GETCONCURRENCY */ + +/* Define to 1 if you have the `thr_setconcurrency' function. */ +/* #undef HAVE_THR_SETCONCURRENCY */ + +/* Define to 1 if you have the `thr_yield' function. */ +/* #undef HAVE_THR_YIELD */ + +/* define if you have TLS */ +#define HAVE_TLS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UTIME_H 1 + +/* define if you have uuid_generate() */ +/* #undef HAVE_UUID_GENERATE */ + +/* define if you have uuid_to_str() */ +/* #undef HAVE_UUID_TO_STR */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_UUID_UUID_H */ + +/* Define to 1 if you have the `vprintf' function. */ +#define HAVE_VPRINTF 1 + +/* Define to 1 if you have the `vsnprintf' function. */ +#define HAVE_VSNPRINTF 1 + +/* Define to 1 if you have the `wait4' function. */ +#define HAVE_WAIT4 1 + +/* Define to 1 if you have the `waitpid' function. */ +#define HAVE_WAITPID 1 + +/* define if you have winsock */ +/* #undef HAVE_WINSOCK */ + +/* define if you have winsock2 */ +/* #undef HAVE_WINSOCK2 */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK2_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINSOCK_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WIREDTIGER_H */ + +/* Define to 1 if you have the `write' function. */ +#define HAVE_WRITE 1 + +/* define if select implicitly yields */ +#define HAVE_YIELDING_SELECT 1 + +/* Define to 1 if you have the `_vsnprintf' function. */ +/* #undef HAVE__VSNPRINTF */ + +/* define to 32-bit or greater integer type */ +#define LBER_INT_T int + +/* define to large integer type */ +#define LBER_LEN_T long + +/* define to socket descriptor type */ +#define LBER_SOCKET_T int + +/* define to large integer type */ +#define LBER_TAG_T long + +/* define to 1 if library is thread safe */ +#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1 + +/* define to LDAP VENDOR VERSION */ +/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */ + +/* define this to add debugging code */ +/* #undef LDAP_DEBUG */ + +/* define if LDAP libs are dynamic */ +/* #undef LDAP_LIBS_DYNAMIC */ + +/* define to support PF_INET6 */ +#define LDAP_PF_INET6 1 + +/* define to support PF_LOCAL */ +#define LDAP_PF_LOCAL 1 + +/* define this to add SLAPI code */ +/* #undef LDAP_SLAPI */ + +/* define this to add syslog code */ +/* #undef LDAP_SYSLOG */ + +/* Version */ +#define LDAP_VENDOR_VERSION 20501 + +/* Major */ +#define LDAP_VENDOR_VERSION_MAJOR 2 + +/* Minor */ +#define LDAP_VENDOR_VERSION_MINOR 5 + +/* Patch */ +#define LDAP_VENDOR_VERSION_PATCH X + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* define if memcmp is not 8-bit clean or is otherwise broken */ +/* #undef NEED_MEMCMP_REPLACEMENT */ + +/* define if you have (or want) no threads */ +/* #undef NO_THREADS */ + +/* define to use the original debug style */ +/* #undef OLD_DEBUG */ + +/* Package */ +#define OPENLDAP_PACKAGE "OpenLDAP" + +/* Version */ +#define OPENLDAP_VERSION "2.5.X" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "" + +/* define if sched_yield yields the entire process */ +/* #undef REPLACE_BROKEN_YIELD */ + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE void + +/* Define to the type of arg 1 for `select'. */ +#define SELECT_TYPE_ARG1 int + +/* Define to the type of args 2, 3 and 4 for `select'. */ +#define SELECT_TYPE_ARG234 (fd_set *) + +/* Define to the type of arg 5 for `select'. */ +#define SELECT_TYPE_ARG5 (struct timeval *) + +/* The size of `int', as computed by sizeof. */ +#define SIZEOF_INT 4 + +/* The size of `long', as computed by sizeof. */ +#define SIZEOF_LONG 8 + +/* The size of `long long', as computed by sizeof. */ +#define SIZEOF_LONG_LONG 8 + +/* The size of `short', as computed by sizeof. */ +#define SIZEOF_SHORT 2 + +/* The size of `wchar_t', as computed by sizeof. */ +#define SIZEOF_WCHAR_T 4 + +/* define to support per-object ACIs */ +/* #undef SLAPD_ACI_ENABLED */ + +/* define to support LDAP Async Metadirectory backend */ +/* #undef SLAPD_ASYNCMETA */ + +/* define to support cleartext passwords */ +/* #undef SLAPD_CLEARTEXT */ + +/* define to support crypt(3) passwords */ +/* #undef SLAPD_CRYPT */ + +/* define to support DNS SRV backend */ +/* #undef SLAPD_DNSSRV */ + +/* define to support LDAP backend */ +/* #undef SLAPD_LDAP */ + +/* define to support MDB backend */ +/* #undef SLAPD_MDB */ + +/* define to support LDAP Metadirectory backend */ +/* #undef SLAPD_META */ + +/* define to support modules */ +/* #undef SLAPD_MODULES */ + +/* dynamically linked module */ +#define SLAPD_MOD_DYNAMIC 2 + +/* statically linked module */ +#define SLAPD_MOD_STATIC 1 + +/* define to support cn=Monitor backend */ +/* #undef SLAPD_MONITOR */ + +/* define to support NDB backend */ +/* #undef SLAPD_NDB */ + +/* define to support NULL backend */ +/* #undef SLAPD_NULL */ + +/* define for In-Directory Access Logging overlay */ +/* #undef SLAPD_OVER_ACCESSLOG */ + +/* define for Audit Logging overlay */ +/* #undef SLAPD_OVER_AUDITLOG */ + +/* define for Automatic Certificate Authority overlay */ +/* #undef SLAPD_OVER_AUTOCA */ + +/* define for Collect overlay */ +/* #undef SLAPD_OVER_COLLECT */ + +/* define for Attribute Constraint overlay */ +/* #undef SLAPD_OVER_CONSTRAINT */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DDS */ + +/* define for Dynamic Directory Services overlay */ +/* #undef SLAPD_OVER_DEREF */ + +/* define for Dynamic Group overlay */ +/* #undef SLAPD_OVER_DYNGROUP */ + +/* define for Dynamic List overlay */ +/* #undef SLAPD_OVER_DYNLIST */ + +/* define for Reverse Group Membership overlay */ +/* #undef SLAPD_OVER_MEMBEROF */ + +/* define for Password Policy overlay */ +/* #undef SLAPD_OVER_PPOLICY */ + +/* define for Proxy Cache overlay */ +/* #undef SLAPD_OVER_PROXYCACHE */ + +/* define for Referential Integrity overlay */ +/* #undef SLAPD_OVER_REFINT */ + +/* define for Return Code overlay */ +/* #undef SLAPD_OVER_RETCODE */ + +/* define for Rewrite/Remap overlay */ +/* #undef SLAPD_OVER_RWM */ + +/* define for Sequential Modify overlay */ +/* #undef SLAPD_OVER_SEQMOD */ + +/* define for ServerSideSort/VLV overlay */ +/* #undef SLAPD_OVER_SSSVLV */ + +/* define for Syncrepl Provider overlay */ +/* #undef SLAPD_OVER_SYNCPROV */ + +/* define for Translucent Proxy overlay */ +/* #undef SLAPD_OVER_TRANSLUCENT */ + +/* define for Attribute Uniqueness overlay */ +/* #undef SLAPD_OVER_UNIQUE */ + +/* define for Value Sorting overlay */ +/* #undef SLAPD_OVER_VALSORT */ + +/* define to support PASSWD backend */ +/* #undef SLAPD_PASSWD */ + +/* define to support PERL backend */ +/* #undef SLAPD_PERL */ + +/* define to support relay backend */ +/* #undef SLAPD_RELAY */ + +/* define to support reverse lookups */ +/* #undef SLAPD_RLOOKUPS */ + +/* define to support SHELL backend */ +/* #undef SLAPD_SHELL */ + +/* define to support SOCK backend */ +/* #undef SLAPD_SOCK */ + +/* define to support SASL passwords */ +/* #undef SLAPD_SPASSWD */ + +/* define to support SQL backend */ +/* #undef SLAPD_SQL */ + +/* define to support WiredTiger backend */ +/* #undef SLAPD_WT */ + +/* define to support run-time loadable ACL */ +/* #undef SLAP_DYNACL */ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME 1 + +/* Define to 1 if your declares `struct tm'. */ +/* #undef TM_IN_SYS_TIME */ + +/* set to urandom device */ +#define URANDOM_DEVICE "/dev/urandom" + +/* define to use OpenSSL BIGNUM for MP */ +/* #undef USE_MP_BIGNUM */ + +/* define to use GMP for MP */ +/* #undef USE_MP_GMP */ + +/* define to use 'long' for MP */ +/* #undef USE_MP_LONG */ + +/* define to use 'long long' for MP */ +/* #undef USE_MP_LONG_LONG */ + +/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most + significant byte first (like Motorola and SPARC, unlike Intel). */ +#if defined AC_APPLE_UNIVERSAL_BUILD +# if defined __BIG_ENDIAN__ +# define WORDS_BIGENDIAN 1 +# endif +#else +# ifndef WORDS_BIGENDIAN +/* # undef WORDS_BIGENDIAN */ +# endif +#endif + +/* Define to the type of arg 3 for `accept'. */ +#define ber_socklen_t socklen_t + +/* Define to `char *' if does not define. */ +/* #undef caddr_t */ + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to `int' if doesn't define. */ +/* #undef gid_t */ + +/* Define to `int' if does not define. */ +/* #undef mode_t */ + +/* Define to `long' if does not define. */ +/* #undef off_t */ + +/* Define to `int' if does not define. */ +/* #undef pid_t */ + +/* Define to `int' if does not define. */ +/* #undef sig_atomic_t */ + +/* Define to `unsigned' if does not define. */ +/* #undef size_t */ + +/* define to snprintf routine */ +/* #undef snprintf */ + +/* Define like ber_socklen_t if does not define. */ +/* #undef socklen_t */ + +/* Define to `signed int' if does not define. */ +/* #undef ssize_t */ + +/* Define to `int' if doesn't define. */ +/* #undef uid_t */ + +/* define as empty if volatile is not supported */ +/* #undef volatile */ + +/* define to snprintf routine */ +/* #undef vsnprintf */ + + +/* begin of portable.h.post */ + +#ifdef _WIN32 +/* don't suck in all of the win32 api */ +# define WIN32_LEAN_AND_MEAN 1 +#endif + +#ifndef LDAP_NEEDS_PROTOTYPES +/* force LDAP_P to always include prototypes */ +#define LDAP_NEEDS_PROTOTYPES 1 +#endif + +#ifndef LDAP_REL_ENG +#if (LDAP_VENDOR_VERSION == 000000) && !defined(LDAP_DEVEL) +#define LDAP_DEVEL +#endif +#if defined(LDAP_DEVEL) && !defined(LDAP_TEST) +#define LDAP_TEST +#endif +#endif + +#ifdef HAVE_STDDEF_H +# include +#endif + +#ifdef HAVE_EBCDIC +/* ASCII/EBCDIC converting replacements for stdio funcs + * vsnprintf and snprintf are used too, but they are already + * checked by the configure script + */ +#define fputs ber_pvt_fputs +#define fgets ber_pvt_fgets +#define printf ber_pvt_printf +#define fprintf ber_pvt_fprintf +#define vfprintf ber_pvt_vfprintf +#define vsprintf ber_pvt_vsprintf +#endif + +#include "ac/fdset.h" + +#include "ldap_cdefs.h" +#include "ldap_features.h" + +#include "ac/assert.h" +#include "ac/localize.h" + +#endif /* _LDAP_PORTABLE_H */ +/* end of portable.h.post */ + diff --git a/docs/en/getting-started/playground.md b/docs/en/getting-started/playground.md index 353724b6e3e..36595d1998a 100644 --- a/docs/en/getting-started/playground.md +++ b/docs/en/getting-started/playground.md @@ -5,8 +5,31 @@ toc_title: Playground # ClickHouse Playground {#clickhouse-playground} -[ClickHouse Playground](https://play.clickhouse.tech?file=welcome) allows people to experiment with ClickHouse by running queries instantly, without setting up their server or cluster. -Several example datasets are available in the Playground as well as sample queries that show ClickHouse features. +[ClickHouse Playground](https://play.clickhouse.tech) allows people to experiment with ClickHouse by running queries instantly, without setting up their server or cluster. +Several example datasets are available in the Playground as well as sample queries that show ClickHouse features. There's also a selection of ClickHouse LTS releases to experiment with. + +ClickHouse Playground gives the experience of m2.small [Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) instance hosted in [Yandex.Cloud](https://cloud.yandex.com/). More information about [cloud providers](../commercial/cloud.md). + +You can make queries to playground using any HTTP client, for example [curl](https://curl.haxx.se) or [wget](https://www.gnu.org/software/wget/), or set up a connection using [JDBC](../interfaces/jdbc.md) or [ODBC](../interfaces/odbc.md) drivers. More information about software products that support ClickHouse is available [here](../interfaces/index.md). + +## Credentials + +| Parameter | Value | +|:------------------|:----------------------------------------| +| HTTPS endpoint | `https://play-api.clickhouse.tech:8443` | +| Native endpoint | `play-api.clickhouse.tech:9440` | +| User | `playground` | +| Password | `clickhouse` | + +!!! note "Note" + Note that all endpoints require a secure TLS connection. + +There are additional endpoints with specific ClickHouse releases to experiment with their differences (ports and user/password are the same as above): + +* 20.3 LTS: `play-api-v20-3.clickhouse.tech` +* 19.14 LTS: `play-api-v19-14.clickhouse.tech` + +## Limitations The queries are executed as a read-only user. It implies some limitations: @@ -14,33 +37,28 @@ The queries are executed as a read-only user. It implies some limitations: - INSERT queries are not allowed The following settings are also enforced: -- [`max_result_bytes=10485760`](../operations/settings/query_complexity/#max-result-bytes) -- [`max_result_rows=2000`](../operations/settings/query_complexity/#setting-max_result_rows) -- [`result_overflow_mode=break`](../operations/settings/query_complexity/#result-overflow-mode) -- [`max_execution_time=60000`](../operations/settings/query_complexity/#max-execution-time) +- [max_result_bytes=10485760](../operations/settings/query_complexity/#max-result-bytes) +- [max_result_rows=2000](../operations/settings/query_complexity/#setting-max_result_rows) +- [result_overflow_mode=break](../operations/settings/query_complexity/#result-overflow-mode) +- [max_execution_time=60000](../operations/settings/query_complexity/#max-execution-time) -ClickHouse Playground gives the experience of m2.small -[Managed Service for ClickHouse](https://cloud.yandex.com/services/managed-clickhouse) -instance hosted in [Yandex.Cloud](https://cloud.yandex.com/). -More information about [cloud providers](../commercial/cloud.md). +## Examples -ClickHouse Playground web interface makes requests via ClickHouse [HTTP API](../interfaces/http.md). -The Playground backend is just a ClickHouse cluster without any additional server-side application. -ClickHouse HTTPS endpoint is also available as a part of the Playground. - -You can make queries to playground using any HTTP client, for example [curl](https://curl.haxx.se) or [wget](https://www.gnu.org/software/wget/), or set up a connection using [JDBC](../interfaces/jdbc.md) or [ODBC](../interfaces/odbc.md) drivers. -More information about software products that support ClickHouse is available [here](../interfaces/index.md). - -| Parameter | Value | -|:----------|:--------------------------------------| -| Endpoint | https://play-api.clickhouse.tech:8443 | -| User | `playground` | -| Password | `clickhouse` | - -Note that this endpoint requires a secure connection. - -Example: +HTTPS endpoint example with `curl`: ``` bash curl "https://play-api.clickhouse.tech:8443/?query=SELECT+'Play+ClickHouse!';&user=playground&password=clickhouse&database=datasets" ``` + +TCP endpoint example with [../interfaces/cli.md]: +``` bash +clickhouse client --secure -h play-api.clickhouse.tech --port 9440 -u playground --password clickhouse -q "SELECT 'Play ClickHouse!'" +``` + +## Implementation Details + +ClickHouse Playground web interface makes requests via ClickHouse [HTTP API](../interfaces/http.md). +The Playground backend is just a ClickHouse cluster without any additional server-side application. As mentioned above, ClickHouse HTTPS and TCP/TLS endpoints are also publicly available as a part of the Playground, both are proxied through [Cloudflare Spectrum](https://www.cloudflare.com/products/cloudflare-spectrum/) to add extra layer of protection and improved global connectivity. + +!!! warning "Warning" + Exposing ClickHouse server to public internet in any other situation is **strongly not recommended**. Make sure it listens only on private network and is covered by properly configured firewall. diff --git a/docs/en/operations/access-rights.md b/docs/en/operations/access-rights.md index aac2befab4c..001afd29fcb 100644 --- a/docs/en/operations/access-rights.md +++ b/docs/en/operations/access-rights.md @@ -22,7 +22,7 @@ You can configure access entities using: - Server [configuration files](configuration-files.md) `users.xml` and `config.xml`. -We recommend using SQL-driven workflow. Both of the configuration methods work simultaneously, so if you use the server configuration files for managing accounts and access rights, you can softly move to SQL-driven workflow. +We recommend using SQL-driven workflow. Both of the configuration methods work simultaneously, so if you use the server configuration files for managing accounts and access rights, you can smoothly switch to SQL-driven workflow. !!! note "Warning" You can't manage the same access entity by both configuration methods simultaneously. @@ -30,32 +30,32 @@ We recommend using SQL-driven workflow. Both of the configuration methods work s ## Usage {#access-control-usage} -By default, the ClickHouse server provides the user account `default` which is not allowed using SQL-driven access control and account management but have all the rights and permissions. The `default` user account is used in any cases when the username is not defined, for example, at login from client or in distributed queries. In distributed query processing a default user account is used, if the configuration of the server or cluster doesn’t specify the [user and password](../engines/table-engines/special/distributed.md) properties. +By default, the ClickHouse server provides the `default` user account which is not allowed using SQL-driven access control and account management but has all the rights and permissions. The `default` user account is used in any cases when the username is not defined, for example, at login from client or in distributed queries. In distributed query processing a default user account is used, if the configuration of the server or cluster doesn’t specify the [user and password](../engines/table-engines/special/distributed.md) properties. -If you just start using ClickHouse, you can use the following scenario: +If you just started using ClickHouse, consider the following scenario: 1. [Enable](#enabling-access-control) SQL-driven access control and account management for the `default` user. -2. Login under the `default` user account and create all the required users. Don't forget to create an administrator account (`GRANT ALL ON *.* WITH GRANT OPTION TO admin_user_account`). +2. Log in to the `default` user account and create all the required users. Don't forget to create an administrator account (`GRANT ALL ON *.* WITH GRANT OPTION TO admin_user_account`). 3. [Restrict permissions](settings/permissions-for-queries.md#permissions_for_queries) for the `default` user and disable SQL-driven access control and account management for it. ### Properties of Current Solution {#access-control-properties} -- You can grant permissions for databases and tables even if they are not exist. -- If a table was deleted, all the privileges that correspond to this table are not revoked. So, if a new table is created later with the same name all the privileges become again actual. To revoke privileges corresponding to the deleted table, you need to perform, for example, the `REVOKE ALL PRIVILEGES ON db.table FROM ALL` query. -- There is no lifetime settings for privileges. +- You can grant permissions for databases and tables even if they do not exist. +- If a table was deleted, all the privileges that correspond to this table are not revoked. This means that even if you create a new table with the same name later, all the privileges remain valid. To revoke privileges corresponding to the deleted table, you need to execute, for example, the `REVOKE ALL PRIVILEGES ON db.table FROM ALL` query. +- There are no lifetime settings for privileges. ## User account {#user-account-management} A user account is an access entity that allows to authorize someone in ClickHouse. A user account contains: - Identification information. -- [Privileges](../sql-reference/statements/grant.md#grant-privileges) that define a scope of queries the user can perform. -- Hosts from which connection to the ClickHouse server is allowed. -- Granted and default roles. -- Settings with their constraints that apply by default at the user's login. +- [Privileges](../sql-reference/statements/grant.md#grant-privileges) that define a scope of queries the user can execute. +- Hosts allowed to connect to the ClickHouse server. +- Assigned and default roles. +- Settings with their constraints applied by default at user login. - Assigned settings profiles. -Privileges to a user account can be granted by the [GRANT](../sql-reference/statements/grant.md) query or by assigning [roles](#role-management). To revoke privileges from a user, ClickHouse provides the [REVOKE](../sql-reference/statements/revoke.md) query. To list privileges for a user, use the - [SHOW GRANTS](../sql-reference/statements/show.md#show-grants-statement) statement. +Privileges can be granted to a user account by the [GRANT](../sql-reference/statements/grant.md) query or by assigning [roles](#role-management). To revoke privileges from a user, ClickHouse provides the [REVOKE](../sql-reference/statements/revoke.md) query. To list privileges for a user, use the [SHOW GRANTS](../sql-reference/statements/show.md#show-grants-statement) statement. Management queries: @@ -66,11 +66,11 @@ Management queries: ### Settings Applying {#access-control-settings-applying} -Settings can be set by different ways: for a user account, in its granted roles and settings profiles. At a user login, if a setting is set in different access entities, the value and constrains of this setting are applied by the following priorities (from higher to lower): +Settings can be configured differently: for a user account, in its granted roles and in settings profiles. At user login, if a setting is configured for different access entities, the value and constraints of this setting are applied as follows (from higher to lower priority): -1. User account setting. -2. The settings of default roles of the user account. If a setting is set in some roles, then order of the setting applying is undefined. -3. The settings in settings profiles assigned to a user or to its default roles. If a setting is set in some profiles, then order of setting applying is undefined. +1. User account settings. +2. The settings of default roles of the user account. If a setting is configured in some roles, then order of the setting application is undefined. +3. The settings from settings profiles assigned to a user or to its default roles. If a setting is configured in some profiles, then order of setting application is undefined. 4. Settings applied to all the server by default or from the [default profile](server-configuration-parameters/settings.md#default-profile). @@ -82,7 +82,7 @@ Role contains: - [Privileges](../sql-reference/statements/grant.md#grant-privileges) - Settings and constraints -- List of granted roles +- List of assigned roles Management queries: @@ -93,11 +93,11 @@ Management queries: - [SET DEFAULT ROLE](../sql-reference/statements/misc.md#set-default-role-statement) - [SHOW CREATE ROLE](../sql-reference/statements/show.md#show-create-role-statement) -Privileges to a role can be granted by the [GRANT](../sql-reference/statements/grant.md) query. To revoke privileges from a role ClickHouse provides the [REVOKE](../sql-reference/statements/revoke.md) query. +Privileges can be granted to a role by the [GRANT](../sql-reference/statements/grant.md) query. To revoke privileges from a role ClickHouse provides the [REVOKE](../sql-reference/statements/revoke.md) query. ## Row Policy {#row-policy-management} -Row policy is a filter that defines which or rows is available for a user or for a role. Row policy contains filters for one specific table and list of roles and/or users which should use this row policy. +Row policy is a filter that defines which of the rows are available to a user or a role. Row policy contains filters for one particular table, as well as a list of roles and/or users which should use this row policy. Management queries: @@ -109,7 +109,7 @@ Management queries: ## Settings Profile {#settings-profiles-management} -Settings profile is a collection of [settings](settings/index.md). Settings profile contains settings and constraints, and list of roles and/or users to which this quota is applied. +Settings profile is a collection of [settings](settings/index.md). Settings profile contains settings and constraints, as well as a list of roles and/or users to which this profile is applied. Management queries: @@ -123,7 +123,7 @@ Management queries: Quota limits resource usage. See [Quotas](quotas.md). -Quota contains a set of limits for some durations, and list of roles and/or users which should use this quota. +Quota contains a set of limits for some durations, as well as a list of roles and/or users which should use this quota. Management queries: @@ -141,7 +141,7 @@ Management queries: - Enable SQL-driven access control and account management for at least one user account. - By default, SQL-driven access control and account management is turned off for all users. You need to configure at least one user in the `users.xml` configuration file and assign 1 to the [access_management](settings/settings-users.md#access_management-user-setting) setting. + By default, SQL-driven access control and account management is disabled for all users. You need to configure at least one user in the `users.xml` configuration file and set the value of the [access_management](settings/settings-users.md#access_management-user-setting) setting to 1. [Original article](https://clickhouse.tech/docs/en/operations/access_rights/) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 3a11785d6ba..41ee085d8d3 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -207,7 +207,7 @@ If `http_port` is specified, the OpenSSL configuration is ignored even if it is **Example** ``` xml -0000 +9999 ``` ## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} diff --git a/docs/en/operations/settings/settings-profiles.md b/docs/en/operations/settings/settings-profiles.md index 0543f0bd954..3e5d1a02cbd 100644 --- a/docs/en/operations/settings/settings-profiles.md +++ b/docs/en/operations/settings/settings-profiles.md @@ -11,7 +11,7 @@ A settings profile is a collection of settings grouped under the same name. ClickHouse also supports [SQL-driven workflow](../access-rights.md#access-control) for managing settings profiles. We recommend using it. -A profile can have any name. The profile can have any name. You can specify the same profile for different users. The most important thing you can write in the settings profile is `readonly=1`, which ensures read-only access. +The profile can have any name. You can specify the same profile for different users. The most important thing you can write in the settings profile is `readonly=1`, which ensures read-only access. Settings profiles can inherit from each other. To use inheritance, indicate one or multiple `profile` settings before the other settings that are listed in the profile. In case when one setting is defined in different profiles, the latest defined is used. diff --git a/docs/en/operations/settings/settings-users.md b/docs/en/operations/settings/settings-users.md index 3f472ad4879..3c104202801 100644 --- a/docs/en/operations/settings/settings-users.md +++ b/docs/en/operations/settings/settings-users.md @@ -76,7 +76,7 @@ Password can be specified in plaintext or in SHA256 (hex format). ### access_management {#access_management-user-setting} -This setting enables of disables using of SQL-driven [access control and account management](../access-rights.md#access-control) for the user. +This setting enables or disables using of SQL-driven [access control and account management](../access-rights.md#access-control) for the user. Possible values: diff --git a/docs/en/sql-reference/statements/alter.md b/docs/en/sql-reference/statements/alter.md index eb0c42ad9c3..6b1adcdb033 100644 --- a/docs/en/sql-reference/statements/alter.md +++ b/docs/en/sql-reference/statements/alter.md @@ -520,23 +520,23 @@ To use `ALTER USER` you must have the [ALTER USER](grant.md#grant-access-managem ### Examples {#alter-user-examples} -Set granted roles as default: +Set assigned roles as default: ``` sql ALTER USER user DEFAULT ROLE role1, role2 ``` -If roles aren't previously granted to a user, ClickHouse throws an exception. +If roles aren't previously assigned to a user, ClickHouse throws an exception. -Set all the granted roles to default: +Set all the assigned roles to default: ``` sql ALTER USER user DEFAULT ROLE ALL ``` -If a role will be granted to a user in the future it will become default automatically. +If a role is assigned to a user in the future, it will become default automatically. -Set all the granted roles to default excepting `role1` and `role2`: +Set all the assigned roles to default, excepting `role1` and `role2`: ``` sql ALTER USER user DEFAULT ROLE ALL EXCEPT role1, role2 @@ -591,7 +591,7 @@ ALTER QUOTA [IF EXISTS] name [ON CLUSTER cluster_name] ## ALTER SETTINGS PROFILE {#alter-settings-profile-statement} -Changes quotas. +Changes settings profiles. ### Syntax {#alter-settings-profile-syntax} diff --git a/docs/en/sql-reference/statements/create.md b/docs/en/sql-reference/statements/create.md index ea03983e2f9..7a7f4635ec3 100644 --- a/docs/en/sql-reference/statements/create.md +++ b/docs/en/sql-reference/statements/create.md @@ -327,23 +327,23 @@ There are multiple ways of user identification: #### User Host -User host is a host from which a connection to ClickHouse server could be established. Host can be specified in the `HOST` section of query by the following ways: +User host is a host from which a connection to ClickHouse server could be established. The host can be specified in the `HOST` query section in the following ways: - `HOST IP 'ip_address_or_subnetwork'` — User can connect to ClickHouse server only from the specified IP address or a [subnetwork](https://en.wikipedia.org/wiki/Subnetwork). Examples: `HOST IP '192.168.0.0/16'`, `HOST IP '2001:DB8::/32'`. For use in production, only specify `HOST IP` elements (IP addresses and their masks), since using `host` and `host_regexp` might cause extra latency. -- `HOST ANY` — User can connect from any location. This is default option. +- `HOST ANY` — User can connect from any location. This is a default option. - `HOST LOCAL` — User can connect only locally. - `HOST NAME 'fqdn'` — User host can be specified as FQDN. For example, `HOST NAME 'mysite.com'`. - `HOST NAME REGEXP 'regexp'` — You can use [pcre](http://www.pcre.org/) regular expressions when specifying user hosts. For example, `HOST NAME REGEXP '.*\.mysite\.com'`. -- `HOST LIKE 'template'` — Allows you use the [LIKE](../functions/string-search-functions.md#function-like) operator to filter the user hosts. For example, `HOST LIKE '%'` is equivalent to `HOST ANY`, `HOST LIKE '%.mysite.com'` filters all the hosts in the `mysite.com` domain. +- `HOST LIKE 'template'` — Allows you to use the [LIKE](../functions/string-search-functions.md#function-like) operator to filter the user hosts. For example, `HOST LIKE '%'` is equivalent to `HOST ANY`, `HOST LIKE '%.mysite.com'` filters all the hosts in the `mysite.com` domain. -Another way of specifying host is to use `@` syntax with the user name. Examples: +Another way of specifying host is to use `@` syntax following the username. Examples: - `CREATE USER mira@'127.0.0.1'` — Equivalent to the `HOST IP` syntax. - `CREATE USER mira@'localhost'` — Equivalent to the `HOST LOCAL` syntax. - `CREATE USER mira@'192.168.%.%'` — Equivalent to the `HOST LIKE` syntax. !!! info "Warning" - ClickHouse treats `user_name@'address'` as a user name as a whole. Thus, technically you can create multiple users with `user_name` and different constructions after `@`. We don't recommend to do so. + ClickHouse treats `user_name@'address'` as a username as a whole. Thus, technically you can create multiple users with the same `user_name` and different constructions after `@`. However, we don't recommend to do so. ### Examples {#create-user-examples} @@ -369,7 +369,7 @@ Create the user account `john` and make all his future roles default: ALTER USER user DEFAULT ROLE ALL ``` -When some role will be assigned to `john` in the future it will become default automatically. +When some role is assigned to `john` in the future, it will become default automatically. Create the user account `john` and make all his future roles default excepting `role1` and `role2`: @@ -391,15 +391,15 @@ CREATE ROLE [IF NOT EXISTS | OR REPLACE] name ### Description {#create-role-description} -Role is a set of [privileges](grant.md#grant-privileges). A user granted with a role gets all the privileges of this role. +Role is a set of [privileges](grant.md#grant-privileges). A user assigned a role gets all the privileges of this role. -A user can be assigned with multiple roles. Users can apply their granted roles in arbitrary combinations by the [SET ROLE](misc.md#set-role-statement) statement. The final scope of privileges is a combined set of all the privileges of all the applied roles. If a user has privileges granted directly to it's user account, they are also combined with the privileges granted by roles. +A user can be assigned multiple roles. Users can apply their assigned roles in arbitrary combinations by the [SET ROLE](misc.md#set-role-statement) statement. The final scope of privileges is a combined set of all the privileges of all the applied roles. If a user has privileges granted directly to it's user account, they are also combined with the privileges granted by roles. User can have default roles which apply at user login. To set default roles, use the [SET DEFAULT ROLE](misc.md#set-default-role-statement) statement or the [ALTER USER](alter.md#alter-user-statement) statement. To revoke a role, use the [REVOKE](revoke.md) statement. -To delete role, use the [DROP ROLE](misc.md#drop-role-statement) statement. The deleted role is being automatically revoked from all the users and roles to which it was granted. +To delete role, use the [DROP ROLE](misc.md#drop-role-statement) statement. The deleted role is being automatically revoked from all the users and roles to which it was assigned. ### Examples {#create-role-examples} @@ -410,13 +410,13 @@ GRANT SELECT ON db.* TO accountant; This sequence of queries creates the role `accountant` that has the privilege of reading data from the `accounting` database. -Granting the role to the user `mira`: +Assigning the role to the user `mira`: ```sql GRANT accountant TO mira; ``` -After the role is granted, the user can use it and perform the allowed queries. For example: +After the role is assigned, the user can apply it and execute the allowed queries. For example: ```sql SET ROLE accountant; @@ -443,15 +443,15 @@ Using this section you can create permissive or restrictive policies. Permissive policy grants access to rows. Permissive policies which apply to the same table are combined together using the boolean `OR` operator. Policies are permissive by default. -Restrictive policy restricts access to row. Restrictive policies which apply to the same table are combined together using the boolean `AND` operator. +Restrictive policy restricts access to rows. Restrictive policies which apply to the same table are combined together using the boolean `AND` operator. Restrictive policies apply to rows that passed the permissive filters. If you set restrictive policies but no permissive policies, the user can't get any row from the table. #### Section TO {#create-row-policy-to} -In the section `TO` you can give a mixed list of roles and users, for example, `CREATE ROW POLICY ... TO accountant, john@localhost`. +In the section `TO` you can provide a mixed list of roles and users, for example, `CREATE ROW POLICY ... TO accountant, john@localhost`. -Keyword `ALL` means all the ClickHouse users including current user. Keywords `ALL EXCEPT` allow to to exclude some users from the all users list, for example `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost` +Keyword `ALL` means all the ClickHouse users including current user. Keywords `ALL EXCEPT` allow to exclude some users from the all users list, for example, `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost` ### Examples @@ -494,7 +494,7 @@ CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_na [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] ``` -# Example {#create-settings-profile-syntax} +### Example {#create-settings-profile-syntax} Create the `max_memory_usage_profile` settings profile with value and constraints for the `max_memory_usage` setting. Assign it to `robin`: diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index f0a1508f1e9..d9e4f2f9309 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -7,9 +7,9 @@ toc_title: GRANT # GRANT - Grants [privileges](#grant-privileges) to ClickHouse user accounts or roles. -- Assigns roles to user accounts or to another roles. +- Assigns roles to user accounts or to the other roles. -To revoke privileges, use the [REVOKE](revoke.md) statement. Also you can list granted privileges by the [SHOW GRANTS](show.md#show-grants-statement) statement. +To revoke privileges, use the [REVOKE](revoke.md) statement. Also you can list granted privileges with the [SHOW GRANTS](show.md#show-grants-statement) statement. ## Granting Privilege Syntax {#grant-privigele-syntax} @@ -21,10 +21,10 @@ GRANT [ON CLUSTER cluster_name] privilege[(column_name [,...])] [,...] ON {db.ta - `role` — ClickHouse user role. - `user` — ClickHouse user account. -The `WITH GRANT OPTION` clause grants `user` or `role` with permission to perform the `GRANT` query. Users can grant privileges of the same scope they have and less. +The `WITH GRANT OPTION` clause grants `user` or `role` with permission to execute the `GRANT` query. Users can grant privileges of the same scope they have and less. -## Granting Role Syntax {#assign-role-syntax} +## Assigning Role Syntax {#assign-role-syntax} ```sql GRANT [ON CLUSTER cluster_name] role [,...] TO {user | another_role | CURRENT_USER} [,...] [WITH ADMIN OPTION] @@ -33,7 +33,7 @@ GRANT [ON CLUSTER cluster_name] role [,...] TO {user | another_role | CURRENT_US - `role` — ClickHouse user role. - `user` — ClickHouse user account. -The `WITH ADMIN OPTION` clause sets [ADMIN OPTION](#admin-option-privilege) privilege for `user` or `role`. +The `WITH ADMIN OPTION` clause grants [ADMIN OPTION](#admin-option-privilege) privilege to `user` or `role`. ## Usage {#grant-usage} @@ -45,28 +45,28 @@ For example, administrator has granted privileges to the `john` account by the q GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION ``` -It means that `john` has the permission to perform: +It means that `john` has the permission to execute: - `SELECT x,y FROM db.table`. - `SELECT x FROM db.table`. - `SELECT y FROM db.table`. -`john` can't perform `SELECT z FROM db.table`. The `SELECT * FROM db.table` also is not available. Processing this query, ClickHouse doesn't return any data, even `x` and `y`. The only exception is if a table contains only `x` and `y` columns, in this case ClickHouse returns all the data. +`john` can't execute `SELECT z FROM db.table`. The `SELECT * FROM db.table` also is not available. Processing this query, ClickHouse doesn't return any data, even `x` and `y`. The only exception is if a table contains only `x` and `y` columns. In this case ClickHouse returns all the data. -Also `john` has the `GRANT OPTION` privilege, so it can grant other users with privileges of the same or the smaller scope. +Also `john` has the `GRANT OPTION` privilege, so it can grant other users with privileges of the same or smaller scope. -Specifying privileges you can use asterisk (`*`) instead of a table or a database name. For example, the `GRANT SELECT ON db.* TO john` query allows `john` to perform the `SELECT` query over all the tables in `db` database. Also, you can omit database name. In this case privileges are granted for current database, for example: `GRANT SELECT ON * TO john` grants the privilege on all the tables in the current database, `GRANT SELECT ON mytable TO john` grants the privilege on the `mytable` table in the current database. +Specifying privileges you can use asterisk (`*`) instead of a table or a database name. For example, the `GRANT SELECT ON db.* TO john` query allows `john` to execute the `SELECT` query over all the tables in `db` database. Also, you can omit database name. In this case privileges are granted for current database. For example, `GRANT SELECT ON * TO john` grants the privilege on all the tables in the current database, `GRANT SELECT ON mytable TO john` grants the privilege on the `mytable` table in the current database. Access to the `system` database is always allowed (since this database is used for processing queries). -You can grant multiple privileges to multiple accounts in one query. The query `GRANT SELECT, INSERT ON *.* TO john, robin` allows accounts `john` and `robin` to perform the `INSERT` and `SELECT` queries over all the tables in all the databases on the server. +You can grant multiple privileges to multiple accounts in one query. The query `GRANT SELECT, INSERT ON *.* TO john, robin` allows accounts `john` and `robin` to execute the `INSERT` and `SELECT` queries over all the tables in all the databases on the server. ## Privileges {#grant-privileges} -Privilege is a permission to perform specific kind of queries. +Privilege is a permission to execute specific kind of queries. -Privileges have an hierarchical structure. A set of permitted queries depends on the privilege scope. +Privileges have a hierarchical structure. A set of permitted queries depends on the privilege scope. Hierarchy of privileges: @@ -212,20 +212,20 @@ The special privilege [ALL](#grant-all) grants all the privileges to a user acco By default, a user account or a role has no privileges. -If a user or role have no privileges it displayed as [NONE](#grant-none) privilege. +If a user or a role has no privileges, it is displayed as [NONE](#grant-none) privilege. -Some queries by their implementation require a set of privileges. For example, to perform the [RENAME](misc.md#misc_operations-rename) query you need the following privileges: `SELECT`, `CREATE TABLE`, `INSERT` and `DROP TABLE`. +Some queries by their implementation require a set of privileges. For example, to execute the [RENAME](misc.md#misc_operations-rename) query you need the following privileges: `SELECT`, `CREATE TABLE`, `INSERT` and `DROP TABLE`. ### SELECT {#grant-select} -Allows to perform [SELECT](select/index.md) queries. +Allows executing [SELECT](select/index.md) queries. Privilege level: `COLUMN`. **Description** -User granted with this privilege can perform `SELECT` queries over a specified list of columns in the specified table and database. If user includes other columns then specified a query returns no data. +User granted with this privilege can execute `SELECT` queries over a specified list of columns in the specified table and database. If user includes other columns then specified a query returns no data. Consider the following privilege: @@ -233,17 +233,17 @@ Consider the following privilege: GRANT SELECT(x,y) ON db.table TO john ``` -This privilege allows `john` to perform any `SELECT` query that involves data from the `x` and/or `y` columns in `db.table`. For example, `SELECT x FROM db.table`. `john` can't perform `SELECT z FROM db.table`. The `SELECT * FROM db.table` also is not available. Processing this query, ClickHouse doesn't return any data, even `x` and `y`. The only exception is if a table contains only `x` and `y` columns, in this case ClickHouse returns all the data. +This privilege allows `john` to execute any `SELECT` query that involves data from the `x` and/or `y` columns in `db.table`, for example, `SELECT x FROM db.table`. `john` can't execute `SELECT z FROM db.table`. The `SELECT * FROM db.table` also is not available. Processing this query, ClickHouse doesn't return any data, even `x` and `y`. The only exception is if a table contains only `x` and `y` columns, in this case ClickHouse returns all the data. ### INSERT {#grant-insert} -Allows performing [INSERT](insert-into.md) queries. +Allows executing [INSERT](insert-into.md) queries. Privilege level: `COLUMN`. **Description** -User granted with this privilege can perform `INSERT` queries over a specified list of columns in the specified table and database. If user includes other columns then specified a query doesn't insert any data. +User granted with this privilege can execute `INSERT` queries over a specified list of columns in the specified table and database. If user includes other columns then specified a query doesn't insert any data. **Example** @@ -255,7 +255,7 @@ The granted privilege allows `john` to insert data to the `x` and/or `y` columns ### ALTER {#grant-alter} -Allows performing [ALTER](alter.md) queries corresponding to the following hierarchy of privileges: +Allows executing [ALTER](alter.md) queries according to the following hierarchy of privileges: - `ALTER`. Level: `COLUMN`. - `ALTER TABLE`. Level: `GROUP` @@ -294,14 +294,14 @@ Examples of how this hierarchy is treated: **Notes** -- The `MODIFY SETTING` privilege allows to modify table engine settings. In doesn't affect settings or server configuration parameters. +- The `MODIFY SETTING` privilege allows modifying table engine settings. It doesn't affect settings or server configuration parameters. - The `ATTACH` operation needs the [CREATE](#grant-create) privilege. - The `DETACH` operation needs the [DROP](#grant-drop) privilege. - To stop mutation by the [KILL MUTATION](misc.md#kill-mutation) query, you need to have a privilege to start this mutation. For example, if you want to stop the `ALTER UPDATE` query, you need the `ALTER UPDATE`, `ALTER TABLE`, or `ALTER` privilege. ### CREATE {#grant-create} -Allows to perform [CREATE](create.md) and [ATTACH](misc.md#attach) DDL-queries corresponding to the following hierarchy of privileges: +Allows executing [CREATE](create.md) and [ATTACH](misc.md#attach) DDL-queries according to the following hierarchy of privileges: - `CREATE`. Level: `GROUP` - `CREATE DATABASE`. Level: `DATABASE` @@ -316,7 +316,7 @@ Allows to perform [CREATE](create.md) and [ATTACH](misc.md#attach) DDL-queries c ### DROP {#grant-drop} -Allows to perform [DROP](misc.md#drop) and [DETACH](misc.md#detach) queries corresponding to the following hierarchy of privileges: +Allows executing [DROP](misc.md#drop) and [DETACH](misc.md#detach) queries according to the following hierarchy of privileges: - `DROP`. Level: - `DROP DATABASE`. Level: `DATABASE` @@ -327,19 +327,19 @@ Allows to perform [DROP](misc.md#drop) and [DETACH](misc.md#detach) queries corr ### TRUNCATE {#grant-truncate} -Allows to perform [TRUNCATE](misc.md#truncate-statement) queries. +Allows executing [TRUNCATE](misc.md#truncate-statement) queries. Privilege level: `TABLE`. ### OPTIMIZE {#grant-optimize} -Allows to perform the [OPTIMIZE TABLE](misc.md#misc_operations-optimize) queries. +Allows executing [OPTIMIZE TABLE](misc.md#misc_operations-optimize) queries. Privilege level: `TABLE`. ### SHOW {#grant-show} -Allows to perform `SHOW`, `DESCRIBE`, `USE`, and `EXISTS` queries, corresponding to the following hierarchy of privileges: +Allows executing `SHOW`, `DESCRIBE`, `USE`, and `EXISTS` queries according to the following hierarchy of privileges: - `SHOW`. Level: `GROUP` - `SHOW DATABASES`. Level: `DATABASE`. Allows to execute `SHOW DATABASES`, `SHOW CREATE DATABASE`, `USE ` queries. @@ -349,12 +349,12 @@ Allows to perform `SHOW`, `DESCRIBE`, `USE`, and `EXISTS` queries, corresponding **Notes** -A user has the `SHOW` privilege if it has any another privilege concerning the specified table, dictionary or database. +A user has the `SHOW` privilege if it has any other privilege concerning the specified table, dictionary or database. ### KILL QUERY {#grant-kill-query} -Allows to perform the [KILL](misc.md#kill-query-statement) queries corresponding to the following hierarchy of privileges: +Allows executing [KILL](misc.md#kill-query-statement) queries according to the following hierarchy of privileges: Privilege level: `GLOBAL`. @@ -365,7 +365,7 @@ Privilege level: `GLOBAL`. ### ACCESS MANAGEMENT {#grant-access-management} -Allows a user to perform queries that manage users, roles and row policies. +Allows a user to execute queries that manage users, roles and row policies. - `ACCESS MANAGEMENT`. Level: `GROUP` - `CREATE USER`. Level: `GLOBAL` @@ -391,11 +391,11 @@ Allows a user to perform queries that manage users, roles and row policies. - `SHOW_QUOTAS`. Level: `GLOBAL`. Aliases: `SHOW CREATE QUOTA` - `SHOW_SETTINGS_PROFILES`. Level: `GLOBAL`. Aliases: `SHOW PROFILES`, `SHOW CREATE SETTINGS PROFILE`, `SHOW CREATE PROFILE` -The `ROLE ADMIN` privilege allows a user to grant and revoke any roles including those which are not granted to the user with the admin option. +The `ROLE ADMIN` privilege allows a user to assign and revoke any roles including those which are not assigned to the user with the admin option. ### SYSTEM {#grant-system} -Allows a user to perform the [SYSTEM](system.md) queries corresponding to the following hierarchy of privileges. +Allows a user to execute [SYSTEM](system.md) queries according to the following hierarchy of privileges. - `SYSTEM`. Level: `GROUP` - `SYSTEM SHUTDOWN`. Level: `GLOBAL`. Aliases: `SYSTEM KILL`, `SHUTDOWN` @@ -461,7 +461,7 @@ Examples: Allows a user to execute [dictGet](../functions/ext-dict-functions.md#dictget), [dictHas](../functions/ext-dict-functions.md#dicthas), [dictGetHierarchy](../functions/ext-dict-functions.md#dictgethierarchy), [dictIsIn](../functions/ext-dict-functions.md#dictisin) functions. -Level of privilege: `DICTIONARY`. +Privilege level: `DICTIONARY`. **Examples** @@ -480,6 +480,6 @@ Doesn't grant any privileges. ### ADMIN OPTION {#admin-option-privilege} -The `ADMIN OPTION` privilege allows a user granting their role to another user. +The `ADMIN OPTION` privilege allows a user to grant their role to another user. [Original article](https://clickhouse.tech/docs/en/query_language/grant/) diff --git a/docs/en/sql-reference/statements/misc.md b/docs/en/sql-reference/statements/misc.md index 56a82771e2a..18cbf1a90e8 100644 --- a/docs/en/sql-reference/statements/misc.md +++ b/docs/en/sql-reference/statements/misc.md @@ -126,7 +126,7 @@ DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] Deletes a role. -Deleted role is revoked from all the entities where it was granted. +Deleted role is revoked from all the entities where it was assigned. ### Syntax {#drop-role-syntax} @@ -162,9 +162,9 @@ DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ## DROP SETTINGS PROFILE {#drop-settings-profile-statement} -Deletes a quota. +Deletes a settings profile. -Deleted quota is revoked from all the entities where it was assigned. +Deleted settings profile is revoked from all the entities where it was assigned. ### Syntax {#drop-settings-profile-syntax} @@ -360,4 +360,4 @@ Lets you set the current database for the session. The current database is used for searching for tables if the database is not explicitly defined in the query with a dot before the table name. This query can’t be made when using the HTTP protocol, since there is no concept of a session. -[Original article](https://clickhouse.tech/docs/en/query_language/misc/) +[Original article](https://clickhouse.tech/docs/en/query_language/misc/) diff --git a/docs/en/sql-reference/statements/revoke.md b/docs/en/sql-reference/statements/revoke.md index 6bb5fc19c28..66ff978ddfb 100644 --- a/docs/en/sql-reference/statements/revoke.md +++ b/docs/en/sql-reference/statements/revoke.md @@ -23,7 +23,7 @@ REVOKE [ON CLUSTER cluster_name] [ADMIN OPTION FOR] role [,...] FROM {user | rol ## Description {#revoke-description} -To revoke some privilege you can use a privilege of wider scope then you plan to revoke. For example, if a user has the `SELECT (x,y)` privilege, administrator can perform `REVOKE SELECT(x,y) ...`, or `REVOKE SELECT * ...`, or even `REVOKE ALL PRIVILEGES ...` query to revoke this privilege. +To revoke some privilege you can use a privilege of a wider scope than you plan to revoke. For example, if a user has the `SELECT (x,y)` privilege, administrator can execute `REVOKE SELECT(x,y) ...`, or `REVOKE SELECT * ...`, or even `REVOKE ALL PRIVILEGES ...` query to revoke this privilege. ### Partial Revokes {#partial-revokes-dscr} @@ -32,14 +32,14 @@ You can revoke a part of a privilege. For example, if a user has the `SELECT *.* ## Examples {#revoke-example} -Grant the `john` user account with a privilege to select from all the databases excepting the `accounts` one: +Grant the `john` user account with a privilege to select from all the databases, excepting the `accounts` one: ``` sql GRANT SELECT ON *.* TO john; REVOKE SELECT ON accounts.* FROM john; ``` -Grant the `mira` user account with a privilege to select from all the columns of the `accounts.staff` table excepting the `wage` one. +Grant the `mira` user account with a privilege to select from all the columns of the `accounts.staff` table, excepting the `wage` one. ``` sql GRANT SELECT ON accounts.staff TO mira; diff --git a/docs/en/sql-reference/statements/select/array-join.md b/docs/en/sql-reference/statements/select/array-join.md index 8e05cf51232..21f6ea40492 100644 --- a/docs/en/sql-reference/statements/select/array-join.md +++ b/docs/en/sql-reference/statements/select/array-join.md @@ -1,3 +1,7 @@ +--- +toc_title: ARRAY JOIN +--- + # ARRAY JOIN Clause {#select-array-join-clause} It is a common operation for tables that contain an array column to produce a new table that has a column with each individual array element of that initial column, while values of other columns are duplicated. This is the basic case of what `ARRAY JOIN` clause does. diff --git a/docs/en/sql-reference/statements/select/distinct.md b/docs/en/sql-reference/statements/select/distinct.md index 1f500bf469b..d7799a8343e 100644 --- a/docs/en/sql-reference/statements/select/distinct.md +++ b/docs/en/sql-reference/statements/select/distinct.md @@ -1,3 +1,7 @@ +--- +toc_title: DISTINCT +--- + # DISTINCT Clause {#select-distinct} If `SELECT DISTINCT` is specified, only unique rows will remain in a query result. Thus only a single row will remain out of all the sets of fully matching rows in the result. diff --git a/docs/en/sql-reference/statements/select/format.md b/docs/en/sql-reference/statements/select/format.md index 75fd7256bfc..ca4b89fae71 100644 --- a/docs/en/sql-reference/statements/select/format.md +++ b/docs/en/sql-reference/statements/select/format.md @@ -1,3 +1,7 @@ +--- +toc_title: FORMAT +--- + # FORMAT Clause {#format-clause} ClickHouse supports a wide range of [serialization formats](../../../interfaces/formats.md) that can be used on query results among other things. There are multiple ways to choose a format for `SELECT` output, one of them is to specify `FORMAT format` at the end of query to get resulting data in any specific format. diff --git a/docs/en/sql-reference/statements/select/from.md b/docs/en/sql-reference/statements/select/from.md index d017537557d..fa29576276c 100644 --- a/docs/en/sql-reference/statements/select/from.md +++ b/docs/en/sql-reference/statements/select/from.md @@ -1,3 +1,7 @@ +--- +toc_title: FROM +--- + # FROM Clause {#select-from} The `FROM` clause specifies the source to read data from: diff --git a/docs/en/sql-reference/statements/select/group-by.md b/docs/en/sql-reference/statements/select/group-by.md index c9cb275129e..49d3a2da8a4 100644 --- a/docs/en/sql-reference/statements/select/group-by.md +++ b/docs/en/sql-reference/statements/select/group-by.md @@ -1,3 +1,7 @@ +--- +toc_title: GROUP BY +--- + # GROUP BY Clause {#select-group-by-clause} `GROUP BY` clause switches the `SELECT` query into an aggregation mode, which works as follows: diff --git a/docs/en/sql-reference/statements/select/having.md b/docs/en/sql-reference/statements/select/having.md index 17413645449..7a42f43bf53 100644 --- a/docs/en/sql-reference/statements/select/having.md +++ b/docs/en/sql-reference/statements/select/having.md @@ -1,3 +1,7 @@ +--- +toc_title: HAVING +--- + # HAVING Clause {#having-clause} Allows filtering the aggregation results produced by [GROUP BY](group-by.md). It is similar to the [WHERE](where.md) clause, but the difference is that `WHERE` is performed before aggregation, while `HAVING` is performed after it. diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index 4ee75f7b575..8224bf1e798 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -1,6 +1,7 @@ --- toc_priority: 33 -toc_title: SELECT +toc_folder_title: SELECT +toc_title: Queries Syntax --- # SELECT Queries Syntax {#select-queries-syntax} diff --git a/docs/en/sql-reference/statements/select/into-outfile.md b/docs/en/sql-reference/statements/select/into-outfile.md index 4385d31438a..26b9c2cf8cb 100644 --- a/docs/en/sql-reference/statements/select/into-outfile.md +++ b/docs/en/sql-reference/statements/select/into-outfile.md @@ -1,3 +1,7 @@ +--- +toc_title: INTO OUTFILE +--- + # INTO OUTFILE Clause {#into-outfile-clause} Add the `INTO OUTFILE filename` clause (where filename is a string literal) to `SELECT query` to redirect its output to the specified file on the client-side. diff --git a/docs/en/sql-reference/statements/select/join.md b/docs/en/sql-reference/statements/select/join.md index bc160d1fc8d..c636e1dab8b 100644 --- a/docs/en/sql-reference/statements/select/join.md +++ b/docs/en/sql-reference/statements/select/join.md @@ -1,3 +1,7 @@ +--- +toc_title: JOIN +--- + # JOIN Clause {#select-join} Join produces a new table by combining columns from one or multiple tables by using values common to each. It is a common operation in databases with SQL support, which corresponds to [relational algebra](https://en.wikipedia.org/wiki/Relational_algebra#Joins_and_join-like_operators) join. The special case of one table join is often referred to as "self-join". diff --git a/docs/en/sql-reference/statements/select/limit-by.md b/docs/en/sql-reference/statements/select/limit-by.md index b6af13fb8d2..05b9e7b9151 100644 --- a/docs/en/sql-reference/statements/select/limit-by.md +++ b/docs/en/sql-reference/statements/select/limit-by.md @@ -1,3 +1,7 @@ +--- +toc_title: LIMIT BY +--- + # LIMIT BY Clause {#limit-by-clause} A query with the `LIMIT n BY expressions` clause selects the first `n` rows for each distinct value of `expressions`. The key for `LIMIT BY` can contain any number of [expressions](../../syntax.md#syntax-expressions). diff --git a/docs/en/sql-reference/statements/select/limit.md b/docs/en/sql-reference/statements/select/limit.md index 8f47904cecb..3f43e1b155e 100644 --- a/docs/en/sql-reference/statements/select/limit.md +++ b/docs/en/sql-reference/statements/select/limit.md @@ -1,3 +1,7 @@ +--- +toc_title: LIMIT +--- + # LIMIT Clause {#limit-clause} `LIMIT m` allows to select the first `m` rows from the result. diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md index 39bd19f8eab..10c1f234057 100644 --- a/docs/en/sql-reference/statements/select/order-by.md +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -1,3 +1,7 @@ +--- +toc_title: ORDER BY +--- + # ORDER BY Clause {#select-order-by} The `ORDER BY` clause contains a list of expressions, which can each be attributed with `DESC` (descending) or `ASC` (ascending) modifier which determine the sorting direction. If the direction is not specified, `ASC` is assumed, so it's usually omitted. The sorting direction applies to a single expression, not to the entire list. Example: `ORDER BY Visits DESC, SearchPhrase` diff --git a/docs/en/sql-reference/statements/select/prewhere.md b/docs/en/sql-reference/statements/select/prewhere.md index 765514a6835..38ff11dc548 100644 --- a/docs/en/sql-reference/statements/select/prewhere.md +++ b/docs/en/sql-reference/statements/select/prewhere.md @@ -1,3 +1,7 @@ +--- +toc_title: PREWHERE +--- + # PREWHERE Clause {#prewhere-clause} Prewhere is an optimization to apply filtering more efficiently. It is enabled by default even if `PREWHERE` clause is not specified explicitly. It works by automatically moving part of [WHERE](where.md) condition to prewhere stage. The role of `PREWHERE` clause is only to control this optimization if you think that you know how to do it better than it happens by default. diff --git a/docs/en/sql-reference/statements/select/sample.md b/docs/en/sql-reference/statements/select/sample.md index 955711e74c0..00431d8e44a 100644 --- a/docs/en/sql-reference/statements/select/sample.md +++ b/docs/en/sql-reference/statements/select/sample.md @@ -1,3 +1,7 @@ +--- +toc_title: SAMPLE +--- + # SAMPLE Clause {#select-sample-clause} The `SAMPLE` clause allows for approximated `SELECT` query processing. diff --git a/docs/en/sql-reference/statements/select/union-all.md b/docs/en/sql-reference/statements/select/union-all.md index f89d15e0539..9c5be93a345 100644 --- a/docs/en/sql-reference/statements/select/union-all.md +++ b/docs/en/sql-reference/statements/select/union-all.md @@ -1,3 +1,7 @@ +--- +toc_title: UNION ALL +--- + # UNION ALL Clause {#union-all-clause} You can use `UNION ALL` to combine any number of `SELECT` queries by extending their results. Example: diff --git a/docs/en/sql-reference/statements/select/where.md b/docs/en/sql-reference/statements/select/where.md index c5a6c6add0f..84618be5002 100644 --- a/docs/en/sql-reference/statements/select/where.md +++ b/docs/en/sql-reference/statements/select/where.md @@ -1,3 +1,7 @@ +--- +toc_title: WHERE +--- + # WHERE Clause {#select-where} `WHERE` clause allows to filter the data that is coming from [FROM](from.md) clause of `SELECT`. diff --git a/docs/en/sql-reference/statements/select/with.md b/docs/en/sql-reference/statements/select/with.md index d6530964e64..ac04ce69ae3 100644 --- a/docs/en/sql-reference/statements/select/with.md +++ b/docs/en/sql-reference/statements/select/with.md @@ -1,3 +1,7 @@ +--- +toc_title: WITH +--- + # WITH Clause {#with-clause} This section provides support for Common Table Expressions ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)), so the results of `WITH` clause can be used in the rest of `SELECT` query. diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index 24db87c6171..e892ee7e51b 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -131,7 +131,7 @@ SHOW CREATE USER [name | CURRENT_USER] ## SHOW CREATE ROLE {#show-create-role-statement} -Shows parameters that were used at a [role creation](create.md#create-role-statement) +Shows parameters that were used at a [role creation](create.md#create-role-statement). ### Syntax {#show-create-role-syntax} @@ -143,7 +143,7 @@ SHOW CREATE ROLE name ## SHOW CREATE ROW POLICY {#show-create-row-policy-statement} -Shows parameters that were used at a [row policy creation](create.md#create-row-policy-statement) +Shows parameters that were used at a [row policy creation](create.md#create-row-policy-statement). ### Syntax {#show-create-row-policy-syntax} @@ -154,7 +154,7 @@ SHOW CREATE [ROW] POLICY name ON [database.]table ## SHOW CREATE QUOTA {#show-create-quota-statement} -Shows parameters that were used at a [quota creation](create.md#create-quota-statement) +Shows parameters that were used at a [quota creation](create.md#create-quota-statement). ### Syntax {#show-create-row-policy-syntax} @@ -165,7 +165,7 @@ SHOW CREATE QUOTA [name | CURRENT] ## SHOW CREATE SETTINGS PROFILE {#show-create-settings-profile-statement} -Shows parameters that were used at a [settings profile creation](create.md#create-settings-profile-statement) +Shows parameters that were used at a [settings profile creation](create.md#create-settings-profile-statement). ### Syntax {#show-create-row-policy-syntax} diff --git a/docs/es/operations/server-configuration-parameters/settings.md b/docs/es/operations/server-configuration-parameters/settings.md index 7ca3b628bbc..29f726ee635 100644 --- a/docs/es/operations/server-configuration-parameters/settings.md +++ b/docs/es/operations/server-configuration-parameters/settings.md @@ -209,7 +209,7 @@ Si `http_port` se especifica, la configuración de OpenSSL se ignora incluso si **Ejemplo** ``` xml -0000 +9999 ``` ## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} diff --git a/docs/fa/operations/server-configuration-parameters/settings.md b/docs/fa/operations/server-configuration-parameters/settings.md index 673f24c1494..1459c20dd5c 100644 --- a/docs/fa/operations/server-configuration-parameters/settings.md +++ b/docs/fa/operations/server-configuration-parameters/settings.md @@ -210,7 +210,7 @@ toc_title: "\u062A\u0646\u0638\u06CC\u0645\u0627\u062A \u06A9\u0627\u0631\u06AF\ **مثال** ``` xml -0000 +9999 ``` ## نقلقولهای جدید از این نویسنده {#server_configuration_parameters-http_server_default_response} diff --git a/docs/fr/operations/server-configuration-parameters/settings.md b/docs/fr/operations/server-configuration-parameters/settings.md index 741bec44421..45be3c5c009 100644 --- a/docs/fr/operations/server-configuration-parameters/settings.md +++ b/docs/fr/operations/server-configuration-parameters/settings.md @@ -209,7 +209,7 @@ Si `http_port` est spécifié, la configuration OpenSSL est ignorée même si el **Exemple** ``` xml -0000 +9999 ``` ## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} diff --git a/docs/ja/operations/server-configuration-parameters/settings.md b/docs/ja/operations/server-configuration-parameters/settings.md index 2b460e8aca6..98a31fa5f60 100644 --- a/docs/ja/operations/server-configuration-parameters/settings.md +++ b/docs/ja/operations/server-configuration-parameters/settings.md @@ -209,7 +209,7 @@ HTTP経由でサーバーに接続するためのポート。 **例** ``` xml -0000 +9999 ``` ## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} diff --git a/docs/ru/operations/access-rights.md b/docs/ru/operations/access-rights.md index 8c131aa34e4..99da2550e70 100644 --- a/docs/ru/operations/access-rights.md +++ b/docs/ru/operations/access-rights.md @@ -1,101 +1,143 @@ -# Права доступа {#prava-dostupa} +# Управление доступом {#access-control} -Пользователи и права доступа настраиваются в конфиге пользователей. Обычно это `users.xml`. +ClickHouse поддерживает управление доступом на основе подхода [RBAC](https://ru.wikipedia.org/wiki/Управление_доступом_на_основе_ролей). -Пользователи прописаны в секции `users`. Рассмотрим фрагмент файла `users.xml`: +Объекты системы доступа в ClickHouse: -``` xml - - - - - - + Функциональность необходимо [включить](#enabling-access-control). - - +Рекомендуется использовать SQL-воркфлоу. Оба метода конфигурации работают одновременно, поэтому, если для управления доступом вы используете конфигурационные файлы, вы можете плавно перейти на SQL-воркфлоу. - - default +!!! note "Внимание" + Нельзя одновременно использовать оба метода для управления одним и тем же объектом системы доступа. - - default - - - - - - web - default - - test - - -``` +## Использование {#access-control-usage} -Здесь видно объявление двух пользователей - `default` и `web`. Пользователя `web` мы добавили самостоятельно. +По умолчанию сервер ClickHouse предоставляет аккаунт пользователя `default`, для которого выключена функция SQL-ориентированного управления доступом, но у него есть все права и разрешения. Аккаунт `default` используется во всех случаях, когда имя пользователя не определено. Например, при входе с клиента или в распределенных запросах. При распределенной обработке запроса `default` используется, если в конфигурации сервера или кластера не указаны свойства [user и password](../engines/table-engines/special/distributed.md). -Пользователь `default` выбирается в случаях, когда имя пользователя не передаётся. Также пользователь `default` может использоваться при распределённой обработке запроса - если в конфигурации кластера для сервера не указаны `user` и `password`. (см. раздел о движке [Distributed](../engines/table-engines/special/distributed.md)). +Если вы начали пользоваться ClickHouse недавно, попробуйте следующий сценарий: -Пользователь, который используется для обмена информацией между серверами, объединенными в кластер, не должен иметь существенных ограничений или квот - иначе распределённые запросы сломаются. +1. [Включите](#enabling-access-control) SQL-ориентированное управление доступом для пользователя `default`. +2. Войдите под пользователем `default` и создайте всех необходимых пользователей. Не забудьте создать аккаунт администратора (`GRANT ALL ON *.* WITH GRANT OPTION TO admin_user_account`). +3. [Ограничьте разрешения](settings/permissions-for-queries.md#permissions_for_queries) для пользователя `default` и отключите для него SQL-ориентированное управление доступом. -Пароль указывается либо в открытом виде (не рекомендуется), либо в виде SHA-256. Хэш не содержит соль. В связи с этим, не следует рассматривать такие пароли, как защиту от потенциального злоумышленника. Скорее, они нужны для защиты от сотрудников. +### Особенности реализации {#access-control-properties} -Указывается список сетей, из которых разрешён доступ. В этом примере, список сетей для обеих пользователей, загружается из отдельного файла (`/etc/metrika.xml`), содержащего подстановку `networks`. Вот его фрагмент: +- Вы можете выдавать разрешения на базы данных или таблицы, даже если они не существуют. +- При удалении таблицы все связанные с ней привилегии не отзываются. Если вы затем создадите новую таблицу с таким же именем, все привилегии останутся действительными. Чтобы отозвать привилегии, связанные с удаленной таблицей, необходимо выполнить, например, запрос `REVOKE ALL PRIVILEGES ON db.table FROM ALL`. +- У привилегий нет настроек времени жизни. -``` xml - - ... - - ::/64 - 203.0.113.0/24 - 2001:DB8::/32 - ... - - -``` +## Аккаунт пользователя {#user-account-management} -Можно было бы указать этот список сетей непосредственно в `users.xml`, или в файле в директории `users.d` (подробнее смотрите раздел «[Конфигурационные файлы](configuration-files.md#configuration_files)»). +Аккаунт пользователя — это объект системы доступа, позволяющий авторизовать кого-либо в ClickHouse. Аккаунт содержит: -В конфиге приведён комментарий, указывающий, как можно открыть доступ отовсюду. +- Идентификационную информацию. +- [Привилегии](../sql-reference/statements/grant.md#grant-privileges), определяющие область действия запросов, которые могут быть выполнены пользователем. +- Хосты, которые могут подключаться к серверу ClickHouse. +- Назначенные роли и роли по умолчанию. +- Настройки и их ограничения, которые применяются по умолчанию при входе пользователя. +- Присвоенные профили настроек. -Для продакшен использования, указывайте только элементы вида `ip` (IP-адреса и их маски), так как использование `host` и `host_regexp` может вызывать лишние задержки. +Привилегии присваиваются аккаунту пользователя с помощью запроса [GRANT](../sql-reference/statements/grant.md) или через назначение [ролей](#role-management). Отозвать привилегию можно с помощью запроса [REVOKE](../sql-reference/statements/revoke.md). Чтобы вывести список присвоенных привилегий, используется выражение [SHOW GRANTS](../sql-reference/statements/show.md#show-grants-statement). -Далее указывается используемый профиль настроек пользователя (смотрите раздел «[Профили настроек](settings/settings-profiles.md)»). Вы можете указать профиль по умолчанию - `default`. Профиль может называться как угодно; один и тот же профиль может быть указан для разных пользователей. Наиболее важная вещь, которую вы можете прописать в профиле настроек `readonly=1`, что обеспечивает доступ только на чтение. -Затем указывается используемая квота (смотрите раздел «[Квоты](quotas.md#quotas)»). Вы можете указать квоту по умолчанию — `default`. Она настроена в конфиге по умолчанию так, что только считает использование ресурсов, но никак их не ограничивает. Квота может называться как угодно. Одна и та же квота может быть указана для разных пользователей, в этом случае подсчёт использования ресурсов делается для каждого пользователя по отдельности. +Запросы управления: -Также, в необязательном разделе `` можно указать перечень баз, к которым у пользователя будет доступ. По умолчанию пользователю доступны все базы. Можно указать базу данных `default`, в этом случае пользователь получит доступ к базе данных по умолчанию. +- [CREATE USER](../sql-reference/statements/create.md#create-user-statement) +- [ALTER USER](../sql-reference/statements/alter.md#alter-user-statement) +- [DROP USER](../sql-reference/statements/misc.md#drop-user-statement) +- [SHOW CREATE USER](../sql-reference/statements/show.md#show-create-user-statement) -Доступ к БД `system` всегда считается разрешённым (так как эта БД используется для выполнения запросов). +### Применение настроек {#access-control-settings-applying} -Пользователь может получить список всех БД и таблиц в них с помощью запросов `SHOW` или системных таблиц, даже если у него нет доступа к отдельным БД. +Настройки могут быть заданы разными способами: для аккаунта пользователя, для назначенных ему ролей или в профилях настроек. При входе пользователя, если настройка задана для разных объектов системы доступа, значение настройки и ее ограничения применяются в следующем порядке (от высшего приоритета к низшему): + +1. Настройки аккаунта. +2. Настройки ролей по умолчанию для аккаунта. Если настройка задана для нескольких ролей, порядок применения не определен. +3. Настройки из профилей настроек, присвоенных пользователю или его ролям по умолчанию. Если настройка задана в нескольких профилях, порядок применения не определен. +4. Настройки, которые по умолчанию применяются ко всему серверу, или настройки из [профиля по умолчанию](server-configuration-parameters/settings.md#default-profile). + + +## Роль {#role-management} + +Роль — это контейнер объектов системы доступа, которые можно присвоить аккаунту пользователя. + +Роль содержит: + +- [Привилегии](../sql-reference/statements/grant.md#grant-privileges) +- Настройки и ограничения +- Список назначенных ролей + +Запросы управления: + +- [CREATE ROLE](../sql-reference/statements/create.md#create-role-statement) +- [ALTER ROLE](../sql-reference/statements/alter.md#alter-role-statement) +- [DROP ROLE](../sql-reference/statements/misc.md#drop-role-statement) +- [SET ROLE](../sql-reference/statements/misc.md#set-role-statement) +- [SET DEFAULT ROLE](../sql-reference/statements/misc.md#set-default-role-statement) +- [SHOW CREATE ROLE](../sql-reference/statements/show.md#show-create-role-statement) + +Привилегии можно присвоить роли с помощью запроса [GRANT](../sql-reference/statements/grant.md). Для отзыва привилегий у роли ClickHouse предоставляет запрос [REVOKE](../sql-reference/statements/revoke.md). + +## Политика доступа к строкам {#row-policy-management} + +Политика доступа к строкам — это фильтр, определяющий, какие строки доступны пользователю или роли. Политика содержит фильтры для конкретной таблицы, а также список ролей и/или пользователей, которые должны использовать данную политику. + +Запросы управления: + +- [CREATE ROW POLICY](../sql-reference/statements/create.md#create-row-policy-statement) +- [ALTER ROW POLICY](../sql-reference/statements/alter.md#alter-row-policy-statement) +- [DROP ROW POLICY](../sql-reference/statements/misc.md#drop-row-policy-statement) +- [SHOW CREATE ROW POLICY](../sql-reference/statements/show.md#show-create-row-policy-statement) + + +## Профиль настроек {#settings-profiles-management} + +Профиль настроек — это набор [настроек](settings/index.md). Профиль настроек содержит настройки и ограничения, а также список ролей и/или пользователей, по отношению к которым применяется данный профиль. + +Запросы управления: + +- [CREATE SETTINGS PROFILE](../sql-reference/statements/create.md#create-settings-profile-statement) +- [ALTER SETTINGS PROFILE](../sql-reference/statements/alter.md#alter-settings-profile-statement) +- [DROP SETTINGS PROFILE](../sql-reference/statements/misc.md#drop-settings-profile-statement) +- [SHOW CREATE SETTINGS PROFILE](../sql-reference/statements/show.md#show-create-settings-profile-statement) + + +## Квота {#quotas-management} + +Квота ограничивает использование ресурсов. См. [Квоты](quotas.md). + +Квота содержит набор ограничений определенной длительности, а также список ролей и/или пользователей, на которых распространяется данная квота. + +Запросы управления: + +- [CREATE QUOTA](../sql-reference/statements/create.md#create-quota-statement) +- [ALTER QUOTA](../sql-reference/statements/alter.md#alter-quota-statement) +- [DROP QUOTA](../sql-reference/statements/misc.md#drop-quota-statement) +- [SHOW CREATE QUOTA](../sql-reference/statements/show.md#show-create-quota-statement) + + +## Включение SQL-ориентированного управления доступом {#enabling-access-control} + +- Настройте каталог для хранения конфигураций. + + ClickHouse хранит конфигурации объектов системы доступа в каталоге, установленном в конфигурационном параметре сервера [access_control_path](server-configuration-parameters/settings.md#access_control_path). + +- Включите SQL-ориентированное управление доступом как минимум для одного аккаунта. + + По умолчанию управление доступом на основе SQL выключено для всех пользователей. Вам необходимо настроить хотя бы одного пользователя в файле конфигурации `users.xml` и присвоить значение 1 параметру [access_management](settings/settings-users.md#access_management-user-setting). -Доступ к БД не связан с настройкой [readonly](settings/permissions-for-queries.md#settings_readonly). Невозможно дать полный доступ к одной БД и `readonly` к другой. [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/access_rights/) diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 43e7eaac7d7..f86fc1baa75 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -195,7 +195,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat **Пример** ``` xml -0000 +9999 ``` ## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} @@ -832,4 +832,14 @@ ClickHouse использует ZooKeeper для хранения метадан **Значение по умолчанию**: 15. +## access_control_path {#access_control_path} + +Путь к каталогу, где сервер ClickHouse хранит конфигурации пользователей и ролей, созданные командами SQL. + +Значение по умолчанию: `/var/lib/clickhouse/access/`. + +**Смотрите также** + +- [Управление доступом](../access-rights.md#access-control) + [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/server_configuration_parameters/settings/) diff --git a/docs/ru/operations/settings/settings-profiles.md b/docs/ru/operations/settings/settings-profiles.md index a0a8b4ba0ba..d1e24490120 100644 --- a/docs/ru/operations/settings/settings-profiles.md +++ b/docs/ru/operations/settings/settings-profiles.md @@ -1,6 +1,15 @@ -# Профили настроек {#profili-nastroek} +# Профили настроек {#settings-profiles} + +Профиль настроек — это набор настроек, сгруппированных под одним именем. + +!!! note "Информация" + Для управления профилями настроек рекомендуется использовать [SQL-ориентированный воркфлоу](../access-rights.md#access-control), который также поддерживается в ClickHouse. + + +Название профиля может быть любым. Вы можете указать один и тот же профиль для разных пользователей. Самое важное, что можно прописать в профиле — `readonly=1`, это обеспечит доступ только на чтение. + +Профили настроек поддерживают наследование. Это реализуется указанием одной или нескольких настроек `profile` перед остальными настройками, перечисленными в профиле. Если одна настройка указана в нескольких профилях, используется последнее из значений. -Профили настроек - это множество настроек, сгруппированных под одним именем. Для каждого пользователя ClickHouse указывается некоторый профиль. Все настройки профиля можно применить, установив настройку `profile`. Пример: @@ -57,8 +66,10 @@ SET profile = 'web' ``` -В примере задано два профиля: `default` и `web`. Профиль `default` имеет специальное значение - он всегда обязан присутствовать и применяется при запуске сервера. То есть, профиль `default` содержит настройки по умолчанию. Профиль `web` - обычный профиль, который может быть установлен с помощью запроса `SET` или с помощью параметра URL при запросе по HTTP. +В примере задано два профиля: `default` и `web`. -Профили настроек могут наследоваться от друг-друга - это реализуется указанием одной или нескольких настроек `profile` перед остальными настройками, перечисленными в профиле. Если одна настройка указана в нескольких профилях, используется последнее из значений. +Профиль `default` имеет специальное значение — он обязателен и применяется при запуске сервера. Профиль `default` содержит настройки по умолчанию. + +Профиль `web` — обычный профиль, который может быть установлен с помощью запроса `SET` или параметра URL при запросе по HTTP. [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings_profiles/) diff --git a/docs/ru/operations/settings/settings-users.md b/docs/ru/operations/settings/settings-users.md index 2d2e2dec506..95701f0f639 100644 --- a/docs/ru/operations/settings/settings-users.md +++ b/docs/ru/operations/settings/settings-users.md @@ -2,6 +2,9 @@ Раздел `users` конфигурационного файла `user.xml` содержит настройки для пользователей. +!!! note "Информация" + Для управления пользователями рекомендуется использовать [SQL-ориентированный воркфлоу](../access-rights.md#access-control), который также поддерживается в ClickHouse. + Структура раздела `users`: ``` xml @@ -12,6 +15,8 @@ + 0|1 + @@ -67,6 +72,17 @@ Первая строка результата — пароль. Вторая строка — соответствующий ему двойной хэш SHA1. +### access_management {#access_management-user-setting} + +Включает или выключает SQL-ориентированное [управление доступом](../access-rights.md#access-control) для пользователя. + +Возможные значения: + +- 0 — Выключено. +- 1 — Включено. + +Значение по умолчанию: 0. + ### user\_name/networks {#user-namenetworks} Список сетей, из которых пользователь может подключиться к серверу ClickHouse. diff --git a/docs/ru/sql-reference/statements/alter.md b/docs/ru/sql-reference/statements/alter.md index 1b35f411df2..c4afb6981c4 100644 --- a/docs/ru/sql-reference/statements/alter.md +++ b/docs/ru/sql-reference/statements/alter.md @@ -498,8 +498,112 @@ ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name Мутации линейно упорядочены между собой и накладываются на каждый кусок в порядке добавления. Мутации также упорядочены со вставками - гарантируется, что данные, вставленные в таблицу до начала выполнения запроса мутации, будут изменены, а данные, вставленные после окончания запроса мутации, изменены не будут. При этом мутации никак не блокируют вставки. -Запрос завершается немедленно после добавления информации о мутации (для реплицированных таблиц - в ZooKeeper, для нереплицированных - на файловую систему). Сама мутация выполняется асинхронно, используя настройки системного профиля. Следить за ходом её выполнения можно по таблице [`system.mutations`](../../operations/system-tables.md#system_tables-mutations). Добавленные мутации будут выполняться до конца даже в случае перезапуска серверов ClickHouse. Откатить мутацию после её добавления нельзя, но если мутация по какой-то причине не может выполниться до конца, её можно остановить с помощью запроса [`KILL MUTATION`](misc.md#kill-mutation). +Запрос завершается немедленно после добавления информации о мутации (для реплицированных таблиц - в ZooKeeper, для нереплицированных - на файловую систему). Сама мутация выполняется асинхронно, используя настройки системного профиля. Следить за ходом её выполнения можно по таблице [`system.mutations`](../../operations/system-tables.md#system_tables-mutations). Добавленные мутации будут выполняться до конца даже в случае перезапуска серверов ClickHouse. Откатить мутацию после её добавления нельзя, но если мутация по какой-то причине не может выполниться до конца, её можно остановить с помощью запроса [`KILL MUTATION`](misc.md#kill-mutation-statement). Записи о последних выполненных мутациях удаляются не сразу (количество сохраняемых мутаций определяется параметром движка таблиц `finished_mutations_to_keep`). Более старые записи удаляются. +## ALTER USER {#alter-user-statement} + +Изменяет аккаунт пользователя ClickHouse. + +### Синтаксис {#alter-user-syntax} + +``` sql +ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] + [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + +### Описание {#alter-user-dscr} + +Для выполнения `ALTER USER` необходима привилегия [ALTER USER](grant.md#grant-access-management). + +### Примеры {#alter-user-examples} + +Установить ролями по умолчанию роли, назначенные пользователю: + +``` sql +ALTER USER user DEFAULT ROLE role1, role2 +``` + +Если роли не были назначены пользователю, ClickHouse выбрасывает исключение. + +Установить ролями по умолчанию все роли, назначенные пользователю: + +``` sql +ALTER USER user DEFAULT ROLE ALL +``` + +Если роль будет впоследствии назначена пользователю, она автоматически станет ролью по умолчанию. + +Установить ролями по умолчанию все назначенные пользователю роли кроме `role1` и `role2`: + +``` sql +ALTER USER user DEFAULT ROLE ALL EXCEPT role1, role2 +``` + + +## ALTER ROLE {#alter-role-statement} + +Изменяет роль. + +### Синтаксис {#alter-role-syntax} + +``` sql +ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + + +## ALTER ROW POLICY {#alter-row-policy-statement} + +Изменяет политику доступа к строкам. + +### Синтаксис {#alter-row-policy-syntax} + +``` sql +ALTER [ROW] POLICY [IF EXISTS] name [ON CLUSTER cluster_name] ON [database.]table + [RENAME TO new_name] + [AS {PERMISSIVE | RESTRICTIVE}] + [FOR SELECT] + [USING {condition | NONE}][,...] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + + +## ALTER QUOTA {#alter-quota-statement} + +Изменяет квоту. + +### Синтаксис {#alter-quota-syntax} + +``` sql +ALTER QUOTA [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] + [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} + {MAX { {QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number } [,...] | + NO LIMITS | TRACKING ONLY} [,...]] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + + +## ALTER SETTINGS PROFILE {#alter-settings-profile-statement} + +Изменяет профили настроек. + +### Синтаксис {#alter-settings-profile-syntax} + +``` sql +ALTER SETTINGS PROFILE [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] +``` + + + [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/) diff --git a/docs/ru/sql-reference/statements/create.md b/docs/ru/sql-reference/statements/create.md index 2a57ac0a143..70544c57dfd 100644 --- a/docs/ru/sql-reference/statements/create.md +++ b/docs/ru/sql-reference/statements/create.md @@ -302,4 +302,206 @@ LIFETIME([MIN val1] MAX val2) Смотрите [Внешние словари](../../sql-reference/statements/create.md). +## CREATE USER {#create-user-statement} + +Создает [аккаунт пользователя](../../operations/access-rights.md#user-account-management). + +### Синтаксис {#create-user-syntax} + +```sql +CREATE USER [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name] + [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}] + [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + [DEFAULT ROLE role [,...]] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + +#### Идентификация + +Существует несколько способов идентификации пользователя: + +- `IDENTIFIED WITH no_password` +- `IDENTIFIED WITH plaintext_password BY 'qwerty'` +- `IDENTIFIED WITH sha256_password BY 'qwerty'` or `IDENTIFIED BY 'password'` +- `IDENTIFIED WITH sha256_hash BY 'hash'` +- `IDENTIFIED WITH double_sha1_password BY 'qwerty'` +- `IDENTIFIED WITH double_sha1_hash BY 'hash'` + +#### Пользовательский хост + +Пользовательский хост — это хост, с которого можно установить соединение с сервером ClickHouse. Хост задается в секции `HOST` следующими способами: + +- `HOST IP 'ip_address_or_subnetwork'` — Пользователь может подключиться к серверу ClickHouse только с указанного IP-адреса или [подсети](https://ru.wikipedia.org/wiki/Подсеть). Примеры: `HOST IP '192.168.0.0/16'`, `HOST IP '2001:DB8::/32'`. При использовании в эксплуатации указывайте только элементы `HOST IP` (IP-адреса и маски подсети), так как использование `host` и `host_regexp` может привести к дополнительной задержке. +- `HOST ANY` — Пользователь может подключиться с любого хоста. Используется по умолчанию. +- `HOST LOCAL` — Пользователь может подключиться только локально. +- `HOST NAME 'fqdn'` — Хост задается через FQDN. Например, `HOST NAME 'mysite.com'`. +- `HOST NAME REGEXP 'regexp'` — Позволяет использовать регулярные выражения [pcre](http://www.pcre.org/), чтобы задать хосты. Например, `HOST NAME REGEXP '.*\.mysite\.com'`. +- `HOST LIKE 'template'` — Позволяет использовать оператор [LIKE](../functions/string-search-functions.md#function-like) для фильтрации хостов. Например, `HOST LIKE '%'` эквивалентен `HOST ANY`; `HOST LIKE '%.mysite.com'` разрешает подключение со всех хостов в домене `mysite.com`. + +Также, чтобы задать хост, вы можете использовать `@` вместе с именем пользователя. Примеры: + +- `CREATE USER mira@'127.0.0.1'` — Эквивалентно `HOST IP`. +- `CREATE USER mira@'localhost'` — Эквивалентно `HOST LOCAL`. +- `CREATE USER mira@'192.168.%.%'` — Эквивалентно `HOST LIKE`. + +!!! info "Внимание" + ClickHouse трактует конструкцию `user_name@'address'` как имя пользователя целиком. То есть технически вы можете создать несколько пользователей с одинаковыми `user_name`, но разными частями конструкции после `@`, но лучше так не делать. + + +### Примеры {#create-user-examples} + + +Создать аккаунт `mira`, защищенный паролем `qwerty`: + +```sql +CREATE USER mira HOST IP '127.0.0.1' IDENTIFIED WITH sha256_password BY 'qwerty' +``` + +Пользователь `mira` должен запустить клиентское приложение на хосте, где запущен ClickHouse. + +Создать аккаунт `john`, назначить на него роли, сделать данные роли ролями по умолчанию: + +``` sql +CREATE USER john DEFAULT ROLE role1, role2 +``` + +Создать аккаунт `john` и установить ролями по умолчанию все его будущие роли: + +``` sql +ALTER USER user DEFAULT ROLE ALL +``` + +Когда роль будет назначена аккаунту `john`, она автоматически станет ролью по умолчанию. + +Создать аккаунт `john` и установить ролями по умолчанию все его будущие роли, кроме `role1` и `role2`: + +``` sql +ALTER USER john DEFAULT ROLE ALL EXCEPT role1, role2 +``` + + +## CREATE ROLE {#create-role-statement} + +Создает [роль](../../operations/access-rights.md#role-management). + +### Синтаксис {#create-role-syntax} + +```sql +CREATE ROLE [IF NOT EXISTS | OR REPLACE] name + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + +### Описание {#create-role-description} + +Роль — это набор [привилегий](grant.md#grant-privileges). Пользователь, которому назначена роль, получает все привилегии этой роли. + +Одному пользователю можно назначить несколько ролей. Пользователи могут применять назначенные роли в произвольных комбинациях с помощью выражения [SET ROLE](misc.md#set-role-statement). Конечный объем привилегий — это комбинация всех привилегий всех примененных ролей. Если у пользователя имеются привилегии, присвоенные его аккаунту напрямую, они также прибавляются к привилегиям, присвоенным через роли. + +Роли по умолчанию применяются при входе пользователя в систему. Установить роли по умолчанию можно с помощью выражений [SET DEFAULT ROLE](misc.md#set-default-role-statement) или [ALTER USER](alter.md#alter-user-statement). + +Для отзыва роли используется выражение [REVOKE](revoke.md). + +Для удаления роли используется выражение [DROP ROLE](misc.md#drop-role-statement). Удаленная роль автоматически отзывается у всех пользователей, которым была назначена. + +### Примеры {#create-role-examples} + +```sql +CREATE ROLE accountant; +GRANT SELECT ON db.* TO accountant; +``` + +Такая последовательность запросов создаст роль `accountant`, у которой есть привилегия на чтение из базы данных `accounting`. + +Назначить роль `accountant` аккаунту `mira`: + +```sql +GRANT accountant TO mira; +``` + +После назначения роли пользователь может ее применить и выполнять разрешенные ей запросы. Например: + +```sql +SET ROLE accountant; +SELECT * FROM db.*; +``` + +## CREATE ROW POLICY {#create-row-policy-statement} + +Создает [фильтр для строк](../../operations/access-rights.md#row-policy-management), которые пользователь может прочесть из таблицы. + +### Синтаксис {#create-row-policy-syntax} + +``` sql +CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name [ON CLUSTER cluster_name] ON [db.]table + [AS {PERMISSIVE | RESTRICTIVE}] + [FOR SELECT] + [USING condition] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + +#### Секция AS {#create-row-policy-as} + +С помощью данной секции можно создать политику разрешения или ограничения. + +Политика разрешения предоставляет доступ к строкам. Разрешительные политики, которые применяются к одной таблице, объединяются с помощью логического оператора `OR`. Политики являются разрешительными по умолчанию. + +Политика ограничения запрещает доступ к строкам. Ограничительные политики, которые применяются к одной таблице, объединяются логическим оператором `AND`. + +Ограничительные политики применяются к строкам, прошедшим фильтр разрешительной политики. Если вы не зададите разрешительные политики, пользователь не сможет обращаться ни к каким строкам из таблицы. + +#### Секция TO {#create-row-policy-to} + +В секции `TO` вы можете перечислить как роли, так и пользователей. Например, `CREATE ROW POLICY ... TO accountant, john@localhost`. + +Ключевым словом `ALL` обозначаются все пользователи, включая текущего. Ключевые слова `ALL EXCEPT` позволяют исключить пользователей из списка всех пользователей. Например, `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost` + +### Примеры + +- `CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO accountant, john@localhost` +- `CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO ALL EXCEPT mira` + + +## CREATE QUOTA {#create-quota-statement} + +Создает [квоту](../../operations/access-rights.md#quotas-management), которая может быть присвоена пользователю или роли. + +### Синтаксис {#create-quota-syntax} + +``` sql +CREATE QUOTA [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name] + [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] + [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY} + {MAX { {QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number } [,...] | + NO LIMITS | TRACKING ONLY} [,...]] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + +### Пример {#create-quota-example} + +Ограничить максимальное количество запросов для текущего пользователя до 123 запросов каждые 15 месяцев: + +``` sql +CREATE QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 123 TO CURRENT_USER +``` + + +## CREATE SETTINGS PROFILE {#create-settings-profile-statement} + +Создает [профиль настроек](../../operations/access-rights.md#settings-profiles-management), который может быть присвоен пользователю или роли. + +### Синтаксис {#create-settings-profile-syntax} + +``` sql +CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name [ON CLUSTER cluster_name] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] +``` + +### Пример {#create-settings-profile-syntax} + +Создать профиль настроек `max_memory_usage_profile`, который содержит значение и ограничения для настройки `max_memory_usage`. Присвоить профиль пользователю `robin`: + +``` sql +CREATE SETTINGS PROFILE max_memory_usage_profile SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin +``` + [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/create/) diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md deleted file mode 120000 index f2acbe125b4..00000000000 --- a/docs/ru/sql-reference/statements/grant.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/sql-reference/statements/grant.md \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md new file mode 100644 index 00000000000..a328975ec12 --- /dev/null +++ b/docs/ru/sql-reference/statements/grant.md @@ -0,0 +1,479 @@ +# GRANT + +- Присваивает [привилегии](#grant-privileges) пользователям или ролям ClickHouse. +- Назначает роли пользователям или другим ролям. + +Отозвать привилегию можно с помощью выражения [REVOKE](revoke.md). Чтобы вывести список присвоенных привилегий, воспользуйтесь выражением [SHOW GRANTS](show.md#show-grants-statement). + +## Синтаксис присвоения привилегий {#grant-privigele-syntax} + +```sql +GRANT [ON CLUSTER cluster_name] privilege[(column_name [,...])] [,...] ON {db.table|db.*|*.*|table|*} TO {user | role | CURRENT_USER} [,...] [WITH GRANT OPTION] +``` + +- `privilege` — Тип привилегии +- `role` — Роль пользователя ClickHouse. +- `user` — Пользователь ClickHouse. + +`WITH GRANT OPTION` разрешает пользователю или роли выполнять запрос `GRANT`. Пользователь может выдавать только те привилегии, которые есть у него, той же или меньшей области действий. + + +## Синтаксис назначения ролей {#assign-role-syntax} + +```sql +GRANT [ON CLUSTER cluster_name] role [,...] TO {user | another_role | CURRENT_USER} [,...] [WITH ADMIN OPTION] +``` + +- `role` — Роль пользователя ClickHouse. +- `user` — Пользователь ClickHouse. + +`WITH ADMIN OPTION` присваивает привилегию [ADMIN OPTION](#admin-option-privilege) пользователю или роли. + +## Использование {#grant-usage} + +Для использования `GRANT` пользователь должен иметь привилегию `GRANT OPTION`. Пользователь может выдавать привилегии только внутри области действий назначенных ему самому привилегий. + +Например, администратор выдал привилегию пользователю `john`: + +```sql +GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION +``` + +Это означает, что пользователю `john` разрешено выполнять: + +- `SELECT x,y FROM db.table`. +- `SELECT x FROM db.table`. +- `SELECT y FROM db.table`. + +`john` не может выполнить `SELECT z FROM db.table` или `SELECT * FROM db.table`. После обработки данных запросов ClickHouse ничего не вернет — даже `x` или `y`. Единственное исключение — если таблица содержит только столбцы `x` и `y`. В таком случае ClickHouse вернет все данные. + +Также у `john` есть привилегия `GRANT OPTION`. `john` может выдать другим пользователям привилегии той же или меньшей области действий из тех, которые есть у него. + +При присвоении привилегий допускается использовать астериск (`*`) вместо имени таблицы или базы данных. Например, запрос `GRANT SELECT ON db.* TO john` позволит пользователю `john` выполнять `SELECT` над всеми таблицам в базе данных `db`. Также вы можете опускать имя базы данных. В таком случае привилегии позволят совершать операции над текущей базой данных. Например, запрос `GRANT SELECT ON * TO john` выдаст привилегию на выполнение `SELECT` над всеми таблицами в текущей базе данных; `GRANT SELECT ON mytable TO john` — только над таблицей `mytable` в текущей базе данных. + +Доступ к базе данных `system` разрешен всегда (данная база данных используется при обработке запросов). + +Вы можете присвоить несколько привилегий нескольким пользователям в одном запросе. Запрос `GRANT SELECT, INSERT ON *.* TO john, robin` позволит пользователям `john` и `robin` выполнять `INSERT` и `SELECT` над всеми таблицами всех баз данных на сервере. + + +## Привилегии {#grant-privileges} + +Привилегия — это разрешение на выполнение определенного типа запросов. + +Привилегии имеют иерархическую структуру. Набор разрешенных запросов зависит от области действия привилегии. + +Иерархия привилегий: + +- [SELECT](#grant-select) +- [INSERT](#grant-insert) +- [ALTER](#grant-alter) + - `ALTER TABLE` + - `ALTER UPDATE` + - `ALTER DELETE` + - `ALTER COLUMN` + - `ALTER ADD COLUMN` + - `ALTER DROP COLUMN` + - `ALTER MODIFY COLUMN` + - `ALTER COMMENT COLUMN` + - `ALTER CLEAR COLUMN` + - `ALTER RENAME COLUMN` + - `ALTER INDEX` + - `ALTER ORDER BY` + - `ALTER ADD INDEX` + - `ALTER DROP INDEX` + - `ALTER MATERIALIZE INDEX` + - `ALTER CLEAR INDEX` + - `ALTER CONSTRAINT` + - `ALTER ADD CONSTRAINT` + - `ALTER DROP CONSTRAINT` + - `ALTER TTL` + - `ALTER MATERIALIZE TTL` + - `ALTER SETTINGS` + - `ALTER MOVE PARTITION` + - `ALTER FETCH PARTITION` + - `ALTER FREEZE PARTITION` + - `ALTER VIEW` + - `ALTER VIEW REFRESH ` + - `ALTER VIEW MODIFY QUERY` +- [CREATE](#grant-create) + - `CREATE DATABASE` + - `CREATE TABLE` + - `CREATE VIEW` + - `CREATE DICTIONARY` + - `CREATE TEMPORARY TABLE` +- [DROP](#grant-drop) + - `DROP DATABASE` + - `DROP TABLE` + - `DROP VIEW` + - `DROP DICTIONARY` +- [TRUNCATE](#grant-truncate) +- [OPTIMIZE](#grant-optimize) +- [SHOW](#grant-show) + - `SHOW DATABASES` + - `SHOW TABLES` + - `SHOW COLUMNS` + - `SHOW DICTIONARIES` +- [KILL QUERY](#grant-kill-query) +- [ACCESS MANAGEMENT](#grant-access-management) + - `CREATE USER` + - `ALTER USER` + - `DROP USER` + - `CREATE ROLE` + - `ALTER ROLE` + - `DROP ROLE` + - `CREATE ROW POLICY` + - `ALTER ROW POLICY` + - `DROP ROW POLICY` + - `CREATE QUOTA` + - `ALTER QUOTA` + - `DROP QUOTA` + - `CREATE SETTINGS PROFILE` + - `ALTER SETTINGS PROFILE` + - `DROP SETTINGS PROFILE` + - `SHOW ACCESS` + - `SHOW_USERS` + - `SHOW_ROLES` + - `SHOW_ROW_POLICIES` + - `SHOW_QUOTAS` + - `SHOW_SETTINGS_PROFILES` + - `ROLE ADMIN` +- [SYSTEM](#grant-system) + - `SYSTEM SHUTDOWN` + - `SYSTEM DROP CACHE` + - `SYSTEM DROP DNS CACHE` + - `SYSTEM DROP MARK CACHE` + - `SYSTEM DROP UNCOMPRESSED CACHE` + - `SYSTEM RELOAD` + - `SYSTEM RELOAD CONFIG` + - `SYSTEM RELOAD DICTIONARY` + - `SYSTEM RELOAD EMBEDDED DICTIONARIES` + - `SYSTEM MERGES` + - `SYSTEM TTL MERGES` + - `SYSTEM FETCHES` + - `SYSTEM MOVES` + - `SYSTEM SENDS` + - `SYSTEM DISTRIBUTED SENDS` + - `SYSTEM REPLICATED SENDS` + - `SYSTEM REPLICATION QUEUES` + - `SYSTEM SYNC REPLICA` + - `SYSTEM RESTART REPLICA` + - `SYSTEM FLUSH` + - `SYSTEM FLUSH DISTRIBUTED` + - `SYSTEM FLUSH LOGS` +- [INTROSPECTION](#grant-introspection) + - `addressToLine` + - `addressToSymbol` + - `demangle` +- [SOURCES](#grant-sources) + - `FILE` + - `URL` + - `REMOTE` + - `YSQL` + - `ODBC` + - `JDBC` + - `HDFS` + - `S3` +- [dictGet](#grant-dictget) + +Примеры того, как трактуется данная иерархия: + +- Привилегия `ALTER` включает все остальные `ALTER*` привилегии. +- `ALTER CONSTRAINT` включает `ALTER ADD CONSTRAINT` и `ALTER DROP CONSTRAINT`. + +Привилегии применяются на разных уровнях. Уровень определяет синтаксис присваивания привилегии. + +Уровни (от низшего к высшему): + +- `COLUMN` — Привилегия присваивается для столбца, таблицы, базы данных или глобально. +- `TABLE` — Привилегия присваивается для таблицы, базы данных или глобально. +- `VIEW` — Привилегия присваивается для представления, базы данных или глобально. +- `DICTIONARY` — Привилегия присваивается для словаря, базы данных или глобально. +- `DATABASE` — Привилегия присваивается для базы данных или глобально. +- `GLOBAL` — Привилегия присваивается только глобально. +- `GROUP` — Группирует привилегии разных уровней. При присвоении привилегии уровня `GROUP` присваиваются только привилегии из группы в соответствии с используемым синтаксисом. + +Примеры допустимого синтаксиса: + +- `GRANT SELECT(x) ON db.table TO user` +- `GRANT SELECT ON db.* TO user` + +Примеры недопустимого синтаксиса: + +- `GRANT CREATE USER(x) ON db.table TO user` +- `GRANT CREATE USER ON db.* TO user` + +Специальная привилегия [ALL](#grant-all) присваивает все привилегии пользователю или роли. + +По умолчанию пользователь или роль не имеют привилегий. + +Отсутствие привилегий у пользователя или роли отображается как привилегия [NONE](#grant-none). + +Выполнение некоторых запросов требует определенного набора привилегий. Например, чтобы выполнить запрос [RENAME](misc.md#misc_operations-rename), нужны следующие привилегии: `SELECT`, `CREATE TABLE`, `INSERT` и `DROP TABLE`. + + +### SELECT {#grant-select} + +Разрешает выполнять запросы [SELECT](select/index.md). + +Уровень: `COLUMN`. + +**Описание** + +Пользователь с данной привилегией может выполнять запросы `SELECT` над определенными столбцами из определенной таблицы и базы данных. При включении в запрос других столбцов запрос ничего не вернет. + +Рассмотрим следующую привилегию: + +```sql +GRANT SELECT(x,y) ON db.table TO john +``` + +Данная привилегия позволяет пользователю `john` выполнять выборку данных из столбцов `x` и/или `y` в `db.table`, например, `SELECT x FROM db.table`. `john` не может выполнить `SELECT z FROM db.table` или `SELECT * FROM db.table`. После обработки данных запросов ClickHouse ничего не вернет — даже `x` или `y`. Единственное исключение — если таблица содержит только столбцы `x` и `y`. В таком случае ClickHouse вернет все данные. + +### INSERT {#grant-insert} + +Разрешает выполнять запросы [INSERT](insert-into.md). + +Уровень: `COLUMN`. + +**Описание** + +Пользователь с данной привилегией может выполнять запросы `INSERT` над определенными столбцами из определенной таблицы и базы данных. При включении в запрос других столбцов запрос не добавит никаких данных. + +**Пример** + +```sql +GRANT INSERT(x,y) ON db.table TO john +``` + +Присвоенная привилегия позволит пользователю `john` вставить данные в столбцы `x` и/или `y` в `db.table`. + +### ALTER {#grant-alter} + +Разрешает выполнять запросы [ALTER](alter.md) в соответствии со следующей иерархией привилегий: + +- `ALTER`. Уровень: `COLUMN`. + - `ALTER TABLE`. Уровень: `GROUP` + - `ALTER UPDATE`. Уровень: `COLUMN`. Алиасы: `UPDATE` + - `ALTER DELETE`. Уровень: `COLUMN`. Алиасы: `DELETE` + - `ALTER COLUMN`. Уровень: `GROUP` + - `ALTER ADD COLUMN`. Уровень: `COLUMN`. Алиасы: `ADD COLUMN` + - `ALTER DROP COLUMN`. Уровень: `COLUMN`. Алиасы: `DROP COLUMN` + - `ALTER MODIFY COLUMN`. Уровень: `COLUMN`. Алиасы: `MODIFY COLUMN` + - `ALTER COMMENT COLUMN`. Уровень: `COLUMN`. Алиасы: `COMMENT COLUMN` + - `ALTER CLEAR COLUMN`. Уровень: `COLUMN`. Алиасы: `CLEAR COLUMN` + - `ALTER RENAME COLUMN`. Уровень: `COLUMN`. Алиасы: `RENAME COLUMN` + - `ALTER INDEX`. Уровень: `GROUP`. Алиасы: `INDEX` + - `ALTER ORDER BY`. Уровень: `TABLE`. Алиасы: `ALTER MODIFY ORDER BY`, `MODIFY ORDER BY` + - `ALTER ADD INDEX`. Уровень: `TABLE`. Алиасы: `ADD INDEX` + - `ALTER DROP INDEX`. Уровень: `TABLE`. Алиасы: `DROP INDEX` + - `ALTER MATERIALIZE INDEX`. Уровень: `TABLE`. Алиасы: `MATERIALIZE INDEX` + - `ALTER CLEAR INDEX`. Уровень: `TABLE`. Алиасы: `CLEAR INDEX` + - `ALTER CONSTRAINT`. Уровень: `GROUP`. Алиасы: `CONSTRAINT` + - `ALTER ADD CONSTRAINT`. Уровень: `TABLE`. Алиасы: `ADD CONSTRAINT` + - `ALTER DROP CONSTRAINT`. Уровень: `TABLE`. Алиасы: `DROP CONSTRAINT` + - `ALTER TTL`. Уровень: `TABLE`. Алиасы: `ALTER MODIFY TTL`, `MODIFY TTL` + - `ALTER MATERIALIZE TTL`. Уровень: `TABLE`. Алиасы: `MATERIALIZE TTL` + - `ALTER SETTINGS`. Уровень: `TABLE`. Алиасы: `ALTER SETTING`, `ALTER MODIFY SETTING`, `MODIFY SETTING` + - `ALTER MOVE PARTITION`. Уровень: `TABLE`. Алиасы: `ALTER MOVE PART`, `MOVE PARTITION`, `MOVE PART` + - `ALTER FETCH PARTITION`. Уровень: `TABLE`. Алиасы: `FETCH PARTITION` + - `ALTER FREEZE PARTITION`. Уровень: `TABLE`. Алиасы: `FREEZE PARTITION` + - `ALTER VIEW` Уровень: `GROUP` + - `ALTER VIEW REFRESH `. Уровень: `VIEW`. Алиасы: `ALTER LIVE VIEW REFRESH`, `REFRESH VIEW` + - `ALTER VIEW MODIFY QUERY`. Уровень: `VIEW`. Алиасы: `ALTER TABLE MODIFY QUERY` + +Примеры того, как трактуется данная иерархия: + +- Привилегия `ALTER` включает все остальные `ALTER*` привилегии. +- `ALTER CONSTRAINT` включает `ALTER ADD CONSTRAINT` и `ALTER DROP CONSTRAINT`. + +**Дополнительно** + +- Привилегия `MODIFY SETTING` позволяет изменять настройки движков таблиц. Не влияет на настройки или конфигурационные параметры сервера. +- Операция `ATTACH` требует наличие привилегии [CREATE](#grant-create). +- Операция `DETACH` требует наличие привилегии [DROP](#grant-drop). +- Для остановки мутации с помощью [KILL MUTATION](misc.md#kill-mutation-statement), необходима привилегия на выполнение данной мутации. Например, чтобы остановить запрос `ALTER UPDATE`, необходима одна из привилегий: `ALTER UPDATE`, `ALTER TABLE` или `ALTER`. + +### CREATE {#grant-create} + +Разрешает выполнять DDL-запросы [CREATE](create.md) и [ATTACH](misc.md#attach) в соответствии со следующей иерархией привилегий: + +- `CREATE`. Уровень: `GROUP` + - `CREATE DATABASE`. Уровень: `DATABASE` + - `CREATE TABLE`. Уровень: `TABLE` + - `CREATE VIEW`. Уровень: `VIEW` + - `CREATE DICTIONARY`. Уровень: `DICTIONARY` + - `CREATE TEMPORARY TABLE`. Уровень: `GLOBAL` + +**Дополнительно** + +- Для удаления созданной таблицы пользователю необходима привилегия [DROP](#grant-drop). + +### DROP {#grant-drop} + +Разрешает выполнять запросы [DROP](misc.md#drop) и [DETACH](misc.md#detach-statement) в соответствии со следующей иерархией привилегий: + +- `DROP`. Уровень: + - `DROP DATABASE`. Уровень: `DATABASE` + - `DROP TABLE`. Уровень: `TABLE` + - `DROP VIEW`. Уровень: `VIEW` + - `DROP DICTIONARY`. Уровень: `DICTIONARY` + + +### TRUNCATE {#grant-truncate} + +Разрешает выполнять запросы [TRUNCATE](misc.md#truncate-statement). + +Уровень: `TABLE`. + +### OPTIMIZE {#grant-optimize} + +Разрешает выполнять запросы [OPTIMIZE TABLE](misc.md#misc_operations-optimize). + +Уровень: `TABLE`. + +### SHOW {#grant-show} + +Разрешает выполнять запросы `SHOW`, `DESCRIBE`, `USE` и `EXISTS` в соответствии со следующей иерархией привилегий: + +- `SHOW`. Уровень: `GROUP` + - `SHOW DATABASES`. Уровень: `DATABASE`. Разрешает выполнять запросы `SHOW DATABASES`, `SHOW CREATE DATABASE`, `USE `. + - `SHOW TABLES`. Уровень: `TABLE`. Разрешает выполнять запросы `SHOW TABLES`, `EXISTS `, `CHECK
`. + - `SHOW COLUMNS`. Уровень: `COLUMN`. Разрешает выполнять запросы `SHOW CREATE TABLE`, `DESCRIBE`. + - `SHOW DICTIONARIES`. Уровень: `DICTIONARY`. Разрешает выполнять запросы `SHOW DICTIONARIES`, `SHOW CREATE DICTIONARY`, `EXISTS `. + +**Дополнительно** + +У пользователя есть привилегия `SHOW`, если ему присвоена любая другая привилегия по отношению к определенной таблице, словарю или базе данных. + + +### KILL QUERY {#grant-kill-query} + +Разрешает выполнять запросы [KILL](misc.md#kill-query-statement) в соответствии со следующей иерархией привилегий: + +Уровень: `GLOBAL`. + +**Дополнительно** + +`KILL QUERY` позволяет пользователю останавливать запросы других пользователей. + + +### ACCESS MANAGEMENT {#grant-access-management} + +Разрешает пользователю выполнять запросы на управление пользователями, ролями и политиками доступа к строкам. + +- `ACCESS MANAGEMENT`. Уровень: `GROUP` + - `CREATE USER`. Уровень: `GLOBAL` + - `ALTER USER`. Уровень: `GLOBAL` + - `DROP USER`. Уровень: `GLOBAL` + - `CREATE ROLE`. Уровень: `GLOBAL` + - `ALTER ROLE`. Уровень: `GLOBAL` + - `DROP ROLE`. Уровень: `GLOBAL` + - `ROLE ADMIN`. Уровень: `GLOBAL` + - `CREATE ROW POLICY`. Уровень: `GLOBAL`. Алиасы: `CREATE POLICY` + - `ALTER ROW POLICY`. Уровень: `GLOBAL`. Алиасы: `ALTER POLICY` + - `DROP ROW POLICY`. Уровень: `GLOBAL`. Алиасы: `DROP POLICY` + - `CREATE QUOTA`. Уровень: `GLOBAL` + - `ALTER QUOTA`. Уровень: `GLOBAL` + - `DROP QUOTA`. Уровень: `GLOBAL` + - `CREATE SETTINGS PROFILE`. Уровень: `GLOBAL`. Алиасы: `CREATE PROFILE` + - `ALTER SETTINGS PROFILE`. Уровень: `GLOBAL`. Алиасы: `ALTER PROFILE` + - `DROP SETTINGS PROFILE`. Уровень: `GLOBAL`. Алиасы: `DROP PROFILE` + - `SHOW ACCESS`. Уровень: `GROUP` + - `SHOW_USERS`. Уровень: `GLOBAL`. Алиасы: `SHOW CREATE USER` + - `SHOW_ROLES`. Уровень: `GLOBAL`. Алиасы: `SHOW CREATE ROLE` + - `SHOW_ROW_POLICIES`. Уровень: `GLOBAL`. Алиасы: `SHOW POLICIES`, `SHOW CREATE ROW POLICY`, `SHOW CREATE POLICY` + - `SHOW_QUOTAS`. Уровень: `GLOBAL`. Алиасы: `SHOW CREATE QUOTA` + - `SHOW_SETTINGS_PROFILES`. Уровень: `GLOBAL`. Алиасы: `SHOW PROFILES`, `SHOW CREATE SETTINGS PROFILE`, `SHOW CREATE PROFILE` + +Привилегия `ROLE ADMIN` разрешает пользователю назначать и отзывать любые роли, включая те, которые не назначены пользователю с опцией администратора. + +### SYSTEM {#grant-system} + +Разрешает выполнять запросы [SYSTEM](system.md) в соответствии со следующей иерархией привилегий: + +- `SYSTEM`. Уровень: `GROUP` + - `SYSTEM SHUTDOWN`. Уровень: `GLOBAL`. Алиасы: `SYSTEM KILL`, `SHUTDOWN` + - `SYSTEM DROP CACHE`. Алиасы: `DROP CACHE` + - `SYSTEM DROP DNS CACHE`. Уровень: `GLOBAL`. Алиасы: `SYSTEM DROP DNS`, `DROP DNS CACHE`, `DROP DNS` + - `SYSTEM DROP MARK CACHE`. Уровень: `GLOBAL`. Алиасы: `SYSTEM DROP MARK`, `DROP MARK CACHE`, `DROP MARKS` + - `SYSTEM DROP UNCOMPRESSED CACHE`. Уровень: `GLOBAL`. Алиасы: `SYSTEM DROP UNCOMPRESSED`, `DROP UNCOMPRESSED CACHE`, `DROP UNCOMPRESSED` + - `SYSTEM RELOAD`. Уровень: `GROUP` + - `SYSTEM RELOAD CONFIG`. Уровень: `GLOBAL`. Алиасы: `RELOAD CONFIG` + - `SYSTEM RELOAD DICTIONARY`. Уровень: `GLOBAL`. Алиасы: `SYSTEM RELOAD DICTIONARIES`, `RELOAD DICTIONARY`, `RELOAD DICTIONARIES` + - `SYSTEM RELOAD EMBEDDED DICTIONARIES`. Уровень: `GLOBAL`. Алиасы: `RELOAD EMBEDDED DICTIONARIES` + - `SYSTEM MERGES`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP MERGES`, `SYSTEM START MERGES`, `STOP MERGES`, `START MERGES` + - `SYSTEM TTL MERGES`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP TTL MERGES`, `SYSTEM START TTL MERGES`, `STOP TTL MERGES`, `START TTL MERGES` + - `SYSTEM FETCHES`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP FETCHES`, `SYSTEM START FETCHES`, `STOP FETCHES`, `START FETCHES` + - `SYSTEM MOVES`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP MOVES`, `SYSTEM START MOVES`, `STOP MOVES`, `START MOVES` + - `SYSTEM SENDS`. Уровень: `GROUP`. Алиасы: `SYSTEM STOP SENDS`, `SYSTEM START SENDS`, `STOP SENDS`, `START SENDS` + - `SYSTEM DISTRIBUTED SENDS`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP DISTRIBUTED SENDS`, `SYSTEM START DISTRIBUTED SENDS`, `STOP DISTRIBUTED SENDS`, `START DISTRIBUTED SENDS` + - `SYSTEM REPLICATED SENDS`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP REPLICATED SENDS`, `SYSTEM START REPLICATED SENDS`, `STOP REPLICATED SENDS`, `START REPLICATED SENDS` + - `SYSTEM REPLICATION QUEUES`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP REPLICATION QUEUES`, `SYSTEM START REPLICATION QUEUES`, `STOP REPLICATION QUEUES`, `START REPLICATION QUEUES` + - `SYSTEM SYNC REPLICA`. Уровень: `TABLE`. Алиасы: `SYNC REPLICA` + - `SYSTEM RESTART REPLICA`. Уровень: `TABLE`. Алиасы: `RESTART REPLICA` + - `SYSTEM FLUSH`. Уровень: `GROUP` + - `SYSTEM FLUSH DISTRIBUTED`. Уровень: `TABLE`. Алиасы: `FLUSH DISTRIBUTED` + - `SYSTEM FLUSH LOGS`. Уровень: `GLOBAL`. Алиасы: `FLUSH LOGS` + +Привилегия `SYSTEM RELOAD EMBEDDED DICTIONARIES` имплицитно присваивается привилегией `SYSTEM RELOAD DICTIONARY ON *.*`. + + +### INTROSPECTION {#grant-introspection} + +Разрешает использовать функции [интроспекции](../../operations/optimizing-performance/sampling-query-profiler.md). + +- `INTROSPECTION`. Уровень: `GROUP`. Алиасы: `INTROSPECTION FUNCTIONS` + - `addressToLine`. Уровень: `GLOBAL` + - `addressToSymbol`. Уровень: `GLOBAL` + - `demangle`. Уровень: `GLOBAL` + + +### SOURCES {#grant-sources} + +Разрешает использовать внешние источники данных. Применяется к [движкам таблиц](../../engines/table-engines/index.md) и [табличным функциям](../table-functions/index.md#table-functions). + +- `SOURCES`. Уровень: `GROUP` + - `FILE`. Уровень: `GLOBAL` + - `URL`. Уровень: `GLOBAL` + - `REMOTE`. Уровень: `GLOBAL` + - `YSQL`. Уровень: `GLOBAL` + - `ODBC`. Уровень: `GLOBAL` + - `JDBC`. Уровень: `GLOBAL` + - `HDFS`. Уровень: `GLOBAL` + - `S3`. Уровень: `GLOBAL` + +Привилегия `SOURCES` разрешает использование всех источников. Также вы можете присвоить привилегию для каждого источника отдельно. Для использования источников необходимы дополнительные привилегии. + +Примеры: + +- Чтобы создать таблицу с [движком MySQL](../../engines/table-engines/integrations/mysql.md), необходимы привилегии `CREATE TABLE (ON db.table_name)` и `MYSQL`. +- Чтобы использовать [табличную функцию mysql](../table-functions/mysql.md), необходимы привилегии `CREATE TEMPORARY TABLE` и `MYSQL`. + +### dictGet {#grant-dictget} + +- `dictGet`. Алиасы: `dictHas`, `dictGetHierarchy`, `dictIsIn` + +Разрешает вызывать функции [dictGet](../functions/ext-dict-functions.md#dictget), [dictHas](../functions/ext-dict-functions.md#dicthas), [dictGetHierarchy](../functions/ext-dict-functions.md#dictgethierarchy), [dictIsIn](../functions/ext-dict-functions.md#dictisin). + +Уровень: `DICTIONARY`. + +**Примеры** + +- `GRANT dictGet ON mydb.mydictionary TO john` +- `GRANT dictGet ON mydictionary TO john` + +### ALL {#grant-all} + +Присваивает пользователю или роли все привилегии на объект с регулируемым доступом. + + +### NONE {#grant-none} + +Не присваивает никаких привилегий. + + +### ADMIN OPTION {#admin-option-privilege} + +Привилегия `ADMIN OPTION` разрешает пользователю назначать свои роли другому пользователю. + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/grant/) diff --git a/docs/ru/sql-reference/statements/misc.md b/docs/ru/sql-reference/statements/misc.md index cdd53843ab4..97d2ce5818e 100644 --- a/docs/ru/sql-reference/statements/misc.md +++ b/docs/ru/sql-reference/statements/misc.md @@ -70,7 +70,7 @@ DESC|DESCRIBE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] Вложенные структуры данных выводятся в «развёрнутом» виде. То есть, каждый столбец - по отдельности, с именем через точку. -## DETACH {#detach} +## DETACH {#detach-statement} Удаляет из сервера информацию о таблице name. Сервер перестаёт знать о существовании таблицы. @@ -91,9 +91,6 @@ DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] ``` -Удаляет все таблицы внутри базы данных db, а затем саму базу данных db. -Если указано `IF EXISTS` - не выдавать ошибку, если база данных не существует. - ``` sql DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` @@ -101,6 +98,68 @@ DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] Удаляет таблицу. Если указано `IF EXISTS` - не выдавать ошибку, если таблица не существует или база данных не существует. +## DROP USER {#drop-user-statement} + +Удаляет пользователя. + +### Синтаксис {#drop-user-syntax} + +```sql +DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + + +## DROP ROLE {#drop-role-statement} + +Удаляет роль. + +При удалении роль отзывается у всех объектов системы доступа, которым она присвоена. + +### Синтаксис {#drop-role-syntax} + +```sql +DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + +## DROP ROW POLICY {#drop-row-policy-statement} + +Удаляет политику доступа к строкам. + +При удалении политика отзывается у всех объектов системы доступа, которым она присвоена. + +### Синтаксис {#drop-row-policy-syntax} + +``` sql +DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] +``` + + +## DROP QUOTA {#drop-quota-statement} + +Удаляет квоту. + +При удалении квота отзывается у всех объектов системы доступа, которым она присвоена. + +### Синтаксис {#drop-quota-syntax} + +``` sql +DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + + +## DROP SETTINGS PROFILE {#drop-settings-profile-statement} + +Удаляет профиль настроек. + +При удалении профиль отзывается у всех объектов системы доступа, которым он присвоен. + +### Синтаксис {#drop-settings-profile-syntax} + +``` sql +DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] +``` + + ## EXISTS {#exists} ``` sql @@ -109,7 +168,7 @@ EXISTS [TEMPORARY] TABLE [db.]name [INTO OUTFILE filename] [FORMAT format] Возвращает один столбец типа `UInt8`, содержащий одно значение - `0`, если таблицы или БД не существует и `1`, если таблица в указанной БД существует. -## KILL QUERY {#kill-query} +## KILL QUERY {#kill-query-statement} ``` sql KILL QUERY [ON CLUSTER cluster] @@ -144,7 +203,7 @@ Readonly-пользователи могут останавливать толь Тестовый вариант запроса (`TEST`) только проверяет права пользователя и выводит список запросов для остановки. -## KILL MUTATION {#kill-mutation} +## KILL MUTATION {#kill-mutation-statement} ``` sql KILL MUTATION [ON CLUSTER cluster] @@ -215,7 +274,57 @@ SET profile = 'profile-name-from-the-settings-file' Подробности смотрите в разделе [Настройки](../../operations/settings/settings.md). -## TRUNCATE {#truncate} +## SET ROLE {#set-role-statement} + +Активирует роли для текущего пользователя. + +### Синтаксис {#set-role-syntax} + +``` sql +SET ROLE {DEFAULT | NONE | role [,...] | ALL | ALL EXCEPT role [,...]} +``` + +## SET DEFAULT ROLE {#set-default-role-statement} + +Устанавливает роли по умолчанию для пользователя. + +Роли по умолчанию активируются автоматически при входе пользователя. Ролями по умолчанию могут быть установлены только ранее назначенные роли. Если роль не назначена пользователю, ClickHouse выбрасывает исключение. + + +### Синтаксис {#set-default-role-syntax} + +``` sql +SET DEFAULT ROLE {NONE | role [,...] | ALL | ALL EXCEPT role [,...]} TO {user|CURRENT_USER} [,...] +``` + + +### Примеры {#set-default-role-examples} + +Установить несколько ролей по умолчанию для пользователя: + +``` sql +SET DEFAULT ROLE role1, role2, ... TO user +``` + +Установить ролями по умолчанию все назначенные пользователю роли: + +``` sql +SET DEFAULT ROLE ALL TO user +``` + +Удалить роли по умолчанию для пользователя: + +``` sql +SET DEFAULT ROLE NONE TO user +``` + +Установить ролями по умолчанию все назначенные пользователю роли за исключением указанных: + +```sql +SET DEFAULT ROLE ALL EXCEPT role1, role2 TO user +``` + +## TRUNCATE {#truncate-statement} ``` sql TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] diff --git a/docs/ru/sql-reference/statements/revoke.md b/docs/ru/sql-reference/statements/revoke.md deleted file mode 120000 index 4321fdb14a7..00000000000 --- a/docs/ru/sql-reference/statements/revoke.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/sql-reference/statements/revoke.md \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/revoke.md b/docs/ru/sql-reference/statements/revoke.md new file mode 100644 index 00000000000..1d2928bb76e --- /dev/null +++ b/docs/ru/sql-reference/statements/revoke.md @@ -0,0 +1,43 @@ +# REVOKE + +Отзывает привилегии у пользователей или ролей. + +## Синтаксис {#revoke-syntax} + +**Отзыв привилегий у пользователей** + +``` sql +REVOKE [ON CLUSTER cluster_name] privilege[(column_name [,...])] [,...] ON {db.table|db.*|*.*|table|*} FROM {user | CURRENT_USER} [,...] | ALL | ALL EXCEPT {user | CURRENT_USER} [,...] +``` + +**Отзыв ролей у пользователей** + +``` sql +REVOKE [ON CLUSTER cluster_name] [ADMIN OPTION FOR] role [,...] FROM {user | role | CURRENT_USER} [,...] | ALL | ALL EXCEPT {user_name | role_name | CURRENT_USER} [,...] +``` + +## Описание {#revoke-description} + +Для отзыва привилегий можно использовать привилегию более широкой области действия. Например, если у пользователя есть привилегия `SELECT (x,y)`, администратор может отозвать ее с помощью одного из запросов: `REVOKE SELECT(x,y) ...`, `REVOKE SELECT * ...` или даже `REVOKE ALL PRIVILEGES ...`. + +### Частичный отзыв {#partial-revokes-dscr} + +Вы можете отозвать часть привилегии. Например, если у пользователя есть привилегия `SELECT *.*`, вы можете отозвать привилегию на чтение данных из какой-то таблицы или базы данных. + +## Примеры {#revoke-example} + +Присвоить пользователю `john` привилегию на `SELECT` из всех баз данных кроме `accounts`: + +``` sql +GRANT SELECT ON *.* TO john; +REVOKE SELECT ON accounts.* FROM john; +``` + +Присвоить пользователю `mira` привилегию на `SELECT` из всех столбцов таблицы `accounts.staff` кроме столбца `wage`: + +``` sql +GRANT SELECT ON accounts.staff TO mira; +REVOKE SELECT(wage) ON accounts.staff FROM mira; +``` + +[Оригинальная статья](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/ru/sql-reference/statements/select/prewhere.md b/docs/ru/sql-reference/statements/select/prewhere.md index d83b7ca3b18..1c8595d8e0c 100644 --- a/docs/ru/sql-reference/statements/select/prewhere.md +++ b/docs/ru/sql-reference/statements/select/prewhere.md @@ -1,4 +1,4 @@ -# Секции PREWHERE {#prewhere-clause} +# Секция PREWHERE {#prewhere-clause} Prewhere — это оптимизация для более эффективного применения фильтрации. Она включена по умолчанию, даже если секция `PREWHERE` явно не указана. В этом случае работает автоматическое перемещение части выражения из [WHERE](where.md) до стадии prewhere. Роль секции `PREWHERE` только для управления этой оптимизацией, если вы думаете, что знаете, как сделать перемещение условия лучше, чем это происходит по умолчанию. diff --git a/docs/ru/sql-reference/statements/select/where.md b/docs/ru/sql-reference/statements/select/where.md index e38d36e6fa8..63d081db43d 100644 --- a/docs/ru/sql-reference/statements/select/where.md +++ b/docs/ru/sql-reference/statements/select/where.md @@ -1,6 +1,6 @@ -# Предложение WHERE {#select-where} +# Секция WHERE {#select-where} -Позволяет задать выражение, которое ClickHouse использует для фильтрации данных перед всеми другими действиями в запросе кроме выражений, содержащихся в секции [PREWHERE](prewhere.md#select-prewhere). Обычно, это выражение с логическими операторами. +Позволяет задать выражение, которое ClickHouse использует для фильтрации данных перед всеми другими действиями в запросе кроме выражений, содержащихся в секции [PREWHERE](prewhere.md#prewhere-clause). Обычно, это выражение с логическими операторами. Результат выражения должен иметь тип `UInt8`. diff --git a/docs/ru/sql-reference/statements/show.md b/docs/ru/sql-reference/statements/show.md index 86f7abd8a1b..72180435b9c 100644 --- a/docs/ru/sql-reference/statements/show.md +++ b/docs/ru/sql-reference/statements/show.md @@ -95,4 +95,78 @@ SHOW DICTIONARIES FROM db LIKE '%reg%' LIMIT 2 └──────────────┘ ``` + + +## SHOW GRANTS {#show-grants-statement} + +Выводит привилегии пользователя. + +### Синтаксис {#show-grants-syntax} + +``` sql +SHOW GRANTS [FOR user] +``` + +Если пользователь не задан, запрос возвращает привилегии текущего пользователя. + + + +## SHOW CREATE USER {#show-create-user-statement} + +Выводит параметры, использованные при [создании пользователя](create.md#create-user-statement). + +`SHOW CREATE USER` не возвращает пароль пользователя. + +### Синтаксис {#show-create-user-syntax} + +``` sql +SHOW CREATE USER [name | CURRENT_USER] +``` + + + +## SHOW CREATE ROLE {#show-create-role-statement} + +Выводит параметры, использованные при [создании роли](create.md#create-role-statement). + +### Синтаксис {#show-create-role-syntax} + +``` sql +SHOW CREATE ROLE name +``` + + + +## SHOW CREATE ROW POLICY {#show-create-row-policy-statement} + +Выводит параметры, использованные при [создании политики доступа к строкам](create.md#create-row-policy-statement). + +### Синтаксис {#show-create-row-policy-syntax} + +```sql +SHOW CREATE [ROW] POLICY name ON [database.]table +``` + + +## SHOW CREATE QUOTA {#show-create-quota-statement} + +Выводит параметры, использованные при [создании квоты](create.md#create-quota-statement). + +### Синтаксис {#show-create-row-policy-syntax} + +```sql +SHOW CREATE QUOTA [name | CURRENT] +``` + + +## SHOW CREATE SETTINGS PROFILE {#show-create-settings-profile-statement} + +Выводит параметры, использованные при [создании профиля настроек](create.md#create-settings-profile-statement). + +### Синтаксис {#show-create-row-policy-syntax} + +```sql +SHOW CREATE [SETTINGS] PROFILE name +``` + [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/show/) diff --git a/docs/ru/sql-reference/table-functions/index.md b/docs/ru/sql-reference/table-functions/index.md index 28f437a814a..6e09dd4e41c 100644 --- a/docs/ru/sql-reference/table-functions/index.md +++ b/docs/ru/sql-reference/table-functions/index.md @@ -5,7 +5,7 @@ toc_priority: 34 toc_title: "\u0412\u0432\u0435\u0434\u0435\u043D\u0438\u0435" --- -# Табличные функции {#tablichnye-funktsii} +# Табличные функции {#table-functions} Табличные функции — это метод создания таблиц. diff --git a/docs/tr/operations/server-configuration-parameters/settings.md b/docs/tr/operations/server-configuration-parameters/settings.md index a944261de8d..cc5ef3e8e21 100644 --- a/docs/tr/operations/server-configuration-parameters/settings.md +++ b/docs/tr/operations/server-configuration-parameters/settings.md @@ -209,7 +209,7 @@ Eğer `http_port` belirtilmişse, OpenSSL yapılandırması ayarlanmış olsa bi **Örnek** ``` xml -0000 +9999 ``` ## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} diff --git a/docs/zh/operations/server-configuration-parameters/settings.md b/docs/zh/operations/server-configuration-parameters/settings.md index 23db483217a..2c9d611b6a7 100644 --- a/docs/zh/operations/server-configuration-parameters/settings.md +++ b/docs/zh/operations/server-configuration-parameters/settings.md @@ -207,7 +207,7 @@ ClickHouse每x秒重新加载内置字典。 这使得编辑字典 “on the fly **示例** ``` xml -0000 +9999 ``` ## http\_server\_default\_response {#server_configuration_parameters-http_server_default_response} diff --git a/programs/obfuscator/Obfuscator.cpp b/programs/obfuscator/Obfuscator.cpp index 9a80dc8d035..8b5a8c73ca4 100644 --- a/programs/obfuscator/Obfuscator.cpp +++ b/programs/obfuscator/Obfuscator.cpp @@ -858,7 +858,7 @@ public: ColumnPtr new_nested_column = nested_model->generate(nested_column); - return ColumnArray::create((*std::move(new_nested_column)).mutate(), (*std::move(column_array.getOffsetsPtr())).mutate()); + return ColumnArray::create(IColumn::mutate(std::move(new_nested_column)), IColumn::mutate(std::move(column_array.getOffsetsPtr()))); } void updateSeed() override @@ -896,7 +896,7 @@ public: ColumnPtr new_nested_column = nested_model->generate(nested_column); - return ColumnNullable::create((*std::move(new_nested_column)).mutate(), (*std::move(column_nullable.getNullMapColumnPtr())).mutate()); + return ColumnNullable::create(IColumn::mutate(std::move(new_nested_column)), IColumn::mutate(std::move(column_nullable.getNullMapColumnPtr()))); } void updateSeed() override diff --git a/programs/odbc-bridge/CMakeLists.txt b/programs/odbc-bridge/CMakeLists.txt index 87e872b69ec..ab8d94f2a0c 100644 --- a/programs/odbc-bridge/CMakeLists.txt +++ b/programs/odbc-bridge/CMakeLists.txt @@ -5,6 +5,7 @@ set (CLICKHOUSE_ODBC_BRIDGE_SOURCES IdentifierQuoteHandler.cpp MainHandler.cpp ODBCBlockInputStream.cpp + ODBCBlockOutputStream.cpp ODBCBridge.cpp PingHandler.cpp validateODBCConnectionString.cpp diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index 0d103a96344..e62aad48b90 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -16,6 +16,7 @@ # include # include # include +# include # include # include "getIdentifierQuote.h" # include "validateODBCConnectionString.h" @@ -58,11 +59,6 @@ namespace } } -namespace ErrorCodes -{ - extern const int ILLEGAL_TYPE_OF_ARGUMENT; -} - void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) { Poco::Net::HTMLForm params(request, request.stream()); @@ -116,7 +112,7 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques const auto & context_settings = context.getSettingsRef(); /// TODO Why not do SQLColumns instead? - std::string name = schema_name.empty() ? table_name : schema_name + "." + table_name; + std::string name = schema_name.empty() ? backQuoteIfNeed(table_name) : backQuoteIfNeed(schema_name) + "." + backQuoteIfNeed(table_name); std::stringstream ss; std::string input = "SELECT * FROM " + name + " WHERE 1 = 0"; ParserQueryWithOutput parser; @@ -124,17 +120,7 @@ void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & reques IAST::FormatSettings settings(ss, true); settings.always_quote_identifiers = true; - - auto identifier_quote = getIdentifierQuote(hdbc); - if (identifier_quote.length() == 0) - settings.identifier_quoting_style = IdentifierQuotingStyle::None; - else if (identifier_quote[0] == '`') - settings.identifier_quoting_style = IdentifierQuotingStyle::Backticks; - else if (identifier_quote[0] == '"') - settings.identifier_quoting_style = IdentifierQuotingStyle::DoubleQuotes; - else - throw Exception("Can not map quote identifier '" + identifier_quote + "' to IdentifierQuotingStyle value", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - + settings.identifier_quoting_style = getQuotingStyle(hdbc); select->format(settings); std::string query = ss.str(); diff --git a/programs/odbc-bridge/HandlerFactory.cpp b/programs/odbc-bridge/HandlerFactory.cpp index 6ce04ac21a9..156c2fcce61 100644 --- a/programs/odbc-bridge/HandlerFactory.cpp +++ b/programs/odbc-bridge/HandlerFactory.cpp @@ -30,8 +30,10 @@ Poco::Net::HTTPRequestHandler * HandlerFactory::createRequestHandler(const Poco: #else return nullptr; #endif + else if (uri.getPath() == "/write") + return new ODBCHandler(pool_map, keep_alive_timeout, context, "write"); else - return new ODBCHandler(pool_map, keep_alive_timeout, context); + return new ODBCHandler(pool_map, keep_alive_timeout, context, "read"); } return nullptr; } diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 15954c63e47..edc92552c44 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -5,17 +5,25 @@ #include #include #include "ODBCBlockInputStream.h" +#include "ODBCBlockOutputStream.h" #include #include #include #include -#include #include #include #include #include #include #include +#include +#include +#include "getIdentifierQuote.h" + +#if USE_ODBC +#include +#define POCO_SQL_ODBC_CLASS Poco::Data::ODBC +#endif namespace DB { @@ -63,34 +71,36 @@ ODBCHandler::PoolPtr ODBCHandler::getPool(const std::string & connection_str) return pool_map->at(connection_str); } +void ODBCHandler::processError(Poco::Net::HTTPServerResponse & response, const std::string & message) +{ + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); + if (!response.sent()) + response.send() << message << std::endl; + LOG_WARNING(log, message); +} + void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) { - Poco::Net::HTMLForm params(request, request.stream()); + Poco::Net::HTMLForm params(request); + if (mode == "read") + params.read(request.stream()); LOG_TRACE(log, "Request URI: " + request.getURI()); - auto process_error = [&response, this](const std::string & message) + if (mode == "read" && !params.has("query")) { - response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); - if (!response.sent()) - response.send() << message << std::endl; - LOG_WARNING(log, message); - }; - - if (!params.has("query")) - { - process_error("No 'query' in request body"); + processError(response, "No 'query' in request body"); return; } if (!params.has("columns")) { - process_error("No 'columns' in request URL"); + processError(response, "No 'columns' in request URL"); return; } if (!params.has("connection_string")) { - process_error("No 'connection_string' in request URL"); + processError(response, "No 'connection_string' in request URL"); return; } @@ -100,7 +110,7 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne std::string max_block_size_str = params.get("max_block_size", ""); if (max_block_size_str.empty()) { - process_error("Empty max_block_size specified"); + processError(response, "Empty max_block_size specified"); return; } max_block_size = parse(max_block_size_str); @@ -114,33 +124,70 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne } catch (const Exception & ex) { - process_error("Invalid 'columns' parameter in request body '" + ex.message() + "'"); + processError(response, "Invalid 'columns' parameter in request body '" + ex.message() + "'"); LOG_WARNING(log, ex.getStackTraceString()); return; } std::string format = params.get("format", "RowBinary"); - std::string query = params.get("query"); - LOG_TRACE(log, "Query: " << query); std::string connection_string = params.get("connection_string"); LOG_TRACE(log, "Connection string: '" << connection_string << "'"); WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout); + try { - BlockOutputStreamPtr writer = FormatFactory::instance().getOutput(format, out, *sample_block, context); - auto pool = getPool(connection_string); - ODBCBlockInputStream inp(pool->get(), query, *sample_block, max_block_size); - copyData(inp, *writer); + if (mode == "write") + { + if (!params.has("db_name")) + { + processError(response, "No 'db_name' in request URL"); + return; + } + if (!params.has("table_name")) + { + processError(response, "No 'table_name' in request URL"); + return; + } + std::string db_name = params.get("db_name"); + std::string table_name = params.get("table_name"); + LOG_TRACE(log, "DB name: '" << db_name << "', table name: '" << table_name << "'"); + + auto quoting_style = IdentifierQuotingStyle::None; +#if USE_ODBC + POCO_SQL_ODBC_CLASS::SessionImpl session(validateODBCConnectionString(connection_string), DBMS_DEFAULT_CONNECT_TIMEOUT_SEC); + quoting_style = getQuotingStyle(session.dbc().handle()); +#endif + + auto pool = getPool(connection_string); + ReadBufferFromIStream read_buf(request.stream()); + BlockInputStreamPtr input_stream = FormatFactory::instance().getInput(format, read_buf, *sample_block, + context, max_block_size); + ODBCBlockOutputStream output_stream(pool->get(), db_name, table_name, *sample_block, quoting_style); + copyData(*input_stream, output_stream); + writeStringBinary("Ok.", out); + } + else + { + std::string query = params.get("query"); + LOG_TRACE(log, "Query: " << query); + + BlockOutputStreamPtr writer = FormatFactory::instance().getOutput(format, out, *sample_block, context); + auto pool = getPool(connection_string); + ODBCBlockInputStream inp(pool->get(), query, *sample_block, max_block_size); + copyData(inp, *writer); + } } catch (...) { auto message = getCurrentExceptionMessage(true); response.setStatusAndReason( - Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); // can't call process_error, because of too soon response sending + Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); // can't call process_error, because of too soon response sending writeStringBinary(message, out); tryLogCurrentException(log); + } } + } diff --git a/programs/odbc-bridge/MainHandler.h b/programs/odbc-bridge/MainHandler.h index 806313fc9b2..ec5e6693a60 100644 --- a/programs/odbc-bridge/MainHandler.h +++ b/programs/odbc-bridge/MainHandler.h @@ -24,11 +24,13 @@ public: ODBCHandler(std::shared_ptr pool_map_, size_t keep_alive_timeout_, - Context & context_) + Context & context_, + const String & mode_) : log(&Poco::Logger::get("ODBCHandler")) , pool_map(pool_map_) , keep_alive_timeout(keep_alive_timeout_) , context(context_) + , mode(mode_) { } @@ -40,10 +42,12 @@ private: std::shared_ptr pool_map; size_t keep_alive_timeout; Context & context; + String mode; static inline std::mutex mutex; PoolPtr getPool(const std::string & connection_str); + void processError(Poco::Net::HTTPServerResponse & response, const std::string & message); }; } diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.cpp b/programs/odbc-bridge/ODBCBlockOutputStream.cpp new file mode 100644 index 00000000000..c2597805230 --- /dev/null +++ b/programs/odbc-bridge/ODBCBlockOutputStream.cpp @@ -0,0 +1,133 @@ +#include "ODBCBlockOutputStream.h" + +#include +#include +#include +#include +#include +#include +#include +#include "getIdentifierQuote.h" + + +namespace DB +{ + +namespace +{ + using ValueType = ExternalResultDescription::ValueType; + + std::string getInsertQuery(const std::string & db_name, const std::string & table_name, const ColumnsWithTypeAndName & columns, IdentifierQuotingStyle quoting) + { + ASTInsertQuery query; + query.table_id.database_name = db_name; + query.table_id.table_name = table_name; + query.columns = std::make_shared(','); + query.children.push_back(query.columns); + for (size_t i = 0; i < columns.size(); ++i) + query.columns->children.emplace_back(std::make_shared(columns[i].name)); + + std::stringstream ss; + IAST::FormatSettings settings(ss, true); + settings.always_quote_identifiers = true; + settings.identifier_quoting_style = quoting; + query.IAST::format(settings); + return ss.str(); + } + + std::string getQuestionMarks(size_t n) + { + std::string result = "("; + for (size_t i = 0; i < n; ++i) + { + if (i > 0) + result += ","; + result += "?"; + } + return result + ")"; + } + + Poco::Dynamic::Var getVarFromField(const Field & field, const ValueType type) + { + switch (type) + { + case ValueType::vtUInt8: + return Poco::Dynamic::Var(static_cast(field.get())).convert(); + case ValueType::vtUInt16: + return Poco::Dynamic::Var(static_cast(field.get())).convert(); + case ValueType::vtUInt32: + return Poco::Dynamic::Var(static_cast(field.get())).convert(); + case ValueType::vtUInt64: + return Poco::Dynamic::Var(field.get()).convert(); + case ValueType::vtInt8: + return Poco::Dynamic::Var(static_cast(field.get())).convert(); + case ValueType::vtInt16: + return Poco::Dynamic::Var(static_cast(field.get())).convert(); + case ValueType::vtInt32: + return Poco::Dynamic::Var(static_cast(field.get())).convert(); + case ValueType::vtInt64: + return Poco::Dynamic::Var(field.get()).convert(); + case ValueType::vtFloat32: + return Poco::Dynamic::Var(field.get()).convert(); + case ValueType::vtFloat64: + return Poco::Dynamic::Var(field.get()).convert(); + case ValueType::vtString: + return Poco::Dynamic::Var(field.get()).convert(); + case ValueType::vtDate: + return Poco::Dynamic::Var(LocalDate(DayNum(field.get())).toString()).convert(); + case ValueType::vtDateTime: + return Poco::Dynamic::Var(std::to_string(LocalDateTime(time_t(field.get())))).convert(); + case ValueType::vtUUID: + return Poco::Dynamic::Var(UUID(field.get()).toUnderType().toHexString()).convert(); + } + __builtin_unreachable(); + } +} + +ODBCBlockOutputStream::ODBCBlockOutputStream(Poco::Data::Session && session_, + const std::string & remote_database_name_, + const std::string & remote_table_name_, + const Block & sample_block_, + IdentifierQuotingStyle quoting_) + : session(session_) + , db_name(remote_database_name_) + , table_name(remote_table_name_) + , sample_block(sample_block_) + , quoting(quoting_) + , log(&Logger::get("ODBCBlockOutputStream")) +{ + description.init(sample_block); +} + +Block ODBCBlockOutputStream::getHeader() const +{ + return sample_block; +} + +void ODBCBlockOutputStream::write(const Block & block) +{ + ColumnsWithTypeAndName columns; + for (size_t i = 0; i < block.columns(); ++i) + columns.push_back({block.getColumns()[i], sample_block.getDataTypes()[i], sample_block.getNames()[i]}); + + std::vector row_to_insert(block.columns()); + Poco::Data::Statement statement(session << getInsertQuery(db_name, table_name, columns, quoting) + getQuestionMarks(block.columns())); + for (size_t i = 0; i < block.columns(); ++i) + statement.addBind(Poco::Data::Keywords::use(row_to_insert[i])); + + for (size_t i = 0; i < block.rows(); ++i) + { + for (size_t col_idx = 0; col_idx < block.columns(); ++col_idx) + { + Field val; + columns[col_idx].column->get(i, val); + if (val.isNull()) + row_to_insert[col_idx] = Poco::Dynamic::Var(); + else + row_to_insert[col_idx] = getVarFromField(val, description.types[col_idx].first); + } + statement.execute(); + } +} + +} diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.h b/programs/odbc-bridge/ODBCBlockOutputStream.h new file mode 100644 index 00000000000..39e1d6f77ac --- /dev/null +++ b/programs/odbc-bridge/ODBCBlockOutputStream.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace DB +{ +class ODBCBlockOutputStream : public IBlockOutputStream +{ +public: + ODBCBlockOutputStream(Poco::Data::Session && session_, const std::string & remote_database_name_, + const std::string & remote_table_name_, const Block & sample_block_, IdentifierQuotingStyle quoting); + + Block getHeader() const override; + void write(const Block & block) override; + +private: + Poco::Data::Session session; + std::string db_name; + std::string table_name; + Block sample_block; + IdentifierQuotingStyle quoting; + + ExternalResultDescription description; + Poco::Logger * log; +}; + +} diff --git a/programs/odbc-bridge/getIdentifierQuote.cpp b/programs/odbc-bridge/getIdentifierQuote.cpp index 766d5bf6bde..15b3749d37d 100644 --- a/programs/odbc-bridge/getIdentifierQuote.cpp +++ b/programs/odbc-bridge/getIdentifierQuote.cpp @@ -12,6 +12,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + std::string getIdentifierQuote(SQLHDBC hdbc) { std::string identifier; @@ -36,6 +41,19 @@ std::string getIdentifierQuote(SQLHDBC hdbc) return identifier; } +IdentifierQuotingStyle getQuotingStyle(SQLHDBC hdbc) +{ + auto identifier_quote = getIdentifierQuote(hdbc); + if (identifier_quote.length() == 0) + return IdentifierQuotingStyle::None; + else if (identifier_quote[0] == '`') + return IdentifierQuotingStyle::Backticks; + else if (identifier_quote[0] == '"') + return IdentifierQuotingStyle::DoubleQuotes; + else + throw Exception("Can not map quote identifier '" + identifier_quote + "' to IdentifierQuotingStyle value", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); +} + } #endif diff --git a/programs/odbc-bridge/getIdentifierQuote.h b/programs/odbc-bridge/getIdentifierQuote.h index 8bf119209c2..0fb4c3bddb1 100644 --- a/programs/odbc-bridge/getIdentifierQuote.h +++ b/programs/odbc-bridge/getIdentifierQuote.h @@ -8,11 +8,15 @@ # include +#include + namespace DB { std::string getIdentifierQuote(SQLHDBC hdbc); +IdentifierQuotingStyle getQuotingStyle(SQLHDBC hdbc); + } #endif diff --git a/programs/server/TCPHandler.cpp b/programs/server/TCPHandler.cpp index f6903f0820f..93fa78499a1 100644 --- a/programs/server/TCPHandler.cpp +++ b/programs/server/TCPHandler.cpp @@ -28,7 +28,7 @@ #include #include -#include +#include #include "TCPHandler.h" @@ -553,68 +553,23 @@ void TCPHandler::processOrdinaryQueryWithProcessors() /// Send header-block, to allow client to prepare output format for data to send. { - auto & header = pipeline.getHeader(); + const auto & header = pipeline.getHeader(); if (header) sendData(header); } - auto lazy_format = std::make_shared(pipeline.getHeader()); - pipeline.setOutput(lazy_format); - { - auto thread_group = CurrentThread::getGroup(); - ThreadPool pool(1); - auto executor = pipeline.execute(); - std::atomic_bool exception = false; + PullingPipelineExecutor executor(pipeline); + CurrentMetrics::Increment query_thread_metric_increment{CurrentMetrics::QueryThread}; - pool.scheduleOrThrowOnError([&]() - { - /// ThreadStatus thread_status; - - if (thread_group) - CurrentThread::attachTo(thread_group); - - SCOPE_EXIT( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); - - CurrentMetrics::Increment query_thread_metric_increment{CurrentMetrics::QueryThread}; - setThreadName("QueryPipelineEx"); - - try - { - executor->execute(pipeline.getNumThreads()); - } - catch (...) - { - exception = true; - throw; - } - }); - - /// Wait in case of exception happened outside of pool. - SCOPE_EXIT( - lazy_format->finish(); - - try - { - pool.wait(); - } - catch (...) - { - /// If exception was thrown during pipeline execution, skip it while processing other exception. - tryLogCurrentException(log); - } - ); - - while (!lazy_format->isFinished() && !exception) + Block block; + while (executor.pull(block, query_context->getSettingsRef().interactive_delay / 1000)) { if (isQueryCancelled()) { /// A packet was received requesting to stop execution of the request. - executor->cancel(); + executor.cancel(); break; } @@ -627,17 +582,13 @@ void TCPHandler::processOrdinaryQueryWithProcessors() sendLogs(); - if (auto block = lazy_format->getBlock(query_context->getSettingsRef().interactive_delay / 1000)) + if (block) { if (!state.io.null_format) sendData(block); } } - /// Finish lazy_format before waiting. Otherwise some thread may write into it, and waiting will lock. - lazy_format->finish(); - pool.wait(); - /** If data has run out, we will send the profiling data and total values to * the last zero block to be able to use * this information in the suffix output of stream. @@ -647,9 +598,9 @@ void TCPHandler::processOrdinaryQueryWithProcessors() */ if (!isQueryCancelled()) { - sendTotals(lazy_format->getTotals()); - sendExtremes(lazy_format->getExtremes()); - sendProfileInfo(lazy_format->getProfileInfo()); + sendTotals(executor.getTotalsBlock()); + sendExtremes(executor.getExtremesBlock()); + sendProfileInfo(executor.getProfileInfo()); sendProgress(); sendLogs(); } diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index f8f15e425ed..1c1215a0e28 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include @@ -42,7 +42,7 @@ public: std::shared_ptr getContextAccess( const UUID & user_id, - const std::vector & current_roles, + const boost::container::flat_set & current_roles, bool use_default_roles, const Settings & settings, const String & current_database, @@ -113,7 +113,7 @@ void AccessControlManager::setDefaultProfileName(const String & default_profile_ std::shared_ptr AccessControlManager::getContextAccess( const UUID & user_id, - const std::vector & current_roles, + const boost::container::flat_set & current_roles, bool use_default_roles, const Settings & settings, const String & current_database, @@ -124,36 +124,36 @@ std::shared_ptr AccessControlManager::getContextAccess( std::shared_ptr AccessControlManager::getEnabledRoles( - const std::vector & current_roles, - const std::vector & current_roles_with_admin_option) const + const boost::container::flat_set & current_roles, + const boost::container::flat_set & current_roles_with_admin_option) const { return role_cache->getEnabledRoles(current_roles, current_roles_with_admin_option); } -std::shared_ptr AccessControlManager::getEnabledRowPolicies(const UUID & user_id, const std::vector & enabled_roles) const +std::shared_ptr AccessControlManager::getEnabledRowPolicies(const UUID & user_id, const boost::container::flat_set & enabled_roles) const { return row_policy_cache->getEnabledRowPolicies(user_id, enabled_roles); } std::shared_ptr AccessControlManager::getEnabledQuota( - const UUID & user_id, const String & user_name, const std::vector & enabled_roles, const Poco::Net::IPAddress & address, const String & custom_quota_key) const + const UUID & user_id, const String & user_name, const boost::container::flat_set & enabled_roles, const Poco::Net::IPAddress & address, const String & custom_quota_key) const { return quota_cache->getEnabledQuota(user_id, user_name, enabled_roles, address, custom_quota_key); } -std::vector AccessControlManager::getQuotaUsageInfo() const +std::vector AccessControlManager::getAllQuotasUsage() const { - return quota_cache->getUsageInfo(); + return quota_cache->getAllQuotasUsage(); } std::shared_ptr AccessControlManager::getEnabledSettings( const UUID & user_id, const SettingsProfileElements & settings_from_user, - const std::vector & enabled_roles, + const boost::container::flat_set & enabled_roles, const SettingsProfileElements & settings_from_enabled_roles) const { return settings_profiles_cache->getEnabledSettings(user_id, settings_from_user, enabled_roles, settings_from_enabled_roles); diff --git a/src/Access/AccessControlManager.h b/src/Access/AccessControlManager.h index 810970a8379..6bcf8d7c504 100644 --- a/src/Access/AccessControlManager.h +++ b/src/Access/AccessControlManager.h @@ -2,6 +2,7 @@ #include #include +#include #include @@ -28,7 +29,7 @@ class EnabledRowPolicies; class RowPolicyCache; class EnabledQuota; class QuotaCache; -struct QuotaUsageInfo; +struct QuotaUsage; struct SettingsProfile; using SettingsProfilePtr = std::shared_ptr; class EnabledSettings; @@ -51,32 +52,32 @@ public: std::shared_ptr getContextAccess( const UUID & user_id, - const std::vector & current_roles, + const boost::container::flat_set & current_roles, bool use_default_roles, const Settings & settings, const String & current_database, const ClientInfo & client_info) const; std::shared_ptr getEnabledRoles( - const std::vector & current_roles, - const std::vector & current_roles_with_admin_option) const; + const boost::container::flat_set & current_roles, + const boost::container::flat_set & current_roles_with_admin_option) const; std::shared_ptr getEnabledRowPolicies( const UUID & user_id, - const std::vector & enabled_roles) const; + const boost::container::flat_set & enabled_roles) const; std::shared_ptr getEnabledQuota( const UUID & user_id, const String & user_name, - const std::vector & enabled_roles, + const boost::container::flat_set & enabled_roles, const Poco::Net::IPAddress & address, const String & custom_quota_key) const; - std::vector getQuotaUsageInfo() const; + std::vector getAllQuotasUsage() const; std::shared_ptr getEnabledSettings(const UUID & user_id, const SettingsProfileElements & settings_from_user, - const std::vector & enabled_roles, + const boost::container::flat_set & enabled_roles, const SettingsProfileElements & settings_from_enabled_roles) const; std::shared_ptr getProfileSettings(const String & profile_name) const; diff --git a/src/Access/AccessFlags.h b/src/Access/AccessFlags.h index cbba295be1a..9b801fd88a3 100644 --- a/src/Access/AccessFlags.h +++ b/src/Access/AccessFlags.h @@ -61,12 +61,19 @@ public: friend bool operator ==(const AccessFlags & left, const AccessFlags & right) { return left.flags == right.flags; } friend bool operator !=(const AccessFlags & left, const AccessFlags & right) { return !(left == right); } + friend bool operator <(const AccessFlags & left, const AccessFlags & right) { return memcmp(&left.flags, &right.flags, sizeof(Flags)) < 0; } + friend bool operator >(const AccessFlags & left, const AccessFlags & right) { return right < left; } + friend bool operator <=(const AccessFlags & left, const AccessFlags & right) { return !(right < left); } + friend bool operator >=(const AccessFlags & left, const AccessFlags & right) { return !(left < right); } void clear() { flags.reset(); } /// Returns a comma-separated list of keywords, like "SELECT, CREATE USER, UPDATE". String toString() const; + /// Returns a list of access types. + std::vector toAccessTypes() const; + /// Returns a list of keywords. std::vector toKeywords() const; @@ -151,21 +158,27 @@ public: return res; } + std::vector flagsToAccessTypes(const Flags & flags_) const + { + std::vector access_types; + flagsToAccessTypesRec(flags_, access_types, *all_node); + return access_types; + } + std::vector flagsToKeywords(const Flags & flags_) const { std::vector keywords; - flagsToKeywordsRec(flags_, keywords, *flags_to_keyword_tree); - - if (keywords.empty()) - keywords.push_back("USAGE"); - + flagsToKeywordsRec(flags_, keywords, *all_node); return keywords; } String flagsToString(const Flags & flags_) const { + auto keywords = flagsToKeywords(flags_); + if (keywords.empty()) + return "USAGE"; String str; - for (const auto & keyword : flagsToKeywords(flags_)) + for (const auto & keyword : keywords) { if (!str.empty()) str += ", "; @@ -201,7 +214,7 @@ private: { const String keyword; NodeType node_type; - AccessType type = AccessType::NONE; + AccessType access_type = AccessType::NONE; Strings aliases; Flags flags; std::vector children; @@ -233,8 +246,8 @@ private: return aliases; } - static void makeFlagsToKeywordTreeNode( - AccessType type, + static void makeNode( + AccessType access_type, const std::string_view & name, const std::string_view & aliases, NodeType node_type, @@ -259,13 +272,13 @@ private: nodes[node->keyword] = node.get(); } - node->type = type; + node->access_type = access_type; node->node_type = node_type; node->aliases = splitAliases(aliases); if (node_type != GROUP) node->setFlag(next_flag++); - bool has_parent_group = (parent_group_name != "NONE"); + bool has_parent_group = (parent_group_name != std::string_view{"NONE"}); if (!has_parent_group) { std::string_view keyword_as_string_view = node->keyword; @@ -286,25 +299,25 @@ private: it_parent->second->addChild(std::move(node)); } - void makeFlagsToKeywordTree() + void makeNodes() { std::unordered_map owned_nodes; std::unordered_map nodes; size_t next_flag = 0; -#define MAKE_ACCESS_FLAGS_TO_KEYWORD_TREE_NODE(name, aliases, node_type, parent_group_name) \ - makeFlagsToKeywordTreeNode(AccessType::name, #name, aliases, node_type, #parent_group_name, nodes, owned_nodes, next_flag); +#define MAKE_ACCESS_FLAGS_NODE(name, aliases, node_type, parent_group_name) \ + makeNode(AccessType::name, #name, aliases, node_type, #parent_group_name, nodes, owned_nodes, next_flag); - APPLY_FOR_ACCESS_TYPES(MAKE_ACCESS_FLAGS_TO_KEYWORD_TREE_NODE) + APPLY_FOR_ACCESS_TYPES(MAKE_ACCESS_FLAGS_NODE) -#undef MAKE_ACCESS_FLAGS_TO_KEYWORD_TREE_NODE +#undef MAKE_ACCESS_FLAGS_NODE if (!owned_nodes.count("NONE")) throw Exception("'NONE' not declared", ErrorCodes::LOGICAL_ERROR); if (!owned_nodes.count("ALL")) throw Exception("'ALL' not declared", ErrorCodes::LOGICAL_ERROR); - flags_to_keyword_tree = std::move(owned_nodes["ALL"]); + all_node = std::move(owned_nodes["ALL"]); none_node = std::move(owned_nodes["NONE"]); owned_nodes.erase("ALL"); owned_nodes.erase("NONE"); @@ -324,7 +337,7 @@ private: if (!start_node) { makeKeywordToFlagsMap(none_node.get()); - start_node = flags_to_keyword_tree.get(); + start_node = all_node.get(); } start_node->aliases.emplace_back(start_node->keyword); @@ -343,10 +356,10 @@ private: if (!start_node) { makeAccessTypeToFlagsMapping(none_node.get()); - start_node = flags_to_keyword_tree.get(); + start_node = all_node.get(); } - size_t index = static_cast(start_node->type); + size_t index = static_cast(start_node->access_type); access_type_to_flags_mapping.resize(std::max(index + 1, access_type_to_flags_mapping.size())); access_type_to_flags_mapping[index] = start_node->flags; @@ -358,7 +371,7 @@ private: { if (!start_node) { - start_node = flags_to_keyword_tree.get(); + start_node = all_node.get(); all_flags = start_node->flags; } if (start_node->node_type != GROUP) @@ -372,12 +385,29 @@ private: Impl() { - makeFlagsToKeywordTree(); + makeNodes(); makeKeywordToFlagsMap(); makeAccessTypeToFlagsMapping(); collectAllFlags(); } + static void flagsToAccessTypesRec(const Flags & flags_, std::vector & access_types, const Node & start_node) + { + Flags matching_flags = (flags_ & start_node.flags); + if (matching_flags.any()) + { + if (matching_flags == start_node.flags) + { + access_types.push_back(start_node.access_type); + } + else + { + for (const auto & child : start_node.children) + flagsToAccessTypesRec(flags_, access_types, *child); + } + } + } + static void flagsToKeywordsRec(const Flags & flags_, std::vector & keywords, const Node & start_node) { Flags matching_flags = (flags_ & start_node.flags); @@ -395,7 +425,7 @@ private: } } - NodePtr flags_to_keyword_tree; + NodePtr all_node; NodePtr none_node; std::unordered_map keyword_to_flags_map; std::vector access_type_to_flags_mapping; @@ -409,6 +439,7 @@ inline AccessFlags::AccessFlags(const std::string_view & keyword) : flags(Impl<> inline AccessFlags::AccessFlags(const std::vector & keywords) : flags(Impl<>::instance().keywordsToFlags(keywords)) {} inline AccessFlags::AccessFlags(const Strings & keywords) : flags(Impl<>::instance().keywordsToFlags(keywords)) {} inline String AccessFlags::toString() const { return Impl<>::instance().flagsToString(flags); } +inline std::vector AccessFlags::toAccessTypes() const { return Impl<>::instance().flagsToAccessTypes(flags); } inline std::vector AccessFlags::toKeywords() const { return Impl<>::instance().flagsToKeywords(flags); } inline AccessFlags AccessFlags::allFlags() { return Impl<>::instance().getAllFlags(); } inline AccessFlags AccessFlags::allGlobalFlags() { return Impl<>::instance().getGlobalFlags(); } diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 79d77e3f352..086d740ede1 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -606,69 +606,102 @@ void AccessRights::revoke(const AccessRightsElements & elements, std::string_vie } -AccessRights::Elements AccessRights::getElements() const +AccessRightsElements AccessRights::getGrants() const +{ + AccessRightsElements grants; + getGrantsAndPartialRevokesImpl(&grants, nullptr); + return grants; +} + +AccessRightsElements AccessRights::getPartialRevokes() const +{ + AccessRightsElements partial_revokes; + getGrantsAndPartialRevokesImpl(nullptr, &partial_revokes); + return partial_revokes; +} + +AccessRights::GrantsAndPartialRevokes AccessRights::getGrantsAndPartialRevokes() const +{ + GrantsAndPartialRevokes res; + getGrantsAndPartialRevokesImpl(&res.grants, &res.revokes); + return res; +} + + +void AccessRights::getGrantsAndPartialRevokesImpl(AccessRightsElements * out_grants, AccessRightsElements * out_partial_revokes) const { if (!root) - return {}; - Elements res; + return; auto global_access = root->access; - if (global_access) - res.grants.push_back({global_access}); + if (out_grants && global_access) + out_grants->push_back({global_access}); if (root->children) { for (const auto & [db_name, db_node] : *root->children) { - auto db_grants = db_node.access - global_access; - auto db_partial_revokes = global_access - db_node.access; - if (db_partial_revokes) - res.partial_revokes.push_back({db_partial_revokes, db_name}); - if (db_grants) - res.grants.push_back({db_grants, db_name}); + if (out_grants) + { + if (auto db_grants = db_node.access - global_access) + out_grants->push_back({db_grants, db_name}); + } + if (out_partial_revokes) + { + if (auto db_partial_revokes = global_access - db_node.access) + out_partial_revokes->push_back({db_partial_revokes, db_name}); + } if (db_node.children) { for (const auto & [table_name, table_node] : *db_node.children) { - auto table_grants = table_node.access - db_node.access; - auto table_partial_revokes = db_node.access - table_node.access; - if (table_partial_revokes) - res.partial_revokes.push_back({table_partial_revokes, db_name, table_name}); - if (table_grants) - res.grants.push_back({table_grants, db_name, table_name}); + if (out_grants) + { + if (auto table_grants = table_node.access - db_node.access) + out_grants->push_back({table_grants, db_name, table_name}); + } + if (out_partial_revokes) + { + if (auto table_partial_revokes = db_node.access - table_node.access) + out_partial_revokes->push_back({table_partial_revokes, db_name, table_name}); + } if (table_node.children) { for (const auto & [column_name, column_node] : *table_node.children) { - auto column_grants = column_node.access - table_node.access; - auto column_partial_revokes = table_node.access - column_node.access; - if (column_partial_revokes) - res.partial_revokes.push_back({column_partial_revokes, db_name, table_name, column_name}); - if (column_grants) - res.grants.push_back({column_grants, db_name, table_name, column_name}); + if (out_grants) + { + if (auto column_grants = column_node.access - table_node.access) + out_grants->push_back({column_grants, db_name, table_name, column_name}); + } + if (out_partial_revokes) + { + if (auto column_partial_revokes = table_node.access - column_node.access) + out_partial_revokes->push_back({column_partial_revokes, db_name, table_name, column_name}); + } } + } } } } } - return res; } String AccessRights::toString() const { - auto elements = getElements(); String res; - if (!elements.grants.empty()) + auto gr = getGrantsAndPartialRevokes(); + if (!gr.grants.empty()) { res += "GRANT "; - res += elements.grants.toString(); + res += gr.grants.toString(); } - if (!elements.partial_revokes.empty()) + if (!gr.revokes.empty()) { if (!res.empty()) res += ", "; res += "REVOKE "; - res += elements.partial_revokes.toString(); + res += gr.revokes.toString(); } if (res.empty()) res = "GRANT USAGE ON *.*"; diff --git a/src/Access/AccessRights.h b/src/Access/AccessRights.h index 133038f2d44..c32514e8feb 100644 --- a/src/Access/AccessRights.h +++ b/src/Access/AccessRights.h @@ -49,12 +49,14 @@ public: void revoke(const AccessRightsElements & elements, std::string_view current_database = {}); /// Returns the information about all the access granted. - struct Elements + struct GrantsAndPartialRevokes { AccessRightsElements grants; - AccessRightsElements partial_revokes; + AccessRightsElements revokes; }; - Elements getElements() const; + AccessRightsElements getGrants() const; + AccessRightsElements getPartialRevokes() const; + GrantsAndPartialRevokes getGrantsAndPartialRevokes() const; /// Returns the information about all the access granted as a string. String toString() const; @@ -92,6 +94,8 @@ private: template AccessFlags getAccessImpl(const Args &... args) const; + void getGrantsAndPartialRevokesImpl(AccessRightsElements * grants, AccessRightsElements * partial_revokes) const; + void logTree() const; struct Node; diff --git a/src/Access/AccessRightsElement.cpp b/src/Access/AccessRightsElement.cpp index 81237a334e2..db1ea5d3d5c 100644 --- a/src/Access/AccessRightsElement.cpp +++ b/src/Access/AccessRightsElement.cpp @@ -1,10 +1,124 @@ #include #include #include +#include +#include +#include +#include +#include namespace DB { +namespace +{ + size_t groupElements(AccessRightsElements & elements, size_t start) + { + auto & start_element = elements[start]; + auto it = std::find_if(elements.begin() + start + 1, elements.end(), + [&](const AccessRightsElement & element) + { + return (element.database != start_element.database) || + (element.any_database != start_element.any_database) || + (element.table != start_element.table) || + (element.any_table != start_element.any_table) || + (element.any_column != start_element.any_column); + }); + size_t end = it - elements.begin(); + + /// All the elements at indices from start to end here specify + /// the same database and table. + + if (start_element.any_column) + { + /// Easy case: the elements don't specify columns. + /// All we need is to combine the access flags. + for (size_t i = start + 1; i != end; ++i) + { + start_element.access_flags |= elements[i].access_flags; + elements[i].access_flags = {}; + } + return end; + } + + /// Difficult case: the elements specify columns. + /// We have to find groups of columns with common access flags. + for (size_t i = start; i != end; ++i) + { + if (!elements[i].access_flags) + continue; + + AccessFlags common_flags = elements[i].access_flags; + size_t num_elements_with_common_flags = 1; + for (size_t j = i + 1; j != end; ++j) + { + auto new_common_flags = common_flags & elements[j].access_flags; + if (new_common_flags) + { + common_flags = new_common_flags; + ++num_elements_with_common_flags; + } + } + + if (num_elements_with_common_flags == 1) + continue; + + if (elements[i].access_flags != common_flags) + { + elements.insert(elements.begin() + i + 1, elements[i]); + elements[i].access_flags = common_flags; + elements[i].columns.clear(); + ++end; + } + + for (size_t j = i + 1; j != end; ++j) + { + if ((elements[j].access_flags & common_flags) == common_flags) + { + boost::range::push_back(elements[i].columns, elements[j].columns); + elements[j].access_flags -= common_flags; + } + } + } + + return end; + } + + /// Tries to combine elements to decrease their number. + void groupElements(AccessRightsElements & elements) + { + if (!boost::range::is_sorted(elements)) + boost::range::sort(elements); /// Algorithm in groupElement() requires elements to be sorted. + for (size_t start = 0; start != elements.size();) + start = groupElements(elements, start); + } + + /// Removes unnecessary elements, sorts elements and makes them unique. + void sortElementsAndMakeUnique(AccessRightsElements & elements) + { + /// Remove empty elements. + boost::range::remove_erase_if(elements, [](const AccessRightsElement & element) + { + return !element.access_flags || (!element.any_column && element.columns.empty()); + }); + + /// Sort columns and make them unique. + for (auto & element : elements) + { + if (element.any_column) + continue; + + if (!boost::range::is_sorted(element.columns)) + boost::range::sort(element.columns); + element.columns.erase(std::unique(element.columns.begin(), element.columns.end()), element.columns.end()); + } + + /// Sort elements themselves. + boost::range::sort(elements); + elements.erase(std::unique(elements.begin(), elements.end()), elements.end()); + } +} + void AccessRightsElement::setDatabase(const String & new_database) { database = new_database; @@ -26,10 +140,29 @@ bool AccessRightsElement::isEmptyDatabase() const String AccessRightsElement::toString() const +{ + String msg = toStringWithoutON(); + msg += " ON "; + + if (any_database) + msg += "*."; + else if (!database.empty()) + msg += backQuoteIfNeed(database) + "."; + + if (any_table) + msg += "*"; + else + msg += backQuoteIfNeed(table); + return msg; +} + +String AccessRightsElement::toStringWithoutON() const { String columns_in_parentheses; if (!any_column) { + if (columns.empty()) + return "USAGE"; for (const auto & column : columns) { columns_in_parentheses += columns_in_parentheses.empty() ? "(" : ", "; @@ -38,25 +171,17 @@ String AccessRightsElement::toString() const columns_in_parentheses += ")"; } + auto keywords = access_flags.toKeywords(); + if (keywords.empty()) + return "USAGE"; + String msg; - for (const std::string_view & keyword : access_flags.toKeywords()) + for (const std::string_view & keyword : keywords) { if (!msg.empty()) msg += ", "; msg += String{keyword} + columns_in_parentheses; } - - msg += " ON "; - - if (any_database) - msg += "*."; - else if (!database.empty() && (database != IDictionary::NO_DATABASE_TAG)) - msg += backQuoteIfNeed(database) + "."; - - if (any_table) - msg += "*"; - else - msg += backQuoteIfNeed(table); return msg; } @@ -68,19 +193,41 @@ void AccessRightsElements::replaceEmptyDatabase(const String & new_database) } -String AccessRightsElements::toString() const +String AccessRightsElements::toString() { - String res; - bool need_comma = false; - for (const auto & element : *this) - { - if (std::exchange(need_comma, true)) - res += ", "; - res += element.toString(); - } + normalize(); - if (res.empty()) - res = "USAGE ON *.*"; - return res; + if (empty()) + return "USAGE ON *.*"; + + String msg; + bool need_comma = false; + for (size_t i = 0; i != size(); ++i) + { + const auto & element = (*this)[i]; + if (std::exchange(need_comma, true)) + msg += ", "; + bool next_element_on_same_db_and_table = false; + if (i != size() - 1) + { + const auto & next_element = (*this)[i + 1]; + if ((element.database == next_element.database) && (element.any_database == next_element.any_database) + && (element.table == next_element.table) && (element.any_table == next_element.any_table)) + next_element_on_same_db_and_table = true; + } + if (next_element_on_same_db_and_table) + msg += element.toStringWithoutON(); + else + msg += element.toString(); + } + return msg; } + + +void AccessRightsElements::normalize() +{ + groupElements(*this); + sortElementsAndMakeUnique(*this); +} + } diff --git a/src/Access/AccessRightsElement.h b/src/Access/AccessRightsElement.h index 3894b6f5157..70eb95c2d17 100644 --- a/src/Access/AccessRightsElement.h +++ b/src/Access/AccessRightsElement.h @@ -71,6 +71,14 @@ struct AccessRightsElement { } + auto toTuple() const { return std::tie(access_flags, database, any_database, table, any_table, columns, any_column); } + friend bool operator==(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() == right.toTuple(); } + friend bool operator!=(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() != right.toTuple(); } + friend bool operator<(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() < right.toTuple(); } + friend bool operator>(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() > right.toTuple(); } + friend bool operator<=(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() <= right.toTuple(); } + friend bool operator>=(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() >= right.toTuple(); } + /// Sets the database. void setDatabase(const String & new_database); @@ -82,6 +90,7 @@ struct AccessRightsElement /// Returns a human-readable representation like "SELECT, UPDATE(x, y) ON db.table". /// The returned string isn't prefixed with the "GRANT" keyword. String toString() const; + String toStringWithoutON() const; }; @@ -94,7 +103,11 @@ public: /// Returns a human-readable representation like "SELECT, UPDATE(x, y) ON db.table". /// The returned string isn't prefixed with the "GRANT" keyword. - String toString() const; + String toString() const { return AccessRightsElements(*this).toString(); } + String toString(); + + /// Reorder and group elements to show them in more readable form. + void normalize(); }; } diff --git a/src/Access/AccessType.h b/src/Access/AccessType.h index d0665a6e55f..c4fdbc46b71 100644 --- a/src/Access/AccessType.h +++ b/src/Access/AccessType.h @@ -167,50 +167,48 @@ enum class AccessType #undef DECLARE_ACCESS_TYPE_ENUM_CONST }; -std::string_view toString(AccessType type); - namespace impl { template - class AccessTypeToKeywordConverter + class AccessTypeToStringConverter { public: - static const AccessTypeToKeywordConverter & instance() + static const AccessTypeToStringConverter & instance() { - static const AccessTypeToKeywordConverter res; + static const AccessTypeToStringConverter res; return res; } std::string_view convert(AccessType type) const { - return access_type_to_keyword_mapping[static_cast(type)]; + return access_type_to_string_mapping[static_cast(type)]; } private: - AccessTypeToKeywordConverter() + AccessTypeToStringConverter() { -#define INSERT_ACCESS_TYPE_KEYWORD_PAIR_TO_MAPPING(name, aliases, node_type, parent_group_name) \ - insertToMapping(AccessType::name, #name); +#define ACCESS_TYPE_TO_STRING_CONVERTER_ADD_TO_MAPPING(name, aliases, node_type, parent_group_name) \ + addToMapping(AccessType::name, #name); - APPLY_FOR_ACCESS_TYPES(INSERT_ACCESS_TYPE_KEYWORD_PAIR_TO_MAPPING) + APPLY_FOR_ACCESS_TYPES(ACCESS_TYPE_TO_STRING_CONVERTER_ADD_TO_MAPPING) -#undef INSERT_ACCESS_TYPE_KEYWORD_PAIR_TO_MAPPING +#undef ACCESS_TYPE_TO_STRING_CONVERTER_ADD_TO_MAPPING } - void insertToMapping(AccessType type, const std::string_view & str) + void addToMapping(AccessType type, const std::string_view & str) { String str2{str}; boost::replace_all(str2, "_", " "); size_t index = static_cast(type); - access_type_to_keyword_mapping.resize(std::max(index + 1, access_type_to_keyword_mapping.size())); - access_type_to_keyword_mapping[index] = str2; + access_type_to_string_mapping.resize(std::max(index + 1, access_type_to_string_mapping.size())); + access_type_to_string_mapping[index] = str2; } - Strings access_type_to_keyword_mapping; + Strings access_type_to_string_mapping; }; } -inline std::string_view toKeyword(AccessType type) { return impl::AccessTypeToKeywordConverter<>::instance().convert(type); } +inline std::string_view toString(AccessType type) { return impl::AccessTypeToStringConverter<>::instance().convert(type); } } diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index ab504e32579..f973e93c76b 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -121,7 +122,6 @@ void ContextAccess::setUser(const UserPtr & user_) const subscription_for_roles_changes = {}; enabled_roles = nullptr; roles_info = nullptr; - roles_with_admin_option = nullptr; enabled_row_policies = nullptr; enabled_quota = nullptr; enabled_settings = nullptr; @@ -131,29 +131,28 @@ void ContextAccess::setUser(const UserPtr & user_) const user_name = user->getName(); trace_log = &Poco::Logger::get("ContextAccess (" + user_name + ")"); - std::vector current_roles, current_roles_with_admin_option; + boost::container::flat_set current_roles, current_roles_with_admin_option; if (params.use_default_roles) { - for (const UUID & id : user->granted_roles) + for (const UUID & id : user->granted_roles.roles) { if (user->default_roles.match(id)) - current_roles.push_back(id); + current_roles.emplace(id); } - boost::range::set_intersection(current_roles, user->granted_roles_with_admin_option, - std::back_inserter(current_roles_with_admin_option)); } else { - current_roles.reserve(params.current_roles.size()); - for (const auto & id : params.current_roles) - { - if (user->granted_roles.count(id)) - current_roles.push_back(id); - if (user->granted_roles_with_admin_option.count(id)) - current_roles_with_admin_option.push_back(id); - } + boost::range::set_intersection( + params.current_roles, + user->granted_roles.roles, + std::inserter(current_roles, current_roles.end())); } + boost::range::set_intersection( + current_roles, + user->granted_roles.roles_with_admin_option, + std::inserter(current_roles_with_admin_option, current_roles_with_admin_option.end())); + subscription_for_roles_changes = {}; enabled_roles = manager->getEnabledRoles(current_roles, current_roles_with_admin_option); subscription_for_roles_changes = enabled_roles->subscribeForChanges([this](const std::shared_ptr & roles_info_) @@ -170,7 +169,6 @@ void ContextAccess::setRolesInfo(const std::shared_ptr & { assert(roles_info_); roles_info = roles_info_; - roles_with_admin_option.store(boost::make_shared>(roles_info->enabled_roles_with_admin_option.begin(), roles_info->enabled_roles_with_admin_option.end())); boost::range::fill(result_access, nullptr /* need recalculate */); enabled_row_policies = manager->getEnabledRowPolicies(*params.user_id, roles_info->enabled_roles); enabled_quota = manager->getEnabledQuota(*params.user_id, user_name, roles_info->enabled_roles, params.address, params.quota_key); @@ -357,10 +355,13 @@ void ContextAccess::checkAdminOption(const UUID & role_id) const if (isGranted(AccessType::ROLE_ADMIN)) return; - auto roles_with_admin_option_loaded = roles_with_admin_option.load(); - if (roles_with_admin_option_loaded && roles_with_admin_option_loaded->count(role_id)) + auto info = getRolesInfo(); + if (info && info->enabled_roles_with_admin_option.count(role_id)) return; + if (!user) + throw Exception(user_name + ": User has been dropped", ErrorCodes::UNKNOWN_USER); + std::optional role_name = manager->readName(role_id); if (!role_name) role_name = "ID {" + toString(role_id) + "}"; @@ -397,13 +398,13 @@ boost::shared_ptr ContextAccess::calculateResultAccess(bool if (grant_option) { - *merged_access = user->access_with_grant_option; + *merged_access = user->access.access_with_grant_option; if (roles_info) merged_access->merge(roles_info->access_with_grant_option); } else { - *merged_access = user->access; + *merged_access = user->access.access; if (roles_info) merged_access->merge(roles_info->access); } @@ -485,31 +486,7 @@ std::shared_ptr ContextAccess::getRolesInfo() const return roles_info; } -std::vector ContextAccess::getCurrentRoles() const -{ - std::lock_guard lock{mutex}; - return roles_info ? roles_info->current_roles : std::vector{}; -} - -Strings ContextAccess::getCurrentRolesNames() const -{ - std::lock_guard lock{mutex}; - return roles_info ? roles_info->getCurrentRolesNames() : Strings{}; -} - -std::vector ContextAccess::getEnabledRoles() const -{ - std::lock_guard lock{mutex}; - return roles_info ? roles_info->enabled_roles : std::vector{}; -} - -Strings ContextAccess::getEnabledRolesNames() const -{ - std::lock_guard lock{mutex}; - return roles_info ? roles_info->getEnabledRolesNames() : Strings{}; -} - -std::shared_ptr ContextAccess::getRowPolicies() const +std::shared_ptr ContextAccess::getEnabledRowPolicies() const { std::lock_guard lock{mutex}; return enabled_row_policies; @@ -528,6 +505,13 @@ std::shared_ptr ContextAccess::getQuota() const } +std::optional ContextAccess::getQuotaUsage() const +{ + std::lock_guard lock{mutex}; + return enabled_quota ? enabled_quota->getUsage() : std::optional{}; +} + + std::shared_ptr ContextAccess::getFullAccess() { static const std::shared_ptr res = [] diff --git a/src/Access/ContextAccess.h b/src/Access/ContextAccess.h index e0fbf58dbe8..27bb29a878c 100644 --- a/src/Access/ContextAccess.h +++ b/src/Access/ContextAccess.h @@ -22,6 +22,7 @@ class EnabledRoles; class EnabledRowPolicies; class EnabledQuota; class EnabledSettings; +struct QuotaUsage; struct Settings; class SettingsConstraints; class AccessControlManager; @@ -35,7 +36,7 @@ public: struct Params { std::optional user_id; - std::vector current_roles; + boost::container::flat_set current_roles; bool use_default_roles = false; UInt64 readonly = 0; bool allow_ddl = false; @@ -56,22 +57,37 @@ public: }; const Params & getParams() const { return params; } + + /// Returns the current user. The function can return nullptr. UserPtr getUser() const; String getUserName() const; bool isCorrectPassword(const String & password) const; bool isClientHostAllowed() const; + /// Returns information about current and enabled roles. + /// The function can return nullptr. std::shared_ptr getRolesInfo() const; - std::vector getCurrentRoles() const; - Strings getCurrentRolesNames() const; - std::vector getEnabledRoles() const; - Strings getEnabledRolesNames() const; - std::shared_ptr getRowPolicies() const; + /// Returns information about enabled row policies. + /// The function can return nullptr. + std::shared_ptr getEnabledRowPolicies() const; + + /// Returns the row policy filter for a specified table. + /// The function returns nullptr if there is no filter to apply. ASTPtr getRowPolicyCondition(const String & database, const String & table_name, RowPolicy::ConditionType index, const ASTPtr & extra_condition = nullptr) const; + + /// Returns the quota to track resource consumption. + /// The function returns nullptr if no tracking or limitation is needed. std::shared_ptr getQuota() const; + std::optional getQuotaUsage() const; + + /// Returns the default settings, i.e. the settings to apply on user's login. + /// The function returns nullptr if it's no need to apply settings. std::shared_ptr getDefaultSettings() const; + + /// Returns the settings' constraints. + /// The function returns nullptr if there are no constraints. std::shared_ptr getSettingsConstraints() const; /// Checks if a specified access is granted, and throws an exception if not. @@ -118,7 +134,8 @@ public: /// Checks if a specified role is granted with admin option, and throws an exception if not. void checkAdminOption(const UUID & role_id) const; - /// Returns an instance of ContextAccess which has full access to everything. + /// Makes an instance of ContextAccess which provides full access to everything + /// without any limitations. This is used for the global context. static std::shared_ptr getFullAccess(); private: @@ -157,7 +174,6 @@ private: mutable std::shared_ptr enabled_roles; mutable ext::scope_guard subscription_for_roles_changes; mutable std::shared_ptr roles_info; - mutable boost::atomic_shared_ptr> roles_with_admin_option; mutable boost::atomic_shared_ptr result_access[7]; mutable std::shared_ptr enabled_row_policies; mutable std::shared_ptr enabled_quota; diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index bf4316cc195..bc3e35a4fc7 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -53,6 +53,9 @@ namespace ErrorCodes namespace { + using EntityType = IAccessStorage::EntityType; + using EntityTypeInfo = IAccessStorage::EntityTypeInfo; + /// Special parser for the 'ATTACH access entity' queries. class ParserAttachAccessEntity : public IParserBase { @@ -79,7 +82,7 @@ namespace /// Reads a file containing ATTACH queries and then parses it to build an access entity. - AccessEntityPtr readAccessEntityFile(const std::filesystem::path & file_path) + AccessEntityPtr readEntityFile(const std::filesystem::path & file_path) { /// Read the file. ReadBufferFromFile in{file_path}; @@ -164,11 +167,11 @@ namespace } - AccessEntityPtr tryReadAccessEntityFile(const std::filesystem::path & file_path, Poco::Logger & log) + AccessEntityPtr tryReadEntityFile(const std::filesystem::path & file_path, Poco::Logger & log) { try { - return readAccessEntityFile(file_path); + return readEntityFile(file_path); } catch (...) { @@ -179,12 +182,12 @@ namespace /// Writes ATTACH queries for building a specified access entity to a file. - void writeAccessEntityFile(const std::filesystem::path & file_path, const IAccessEntity & entity) + void writeEntityFile(const std::filesystem::path & file_path, const IAccessEntity & entity) { /// Build list of ATTACH queries. ASTs queries; queries.push_back(InterpreterShowCreateAccessEntityQuery::getAttachQuery(entity)); - if (entity.getType() == typeid(User) || entity.getType() == typeid(Role)) + if ((entity.getType() == EntityType::USER) || (entity.getType() == EntityType::ROLE)) boost::range::push_back(queries, InterpreterShowGrantsQuery::getAttachGrantQueries(entity)); /// Serialize the list of ATTACH queries to a string. @@ -213,21 +216,21 @@ namespace /// Calculates the path to a file named .sql for saving an access entity. - std::filesystem::path getAccessEntityFilePath(const String & directory_path, const UUID & id) + std::filesystem::path getEntityFilePath(const String & directory_path, const UUID & id) { return std::filesystem::path(directory_path).append(toString(id)).replace_extension(".sql"); } /// Reads a map of name of access entity to UUID for access entities of some type from a file. - std::unordered_map readListFile(const std::filesystem::path & file_path) + std::vector> readListFile(const std::filesystem::path & file_path) { ReadBufferFromFile in(file_path); size_t num; readVarUInt(num, in); - std::unordered_map res; - res.reserve(num); + std::vector> id_name_pairs; + id_name_pairs.reserve(num); for (size_t i = 0; i != num; ++i) { @@ -235,19 +238,19 @@ namespace readStringBinary(name, in); UUID id; readUUIDText(id, in); - res[name] = id; + id_name_pairs.emplace_back(id, std::move(name)); } - return res; + return id_name_pairs; } /// Writes a map of name of access entity to UUID for access entities of some type to a file. - void writeListFile(const std::filesystem::path & file_path, const std::unordered_map & map) + void writeListFile(const std::filesystem::path & file_path, const std::vector> & id_name_pairs) { WriteBufferFromFile out(file_path); - writeVarUInt(map.size(), out); - for (const auto & [name, id] : map) + writeVarUInt(id_name_pairs.size(), out); + for (const auto & [id, name] : id_name_pairs) { writeStringBinary(name, out); writeUUIDText(id, out); @@ -256,24 +259,10 @@ namespace /// Calculates the path for storing a map of name of access entity to UUID for access entities of some type. - std::filesystem::path getListFilePath(const String & directory_path, std::type_index type) + std::filesystem::path getListFilePath(const String & directory_path, EntityType type) { - std::string_view file_name; - if (type == typeid(User)) - file_name = "users"; - else if (type == typeid(Role)) - file_name = "roles"; - else if (type == typeid(Quota)) - file_name = "quotas"; - else if (type == typeid(RowPolicy)) - file_name = "row_policies"; - else if (type == typeid(SettingsProfile)) - file_name = "settings_profiles"; - else - throw Exception("Unexpected type of access entity: " + IAccessEntity::getTypeName(type), - ErrorCodes::LOGICAL_ERROR); - - return std::filesystem::path(directory_path).append(file_name).replace_extension(".list"); + std::string_view file_name = EntityTypeInfo::get(type).list_filename; + return std::filesystem::path(directory_path).append(file_name); } @@ -297,21 +286,12 @@ namespace return false; } } - - - const std::vector & getAllAccessEntityTypes() - { - static const std::vector res = {typeid(User), typeid(Role), typeid(RowPolicy), typeid(Quota), typeid(SettingsProfile)}; - return res; - } } DiskAccessStorage::DiskAccessStorage() : IAccessStorage("disk") { - for (auto type : getAllAccessEntityTypes()) - name_to_id_maps[type]; } @@ -363,18 +343,27 @@ void DiskAccessStorage::initialize(const String & directory_path_, Notifications writeLists(); } - for (const auto & [id, entry] : id_to_entry_map) + for (const auto & [id, entry] : entries_by_id) prepareNotifications(id, entry, false, notifications); } +void DiskAccessStorage::clear() +{ + entries_by_id.clear(); + for (auto type : ext::range(EntityType::MAX)) + entries_by_name_and_type[static_cast(type)].clear(); +} + + bool DiskAccessStorage::readLists() { - assert(id_to_entry_map.empty()); + clear(); + bool ok = true; - for (auto type : getAllAccessEntityTypes()) + for (auto type : ext::range(EntityType::MAX)) { - auto & name_to_id_map = name_to_id_maps.at(type); + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; auto file_path = getListFilePath(directory_path, type); if (!std::filesystem::exists(file_path)) { @@ -385,7 +374,14 @@ bool DiskAccessStorage::readLists() try { - name_to_id_map = readListFile(file_path); + for (const auto & [id, name] : readListFile(file_path)) + { + auto & entry = entries_by_id[id]; + entry.id = id; + entry.type = type; + entry.name = name; + entries_by_name[entry.name] = &entry; + } } catch (...) { @@ -393,17 +389,10 @@ bool DiskAccessStorage::readLists() ok = false; break; } - - for (const auto & [name, id] : name_to_id_map) - id_to_entry_map.emplace(id, Entry{name, type}); } if (!ok) - { - id_to_entry_map.clear(); - for (auto & name_to_id_map : name_to_id_maps | boost::adaptors::map_values) - name_to_id_map.clear(); - } + clear(); return ok; } @@ -419,11 +408,15 @@ bool DiskAccessStorage::writeLists() for (const auto & type : types_of_lists_to_write) { - const auto & name_to_id_map = name_to_id_maps.at(type); + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; auto file_path = getListFilePath(directory_path, type); try { - writeListFile(file_path, name_to_id_map); + std::vector> id_name_pairs; + id_name_pairs.reserve(entries_by_name.size()); + for (const auto * entry : entries_by_name | boost::adaptors::map_values) + id_name_pairs.emplace_back(entry->id, entry->name); + writeListFile(file_path, id_name_pairs); } catch (...) { @@ -441,7 +434,7 @@ bool DiskAccessStorage::writeLists() } -void DiskAccessStorage::scheduleWriteLists(std::type_index type) +void DiskAccessStorage::scheduleWriteLists(EntityType type) { if (failed_to_write_lists) return; @@ -504,7 +497,7 @@ void DiskAccessStorage::listsWritingThreadFunc() bool DiskAccessStorage::rebuildLists() { LOG_WARNING(getLogger(), "Recovering lists in directory " + directory_path); - assert(id_to_entry_map.empty()); + clear(); for (const auto & directory_entry : std::filesystem::directory_iterator(directory_path)) { @@ -518,58 +511,64 @@ bool DiskAccessStorage::rebuildLists() if (!tryParseUUID(path.stem(), id)) continue; - const auto access_entity_file_path = getAccessEntityFilePath(directory_path, id); - auto entity = tryReadAccessEntityFile(access_entity_file_path, *getLogger()); + const auto access_entity_file_path = getEntityFilePath(directory_path, id); + auto entity = tryReadEntityFile(access_entity_file_path, *getLogger()); if (!entity) continue; + const String & name = entity->getName(); auto type = entity->getType(); - auto & name_to_id_map = name_to_id_maps.at(type); - auto it_by_name = name_to_id_map.emplace(entity->getFullName(), id).first; - id_to_entry_map.emplace(id, Entry{it_by_name->first, type}); + auto & entry = entries_by_id[id]; + entry.id = id; + entry.type = type; + entry.name = name; + entry.entity = entity; + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + entries_by_name[entry.name] = &entry; } - for (auto type : getAllAccessEntityTypes()) + for (auto type : ext::range(EntityType::MAX)) types_of_lists_to_write.insert(type); return true; } -std::optional DiskAccessStorage::findImpl(std::type_index type, const String & name) const +std::optional DiskAccessStorage::findImpl(EntityType type, const String & name) const { std::lock_guard lock{mutex}; - const auto & name_to_id_map = name_to_id_maps.at(type); - auto it = name_to_id_map.find(name); - if (it == name_to_id_map.end()) + const auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + auto it = entries_by_name.find(name); + if (it == entries_by_name.end()) return {}; - return it->second; + return it->second->id; } -std::vector DiskAccessStorage::findAllImpl(std::type_index type) const +std::vector DiskAccessStorage::findAllImpl(EntityType type) const { std::lock_guard lock{mutex}; - const auto & name_to_id_map = name_to_id_maps.at(type); + const auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; std::vector res; - res.reserve(name_to_id_map.size()); - boost::range::copy(name_to_id_map | boost::adaptors::map_values, std::back_inserter(res)); + res.reserve(entries_by_name.size()); + for (const auto * entry : entries_by_name | boost::adaptors::map_values) + res.emplace_back(entry->id); return res; } bool DiskAccessStorage::existsImpl(const UUID & id) const { std::lock_guard lock{mutex}; - return id_to_entry_map.count(id); + return entries_by_id.count(id); } AccessEntityPtr DiskAccessStorage::readImpl(const UUID & id) const { std::lock_guard lock{mutex}; - auto it = id_to_entry_map.find(id); - if (it == id_to_entry_map.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) throwNotFound(id); const auto & entry = it->second; @@ -582,8 +581,8 @@ AccessEntityPtr DiskAccessStorage::readImpl(const UUID & id) const String DiskAccessStorage::readNameImpl(const UUID & id) const { std::lock_guard lock{mutex}; - auto it = id_to_entry_map.find(id); - if (it == id_to_entry_map.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) throwNotFound(id); return String{it->second.name}; } @@ -609,25 +608,25 @@ UUID DiskAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool repl void DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, Notifications & notifications) { - const String & name = new_entity->getFullName(); - std::type_index type = new_entity->getType(); + const String & name = new_entity->getName(); + EntityType type = new_entity->getType(); if (!initialized) throw Exception( - "Cannot insert " + new_entity->getTypeName() + " " + backQuote(name) + " to " + getStorageName() - + " because the output directory is not set", + "Cannot insert " + new_entity->outputTypeAndName() + " to storage [" + getStorageName() + + "] because the output directory is not set", ErrorCodes::LOGICAL_ERROR); /// Check that we can insert. - auto it_by_id = id_to_entry_map.find(id); - if (it_by_id != id_to_entry_map.end()) + auto it_by_id = entries_by_id.find(id); + if (it_by_id != entries_by_id.end()) { const auto & existing_entry = it_by_id->second; - throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getFullName()); + throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName()); } - auto & name_to_id_map = name_to_id_maps.at(type); - auto it_by_name = name_to_id_map.find(name); - bool name_collision = (it_by_name != name_to_id_map.end()); + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + auto it_by_name = entries_by_name.find(name); + bool name_collision = (it_by_name != entries_by_name.end()); if (name_collision && !replace_if_exists) throwNameCollisionCannotInsert(type, name); @@ -636,13 +635,15 @@ void DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne writeAccessEntityToDisk(id, *new_entity); if (name_collision && replace_if_exists) - removeNoLock(it_by_name->second, notifications); + removeNoLock(it_by_name->second->id, notifications); /// Do insertion. - it_by_name = name_to_id_map.emplace(name, id).first; - it_by_id = id_to_entry_map.emplace(id, Entry{it_by_name->first, type}).first; - auto & entry = it_by_id->second; + auto & entry = entries_by_id[id]; + entry.id = id; + entry.type = type; + entry.name = name; entry.entity = new_entity; + entries_by_name[entry.name] = &entry; prepareNotifications(id, entry, false, notifications); } @@ -659,22 +660,21 @@ void DiskAccessStorage::removeImpl(const UUID & id) void DiskAccessStorage::removeNoLock(const UUID & id, Notifications & notifications) { - auto it = id_to_entry_map.find(id); - if (it == id_to_entry_map.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) throwNotFound(id); Entry & entry = it->second; - String name{it->second.name}; - std::type_index type = it->second.type; + EntityType type = entry.type; scheduleWriteLists(type); deleteAccessEntityOnDisk(id); /// Do removing. prepareNotifications(id, entry, true, notifications); - id_to_entry_map.erase(it); - auto & name_to_id_map = name_to_id_maps.at(type); - name_to_id_map.erase(name); + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + entries_by_name.erase(entry.name); + entries_by_id.erase(it); } @@ -690,8 +690,8 @@ void DiskAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_fu void DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, Notifications & notifications) { - auto it = id_to_entry_map.find(id); - if (it == id_to_entry_map.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) throwNotFound(id); Entry & entry = it->second; @@ -700,18 +700,22 @@ void DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_ auto old_entity = entry.entity; auto new_entity = update_func(old_entity); + if (!new_entity->isTypeOf(old_entity->getType())) + throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType()); + if (*new_entity == *old_entity) return; - String new_name = new_entity->getFullName(); - auto old_name = entry.name; - const std::type_index type = entry.type; + const String & new_name = new_entity->getName(); + const String & old_name = old_entity->getName(); + const EntityType type = entry.type; + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + bool name_changed = (new_name != old_name); if (name_changed) { - const auto & name_to_id_map = name_to_id_maps.at(type); - if (name_to_id_map.count(new_name)) - throwNameCollisionCannotRename(type, String{old_name}, new_name); + if (entries_by_name.count(new_name)) + throwNameCollisionCannotRename(type, old_name, new_name); scheduleWriteLists(type); } @@ -720,10 +724,9 @@ void DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_ if (name_changed) { - auto & name_to_id_map = name_to_id_maps.at(type); - name_to_id_map.erase(String{old_name}); - auto it_by_name = name_to_id_map.emplace(new_name, id).first; - entry.name = it_by_name->first; + entries_by_name.erase(entry.name); + entry.name = new_name; + entries_by_name[entry.name] = &entry; } prepareNotifications(id, entry, false, notifications); @@ -732,19 +735,19 @@ void DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_ AccessEntityPtr DiskAccessStorage::readAccessEntityFromDisk(const UUID & id) const { - return readAccessEntityFile(getAccessEntityFilePath(directory_path, id)); + return readEntityFile(getEntityFilePath(directory_path, id)); } void DiskAccessStorage::writeAccessEntityToDisk(const UUID & id, const IAccessEntity & entity) const { - writeAccessEntityFile(getAccessEntityFilePath(directory_path, id), entity); + writeEntityFile(getEntityFilePath(directory_path, id), entity); } void DiskAccessStorage::deleteAccessEntityOnDisk(const UUID & id) const { - auto file_path = getAccessEntityFilePath(directory_path, id); + auto file_path = getEntityFilePath(directory_path, id); if (!std::filesystem::remove(file_path)) throw Exception("Couldn't delete " + file_path.string(), ErrorCodes::FILE_DOESNT_EXIST); } @@ -759,17 +762,16 @@ void DiskAccessStorage::prepareNotifications(const UUID & id, const Entry & entr for (const auto & handler : entry.handlers_by_id) notifications.push_back({handler, id, entity}); - auto range = handlers_by_type.equal_range(entry.type); - for (auto it = range.first; it != range.second; ++it) - notifications.push_back({it->second, id, entity}); + for (const auto & handler : handlers_by_type[static_cast(entry.type)]) + notifications.push_back({handler, id, entity}); } ext::scope_guard DiskAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const { std::lock_guard lock{mutex}; - auto it = id_to_entry_map.find(id); - if (it == id_to_entry_map.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) return {}; const Entry & entry = it->second; auto handler_it = entry.handlers_by_id.insert(entry.handlers_by_id.end(), handler); @@ -777,8 +779,8 @@ ext::scope_guard DiskAccessStorage::subscribeForChangesImpl(const UUID & id, con return [this, id, handler_it] { std::lock_guard lock2{mutex}; - auto it2 = id_to_entry_map.find(id); - if (it2 != id_to_entry_map.end()) + auto it2 = entries_by_id.find(id); + if (it2 != entries_by_id.end()) { const Entry & entry2 = it2->second; entry2.handlers_by_id.erase(handler_it); @@ -786,23 +788,26 @@ ext::scope_guard DiskAccessStorage::subscribeForChangesImpl(const UUID & id, con }; } -ext::scope_guard DiskAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const +ext::scope_guard DiskAccessStorage::subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const { std::lock_guard lock{mutex}; - auto handler_it = handlers_by_type.emplace(type, handler); + auto & handlers = handlers_by_type[static_cast(type)]; + handlers.push_back(handler); + auto handler_it = std::prev(handlers.end()); - return [this, handler_it] + return [this, type, handler_it] { std::lock_guard lock2{mutex}; - handlers_by_type.erase(handler_it); + auto & handlers2 = handlers_by_type[static_cast(type)]; + handlers2.erase(handler_it); }; } bool DiskAccessStorage::hasSubscriptionImpl(const UUID & id) const { std::lock_guard lock{mutex}; - auto it = id_to_entry_map.find(id); - if (it != id_to_entry_map.end()) + auto it = entries_by_id.find(id); + if (it != entries_by_id.end()) { const Entry & entry = it->second; return !entry.handlers_by_id.empty(); @@ -810,11 +815,11 @@ bool DiskAccessStorage::hasSubscriptionImpl(const UUID & id) const return false; } -bool DiskAccessStorage::hasSubscriptionImpl(std::type_index type) const +bool DiskAccessStorage::hasSubscriptionImpl(EntityType type) const { std::lock_guard lock{mutex}; - auto range = handlers_by_type.equal_range(type); - return range.first != range.second; + const auto & handlers = handlers_by_type[static_cast(type)]; + return !handlers.empty(); } } diff --git a/src/Access/DiskAccessStorage.h b/src/Access/DiskAccessStorage.h index 104c0f1fa38..79a11195318 100644 --- a/src/Access/DiskAccessStorage.h +++ b/src/Access/DiskAccessStorage.h @@ -17,8 +17,8 @@ public: void setDirectory(const String & directory_path_); private: - std::optional findImpl(std::type_index type, const String & name) const override; - std::vector findAllImpl(std::type_index type) const override; + std::optional findImpl(EntityType type, const String & name) const override; + std::vector findAllImpl(EntityType type) const override; bool existsImpl(const UUID & id) const override; AccessEntityPtr readImpl(const UUID & id) const override; String readNameImpl(const UUID & id) const override; @@ -27,14 +27,15 @@ private: void removeImpl(const UUID & id) override; void updateImpl(const UUID & id, const UpdateFunc & update_func) override; ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; - ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override; + ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; bool hasSubscriptionImpl(const UUID & id) const override; - bool hasSubscriptionImpl(std::type_index type) const override; + bool hasSubscriptionImpl(EntityType type) const override; void initialize(const String & directory_path_, Notifications & notifications); + void clear(); bool readLists(); bool writeLists(); - void scheduleWriteLists(std::type_index type); + void scheduleWriteLists(EntityType type); bool rebuildLists(); void startListsWritingThread(); @@ -52,9 +53,9 @@ private: using NameToIDMap = std::unordered_map; struct Entry { - Entry(const std::string_view & name_, std::type_index type_) : name(name_), type(type_) {} - std::string_view name; /// view points to a string in `name_to_id_maps`. - std::type_index type; + UUID id; + String name; + EntityType type; mutable AccessEntityPtr entity; /// may be nullptr, if the entity hasn't been loaded yet. mutable std::list handlers_by_id; }; @@ -63,14 +64,14 @@ private: String directory_path; bool initialized = false; - std::unordered_map name_to_id_maps; - std::unordered_map id_to_entry_map; - boost::container::flat_set types_of_lists_to_write; + std::unordered_map entries_by_id; + std::unordered_map entries_by_name_and_type[static_cast(EntityType::MAX)]; + boost::container::flat_set types_of_lists_to_write; bool failed_to_write_lists = false; /// Whether writing of the list files has been failed since the recent restart of the server. ThreadFromGlobalPool lists_writing_thread; /// List files are written in a separate thread. std::condition_variable lists_writing_thread_should_exit; /// Signals `lists_writing_thread` to exit. std::atomic lists_writing_thread_exited = false; - mutable std::unordered_multimap handlers_by_type; + mutable std::list handlers_by_type[static_cast(EntityType::MAX)]; mutable std::mutex mutex; }; } diff --git a/src/Access/EnabledQuota.cpp b/src/Access/EnabledQuota.cpp index 92257ce0002..e9d586a692f 100644 --- a/src/Access/EnabledQuota.cpp +++ b/src/Access/EnabledQuota.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -30,9 +30,10 @@ struct EnabledQuota::Impl if (resource_type == Quota::EXECUTION_TIME) amount_to_string = [&](UInt64 amount) { return ext::to_string(std::chrono::nanoseconds(amount)); }; + const auto & type_info = Quota::ResourceTypeInfo::get(resource_type); throw Exception( "Quota for user " + backQuote(user_name) + " for " + ext::to_string(duration) + " has been exceeded: " - + Quota::getNameOfResourceType(resource_type) + " = " + amount_to_string(used) + "/" + amount_to_string(max) + ". " + + type_info.outputWithAmount(used) + "/" + type_info.amountToString(max) + ". " + "Interval will end at " + ext::to_string(end_of_interval) + ". " + "Name of quota template: " + backQuote(quota_name), ErrorCodes::QUOTA_EXPIRED); } @@ -83,7 +84,7 @@ struct EnabledQuota::Impl { ResourceAmount used = (interval.used[resource_type] += amount); ResourceAmount max = interval.max[resource_type]; - if (max == Quota::UNLIMITED) + if (!max) continue; if (used > max) { @@ -111,7 +112,7 @@ struct EnabledQuota::Impl { ResourceAmount used = interval.used[resource_type]; ResourceAmount max = interval.max[resource_type]; - if (max == Quota::UNLIMITED) + if (!max) continue; if (used > max) { @@ -128,12 +129,22 @@ struct EnabledQuota::Impl const Intervals & intervals, std::chrono::system_clock::time_point current_time) { - for (auto resource_type : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) checkExceeded(user_name, intervals, resource_type, current_time); } }; +EnabledQuota::Interval::Interval() +{ + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) + { + used[resource_type].store(0); + max[resource_type] = 0; + } +} + + EnabledQuota::Interval & EnabledQuota::Interval::operator =(const Interval & src) { if (this == &src) @@ -151,27 +162,30 @@ EnabledQuota::Interval & EnabledQuota::Interval::operator =(const Interval & src } -QuotaUsageInfo EnabledQuota::Intervals::getUsageInfo(std::chrono::system_clock::time_point current_time) const +std::optional EnabledQuota::Intervals::getUsage(std::chrono::system_clock::time_point current_time) const { - QuotaUsageInfo info; - info.quota_id = quota_id; - info.quota_name = quota_name; - info.quota_key = quota_key; - info.intervals.reserve(intervals.size()); + if (!quota_id) + return {}; + QuotaUsage usage; + usage.quota_id = *quota_id; + usage.quota_name = quota_name; + usage.quota_key = quota_key; + usage.intervals.reserve(intervals.size()); for (const auto & in : intervals) { - info.intervals.push_back({}); - auto & out = info.intervals.back(); + usage.intervals.push_back({}); + auto & out = usage.intervals.back(); out.duration = in.duration; out.randomize_interval = in.randomize_interval; out.end_of_interval = Impl::getEndOfInterval(in, current_time); for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) { - out.max[resource_type] = in.max[resource_type]; + if (in.max[resource_type]) + out.max[resource_type] = in.max[resource_type]; out.used[resource_type] = in.used[resource_type]; } } - return info; + return usage; } @@ -238,10 +252,10 @@ void EnabledQuota::checkExceeded(ResourceType resource_type) const } -QuotaUsageInfo EnabledQuota::getUsageInfo() const +std::optional EnabledQuota::getUsage() const { auto loaded = intervals.load(); - return loaded->getUsageInfo(std::chrono::system_clock::now()); + return loaded->getUsage(std::chrono::system_clock::now()); } diff --git a/src/Access/EnabledQuota.h b/src/Access/EnabledQuota.h index 5a624c651af..25e804dd050 100644 --- a/src/Access/EnabledQuota.h +++ b/src/Access/EnabledQuota.h @@ -12,7 +12,7 @@ namespace DB { -struct QuotaUsageInfo; +struct QuotaUsage; /// Instances of `EnabledQuota` are used to track resource consumption. @@ -23,7 +23,7 @@ public: { UUID user_id; String user_name; - std::vector enabled_roles; + boost::container::flat_set enabled_roles; Poco::Net::IPAddress client_address; String client_key; @@ -53,7 +53,7 @@ public: void checkExceeded(ResourceType resource_type) const; /// Returns the information about quota consumption. - QuotaUsageInfo getUsageInfo() const; + std::optional getUsage() const; /// Returns an instance of EnabledQuota which is never exceeded. static std::shared_ptr getUnlimitedQuota(); @@ -65,17 +65,17 @@ private: const String & getUserName() const { return params.user_name; } - static constexpr size_t MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; + static constexpr auto MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; struct Interval { mutable std::atomic used[MAX_RESOURCE_TYPE]; ResourceAmount max[MAX_RESOURCE_TYPE]; - std::chrono::seconds duration; - bool randomize_interval; + std::chrono::seconds duration = std::chrono::seconds::zero(); + bool randomize_interval = false; mutable std::atomic end_of_interval; - Interval() {} + Interval(); Interval(const Interval & src) { *this = src; } Interval & operator =(const Interval & src); }; @@ -83,11 +83,11 @@ private: struct Intervals { std::vector intervals; - UUID quota_id; + std::optional quota_id; String quota_name; String quota_key; - QuotaUsageInfo getUsageInfo(std::chrono::system_clock::time_point current_time) const; + std::optional getUsage(std::chrono::system_clock::time_point current_time) const; }; struct Impl; diff --git a/src/Access/EnabledRoles.h b/src/Access/EnabledRoles.h index 122b1a16fe3..0c5b799a30b 100644 --- a/src/Access/EnabledRoles.h +++ b/src/Access/EnabledRoles.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -16,8 +17,8 @@ class EnabledRoles public: struct Params { - std::vector current_roles; - std::vector current_roles_with_admin_option; + boost::container::flat_set current_roles; + boost::container::flat_set current_roles_with_admin_option; auto toTuple() const { return std::tie(current_roles, current_roles_with_admin_option); } friend bool operator ==(const Params & lhs, const Params & rhs) { return lhs.toTuple() == rhs.toTuple(); } diff --git a/src/Access/EnabledRolesInfo.h b/src/Access/EnabledRolesInfo.h index 837d4b74ad5..45e1bfd9057 100644 --- a/src/Access/EnabledRolesInfo.h +++ b/src/Access/EnabledRolesInfo.h @@ -3,8 +3,8 @@ #include #include #include +#include #include -#include namespace DB @@ -13,9 +13,9 @@ namespace DB /// Information about a role. struct EnabledRolesInfo { - std::vector current_roles; - std::vector enabled_roles; - std::vector enabled_roles_with_admin_option; + boost::container::flat_set current_roles; + boost::container::flat_set enabled_roles; + boost::container::flat_set enabled_roles_with_admin_option; std::unordered_map names_of_roles; AccessRights access; AccessRights access_with_grant_option; diff --git a/src/Access/EnabledRowPolicies.cpp b/src/Access/EnabledRowPolicies.cpp index 56c73aaf40d..efd5ed4ae10 100644 --- a/src/Access/EnabledRowPolicies.cpp +++ b/src/Access/EnabledRowPolicies.cpp @@ -6,9 +6,9 @@ namespace DB { -size_t EnabledRowPolicies::Hash::operator()(const DatabaseAndTableNameRef & database_and_table_name) const +size_t EnabledRowPolicies::Hash::operator()(const MixedConditionKey & key) const { - return std::hash{}(database_and_table_name.first) - std::hash{}(database_and_table_name.second); + return std::hash{}(key.database) - std::hash{}(key.table_name) + static_cast(key.condition_type); } @@ -20,16 +20,22 @@ EnabledRowPolicies::EnabledRowPolicies(const Params & params_) EnabledRowPolicies::~EnabledRowPolicies() = default; -ASTPtr EnabledRowPolicies::getCondition(const String & database, const String & table_name, ConditionType type) const +ASTPtr EnabledRowPolicies::getCondition(const String & database, const String & table_name, ConditionType condition_type) const { /// We don't lock `mutex` here. auto loaded = map_of_mixed_conditions.load(); - auto it = loaded->find({database, table_name}); + auto it = loaded->find({database, table_name, condition_type}); if (it == loaded->end()) return {}; - return it->second.mixed_conditions[type]; -} + auto condition = it->second.ast; + + bool value; + if (tryGetLiteralBool(condition.get(), value) && value) + return nullptr; /// The condition is always true, no need to check it. + + return condition; +} ASTPtr EnabledRowPolicies::getCondition(const String & database, const String & table_name, ConditionType type, const ASTPtr & extra_condition) const { @@ -41,31 +47,9 @@ ASTPtr EnabledRowPolicies::getCondition(const String & database, const String & bool value; if (tryGetLiteralBool(condition.get(), value) && value) - condition = nullptr; /// The condition is always true, no need to check it. + return nullptr; /// The condition is always true, no need to check it. return condition; } - -std::vector EnabledRowPolicies::getCurrentPolicyIDs() const -{ - /// We don't lock `mutex` here. - auto loaded = map_of_mixed_conditions.load(); - std::vector policy_ids; - for (const auto & mixed_conditions : *loaded | boost::adaptors::map_values) - boost::range::copy(mixed_conditions.policy_ids, std::back_inserter(policy_ids)); - return policy_ids; -} - - -std::vector EnabledRowPolicies::getCurrentPolicyIDs(const String & database, const String & table_name) const -{ - /// We don't lock `mutex` here. - auto loaded = map_of_mixed_conditions.load(); - auto it = loaded->find({database, table_name}); - if (it == loaded->end()) - return {}; - return it->second.policy_ids; -} - } diff --git a/src/Access/EnabledRowPolicies.h b/src/Access/EnabledRowPolicies.h index 9befb65ff0b..b92939afb03 100644 --- a/src/Access/EnabledRowPolicies.h +++ b/src/Access/EnabledRowPolicies.h @@ -4,8 +4,8 @@ #include #include #include -#include #include +#include namespace DB @@ -21,7 +21,7 @@ public: struct Params { UUID user_id; - std::vector enabled_roles; + boost::container::flat_set enabled_roles; auto toTuple() const { return std::tie(user_id, enabled_roles); } friend bool operator ==(const Params & lhs, const Params & rhs) { return lhs.toTuple() == rhs.toTuple(); } @@ -42,31 +42,32 @@ public: ASTPtr getCondition(const String & database, const String & table_name, ConditionType type) const; ASTPtr getCondition(const String & database, const String & table_name, ConditionType type, const ASTPtr & extra_condition) const; - /// Returns IDs of all the policies used by the current user. - std::vector getCurrentPolicyIDs() const; - - /// Returns IDs of the policies used by a concrete table. - std::vector getCurrentPolicyIDs(const String & database, const String & table_name) const; - private: friend class RowPolicyCache; EnabledRowPolicies(const Params & params_); - using DatabaseAndTableName = std::pair; - using DatabaseAndTableNameRef = std::pair; + struct MixedConditionKey + { + std::string_view database; + std::string_view table_name; + ConditionType condition_type; + + auto toTuple() const { return std::tie(database, table_name, condition_type); } + friend bool operator==(const MixedConditionKey & left, const MixedConditionKey & right) { return left.toTuple() == right.toTuple(); } + friend bool operator!=(const MixedConditionKey & left, const MixedConditionKey & right) { return left.toTuple() != right.toTuple(); } + }; + struct Hash { - size_t operator()(const DatabaseAndTableNameRef & database_and_table_name) const; + size_t operator()(const MixedConditionKey & key) const; }; - static constexpr size_t MAX_CONDITION_TYPE = RowPolicy::MAX_CONDITION_TYPE; - using ParsedConditions = std::array; - struct MixedConditions + + struct MixedCondition { - std::unique_ptr database_and_table_name_keeper; - ParsedConditions mixed_conditions; - std::vector policy_ids; + ASTPtr ast; + std::shared_ptr> database_and_table_name; }; - using MapOfMixedConditions = std::unordered_map; + using MapOfMixedConditions = std::unordered_map; const Params params; mutable boost::atomic_shared_ptr map_of_mixed_conditions; diff --git a/src/Access/EnabledSettings.h b/src/Access/EnabledSettings.h index d8e969d685d..8e92298328c 100644 --- a/src/Access/EnabledSettings.h +++ b/src/Access/EnabledSettings.h @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -17,7 +18,7 @@ public: struct Params { UUID user_id; - std::vector enabled_roles; + boost::container::flat_set enabled_roles; SettingsProfileElements settings_from_enabled_roles; SettingsProfileElements settings_from_user; diff --git a/src/Access/ExtendedRoleSet.cpp b/src/Access/ExtendedRoleSet.cpp index 145bd0fe7e5..a29ee40380c 100644 --- a/src/Access/ExtendedRoleSet.cpp +++ b/src/Access/ExtendedRoleSet.cpp @@ -1,3 +1,4 @@ + #include #include #include @@ -43,12 +44,6 @@ ExtendedRoleSet::ExtendedRoleSet(const std::vector & ids_) } -ExtendedRoleSet::ExtendedRoleSet(const boost::container::flat_set & ids_) -{ - add(ids_); -} - - ExtendedRoleSet::ExtendedRoleSet(const ASTExtendedRoleSet & ast) { init(ast, nullptr); @@ -126,6 +121,7 @@ std::shared_ptr ExtendedRoleSet::toAST() const ast->names.reserve(ids.size()); for (const UUID & id : ids) ast->names.emplace_back(::DB::toString(id)); + boost::range::sort(ast->names); } if (!except_ids.empty()) @@ -133,32 +129,13 @@ std::shared_ptr ExtendedRoleSet::toAST() const ast->except_names.reserve(except_ids.size()); for (const UUID & except_id : except_ids) ast->except_names.emplace_back(::DB::toString(except_id)); + boost::range::sort(ast->except_names); } return ast; } -String ExtendedRoleSet::toString() const -{ - auto ast = toAST(); - return serializeAST(*ast); -} - - -Strings ExtendedRoleSet::toStrings() const -{ - if (all || !except_ids.empty()) - return {toString()}; - - Strings names; - names.reserve(ids.size()); - for (const UUID & id : ids) - names.emplace_back(::DB::toString(id)); - return names; -} - - std::shared_ptr ExtendedRoleSet::toASTWithNames(const AccessControlManager & manager) const { auto ast = std::make_shared(); @@ -192,6 +169,13 @@ std::shared_ptr ExtendedRoleSet::toASTWithNames(const Access } +String ExtendedRoleSet::toString() const +{ + auto ast = toAST(); + return serializeAST(*ast); +} + + String ExtendedRoleSet::toStringWithNames(const AccessControlManager & manager) const { auto ast = toASTWithNames(manager); @@ -201,19 +185,39 @@ String ExtendedRoleSet::toStringWithNames(const AccessControlManager & manager) Strings ExtendedRoleSet::toStringsWithNames(const AccessControlManager & manager) const { - if (all || !except_ids.empty()) - return {toStringWithNames(manager)}; + if (!all && ids.empty()) + return {}; - Strings names; - names.reserve(ids.size()); - for (const UUID & id : ids) + Strings res; + res.reserve(ids.size() + except_ids.size()); + + if (all) + res.emplace_back("ALL"); + else { - auto name = manager.tryReadName(id); - if (name) - names.emplace_back(std::move(*name)); + for (const UUID & id : ids) + { + auto name = manager.tryReadName(id); + if (name) + res.emplace_back(std::move(*name)); + } + std::sort(res.begin(), res.end()); } - boost::range::sort(names); - return names; + + if (!except_ids.empty()) + { + res.emplace_back("EXCEPT"); + size_t old_size = res.size(); + for (const UUID & id : except_ids) + { + auto name = manager.tryReadName(id); + if (name) + res.emplace_back(std::move(*name)); + } + std::sort(res.begin() + old_size, res.end()); + } + + return res; } @@ -244,38 +248,12 @@ void ExtendedRoleSet::add(const std::vector & ids_) } -void ExtendedRoleSet::add(const boost::container::flat_set & ids_) -{ - for (const auto & id : ids_) - add(id); -} - - bool ExtendedRoleSet::match(const UUID & id) const { return (all || ids.count(id)) && !except_ids.count(id); } -bool ExtendedRoleSet::match(const UUID & user_id, const std::vector & enabled_roles) const -{ - if (!all && !ids.count(user_id)) - { - bool found_enabled_role = std::any_of( - enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return ids.count(enabled_role); }); - if (!found_enabled_role) - return false; - } - - if (except_ids.count(user_id)) - return false; - - bool in_except_list = std::any_of( - enabled_roles.begin(), enabled_roles.end(), [this](const UUID & enabled_role) { return except_ids.count(enabled_role); }); - return !in_except_list; -} - - bool ExtendedRoleSet::match(const UUID & user_id, const boost::container::flat_set & enabled_roles) const { if (!all && !ids.count(user_id)) diff --git a/src/Access/ExtendedRoleSet.h b/src/Access/ExtendedRoleSet.h index 486b4277337..eeb4af84f78 100644 --- a/src/Access/ExtendedRoleSet.h +++ b/src/Access/ExtendedRoleSet.h @@ -28,7 +28,6 @@ struct ExtendedRoleSet ExtendedRoleSet(const UUID & id); ExtendedRoleSet(const std::vector & ids_); - ExtendedRoleSet(const boost::container::flat_set & ids_); /// The constructor from AST requires the AccessControlManager if `ast.id_mode == false`. ExtendedRoleSet(const ASTExtendedRoleSet & ast); @@ -37,10 +36,9 @@ struct ExtendedRoleSet ExtendedRoleSet(const ASTExtendedRoleSet & ast, const AccessControlManager & manager, const std::optional & current_user_id); std::shared_ptr toAST() const; - String toString() const; - Strings toStrings() const; - std::shared_ptr toASTWithNames(const AccessControlManager & manager) const; + + String toString() const; String toStringWithNames(const AccessControlManager & manager) const; Strings toStringsWithNames(const AccessControlManager & manager) const; @@ -48,11 +46,9 @@ struct ExtendedRoleSet void clear(); void add(const UUID & id); void add(const std::vector & ids_); - void add(const boost::container::flat_set & ids_); /// Checks if a specified ID matches this ExtendedRoleSet. bool match(const UUID & id) const; - bool match(const UUID & user_id, const std::vector & enabled_roles) const; bool match(const UUID & user_id, const boost::container::flat_set & enabled_roles) const; /// Returns a list of matching IDs. The function must not be called if `all` == `true`. diff --git a/src/Access/GrantedAccess.cpp b/src/Access/GrantedAccess.cpp new file mode 100644 index 00000000000..2af1e0b44ec --- /dev/null +++ b/src/Access/GrantedAccess.cpp @@ -0,0 +1,22 @@ +#include + + +namespace DB +{ + +GrantedAccess::GrantsAndPartialRevokes GrantedAccess::getGrantsAndPartialRevokes() const +{ + GrantsAndPartialRevokes res; + res.grants_with_grant_option = access_with_grant_option.getGrants(); + AccessRights access_without_gg = access; + access_without_gg.revoke(res.grants_with_grant_option); + auto gr = access_without_gg.getGrantsAndPartialRevokes(); + res.grants = std::move(gr.grants); + res.revokes = std::move(gr.revokes); + AccessRights access_with_grant_options_without_r = access_with_grant_option; + access_with_grant_options_without_r.grant(res.revokes); + res.revokes_grant_option = access_with_grant_options_without_r.getPartialRevokes(); + return res; +} + +} diff --git a/src/Access/GrantedAccess.h b/src/Access/GrantedAccess.h new file mode 100644 index 00000000000..b8f6bdfe8fb --- /dev/null +++ b/src/Access/GrantedAccess.h @@ -0,0 +1,55 @@ +#pragma once + +#include + + +namespace DB +{ +/// Access rights as they are granted to a role or user. +/// Stores both the access rights themselves and the access rights with grant option. +struct GrantedAccess +{ + AccessRights access; + AccessRights access_with_grant_option; + + template + void grant(const Args &... args) + { + access.grant(args...); + } + + template + void grantWithGrantOption(const Args &... args) + { + access.grant(args...); + access_with_grant_option.grant(args...); + } + + template + void revoke(const Args &... args) + { + access.revoke(args...); + access_with_grant_option.revoke(args...); + } + + template + void revokeGrantOption(const Args &... args) + { + access_with_grant_option.revoke(args...); + } + + struct GrantsAndPartialRevokes + { + AccessRightsElements grants; + AccessRightsElements revokes; + AccessRightsElements grants_with_grant_option; + AccessRightsElements revokes_grant_option; + }; + + /// Retrieves the information about grants and partial revokes. + GrantsAndPartialRevokes getGrantsAndPartialRevokes() const; + + friend bool operator ==(const GrantedAccess & left, const GrantedAccess & right) { return (left.access == right.access) && (left.access_with_grant_option == right.access_with_grant_option); } + friend bool operator !=(const GrantedAccess & left, const GrantedAccess & right) { return !(left == right); } +}; +} diff --git a/src/Access/GrantedRoles.cpp b/src/Access/GrantedRoles.cpp new file mode 100644 index 00000000000..4d7007c4db6 --- /dev/null +++ b/src/Access/GrantedRoles.cpp @@ -0,0 +1,64 @@ +#include +#include + + +namespace DB +{ +void GrantedRoles::grant(const UUID & role) +{ + roles.insert(role); +} + +void GrantedRoles::grant(const std::vector & roles_) +{ + for (const UUID & role : roles_) + grant(role); +} + +void GrantedRoles::grantWithAdminOption(const UUID & role) +{ + roles.insert(role); + roles_with_admin_option.insert(role); +} + +void GrantedRoles::grantWithAdminOption(const std::vector & roles_) +{ + for (const UUID & role : roles_) + grantWithAdminOption(role); +} + + +void GrantedRoles::revoke(const UUID & role) +{ + roles.erase(role); + roles_with_admin_option.erase(role); +} + +void GrantedRoles::revoke(const std::vector & roles_) +{ + for (const UUID & role : roles_) + revoke(role); +} + +void GrantedRoles::revokeAdminOption(const UUID & role) +{ + roles_with_admin_option.erase(role); +} + +void GrantedRoles::revokeAdminOption(const std::vector & roles_) +{ + for (const UUID & role : roles_) + revokeAdminOption(role); +} + + +GrantedRoles::Grants GrantedRoles::getGrants() const +{ + Grants res; + res.grants_with_admin_option.insert(res.grants_with_admin_option.end(), roles_with_admin_option.begin(), roles_with_admin_option.end()); + res.grants.reserve(roles.size() - roles_with_admin_option.size()); + boost::range::set_difference(roles, roles_with_admin_option, std::back_inserter(res.grants)); + return res; +} + +} diff --git a/src/Access/GrantedRoles.h b/src/Access/GrantedRoles.h new file mode 100644 index 00000000000..fd091755a80 --- /dev/null +++ b/src/Access/GrantedRoles.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ +/// Roles when they are granted to a role or user. +/// Stores both the roles themselves and the roles with admin option. +struct GrantedRoles +{ + boost::container::flat_set roles; + boost::container::flat_set roles_with_admin_option; + + void grant(const UUID & role); + void grant(const std::vector & roles_); + void grantWithAdminOption(const UUID & role); + void grantWithAdminOption(const std::vector & roles_); + + void revoke(const UUID & role); + void revoke(const std::vector & roles_); + void revokeAdminOption(const UUID & role); + void revokeAdminOption(const std::vector & roles_); + + struct Grants + { + std::vector grants; + std::vector grants_with_admin_option; + }; + + /// Retrieves the information about grants. + Grants getGrants() const; + + friend bool operator ==(const GrantedRoles & left, const GrantedRoles & right) { return (left.roles == right.roles) && (left.roles_with_admin_option == right.roles_with_admin_option); } + friend bool operator !=(const GrantedRoles & left, const GrantedRoles & right) { return !(left == right); } +}; +} diff --git a/src/Access/IAccessEntity.cpp b/src/Access/IAccessEntity.cpp index 5dbc056b71c..5dc566fe456 100644 --- a/src/Access/IAccessEntity.cpp +++ b/src/Access/IAccessEntity.cpp @@ -1,48 +1,12 @@ #include -#include -#include -#include -#include -#include -#include namespace DB { -String IAccessEntity::getTypeName(std::type_index type) -{ - if (type == typeid(User)) - return "User"; - if (type == typeid(Quota)) - return "Quota"; - if (type == typeid(RowPolicy)) - return "Row policy"; - if (type == typeid(Role)) - return "Role"; - if (type == typeid(SettingsProfile)) - return "Settings profile"; - return demangle(type.name()); -} - - -const char * IAccessEntity::getKeyword(std::type_index type) -{ - if (type == typeid(User)) - return "USER"; - if (type == typeid(Quota)) - return "QUOTA"; - if (type == typeid(RowPolicy)) - return "ROW POLICY"; - if (type == typeid(Role)) - return "ROLE"; - if (type == typeid(SettingsProfile)) - return "SETTINGS PROFILE"; - __builtin_unreachable(); -} - bool IAccessEntity::equal(const IAccessEntity & other) const { - return (full_name == other.full_name) && (getType() == other.getType()); + return (name == other.name) && (getType() == other.getType()); } + } diff --git a/src/Access/IAccessEntity.h b/src/Access/IAccessEntity.h index 9214d64aa8c..39a5cefa7d7 100644 --- a/src/Access/IAccessEntity.h +++ b/src/Access/IAccessEntity.h @@ -2,12 +2,24 @@ #include #include +#include +#include #include -#include namespace DB { +namespace ErrorCodes +{ + extern const int UNKNOWN_USER; + extern const int UNKNOWN_ROLE; + extern const int UNKNOWN_ROW_POLICY; + extern const int UNKNOWN_QUOTA; + extern const int THERE_IS_NO_PROFILE; + extern const int LOGICAL_ERROR; +} + + /// Access entity is a set of data which have a name and a type. Access entity control something related to the access control. /// Entities can be stored to a file or another storage, see IAccessStorage. struct IAccessEntity @@ -17,35 +29,120 @@ struct IAccessEntity virtual ~IAccessEntity() = default; virtual std::shared_ptr clone() const = 0; - std::type_index getType() const { return typeid(*this); } - static String getTypeName(std::type_index type); - const String getTypeName() const { return getTypeName(getType()); } - static const char * getKeyword(std::type_index type); - const char * getKeyword() const { return getKeyword(getType()); } + enum class Type + { + USER, + ROLE, + SETTINGS_PROFILE, + ROW_POLICY, + QUOTA, - template - bool isTypeOf() const { return isTypeOf(typeid(EntityType)); } - bool isTypeOf(std::type_index type) const { return type == getType(); } + MAX, + }; - virtual void setName(const String & name_) { full_name = name_; } - virtual String getName() const { return full_name; } - String getFullName() const { return full_name; } + virtual Type getType() const = 0; + + struct TypeInfo + { + const char * const raw_name; + const String name; /// Uppercased with spaces instead of underscores, e.g. "SETTINGS PROFILE". + const String alias; /// Alias of the keyword or empty string, e.g. "PROFILE". + const String name_for_output_with_entity_name; /// Lowercased with spaces instead of underscores, e.g. "settings profile". + const char unique_char; /// Unique character for this type. E.g. 'P' for SETTINGS_PROFILE. + const String list_filename; /// Name of the file containing list of objects of this type, including the file extension ".list". + const int not_found_error_code; + + static const TypeInfo & get(Type type_); + String outputWithEntityName(const String & entity_name) const; + }; + + const TypeInfo & getTypeInfo() const { return TypeInfo::get(getType()); } + String outputTypeAndName() const { return getTypeInfo().outputWithEntityName(getName()); } + + template + bool isTypeOf() const { return isTypeOf(EntityClassT::TYPE); } + bool isTypeOf(Type type) const { return type == getType(); } + + virtual void setName(const String & name_) { name = name_; } + const String & getName() const { return name; } friend bool operator ==(const IAccessEntity & lhs, const IAccessEntity & rhs) { return lhs.equal(rhs); } friend bool operator !=(const IAccessEntity & lhs, const IAccessEntity & rhs) { return !(lhs == rhs); } protected: - String full_name; + String name; virtual bool equal(const IAccessEntity & other) const; /// Helper function to define clone() in the derived classes. - template + template std::shared_ptr cloneImpl() const { - return std::make_shared(typeid_cast(*this)); + return std::make_shared(typeid_cast(*this)); } }; using AccessEntityPtr = std::shared_ptr; + + +inline const IAccessEntity::TypeInfo & IAccessEntity::TypeInfo::get(Type type_) +{ + static constexpr auto make_info = [](const char * raw_name_, char unique_char_, const char * list_filename_, int not_found_error_code_) + { + String init_name = raw_name_; + boost::to_upper(init_name); + boost::replace_all(init_name, "_", " "); + String init_alias; + if (auto underscore_pos = init_name.find_first_of(" "); underscore_pos != String::npos) + init_alias = init_name.substr(underscore_pos + 1); + String init_name_for_output_with_entity_name = init_name; + boost::to_lower(init_name_for_output_with_entity_name); + return TypeInfo{raw_name_, std::move(init_name), std::move(init_alias), std::move(init_name_for_output_with_entity_name), unique_char_, list_filename_, not_found_error_code_}; + }; + + switch (type_) + { + case Type::USER: + { + static const auto info = make_info("USER", 'U', "users.list", ErrorCodes::UNKNOWN_USER); + return info; + } + case Type::ROLE: + { + static const auto info = make_info("ROLE", 'R', "roles.list", ErrorCodes::UNKNOWN_ROLE); + return info; + } + case Type::SETTINGS_PROFILE: + { + static const auto info = make_info("SETTINGS_PROFILE", 'S', "settings_profiles.list", ErrorCodes::THERE_IS_NO_PROFILE); + return info; + } + case Type::ROW_POLICY: + { + static const auto info = make_info("ROW_POLICY", 'P', "row_policies.list", ErrorCodes::UNKNOWN_ROW_POLICY); + return info; + } + case Type::QUOTA: + { + static const auto info = make_info("QUOTA", 'Q', "quotas.list", ErrorCodes::UNKNOWN_QUOTA); + return info; + } + case Type::MAX: break; + } + throw Exception("Unknown type: " + std::to_string(static_cast(type_)), ErrorCodes::LOGICAL_ERROR); +} + +inline String IAccessEntity::TypeInfo::outputWithEntityName(const String & entity_name) const +{ + String msg = name_for_output_with_entity_name; + msg += " "; + msg += backQuote(entity_name); + return msg; +} + +inline String toString(IAccessEntity::Type type) +{ + return IAccessEntity::TypeInfo::get(type).name; +} + } diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 3dfc3e232ba..8e4314ec7c5 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -13,27 +13,44 @@ namespace DB namespace ErrorCodes { extern const int BAD_CAST; - extern const int ACCESS_ENTITY_NOT_FOUND; extern const int ACCESS_ENTITY_ALREADY_EXISTS; + extern const int ACCESS_ENTITY_NOT_FOUND; extern const int ACCESS_STORAGE_READONLY; - extern const int UNKNOWN_USER; - extern const int UNKNOWN_ROLE; } -std::vector IAccessStorage::findAll(std::type_index type) const +namespace +{ + using EntityType = IAccessStorage::EntityType; + using EntityTypeInfo = IAccessStorage::EntityTypeInfo; + + bool isNotFoundErrorCode(int error_code) + { + if (error_code == ErrorCodes::ACCESS_ENTITY_NOT_FOUND) + return true; + + for (auto type : ext::range(EntityType::MAX)) + if (error_code == EntityTypeInfo::get(type).not_found_error_code) + return true; + + return false; + } +} + + +std::vector IAccessStorage::findAll(EntityType type) const { return findAllImpl(type); } -std::optional IAccessStorage::find(std::type_index type, const String & name) const +std::optional IAccessStorage::find(EntityType type, const String & name) const { return findImpl(type, name); } -std::vector IAccessStorage::find(std::type_index type, const Strings & names) const +std::vector IAccessStorage::find(EntityType type, const Strings & names) const { std::vector ids; ids.reserve(names.size()); @@ -47,7 +64,7 @@ std::vector IAccessStorage::find(std::type_index type, const Strings & nam } -UUID IAccessStorage::getID(std::type_index type, const String & name) const +UUID IAccessStorage::getID(EntityType type, const String & name) const { auto id = findImpl(type, name); if (id) @@ -56,7 +73,7 @@ UUID IAccessStorage::getID(std::type_index type, const String & name) const } -std::vector IAccessStorage::getIDs(std::type_index type, const Strings & names) const +std::vector IAccessStorage::getIDs(EntityType type, const Strings & names) const { std::vector ids; ids.reserve(names.size()); @@ -190,6 +207,7 @@ void IAccessStorage::remove(const UUID & id) void IAccessStorage::remove(const std::vector & ids) { String error_message; + std::optional error_code; for (const auto & id : ids) { try @@ -198,13 +216,17 @@ void IAccessStorage::remove(const std::vector & ids) } catch (Exception & e) { - if (e.code() != ErrorCodes::ACCESS_ENTITY_NOT_FOUND) + if (!isNotFoundErrorCode(e.code())) throw; error_message += (error_message.empty() ? "" : ". ") + e.message(); + if (error_code && (*error_code != e.code())) + error_code = ErrorCodes::ACCESS_ENTITY_NOT_FOUND; + else + error_code = e.code(); } } if (!error_message.empty()) - throw Exception(error_message, ErrorCodes::ACCESS_ENTITY_NOT_FOUND); + throw Exception(error_message, *error_code); } @@ -250,6 +272,7 @@ void IAccessStorage::update(const UUID & id, const UpdateFunc & update_func) void IAccessStorage::update(const std::vector & ids, const UpdateFunc & update_func) { String error_message; + std::optional error_code; for (const auto & id : ids) { try @@ -258,13 +281,17 @@ void IAccessStorage::update(const std::vector & ids, const UpdateFunc & up } catch (Exception & e) { - if (e.code() != ErrorCodes::ACCESS_ENTITY_NOT_FOUND) + if (!isNotFoundErrorCode(e.code())) throw; error_message += (error_message.empty() ? "" : ". ") + e.message(); + if (error_code && (*error_code != e.code())) + error_code = ErrorCodes::ACCESS_ENTITY_NOT_FOUND; + else + error_code = e.code(); } } if (!error_message.empty()) - throw Exception(error_message, ErrorCodes::ACCESS_ENTITY_NOT_FOUND); + throw Exception(error_message, *error_code); } @@ -301,7 +328,7 @@ std::vector IAccessStorage::tryUpdate(const std::vector & ids, const } -ext::scope_guard IAccessStorage::subscribeForChanges(std::type_index type, const OnChangedHandler & handler) const +ext::scope_guard IAccessStorage::subscribeForChanges(EntityType type, const OnChangedHandler & handler) const { return subscribeForChangesImpl(type, handler); } @@ -322,7 +349,7 @@ ext::scope_guard IAccessStorage::subscribeForChanges(const std::vector & i } -bool IAccessStorage::hasSubscription(std::type_index type) const +bool IAccessStorage::hasSubscription(EntityType type) const { return hasSubscriptionImpl(type); } @@ -361,79 +388,72 @@ Poco::Logger * IAccessStorage::getLogger() const void IAccessStorage::throwNotFound(const UUID & id) const { - throw Exception("ID {" + toString(id) + "} not found in " + getStorageName(), ErrorCodes::ACCESS_ENTITY_NOT_FOUND); + throw Exception("ID {" + toString(id) + "} not found in [" + getStorageName() + "]", ErrorCodes::ACCESS_ENTITY_NOT_FOUND); } -void IAccessStorage::throwNotFound(std::type_index type, const String & name) const +void IAccessStorage::throwNotFound(EntityType type, const String & name) const { - int error_code; - if (type == typeid(User)) - error_code = ErrorCodes::UNKNOWN_USER; - else if (type == typeid(Role)) - error_code = ErrorCodes::UNKNOWN_ROLE; - else - error_code = ErrorCodes::ACCESS_ENTITY_NOT_FOUND; - - throw Exception(getTypeName(type) + " " + backQuote(name) + " not found in " + getStorageName(), error_code); + int error_code = EntityTypeInfo::get(type).not_found_error_code; + throw Exception("There is no " + outputEntityTypeAndName(type, name) + " in [" + getStorageName() + "]", error_code); } -void IAccessStorage::throwBadCast(const UUID & id, std::type_index type, const String & name, std::type_index required_type) +void IAccessStorage::throwBadCast(const UUID & id, EntityType type, const String & name, EntityType required_type) { throw Exception( - "ID {" + toString(id) + "}: " + getTypeName(type) + backQuote(name) + " expected to be of type " + getTypeName(required_type), + "ID {" + toString(id) + "}: " + outputEntityTypeAndName(type, name) + " expected to be of type " + toString(required_type), ErrorCodes::BAD_CAST); } -void IAccessStorage::throwIDCollisionCannotInsert(const UUID & id, std::type_index type, const String & name, std::type_index existing_type, const String & existing_name) const +void IAccessStorage::throwIDCollisionCannotInsert(const UUID & id, EntityType type, const String & name, EntityType existing_type, const String & existing_name) const { throw Exception( - getTypeName(type) + " " + backQuote(name) + ": cannot insert because the ID {" + toString(id) + "} is already used by " - + getTypeName(existing_type) + " " + backQuote(existing_name) + " in " + getStorageName(), + outputEntityTypeAndName(type, name) + ": cannot insert because the ID {" + toString(id) + "} is already used by " + + outputEntityTypeAndName(existing_type, existing_name) + " in [" + getStorageName() + "]", ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS); } -void IAccessStorage::throwNameCollisionCannotInsert(std::type_index type, const String & name) const +void IAccessStorage::throwNameCollisionCannotInsert(EntityType type, const String & name) const { throw Exception( - getTypeName(type) + " " + backQuote(name) + ": cannot insert because " + getTypeName(type) + " " + backQuote(name) - + " already exists in " + getStorageName(), + outputEntityTypeAndName(type, name) + ": cannot insert because " + outputEntityTypeAndName(type, name) + " already exists in [" + + getStorageName() + "]", ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS); } -void IAccessStorage::throwNameCollisionCannotRename(std::type_index type, const String & old_name, const String & new_name) const +void IAccessStorage::throwNameCollisionCannotRename(EntityType type, const String & old_name, const String & new_name) const { throw Exception( - getTypeName(type) + " " + backQuote(old_name) + ": cannot rename to " + backQuote(new_name) + " because " + getTypeName(type) + " " - + backQuote(new_name) + " already exists in " + getStorageName(), + outputEntityTypeAndName(type, old_name) + ": cannot rename to " + backQuote(new_name) + " because " + + outputEntityTypeAndName(type, new_name) + " already exists in [" + getStorageName() + "]", ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS); } -void IAccessStorage::throwReadonlyCannotInsert(std::type_index type, const String & name) const +void IAccessStorage::throwReadonlyCannotInsert(EntityType type, const String & name) const { throw Exception( - "Cannot insert " + getTypeName(type) + " " + backQuote(name) + " to " + getStorageName() + " because this storage is readonly", + "Cannot insert " + outputEntityTypeAndName(type, name) + " to [" + getStorageName() + "] because this storage is readonly", ErrorCodes::ACCESS_STORAGE_READONLY); } -void IAccessStorage::throwReadonlyCannotUpdate(std::type_index type, const String & name) const +void IAccessStorage::throwReadonlyCannotUpdate(EntityType type, const String & name) const { throw Exception( - "Cannot update " + getTypeName(type) + " " + backQuote(name) + " in " + getStorageName() + " because this storage is readonly", + "Cannot update " + outputEntityTypeAndName(type, name) + " in [" + getStorageName() + "] because this storage is readonly", ErrorCodes::ACCESS_STORAGE_READONLY); } -void IAccessStorage::throwReadonlyCannotRemove(std::type_index type, const String & name) const +void IAccessStorage::throwReadonlyCannotRemove(EntityType type, const String & name) const { throw Exception( - "Cannot remove " + getTypeName(type) + " " + backQuote(name) + " from " + getStorageName() + " because this storage is readonly", + "Cannot remove " + outputEntityTypeAndName(type, name) + " from [" + getStorageName() + "] because this storage is readonly", ErrorCodes::ACCESS_STORAGE_READONLY); } } diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 30a1a6bdc32..081fed87bd2 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -25,50 +25,53 @@ public: /// Returns the name of this storage. const String & getStorageName() const { return storage_name; } - /// Returns the identifiers of all the entities of a specified type contained in the storage. - std::vector findAll(std::type_index type) const; + using EntityType = IAccessEntity::Type; + using EntityTypeInfo = IAccessEntity::TypeInfo; - template - std::vector findAll() const { return findAll(typeid(EntityType)); } + /// Returns the identifiers of all the entities of a specified type contained in the storage. + std::vector findAll(EntityType type) const; + + template + std::vector findAll() const { return findAll(EntityClassT::TYPE); } /// Searchs for an entity with specified type and name. Returns std::nullopt if not found. - std::optional find(std::type_index type, const String & name) const; + std::optional find(EntityType type, const String & name) const; - template - std::optional find(const String & name) const { return find(typeid(EntityType), name); } + template + std::optional find(const String & name) const { return find(EntityClassT::TYPE, name); } - std::vector find(std::type_index type, const Strings & names) const; + std::vector find(EntityType type, const Strings & names) const; - template - std::vector find(const Strings & names) const { return find(typeid(EntityType), names); } + template + std::vector find(const Strings & names) const { return find(EntityClassT::TYPE, names); } /// Searchs for an entity with specified name and type. Throws an exception if not found. - UUID getID(std::type_index type, const String & name) const; + UUID getID(EntityType type, const String & name) const; - template - UUID getID(const String & name) const { return getID(typeid(EntityType), name); } + template + UUID getID(const String & name) const { return getID(EntityClassT::TYPE, name); } - std::vector getIDs(std::type_index type, const Strings & names) const; + std::vector getIDs(EntityType type, const Strings & names) const; - template - std::vector getIDs(const Strings & names) const { return getIDs(typeid(EntityType), names); } + template + std::vector getIDs(const Strings & names) const { return getIDs(EntityClassT::TYPE, names); } /// Returns whether there is an entity with such identifier in the storage. bool exists(const UUID & id) const; /// Reads an entity. Throws an exception if not found. - template - std::shared_ptr read(const UUID & id) const; + template + std::shared_ptr read(const UUID & id) const; - template - std::shared_ptr read(const String & name) const; + template + std::shared_ptr read(const String & name) const; /// Reads an entity. Returns nullptr if not found. - template - std::shared_ptr tryRead(const UUID & id) const; + template + std::shared_ptr tryRead(const UUID & id) const; - template - std::shared_ptr tryRead(const String & name) const; + template + std::shared_ptr tryRead(const String & name) const; /// Reads only name of an entity. String readName(const UUID & id) const; @@ -118,22 +121,22 @@ public: /// Subscribes for all changes. /// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only). - ext::scope_guard subscribeForChanges(std::type_index type, const OnChangedHandler & handler) const; + ext::scope_guard subscribeForChanges(EntityType type, const OnChangedHandler & handler) const; - template - ext::scope_guard subscribeForChanges(OnChangedHandler handler) const { return subscribeForChanges(typeid(EntityType), handler); } + template + ext::scope_guard subscribeForChanges(OnChangedHandler handler) const { return subscribeForChanges(EntityClassT::TYPE, handler); } /// Subscribes for changes of a specific entry. /// Can return nullptr if cannot subscribe (identifier not found) or if it doesn't make sense (the storage is read-only). ext::scope_guard subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const; ext::scope_guard subscribeForChanges(const std::vector & ids, const OnChangedHandler & handler) const; - bool hasSubscription(std::type_index type) const; + bool hasSubscription(EntityType type) const; bool hasSubscription(const UUID & id) const; protected: - virtual std::optional findImpl(std::type_index type, const String & name) const = 0; - virtual std::vector findAllImpl(std::type_index type) const = 0; + virtual std::optional findImpl(EntityType type, const String & name) const = 0; + virtual std::vector findAllImpl(EntityType type) const = 0; virtual bool existsImpl(const UUID & id) const = 0; virtual AccessEntityPtr readImpl(const UUID & id) const = 0; virtual String readNameImpl(const UUID & id) const = 0; @@ -142,23 +145,23 @@ protected: virtual void removeImpl(const UUID & id) = 0; virtual void updateImpl(const UUID & id, const UpdateFunc & update_func) = 0; virtual ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const = 0; - virtual ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const = 0; + virtual ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const = 0; virtual bool hasSubscriptionImpl(const UUID & id) const = 0; - virtual bool hasSubscriptionImpl(std::type_index type) const = 0; + virtual bool hasSubscriptionImpl(EntityType type) const = 0; static UUID generateRandomID(); Poco::Logger * getLogger() const; - static String getTypeName(std::type_index type) { return IAccessEntity::getTypeName(type); } + static String outputEntityTypeAndName(EntityType type, const String & name) { return EntityTypeInfo::get(type).outputWithEntityName(name); } [[noreturn]] void throwNotFound(const UUID & id) const; - [[noreturn]] void throwNotFound(std::type_index type, const String & name) const; - [[noreturn]] static void throwBadCast(const UUID & id, std::type_index type, const String & name, std::type_index required_type); + [[noreturn]] void throwNotFound(EntityType type, const String & name) const; + [[noreturn]] static void throwBadCast(const UUID & id, EntityType type, const String & name, EntityType required_type); [[noreturn]] void throwIDCollisionCannotInsert( - const UUID & id, std::type_index type, const String & name, std::type_index existing_type, const String & existing_name) const; - [[noreturn]] void throwNameCollisionCannotInsert(std::type_index type, const String & name) const; - [[noreturn]] void throwNameCollisionCannotRename(std::type_index type, const String & old_name, const String & new_name) const; - [[noreturn]] void throwReadonlyCannotInsert(std::type_index type, const String & name) const; - [[noreturn]] void throwReadonlyCannotUpdate(std::type_index type, const String & name) const; - [[noreturn]] void throwReadonlyCannotRemove(std::type_index type, const String & name) const; + const UUID & id, EntityType type, const String & name, EntityType existing_type, const String & existing_name) const; + [[noreturn]] void throwNameCollisionCannotInsert(EntityType type, const String & name) const; + [[noreturn]] void throwNameCollisionCannotRename(EntityType type, const String & old_name, const String & new_name) const; + [[noreturn]] void throwReadonlyCannotInsert(EntityType type, const String & name) const; + [[noreturn]] void throwReadonlyCannotUpdate(EntityType type, const String & name) const; + [[noreturn]] void throwReadonlyCannotRemove(EntityType type, const String & name) const; using Notification = std::tuple; using Notifications = std::vector; @@ -172,38 +175,43 @@ private: }; -template -std::shared_ptr IAccessStorage::read(const UUID & id) const +template +std::shared_ptr IAccessStorage::read(const UUID & id) const { auto entity = readImpl(id); - auto ptr = typeid_cast>(entity); - if (ptr) - return ptr; - throwBadCast(id, entity->getType(), entity->getFullName(), typeid(EntityType)); + if constexpr (std::is_same_v) + return entity; + else + { + auto ptr = typeid_cast>(entity); + if (ptr) + return ptr; + throwBadCast(id, entity->getType(), entity->getName(), EntityClassT::TYPE); + } } -template -std::shared_ptr IAccessStorage::read(const String & name) const +template +std::shared_ptr IAccessStorage::read(const String & name) const { - return read(getID(name)); + return read(getID(name)); } -template -std::shared_ptr IAccessStorage::tryRead(const UUID & id) const +template +std::shared_ptr IAccessStorage::tryRead(const UUID & id) const { auto entity = tryReadBase(id); if (!entity) return nullptr; - return typeid_cast>(entity); + return typeid_cast>(entity); } -template -std::shared_ptr IAccessStorage::tryRead(const String & name) const +template +std::shared_ptr IAccessStorage::tryRead(const String & name) const { - auto id = find(name); - return id ? tryRead(*id) : nullptr; + auto id = find(name); + return id ? tryRead(*id) : nullptr; } } diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index ce668bcd8b3..720b82796b7 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -1,6 +1,8 @@ #include #include -#include +#include +#include +#include namespace DB @@ -11,11 +13,12 @@ MemoryAccessStorage::MemoryAccessStorage(const String & storage_name_) } -std::optional MemoryAccessStorage::findImpl(std::type_index type, const String & name) const +std::optional MemoryAccessStorage::findImpl(EntityType type, const String & name) const { std::lock_guard lock{mutex}; - auto it = names.find({name, type}); - if (it == names.end()) + const auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + auto it = entries_by_name.find(name); + if (it == entries_by_name.end()) return {}; Entry & entry = *(it->second); @@ -23,12 +26,12 @@ std::optional MemoryAccessStorage::findImpl(std::type_index type, const St } -std::vector MemoryAccessStorage::findAllImpl(std::type_index type) const +std::vector MemoryAccessStorage::findAllImpl(EntityType type) const { std::lock_guard lock{mutex}; std::vector result; - result.reserve(entries.size()); - for (const auto & [id, entry] : entries) + result.reserve(entries_by_id.size()); + for (const auto & [id, entry] : entries_by_id) if (entry.entity->isTypeOf(type)) result.emplace_back(id); return result; @@ -38,15 +41,15 @@ std::vector MemoryAccessStorage::findAllImpl(std::type_index type) const bool MemoryAccessStorage::existsImpl(const UUID & id) const { std::lock_guard lock{mutex}; - return entries.count(id); + return entries_by_id.count(id); } AccessEntityPtr MemoryAccessStorage::readImpl(const UUID & id) const { std::lock_guard lock{mutex}; - auto it = entries.find(id); - if (it == entries.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) throwNotFound(id); const Entry & entry = it->second; return entry.entity; @@ -55,7 +58,7 @@ AccessEntityPtr MemoryAccessStorage::readImpl(const UUID & id) const String MemoryAccessStorage::readNameImpl(const UUID & id) const { - return readImpl(id)->getFullName(); + return readImpl(id)->getName(); } @@ -73,19 +76,20 @@ UUID MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool re void MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, Notifications & notifications) { - const String & name = new_entity->getFullName(); - std::type_index type = new_entity->getType(); + const String & name = new_entity->getName(); + EntityType type = new_entity->getType(); /// Check that we can insert. - auto it = entries.find(id); - if (it != entries.end()) + auto it = entries_by_id.find(id); + if (it != entries_by_id.end()) { const auto & existing_entry = it->second; - throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getFullName()); + throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName()); } - auto it2 = names.find({name, type}); - if (it2 != names.end()) + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + auto it2 = entries_by_name.find(name); + if (it2 != entries_by_name.end()) { const auto & existing_entry = *(it2->second); if (replace_if_exists) @@ -95,10 +99,10 @@ void MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & } /// Do insertion. - auto & entry = entries[id]; + auto & entry = entries_by_id[id]; entry.id = id; entry.entity = new_entity; - names[std::pair{name, type}] = &entry; + entries_by_name[name] = &entry; prepareNotifications(entry, false, notifications); } @@ -115,19 +119,20 @@ void MemoryAccessStorage::removeImpl(const UUID & id) void MemoryAccessStorage::removeNoLock(const UUID & id, Notifications & notifications) { - auto it = entries.find(id); - if (it == entries.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) throwNotFound(id); Entry & entry = it->second; - const String & name = entry.entity->getFullName(); - std::type_index type = entry.entity->getType(); + const String & name = entry.entity->getName(); + EntityType type = entry.entity->getType(); prepareNotifications(entry, true, notifications); /// Do removing. - names.erase({name, type}); - entries.erase(it); + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + entries_by_name.erase(name); + entries_by_id.erase(it); } @@ -143,27 +148,30 @@ void MemoryAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_ void MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, Notifications & notifications) { - auto it = entries.find(id); - if (it == entries.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) throwNotFound(id); Entry & entry = it->second; auto old_entity = entry.entity; auto new_entity = update_func(old_entity); + if (!new_entity->isTypeOf(old_entity->getType())) + throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType()); + if (*new_entity == *old_entity) return; entry.entity = new_entity; - if (new_entity->getFullName() != old_entity->getFullName()) + if (new_entity->getName() != old_entity->getName()) { - auto it2 = names.find({new_entity->getFullName(), new_entity->getType()}); - if (it2 != names.end()) - throwNameCollisionCannotRename(old_entity->getType(), old_entity->getFullName(), new_entity->getFullName()); + auto & entries_by_name = entries_by_name_and_type[static_cast(old_entity->getType())]; + auto it2 = entries_by_name.find(new_entity->getName()); + if (it2 != entries_by_name.end()) + throwNameCollisionCannotRename(old_entity->getType(), old_entity->getName(), new_entity->getName()); - names.erase({old_entity->getFullName(), old_entity->getType()}); - names[std::pair{new_entity->getFullName(), new_entity->getType()}] = &entry; + entries_by_name[new_entity->getName()] = &entry; } prepareNotifications(entry, false, notifications); @@ -192,43 +200,47 @@ void MemoryAccessStorage::setAll(const std::vector> & all_entities, Notifications & notifications) { - /// Get list of the currently used IDs. Later we will remove those of them which are not used anymore. - std::unordered_set not_used_ids; - for (const auto & id_and_entry : entries) - not_used_ids.emplace(id_and_entry.first); + boost::container::flat_set not_used_ids; + std::vector conflicting_ids; - /// Remove conflicting entities. + /// Get the list of currently used IDs. Later we will remove those of them which are not used anymore. + for (const auto & id : entries_by_id | boost::adaptors::map_keys) + not_used_ids.emplace(id); + + /// Get the list of conflicting IDs and update the list of currently used ones. for (const auto & [id, entity] : all_entities) { - auto it = entries.find(id); - if (it != entries.end()) + auto it = entries_by_id.find(id); + if (it != entries_by_id.end()) { not_used_ids.erase(id); /// ID is used. + Entry & entry = it->second; if (entry.entity->getType() != entity->getType()) - { - removeNoLock(id, notifications); - continue; - } + conflicting_ids.emplace_back(id); /// Conflict: same ID, different type. } - auto it2 = names.find({entity->getFullName(), entity->getType()}); - if (it2 != names.end()) + + const auto & entries_by_name = entries_by_name_and_type[static_cast(entity->getType())]; + auto it2 = entries_by_name.find(entity->getName()); + if (it2 != entries_by_name.end()) { Entry & entry = *(it2->second); if (entry.id != id) - removeNoLock(id, notifications); + conflicting_ids.emplace_back(entry.id); /// Conflict: same name and type, different ID. } } - /// Remove entities which are not used anymore. - for (const auto & id : not_used_ids) + /// Remove entities which are not used anymore and which are in conflict with new entities. + boost::container::flat_set ids_to_remove = std::move(not_used_ids); + boost::range::copy(conflicting_ids, std::inserter(ids_to_remove, ids_to_remove.end())); + for (const auto & id : ids_to_remove) removeNoLock(id, notifications); /// Insert or update entities. for (const auto & [id, entity] : all_entities) { - auto it = entries.find(id); - if (it != entries.end()) + auto it = entries_by_id.find(id); + if (it != entries_by_id.end()) { if (*(it->second.entity) != *entity) { @@ -244,24 +256,27 @@ void MemoryAccessStorage::setAllNoLock(const std::vectorgetType()); - for (auto it = range.first; it != range.second; ++it) - notifications.push_back({it->second, entry.id, remove ? nullptr : entry.entity}); + for (const auto & handler : handlers_by_type[static_cast(entry.entity->getType())]) + notifications.push_back({handler, entry.id, entity}); } -ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const +ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const { std::lock_guard lock{mutex}; - auto handler_it = handlers_by_type.emplace(type, handler); + auto & handlers = handlers_by_type[static_cast(type)]; + handlers.push_back(handler); + auto handler_it = std::prev(handlers.end()); - return [this, handler_it] + return [this, type, handler_it] { std::lock_guard lock2{mutex}; - handlers_by_type.erase(handler_it); + auto & handlers2 = handlers_by_type[static_cast(type)]; + handlers2.erase(handler_it); }; } @@ -269,8 +284,8 @@ ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(std::type_index ty ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const { std::lock_guard lock{mutex}; - auto it = entries.find(id); - if (it == entries.end()) + auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) return {}; const Entry & entry = it->second; auto handler_it = entry.handlers_by_id.insert(entry.handlers_by_id.end(), handler); @@ -278,8 +293,8 @@ ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(const UUID & id, c return [this, id, handler_it] { std::lock_guard lock2{mutex}; - auto it2 = entries.find(id); - if (it2 != entries.end()) + auto it2 = entries_by_id.find(id); + if (it2 != entries_by_id.end()) { const Entry & entry2 = it2->second; entry2.handlers_by_id.erase(handler_it); @@ -291,8 +306,8 @@ ext::scope_guard MemoryAccessStorage::subscribeForChangesImpl(const UUID & id, c bool MemoryAccessStorage::hasSubscriptionImpl(const UUID & id) const { std::lock_guard lock{mutex}; - auto it = entries.find(id); - if (it != entries.end()) + auto it = entries_by_id.find(id); + if (it != entries_by_id.end()) { const Entry & entry = it->second; return !entry.handlers_by_id.empty(); @@ -301,10 +316,10 @@ bool MemoryAccessStorage::hasSubscriptionImpl(const UUID & id) const } -bool MemoryAccessStorage::hasSubscriptionImpl(std::type_index type) const +bool MemoryAccessStorage::hasSubscriptionImpl(EntityType type) const { std::lock_guard lock{mutex}; - auto range = handlers_by_type.equal_range(type); - return range.first != range.second; + const auto & handlers = handlers_by_type[static_cast(type)]; + return !handlers.empty(); } } diff --git a/src/Access/MemoryAccessStorage.h b/src/Access/MemoryAccessStorage.h index b93c2868d34..a2fdd0d0044 100644 --- a/src/Access/MemoryAccessStorage.h +++ b/src/Access/MemoryAccessStorage.h @@ -20,8 +20,8 @@ public: void setAll(const std::vector> & all_entities); private: - std::optional findImpl(std::type_index type, const String & name) const override; - std::vector findAllImpl(std::type_index type) const override; + std::optional findImpl(EntityType type, const String & name) const override; + std::vector findAllImpl(EntityType type) const override; bool existsImpl(const UUID & id) const override; AccessEntityPtr readImpl(const UUID & id) const override; String readNameImpl(const UUID & id) const override; @@ -30,9 +30,9 @@ private: void removeImpl(const UUID & id) override; void updateImpl(const UUID & id, const UpdateFunc & update_func) override; ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; - ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override; + ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; bool hasSubscriptionImpl(const UUID & id) const override; - bool hasSubscriptionImpl(std::type_index type) const override; + bool hasSubscriptionImpl(EntityType type) const override; struct Entry { @@ -47,18 +47,9 @@ private: void setAllNoLock(const std::vector> & all_entities, Notifications & notifications); void prepareNotifications(const Entry & entry, bool remove, Notifications & notifications) const; - using NameTypePair = std::pair; - struct Hash - { - size_t operator()(const NameTypePair & key) const - { - return std::hash{}(key.first) - std::hash{}(key.second); - } - }; - mutable std::mutex mutex; - std::unordered_map entries; /// We want to search entries both by ID and by the pair of name and type. - std::unordered_map names; /// and by the pair of name and type. - mutable std::unordered_multimap handlers_by_type; + std::unordered_map entries_by_id; /// We want to search entries both by ID and by the pair of name and type. + std::unordered_map entries_by_name_and_type[static_cast(EntityType::MAX)]; + mutable std::list handlers_by_type[static_cast(EntityType::MAX)]; }; } diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 740fe1dac04..0dd1f142f31 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -38,7 +38,7 @@ MultipleAccessStorage::MultipleAccessStorage( } -std::vector MultipleAccessStorage::findMultiple(std::type_index type, const String & name) const +std::vector MultipleAccessStorage::findMultiple(EntityType type, const String & name) const { std::vector ids; for (const auto & nested_storage : nested_storages) @@ -55,7 +55,7 @@ std::vector MultipleAccessStorage::findMultiple(std::type_index type, cons } -std::optional MultipleAccessStorage::findImpl(std::type_index type, const String & name) const +std::optional MultipleAccessStorage::findImpl(EntityType type, const String & name) const { auto ids = findMultiple(type, name); if (ids.empty()) @@ -72,13 +72,13 @@ std::optional MultipleAccessStorage::findImpl(std::type_index type, const } throw Exception( - "Found " + getTypeName(type) + " " + backQuote(name) + " in " + std::to_string(ids.size()) - + " storages: " + joinStorageNames(storages_with_duplicates), + "Found " + outputEntityTypeAndName(type, name) + " in " + std::to_string(ids.size()) + + " storages [" + joinStorageNames(storages_with_duplicates) + "]", ErrorCodes::ACCESS_ENTITY_FOUND_DUPLICATES); } -std::vector MultipleAccessStorage::findAllImpl(std::type_index type) const +std::vector MultipleAccessStorage::findAllImpl(EntityType type) const { std::vector all_ids; for (const auto & nested_storage : nested_storages) @@ -180,11 +180,7 @@ UUID MultipleAccessStorage::insertImpl(const AccessEntityPtr & entity, bool repl } if (!nested_storage_for_insertion) - { - throw Exception( - "Not found a storage to insert " + entity->getTypeName() + backQuote(entity->getName()), - ErrorCodes::ACCESS_STORAGE_FOR_INSERTION_NOT_FOUND); - } + throw Exception("Not found a storage to insert " + entity->outputTypeAndName(), ErrorCodes::ACCESS_STORAGE_FOR_INSERTION_NOT_FOUND); auto id = replace_if_exists ? nested_storage_for_insertion->insertOrReplace(entity) : nested_storage_for_insertion->insert(entity); std::lock_guard lock{ids_cache_mutex}; @@ -214,7 +210,7 @@ ext::scope_guard MultipleAccessStorage::subscribeForChangesImpl(const UUID & id, } -ext::scope_guard MultipleAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const +ext::scope_guard MultipleAccessStorage::subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const { ext::scope_guard subscriptions; for (const auto & nested_storage : nested_storages) @@ -234,7 +230,7 @@ bool MultipleAccessStorage::hasSubscriptionImpl(const UUID & id) const } -bool MultipleAccessStorage::hasSubscriptionImpl(std::type_index type) const +bool MultipleAccessStorage::hasSubscriptionImpl(EntityType type) const { for (const auto & nested_storage : nested_storages) { diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index 898d55d30de..ec8c8f2a101 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -15,7 +15,7 @@ public: MultipleAccessStorage(std::vector> nested_storages_); - std::vector findMultiple(std::type_index type, const String & name) const; + std::vector findMultiple(EntityType type, const String & name) const; template std::vector findMultiple(const String & name) const { return findMultiple(EntityType::TYPE, name); } @@ -29,8 +29,8 @@ public: const Storage & getStorageByIndex(size_t i) const { return *(nested_storages[i]); } protected: - std::optional findImpl(std::type_index type, const String & name) const override; - std::vector findAllImpl(std::type_index type) const override; + std::optional findImpl(EntityType type, const String & name) const override; + std::vector findAllImpl(EntityType type) const override; bool existsImpl(const UUID & id) const override; AccessEntityPtr readImpl(const UUID & id) const override; String readNameImpl(const UUID &id) const override; @@ -39,9 +39,9 @@ protected: void removeImpl(const UUID & id) override; void updateImpl(const UUID & id, const UpdateFunc & update_func) override; ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; - ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override; + ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; bool hasSubscriptionImpl(const UUID & id) const override; - bool hasSubscriptionImpl(std::type_index type) const override; + bool hasSubscriptionImpl(EntityType type) const override; private: std::vector> nested_storages; diff --git a/src/Access/Quota.cpp b/src/Access/Quota.cpp index e3a9e11eb10..dc855599999 100644 --- a/src/Access/Quota.cpp +++ b/src/Access/Quota.cpp @@ -1,16 +1,9 @@ #include #include -#include namespace DB { -Quota::Limits::Limits() -{ - boost::range::fill(max, 0); -} - - bool operator ==(const Quota::Limits & lhs, const Quota::Limits & rhs) { return boost::range::equal(lhs.max, rhs.max) && (lhs.duration == rhs.duration) @@ -26,20 +19,5 @@ bool Quota::equal(const IAccessEntity & other) const return (all_limits == other_quota.all_limits) && (key_type == other_quota.key_type) && (to_roles == other_quota.to_roles); } - -const char * Quota::resourceTypeToColumnName(ResourceType resource_type) -{ - switch (resource_type) - { - case Quota::QUERIES: return "queries"; - case Quota::ERRORS: return "errors"; - case Quota::RESULT_ROWS: return "result_rows"; - case Quota::RESULT_BYTES: return "result_bytes"; - case Quota::READ_ROWS: return "read_rows"; - case Quota::READ_BYTES: return "read_bytes"; - case Quota::EXECUTION_TIME: return "execution_time"; - } - __builtin_unreachable(); -} } diff --git a/src/Access/Quota.h b/src/Access/Quota.h index 714d582e98f..25b56756dc1 100644 --- a/src/Access/Quota.h +++ b/src/Access/Quota.h @@ -2,11 +2,18 @@ #include #include +#include #include namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + /** Quota for resources consumption for specific interval. * Used to limit resource usage by user. * Quota is applied "softly" - could be slightly exceed, because it is checked usually only on each block of processed data. @@ -17,6 +24,8 @@ namespace DB */ struct Quota : public IAccessEntity { + using ResourceAmount = UInt64; + enum ResourceType { QUERIES, /// Number of queries. @@ -26,22 +35,32 @@ struct Quota : public IAccessEntity READ_ROWS, /// Number of rows read from tables. READ_BYTES, /// Number of bytes read from tables. EXECUTION_TIME, /// Total amount of query execution time in nanoseconds. - }; - static constexpr size_t MAX_RESOURCE_TYPE = 7; - using ResourceAmount = UInt64; - static constexpr ResourceAmount UNLIMITED = 0; /// 0 means unlimited. + MAX_RESOURCE_TYPE + }; + + struct ResourceTypeInfo + { + const char * const raw_name; + const String name; /// Lowercased with underscores, e.g. "result_rows". + const String keyword; /// Uppercased with spaces, e.g. "RESULT ROWS". + const bool output_as_float = false; + const UInt64 output_denominator = 1; + String amountToString(ResourceAmount amount) const; + ResourceAmount amountFromString(const String & str) const; + String outputWithAmount(ResourceAmount amount) const; + static const ResourceTypeInfo & get(ResourceType type); + }; /// Amount of resources available to consume for each duration. struct Limits { - ResourceAmount max[MAX_RESOURCE_TYPE]; + std::optional max[MAX_RESOURCE_TYPE]; std::chrono::seconds duration = std::chrono::seconds::zero(); /// Intervals can be randomized (to avoid DoS if intervals for many users end at one time). bool randomize_interval = false; - Limits(); friend bool operator ==(const Limits & lhs, const Limits & rhs); friend bool operator !=(const Limits & lhs, const Limits & rhs) { return !(lhs == rhs); } }; @@ -58,8 +77,17 @@ struct Quota : public IAccessEntity CLIENT_KEY, /// Client should explicitly supply a key to use. CLIENT_KEY_OR_USER_NAME, /// Same as CLIENT_KEY, but use USER_NAME if the client doesn't supply a key. CLIENT_KEY_OR_IP_ADDRESS, /// Same as CLIENT_KEY, but use IP_ADDRESS if the client doesn't supply a key. + + MAX }; - static constexpr size_t MAX_KEY_TYPE = 6; + + struct KeyTypeInfo + { + const char * const raw_name; + const String name; /// Lowercased with spaces, e.g. "client key". + static const KeyTypeInfo & get(KeyType type); + }; + KeyType key_type = KeyType::NONE; /// Which roles or users should use this quota. @@ -67,73 +95,146 @@ struct Quota : public IAccessEntity bool equal(const IAccessEntity & other) const override; std::shared_ptr clone() const override { return cloneImpl(); } - - static const char * getNameOfResourceType(ResourceType resource_type); - static const char * resourceTypeToKeyword(ResourceType resource_type); - static const char * resourceTypeToColumnName(ResourceType resource_type); - static const char * getNameOfKeyType(KeyType key_type); - static double executionTimeToSeconds(ResourceAmount ns); - static ResourceAmount secondsToExecutionTime(double s); + static constexpr const Type TYPE = Type::QUOTA; + Type getType() const override { return TYPE; } }; -inline const char * Quota::getNameOfResourceType(ResourceType resource_type) +inline String Quota::ResourceTypeInfo::amountToString(ResourceAmount amount) const { - switch (resource_type) + if (!(amount % output_denominator)) + return std::to_string(amount / output_denominator); + else + return boost::lexical_cast(static_cast(amount) / output_denominator); +} + +inline Quota::ResourceAmount Quota::ResourceTypeInfo::amountFromString(const String & str) const +{ + if (output_denominator == 1) + return static_cast(std::strtoul(str.c_str(), nullptr, 10)); + else + return static_cast(std::strtod(str.c_str(), nullptr) * output_denominator); +} + +inline String Quota::ResourceTypeInfo::outputWithAmount(ResourceAmount amount) const +{ + String res = name; + res += " = "; + res += amountToString(amount); + return res; +} + +inline String toString(Quota::ResourceType type) +{ + return Quota::ResourceTypeInfo::get(type).raw_name; +} + +inline const Quota::ResourceTypeInfo & Quota::ResourceTypeInfo::get(ResourceType type) +{ + static constexpr auto make_info = [](const char * raw_name_, UInt64 output_denominator_) { - case Quota::QUERIES: return "queries"; - case Quota::ERRORS: return "errors"; - case Quota::RESULT_ROWS: return "result rows"; - case Quota::RESULT_BYTES: return "result bytes"; - case Quota::READ_ROWS: return "read rows"; - case Quota::READ_BYTES: return "read bytes"; - case Quota::EXECUTION_TIME: return "execution time"; - } - __builtin_unreachable(); -} + String init_name = raw_name_; + boost::to_lower(init_name); + String init_keyword = raw_name_; + boost::replace_all(init_keyword, "_", " "); + bool init_output_as_float = (output_denominator_ != 1); + return ResourceTypeInfo{raw_name_, std::move(init_name), std::move(init_keyword), init_output_as_float, output_denominator_}; + }; - -inline const char * Quota::resourceTypeToKeyword(ResourceType resource_type) -{ - switch (resource_type) + switch (type) { - case Quota::QUERIES: return "QUERIES"; - case Quota::ERRORS: return "ERRORS"; - case Quota::RESULT_ROWS: return "RESULT ROWS"; - case Quota::RESULT_BYTES: return "RESULT BYTES"; - case Quota::READ_ROWS: return "READ ROWS"; - case Quota::READ_BYTES: return "READ BYTES"; - case Quota::EXECUTION_TIME: return "EXECUTION TIME"; + case Quota::QUERIES: + { + static const auto info = make_info("QUERIES", 1); + return info; + } + case Quota::ERRORS: + { + static const auto info = make_info("ERRORS", 1); + return info; + } + case Quota::RESULT_ROWS: + { + static const auto info = make_info("RESULT_ROWS", 1); + return info; + } + case Quota::RESULT_BYTES: + { + static const auto info = make_info("RESULT_BYTES", 1); + return info; + } + case Quota::READ_ROWS: + { + static const auto info = make_info("READ_ROWS", 1); + return info; + } + case Quota::READ_BYTES: + { + static const auto info = make_info("READ_BYTES", 1); + return info; + } + case Quota::EXECUTION_TIME: + { + static const auto info = make_info("EXECUTION_TIME", 1000000000 /* execution_time is stored in nanoseconds */); + return info; + } + case Quota::MAX_RESOURCE_TYPE: break; } - __builtin_unreachable(); + throw Exception("Unexpected resource type: " + std::to_string(static_cast(type)), ErrorCodes::LOGICAL_ERROR); } -inline const char * Quota::getNameOfKeyType(KeyType key_type) +inline String toString(Quota::KeyType type) { - switch (key_type) + return Quota::KeyTypeInfo::get(type).raw_name; +} + +inline const Quota::KeyTypeInfo & Quota::KeyTypeInfo::get(KeyType type) +{ + static constexpr auto make_info = [](const char * raw_name_) { - case KeyType::NONE: return "none"; - case KeyType::USER_NAME: return "user name"; - case KeyType::IP_ADDRESS: return "ip address"; - case KeyType::CLIENT_KEY: return "client key"; - case KeyType::CLIENT_KEY_OR_USER_NAME: return "client key or user name"; - case KeyType::CLIENT_KEY_OR_IP_ADDRESS: return "client key or ip address"; + String init_name = raw_name_; + boost::to_lower(init_name); + boost::replace_all(init_name, "_", " "); + return KeyTypeInfo{raw_name_, std::move(init_name)}; + }; + + switch (type) + { + case KeyType::NONE: + { + static const auto info = make_info("NONE"); + return info; + } + case KeyType::USER_NAME: + { + static const auto info = make_info("USER_NAME"); + return info; + } + case KeyType::IP_ADDRESS: + { + static const auto info = make_info("IP_ADDRESS"); + return info; + } + case KeyType::CLIENT_KEY: + { + static const auto info = make_info("CLIENT_KEY"); + return info; + } + case KeyType::CLIENT_KEY_OR_USER_NAME: + { + static const auto info = make_info("CLIENT_KEY_OR_USER_NAME"); + return info; + } + case KeyType::CLIENT_KEY_OR_IP_ADDRESS: + { + static const auto info = make_info("CLIENT_KEY_OR_IP_ADDRESS"); + return info; + } + case KeyType::MAX: break; } - __builtin_unreachable(); + throw Exception("Unexpected quota key type: " + std::to_string(static_cast(type)), ErrorCodes::LOGICAL_ERROR); } - -inline double Quota::executionTimeToSeconds(ResourceAmount ns) -{ - return std::chrono::duration_cast>(std::chrono::nanoseconds{ns}).count(); -} - -inline Quota::ResourceAmount Quota::secondsToExecutionTime(double s) -{ - return std::chrono::duration_cast(std::chrono::duration(s)).count(); -} - - using QuotaPtr = std::shared_ptr; } diff --git a/src/Access/QuotaCache.cpp b/src/Access/QuotaCache.cpp index dca844bf746..d3ebca9529c 100644 --- a/src/Access/QuotaCache.cpp +++ b/src/Access/QuotaCache.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include @@ -17,6 +17,7 @@ namespace DB namespace ErrorCodes { extern const int QUOTA_REQUIRES_CLIENT_KEY; + extern const int LOGICAL_ERROR; } @@ -72,8 +73,9 @@ String QuotaCache::QuotaInfo::calculateKey(const EnabledQuota & enabled) const return params.client_key; return params.client_address.toString(); } + case KeyType::MAX: break; } - __builtin_unreachable(); + throw Exception("Unexpected quota key type: " + std::to_string(static_cast(quota->key_type)), ErrorCodes::LOGICAL_ERROR); } @@ -101,7 +103,7 @@ boost::shared_ptr QuotaCache::QuotaInfo::rebuildI new_intervals->quota_key = key; auto & intervals = new_intervals->intervals; intervals.reserve(quota->all_limits.size()); - static constexpr size_t MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; + static constexpr auto MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; for (const auto & limits : quota->all_limits) { intervals.emplace_back(); @@ -114,7 +116,8 @@ boost::shared_ptr QuotaCache::QuotaInfo::rebuildI interval.end_of_interval = end_of_interval.time_since_epoch(); for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) { - interval.max[resource_type] = limits.max[resource_type]; + if (limits.max[resource_type]) + interval.max[resource_type] = *limits.max[resource_type]; interval.used[resource_type] = 0; } } @@ -167,12 +170,7 @@ QuotaCache::QuotaCache(const AccessControlManager & access_control_manager_) QuotaCache::~QuotaCache() = default; -std::shared_ptr QuotaCache::getEnabledQuota( - const UUID & user_id, - const String & user_name, - const std::vector & enabled_roles, - const Poco::Net::IPAddress & client_address, - const String & client_key) +std::shared_ptr QuotaCache::getEnabledQuota(const UUID & user_id, const String & user_name, const boost::container::flat_set & enabled_roles, const Poco::Net::IPAddress & client_address, const String & client_key) { std::lock_guard lock{mutex}; ensureAllQuotasRead(); @@ -290,16 +288,20 @@ void QuotaCache::chooseQuotaToConsumeFor(EnabledQuota & enabled) } -std::vector QuotaCache::getUsageInfo() const +std::vector QuotaCache::getAllQuotasUsage() const { std::lock_guard lock{mutex}; - std::vector all_infos; + std::vector all_usage; auto current_time = std::chrono::system_clock::now(); for (const auto & info : all_quotas | boost::adaptors::map_values) { for (const auto & intervals : info.key_to_intervals | boost::adaptors::map_values) - all_infos.push_back(intervals->getUsageInfo(current_time)); + { + auto usage = intervals->getUsage(current_time); + if (usage) + all_usage.push_back(std::move(usage).value()); + } } - return all_infos; + return all_usage; } } diff --git a/src/Access/QuotaCache.h b/src/Access/QuotaCache.h index 8399c5f73eb..6e794f0bbd2 100644 --- a/src/Access/QuotaCache.h +++ b/src/Access/QuotaCache.h @@ -20,14 +20,8 @@ public: QuotaCache(const AccessControlManager & access_control_manager_); ~QuotaCache(); - std::shared_ptr getEnabledQuota( - const UUID & user_id, - const String & user_name, - const std::vector & enabled_roles, - const Poco::Net::IPAddress & address, - const String & client_key); - - std::vector getUsageInfo() const; + std::shared_ptr getEnabledQuota(const UUID & user_id, const String & user_name, const boost::container::flat_set & enabled_roles, const Poco::Net::IPAddress & address, const String & client_key); + std::vector getAllQuotasUsage() const; private: using Interval = EnabledQuota::Interval; diff --git a/src/Access/QuotaUsage.cpp b/src/Access/QuotaUsage.cpp new file mode 100644 index 00000000000..cf03c4f478a --- /dev/null +++ b/src/Access/QuotaUsage.cpp @@ -0,0 +1,16 @@ +#include +#include + + +namespace DB +{ +QuotaUsage::QuotaUsage() : quota_id(UUID(UInt128(0))) +{ +} + + +QuotaUsage::Interval::Interval() +{ + boost::range::fill(used, 0); +} +} diff --git a/src/Access/QuotaUsageInfo.h b/src/Access/QuotaUsage.h similarity index 78% rename from src/Access/QuotaUsageInfo.h rename to src/Access/QuotaUsage.h index 94e16fb9f69..9e53c0cf7d7 100644 --- a/src/Access/QuotaUsageInfo.h +++ b/src/Access/QuotaUsage.h @@ -7,16 +7,16 @@ namespace DB { /// The information about a quota consumption. -struct QuotaUsageInfo +struct QuotaUsage { using ResourceType = Quota::ResourceType; using ResourceAmount = Quota::ResourceAmount; - static constexpr size_t MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; + static constexpr auto MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; struct Interval { ResourceAmount used[MAX_RESOURCE_TYPE]; - ResourceAmount max[MAX_RESOURCE_TYPE]; + std::optional max[MAX_RESOURCE_TYPE]; std::chrono::seconds duration = std::chrono::seconds::zero(); bool randomize_interval = false; std::chrono::system_clock::time_point end_of_interval; @@ -27,6 +27,6 @@ struct QuotaUsageInfo UUID quota_id; String quota_name; String quota_key; - QuotaUsageInfo(); + QuotaUsage(); }; } diff --git a/src/Access/QuotaUsageInfo.cpp b/src/Access/QuotaUsageInfo.cpp deleted file mode 100644 index bcdf2b50062..00000000000 --- a/src/Access/QuotaUsageInfo.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include -#include - - -namespace DB -{ -QuotaUsageInfo::QuotaUsageInfo() : quota_id(UUID(UInt128(0))) -{ -} - - -QuotaUsageInfo::Interval::Interval() -{ - boost::range::fill(used, 0); - boost::range::fill(max, 0); -} -} diff --git a/src/Access/Role.cpp b/src/Access/Role.cpp index f20ef9b9bfa..3df562ad1f0 100644 --- a/src/Access/Role.cpp +++ b/src/Access/Role.cpp @@ -9,9 +9,7 @@ bool Role::equal(const IAccessEntity & other) const if (!IAccessEntity::equal(other)) return false; const auto & other_role = typeid_cast(other); - return (access == other_role.access) && (access_with_grant_option == other_role.access_with_grant_option) - && (granted_roles == other_role.granted_roles) && (granted_roles_with_admin_option == other_role.granted_roles_with_admin_option) - && (settings == other_role.settings); + return (access == other_role.access) && (granted_roles == other_role.granted_roles) && (settings == other_role.settings); } } diff --git a/src/Access/Role.h b/src/Access/Role.h index 04330ba85f5..9acb97bdfbd 100644 --- a/src/Access/Role.h +++ b/src/Access/Role.h @@ -1,10 +1,9 @@ #pragma once #include -#include +#include +#include #include -#include -#include namespace DB @@ -12,14 +11,14 @@ namespace DB struct Role : public IAccessEntity { - AccessRights access; - AccessRights access_with_grant_option; - boost::container::flat_set granted_roles; - boost::container::flat_set granted_roles_with_admin_option; + GrantedAccess access; + GrantedRoles granted_roles; SettingsProfileElements settings; bool equal(const IAccessEntity & other) const override; std::shared_ptr clone() const override { return cloneImpl(); } + static constexpr const Type TYPE = Type::ROLE; + Type getType() const override { return TYPE; } }; using RolePtr = std::shared_ptr; diff --git a/src/Access/RoleCache.cpp b/src/Access/RoleCache.cpp index 0263b793017..ca8065145f3 100644 --- a/src/Access/RoleCache.cpp +++ b/src/Access/RoleCache.cpp @@ -2,68 +2,56 @@ #include #include #include -#include +#include namespace DB { namespace { - struct CollectedRoleInfo - { - RolePtr role; - bool is_current_role = false; - bool with_admin_option = false; - }; - - - void collectRoles(boost::container::flat_map & collected_roles, + void collectRoles(EnabledRolesInfo & roles_info, + boost::container::flat_set & skip_ids, const std::function & get_role_function, const UUID & role_id, bool is_current_role, bool with_admin_option) { - auto it = collected_roles.find(role_id); - if (it != collected_roles.end()) + if (roles_info.enabled_roles.count(role_id)) { - it->second.is_current_role |= is_current_role; - it->second.with_admin_option |= with_admin_option; + if (is_current_role) + roles_info.current_roles.emplace(role_id); + if (with_admin_option) + roles_info.enabled_roles_with_admin_option.emplace(role_id); return; } + if (skip_ids.count(role_id)) + return; + auto role = get_role_function(role_id); - collected_roles[role_id] = CollectedRoleInfo{role, is_current_role, with_admin_option}; if (!role) - return; - - for (const auto & granted_role : role->granted_roles) - collectRoles(collected_roles, get_role_function, granted_role, false, false); - - for (const auto & granted_role : role->granted_roles_with_admin_option) - collectRoles(collected_roles, get_role_function, granted_role, false, true); - } - - - std::shared_ptr collectInfoForRoles(const boost::container::flat_map & roles) - { - auto new_info = std::make_shared(); - for (const auto & [role_id, collect_info] : roles) { - const auto & role = collect_info.role; - if (!role) - continue; - if (collect_info.is_current_role) - new_info->current_roles.emplace_back(role_id); - new_info->enabled_roles.emplace_back(role_id); - if (collect_info.with_admin_option) - new_info->enabled_roles_with_admin_option.emplace_back(role_id); - new_info->names_of_roles[role_id] = role->getName(); - new_info->access.merge(role->access); - new_info->access_with_grant_option.merge(role->access_with_grant_option); - new_info->settings_from_enabled_roles.merge(role->settings); + skip_ids.emplace(role_id); + return; } - return new_info; + + roles_info.enabled_roles.emplace(role_id); + if (is_current_role) + roles_info.current_roles.emplace(role_id); + if (with_admin_option) + roles_info.enabled_roles_with_admin_option.emplace(role_id); + + roles_info.names_of_roles[role_id] = role->getName(); + roles_info.access.merge(role->access.access); + roles_info.access_with_grant_option.merge(role->access.access_with_grant_option); + roles_info.settings_from_enabled_roles.merge(role->settings); + + for (const auto & granted_role : role->granted_roles.roles) + collectRoles(roles_info, skip_ids, get_role_function, granted_role, false, false); + + for (const auto & granted_role : role->granted_roles.roles_with_admin_option) + collectRoles(roles_info, skip_ids, get_role_function, granted_role, false, true); } } @@ -75,8 +63,8 @@ RoleCache::RoleCache(const AccessControlManager & manager_) RoleCache::~RoleCache() = default; -std::shared_ptr RoleCache::getEnabledRoles( - const std::vector & roles, const std::vector & roles_with_admin_option) +std::shared_ptr +RoleCache::getEnabledRoles(const boost::container::flat_set & roles, const boost::container::flat_set & roles_with_admin_option) { std::lock_guard lock{mutex}; @@ -93,13 +81,13 @@ std::shared_ptr RoleCache::getEnabledRoles( } auto res = std::shared_ptr(new EnabledRoles(params)); - collectRolesInfoFor(*res); + collectEnabledRoles(*res); enabled_roles.emplace(std::move(params), res); return res; } -void RoleCache::collectRolesInfo() +void RoleCache::collectEnabledRoles() { /// `mutex` is already locked. @@ -110,28 +98,29 @@ void RoleCache::collectRolesInfo() i = enabled_roles.erase(i); else { - collectRolesInfoFor(*elem); + collectEnabledRoles(*elem); ++i; } } } -void RoleCache::collectRolesInfoFor(EnabledRoles & enabled) +void RoleCache::collectEnabledRoles(EnabledRoles & enabled) { /// `mutex` is already locked. - /// Collect roles in use. That includes the current roles, the roles granted to the current roles, and so on. - boost::container::flat_map collected_roles; + /// Collect enabled roles. That includes the current roles, the roles granted to the current roles, and so on. + auto new_info = std::make_shared(); + boost::container::flat_set skip_ids; auto get_role_function = [this](const UUID & id) { return getRole(id); }; for (const auto & current_role : enabled.params.current_roles) - collectRoles(collected_roles, get_role_function, current_role, true, false); + collectRoles(*new_info, skip_ids, get_role_function, current_role, true, false); for (const auto & current_role : enabled.params.current_roles_with_admin_option) - collectRoles(collected_roles, get_role_function, current_role, true, true); + collectRoles(*new_info, skip_ids, get_role_function, current_role, true, true); /// Collect data from the collected roles. - enabled.setRolesInfo(collectInfoForRoles(collected_roles)); + enabled.setRolesInfo(new_info); } @@ -174,7 +163,7 @@ void RoleCache::roleChanged(const UUID & role_id, const RolePtr & changed_role) return; role_from_cache->first = changed_role; cache.update(role_id, role_from_cache); - collectRolesInfo(); + collectEnabledRoles(); } @@ -182,7 +171,7 @@ void RoleCache::roleRemoved(const UUID & role_id) { std::lock_guard lock{mutex}; cache.remove(role_id); - collectRolesInfo(); + collectEnabledRoles(); } } diff --git a/src/Access/RoleCache.h b/src/Access/RoleCache.h index 69f4cb2ebe8..f1ef783b5c4 100644 --- a/src/Access/RoleCache.h +++ b/src/Access/RoleCache.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -18,11 +19,12 @@ public: RoleCache(const AccessControlManager & manager_); ~RoleCache(); - std::shared_ptr getEnabledRoles(const std::vector & current_roles, const std::vector & current_roles_with_admin_option); + std::shared_ptr getEnabledRoles( + const boost::container::flat_set & current_roles, const boost::container::flat_set & current_roles_with_admin_option); private: - void collectRolesInfo(); - void collectRolesInfoFor(EnabledRoles & enabled); + void collectEnabledRoles(); + void collectEnabledRoles(EnabledRoles & enabled); RolePtr getRole(const UUID & role_id); void roleChanged(const UUID & role_id, const RolePtr & changed_role); void roleRemoved(const UUID & role_id); diff --git a/src/Access/RowPolicy.cpp b/src/Access/RowPolicy.cpp index 65b9451a453..4249f351eae 100644 --- a/src/Access/RowPolicy.cpp +++ b/src/Access/RowPolicy.cpp @@ -1,72 +1,67 @@ #include -#include #include #include namespace DB { -namespace +namespace ErrorCodes { - void generateFullNameImpl(const String & database_, const String & table_name_, const String & policy_name_, String & full_name_) + extern const int NOT_IMPLEMENTED; +} + + +String RowPolicy::NameParts::getName() const +{ + String name; + name.reserve(database.length() + table_name.length() + short_name.length() + 6); + name += backQuoteIfNeed(short_name); + name += " ON "; + if (!name.empty()) { - full_name_.clear(); - full_name_.reserve(database_.length() + table_name_.length() + policy_name_.length() + 6); - full_name_ += backQuoteIfNeed(policy_name_); - full_name_ += " ON "; - if (!database_.empty()) - { - full_name_ += backQuoteIfNeed(database_); - full_name_ += '.'; - } - full_name_ += backQuoteIfNeed(table_name_); + name += backQuoteIfNeed(database); + name += '.'; } + name += backQuoteIfNeed(table_name); + return name; } -String RowPolicy::FullNameParts::getFullName() const +void RowPolicy::setDatabase(const String & database) { - String full_name; - generateFullNameImpl(database, table_name, policy_name, full_name); - return full_name; + name_parts.database = database; + IAccessEntity::setName(name_parts.getName()); } - -String RowPolicy::FullNameParts::getFullName(const Context & context) const +void RowPolicy::setTableName(const String & table_name) { - String full_name; - generateFullNameImpl(database.empty() ? context.getCurrentDatabase() : database, table_name, policy_name, full_name); - return full_name; + name_parts.table_name = table_name; + IAccessEntity::setName(name_parts.getName()); } - -void RowPolicy::setDatabase(const String & database_) +void RowPolicy::setShortName(const String & short_name) { - database = database_; - generateFullNameImpl(database, table_name, policy_name, full_name); + name_parts.short_name = short_name; + IAccessEntity::setName(name_parts.getName()); } - -void RowPolicy::setTableName(const String & table_name_) +void RowPolicy::setNameParts(const String & short_name, const String & database, const String & table_name) { - table_name = table_name_; - generateFullNameImpl(database, table_name, policy_name, full_name); + name_parts.short_name = short_name; + name_parts.database = database; + name_parts.table_name = table_name; + IAccessEntity::setName(name_parts.getName()); } - -void RowPolicy::setName(const String & policy_name_) +void RowPolicy::setNameParts(const NameParts & name_parts_) { - policy_name = policy_name_; - generateFullNameImpl(database, table_name, policy_name, full_name); + name_parts = name_parts_; + IAccessEntity::setName(name_parts.getName()); } - -void RowPolicy::setFullName(const String & database_, const String & table_name_, const String & policy_name_) +void RowPolicy::setName(const String &) { - database = database_; - table_name = table_name_; - policy_name = policy_name_; - generateFullNameImpl(database, table_name, policy_name, full_name); + throw Exception("RowPolicy::setName() is not implemented", ErrorCodes::NOT_IMPLEMENTED); } @@ -75,37 +70,8 @@ bool RowPolicy::equal(const IAccessEntity & other) const if (!IAccessEntity::equal(other)) return false; const auto & other_policy = typeid_cast(other); - return (database == other_policy.database) && (table_name == other_policy.table_name) && (policy_name == other_policy.policy_name) - && boost::range::equal(conditions, other_policy.conditions) && restrictive == other_policy.restrictive - && (to_roles == other_policy.to_roles); -} - - -const char * RowPolicy::conditionTypeToString(ConditionType index) -{ - switch (index) - { - case SELECT_FILTER: return "SELECT_FILTER"; - case INSERT_CHECK: return "INSERT_CHECK"; - case UPDATE_FILTER: return "UPDATE_FILTER"; - case UPDATE_CHECK: return "UPDATE_CHECK"; - case DELETE_FILTER: return "DELETE_FILTER"; - } - __builtin_unreachable(); -} - - -const char * RowPolicy::conditionTypeToColumnName(ConditionType index) -{ - switch (index) - { - case SELECT_FILTER: return "select_filter"; - case INSERT_CHECK: return "insert_check"; - case UPDATE_FILTER: return "update_filter"; - case UPDATE_CHECK: return "update_check"; - case DELETE_FILTER: return "delete_filter"; - } - __builtin_unreachable(); + return (name_parts == other_policy.name_parts) && boost::range::equal(conditions, other_policy.conditions) + && restrictive == other_policy.restrictive && (to_roles == other_policy.to_roles); } } diff --git a/src/Access/RowPolicy.h b/src/Access/RowPolicy.h index 08219edb46b..7febf5991fb 100644 --- a/src/Access/RowPolicy.h +++ b/src/Access/RowPolicy.h @@ -2,35 +2,44 @@ #include #include +#include namespace DB { -class Context; +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} /** Represents a row level security policy for a table. */ struct RowPolicy : public IAccessEntity { - void setDatabase(const String & database_); - void setTableName(const String & table_name_); - void setName(const String & policy_name_) override; - void setFullName(const String & database_, const String & table_name_, const String & policy_name_); - - String getDatabase() const { return database; } - String getTableName() const { return table_name; } - String getName() const override { return policy_name; } - - struct FullNameParts + struct NameParts { + String short_name; String database; String table_name; - String policy_name; - String getFullName() const; - String getFullName(const Context & context) const; + + String getName() const; + auto toTuple() const { return std::tie(short_name, database, table_name); } + friend bool operator ==(const NameParts & left, const NameParts & right) { return left.toTuple() == right.toTuple(); } + friend bool operator !=(const NameParts & left, const NameParts & right) { return left.toTuple() != right.toTuple(); } }; + void setShortName(const String & short_name); + void setDatabase(const String & database); + void setTableName(const String & table_name); + void setNameParts(const String & short_name, const String & database, const String & table_name); + void setNameParts(const NameParts & name_parts); + + const String & getDatabase() const { return name_parts.database; } + const String & getTableName() const { return name_parts.table_name; } + const String & getShortName() const { return name_parts.short_name; } + const NameParts & getNameParts() const { return name_parts; } + /// Filter is a SQL conditional expression used to figure out which rows should be visible /// for user or available for modification. If the expression returns NULL or false for some rows /// those rows are silently suppressed. @@ -40,16 +49,27 @@ struct RowPolicy : public IAccessEntity enum ConditionType { SELECT_FILTER, + +#if 0 /// Row-level security for INSERT, UPDATE, DELETE is not implemented yet. INSERT_CHECK, UPDATE_FILTER, UPDATE_CHECK, DELETE_FILTER, - }; - static constexpr size_t MAX_CONDITION_TYPE = 5; - static const char * conditionTypeToString(ConditionType index); - static const char * conditionTypeToColumnName(ConditionType index); +#endif - String conditions[MAX_CONDITION_TYPE]; + MAX_CONDITION_TYPE + }; + + struct ConditionTypeInfo + { + const char * const raw_name; + const String name; /// Lowercased with underscores, e.g. "select_filter". + const String command; /// Uppercased without last word, e.g. "SELECT". + const bool is_check; /// E.g. false for SELECT_FILTER. + static const ConditionTypeInfo & get(ConditionType type); + }; + + std::array conditions; /// Sets that the policy is permissive. /// A row is only accessible if at least one of the permissive policies passes, @@ -65,16 +85,72 @@ struct RowPolicy : public IAccessEntity bool equal(const IAccessEntity & other) const override; std::shared_ptr clone() const override { return cloneImpl(); } + static constexpr const Type TYPE = Type::ROW_POLICY; + Type getType() const override { return TYPE; } /// Which roles or users should use this row policy. ExtendedRoleSet to_roles; private: - String database; - String table_name; - String policy_name; + void setName(const String & name_) override; + + NameParts name_parts; bool restrictive = false; }; using RowPolicyPtr = std::shared_ptr; + + +inline const RowPolicy::ConditionTypeInfo & RowPolicy::ConditionTypeInfo::get(ConditionType type_) +{ + static constexpr auto make_info = [](const char * raw_name_) + { + String init_name = raw_name_; + boost::to_lower(init_name); + size_t underscore_pos = init_name.find('_'); + String init_command = init_name.substr(0, underscore_pos); + boost::to_upper(init_command); + bool init_is_check = (std::string_view{init_name}.substr(underscore_pos + 1) == "check"); + return ConditionTypeInfo{raw_name_, std::move(init_name), std::move(init_command), init_is_check}; + }; + + switch (type_) + { + case SELECT_FILTER: + { + static const ConditionTypeInfo info = make_info("SELECT_FILTER"); + return info; + } +#if 0 /// Row-level security for INSERT, UPDATE, DELETE is not implemented yet. + case INSERT_CHECK: + { + static const ConditionTypeInfo info = make_info("INSERT_CHECK"); + return info; + } + case UPDATE_FILTER: + { + static const ConditionTypeInfo info = make_info("UPDATE_FILTER"); + return info; + } + case UPDATE_CHECK: + { + static const ConditionTypeInfo info = make_info("UPDATE_CHECK"); + return info; + } + case DELETE_FILTER: + { + static const ConditionTypeInfo info = make_info("DELETE_FILTER"); + return info; + } +#endif + case MAX_CONDITION_TYPE: break; + } + throw Exception("Unknown type: " + std::to_string(static_cast(type_)), ErrorCodes::LOGICAL_ERROR); +} + +inline String toString(RowPolicy::ConditionType type) +{ + return RowPolicy::ConditionTypeInfo::get(type).raw_name; +} + } diff --git a/src/Access/RowPolicyCache.cpp b/src/Access/RowPolicyCache.cpp index 0c9fea69dc4..243f9c85d70 100644 --- a/src/Access/RowPolicyCache.cpp +++ b/src/Access/RowPolicyCache.cpp @@ -16,7 +16,7 @@ namespace DB namespace { using ConditionType = RowPolicy::ConditionType; - constexpr size_t MAX_CONDITION_TYPE = RowPolicy::MAX_CONDITION_TYPE; + constexpr auto MAX_CONDITION_TYPE = RowPolicy::MAX_CONDITION_TYPE; /// Accumulates conditions from multiple row policies and joins them using the AND logical operation. @@ -57,8 +57,9 @@ void RowPolicyCache::PolicyInfo::setPolicy(const RowPolicyPtr & policy_) { policy = policy_; roles = &policy->to_roles; + database_and_table_name = std::make_shared>(policy->getDatabase(), policy->getTableName()); - for (auto type : ext::range_with_static_cast(0, MAX_CONDITION_TYPE)) + for (auto type : ext::range(0, MAX_CONDITION_TYPE)) { parsed_conditions[type] = nullptr; const String & condition = policy->conditions[type]; @@ -84,8 +85,8 @@ void RowPolicyCache::PolicyInfo::setPolicy(const RowPolicyPtr & policy_) { tryLogCurrentException( &Poco::Logger::get("RowPolicy"), - String("Could not parse the condition ") + RowPolicy::conditionTypeToString(type) + " of row policy " - + backQuote(policy->getFullName())); + String("Could not parse the condition ") + toString(type) + " of row policy " + + backQuote(policy->getName())); } } } @@ -99,7 +100,7 @@ RowPolicyCache::RowPolicyCache(const AccessControlManager & access_control_manag RowPolicyCache::~RowPolicyCache() = default; -std::shared_ptr RowPolicyCache::getEnabledRowPolicies(const UUID & user_id, const std::vector & enabled_roles) +std::shared_ptr RowPolicyCache::getEnabledRowPolicies(const UUID & user_id, const boost::container::flat_set & enabled_roles) { std::lock_guard lock{mutex}; ensureAllRowPoliciesRead(); @@ -196,43 +197,45 @@ void RowPolicyCache::mixConditions() void RowPolicyCache::mixConditionsFor(EnabledRowPolicies & enabled) { /// `mutex` is already locked. - struct Mixers - { - ConditionsMixer mixers[MAX_CONDITION_TYPE]; - std::vector policy_ids; - }; + using MapOfMixedConditions = EnabledRowPolicies::MapOfMixedConditions; - using DatabaseAndTableName = EnabledRowPolicies::DatabaseAndTableName; - using DatabaseAndTableNameRef = EnabledRowPolicies::DatabaseAndTableNameRef; + using MixedConditionKey = EnabledRowPolicies::MixedConditionKey; using Hash = EnabledRowPolicies::Hash; - std::unordered_map map_of_mixers; + struct MixerWithNames + { + ConditionsMixer mixer; + std::shared_ptr> database_and_table_name; + }; + + std::unordered_map map_of_mixers; for (const auto & [policy_id, info] : all_policies) { const auto & policy = *info.policy; - auto & mixers = map_of_mixers[std::pair{policy.getDatabase(), policy.getTableName()}]; - if (info.roles->match(enabled.params.user_id, enabled.params.enabled_roles)) + bool match = info.roles->match(enabled.params.user_id, enabled.params.enabled_roles); + MixedConditionKey key; + key.database = info.database_and_table_name->first; + key.table_name = info.database_and_table_name->second; + for (auto type : ext::range(0, MAX_CONDITION_TYPE)) { - mixers.policy_ids.push_back(policy_id); - for (auto type : ext::range(0, MAX_CONDITION_TYPE)) - if (info.parsed_conditions[type]) - mixers.mixers[type].add(info.parsed_conditions[type], policy.isRestrictive()); + if (info.parsed_conditions[type]) + { + key.condition_type = type; + auto & mixer = map_of_mixers[key]; + mixer.database_and_table_name = info.database_and_table_name; + if (match) + mixer.mixer.add(info.parsed_conditions[type], policy.isRestrictive()); + } } } auto map_of_mixed_conditions = boost::make_shared(); - for (auto & [database_and_table_name, mixers] : map_of_mixers) + for (auto & [key, mixer] : map_of_mixers) { - auto database_and_table_name_keeper = std::make_unique(); - database_and_table_name_keeper->first = database_and_table_name.first; - database_and_table_name_keeper->second = database_and_table_name.second; - auto & mixed_conditions = (*map_of_mixed_conditions)[DatabaseAndTableNameRef{database_and_table_name_keeper->first, - database_and_table_name_keeper->second}]; - mixed_conditions.database_and_table_name_keeper = std::move(database_and_table_name_keeper); - mixed_conditions.policy_ids = std::move(mixers.policy_ids); - for (auto type : ext::range(0, MAX_CONDITION_TYPE)) - mixed_conditions.mixed_conditions[type] = std::move(mixers.mixers[type]).getResult(); + auto & mixed_condition = (*map_of_mixed_conditions)[key]; + mixed_condition.database_and_table_name = mixer.database_and_table_name; + mixed_condition.ast = std::move(mixer.mixer).getResult(); } enabled.map_of_mixed_conditions.store(map_of_mixed_conditions); diff --git a/src/Access/RowPolicyCache.h b/src/Access/RowPolicyCache.h index d0ec74b9ab8..139949ae815 100644 --- a/src/Access/RowPolicyCache.h +++ b/src/Access/RowPolicyCache.h @@ -18,11 +18,9 @@ public: RowPolicyCache(const AccessControlManager & access_control_manager_); ~RowPolicyCache(); - std::shared_ptr getEnabledRowPolicies(const UUID & user_id, const std::vector & enabled_roles); + std::shared_ptr getEnabledRowPolicies(const UUID & user_id, const boost::container::flat_set & enabled_roles); private: - using ParsedConditions = EnabledRowPolicies::ParsedConditions; - struct PolicyInfo { PolicyInfo(const RowPolicyPtr & policy_) { setPolicy(policy_); } @@ -30,7 +28,8 @@ private: RowPolicyPtr policy; const ExtendedRoleSet * roles = nullptr; - ParsedConditions parsed_conditions; + std::shared_ptr> database_and_table_name; + ASTPtr parsed_conditions[RowPolicy::MAX_CONDITION_TYPE]; }; void ensureAllRowPoliciesRead(); diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index 21ca2c8ab26..11dbd016e64 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -30,16 +30,23 @@ void SettingsConstraints::clear() } -void SettingsConstraints::setMinValue(const StringRef & name, const Field & min_value) +void SettingsConstraints::setMinValue(const StringRef & setting_name, const Field & min_value) +{ + setMinValue(Settings::findIndexStrict(setting_name), min_value); +} + +void SettingsConstraints::setMinValue(size_t setting_index, const Field & min_value) { - size_t setting_index = Settings::findIndexStrict(name); getConstraintRef(setting_index).min_value = Settings::valueToCorrespondingType(setting_index, min_value); } - -Field SettingsConstraints::getMinValue(const StringRef & name) const +Field SettingsConstraints::getMinValue(const StringRef & setting_name) const +{ + return getMinValue(Settings::findIndexStrict(setting_name)); +} + +Field SettingsConstraints::getMinValue(size_t setting_index) const { - size_t setting_index = Settings::findIndexStrict(name); const auto * ptr = tryGetConstraint(setting_index); if (ptr) return ptr->min_value; @@ -50,14 +57,21 @@ Field SettingsConstraints::getMinValue(const StringRef & name) const void SettingsConstraints::setMaxValue(const StringRef & name, const Field & max_value) { - size_t setting_index = Settings::findIndexStrict(name); + setMaxValue(Settings::findIndexStrict(name), max_value); +} + +void SettingsConstraints::setMaxValue(size_t setting_index, const Field & max_value) +{ getConstraintRef(setting_index).max_value = Settings::valueToCorrespondingType(setting_index, max_value); } - -Field SettingsConstraints::getMaxValue(const StringRef & name) const +Field SettingsConstraints::getMaxValue(const StringRef & setting_name) const +{ + return getMaxValue(Settings::findIndexStrict(setting_name)); +} + +Field SettingsConstraints::getMaxValue(size_t setting_index) const { - size_t setting_index = Settings::findIndexStrict(name); const auto * ptr = tryGetConstraint(setting_index); if (ptr) return ptr->max_value; @@ -66,16 +80,23 @@ Field SettingsConstraints::getMaxValue(const StringRef & name) const } -void SettingsConstraints::setReadOnly(const StringRef & name, bool read_only) +void SettingsConstraints::setReadOnly(const StringRef & setting_name, bool read_only) +{ + setReadOnly(Settings::findIndexStrict(setting_name), read_only); +} + +void SettingsConstraints::setReadOnly(size_t setting_index, bool read_only) { - size_t setting_index = Settings::findIndexStrict(name); getConstraintRef(setting_index).read_only = read_only; } - -bool SettingsConstraints::isReadOnly(const StringRef & name) const +bool SettingsConstraints::isReadOnly(const StringRef & setting_name) const +{ + return isReadOnly(Settings::findIndexStrict(setting_name)); +} + +bool SettingsConstraints::isReadOnly(size_t setting_index) const { - size_t setting_index = Settings::findIndexStrict(name); const auto * ptr = tryGetConstraint(setting_index); if (ptr) return ptr->read_only; @@ -83,20 +104,26 @@ bool SettingsConstraints::isReadOnly(const StringRef & name) const return false; } - -void SettingsConstraints::set(const StringRef & name, const Field & min_value, const Field & max_value, bool read_only) +void SettingsConstraints::set(const StringRef & setting_name, const Field & min_value, const Field & max_value, bool read_only) +{ + set(Settings::findIndexStrict(setting_name), min_value, max_value, read_only); +} + +void SettingsConstraints::set(size_t setting_index, const Field & min_value, const Field & max_value, bool read_only) { - size_t setting_index = Settings::findIndexStrict(name); auto & ref = getConstraintRef(setting_index); ref.min_value = min_value; ref.max_value = max_value; ref.read_only = read_only; } - -void SettingsConstraints::get(const StringRef & name, Field & min_value, Field & max_value, bool & read_only) const +void SettingsConstraints::get(const StringRef & setting_name, Field & min_value, Field & max_value, bool & read_only) const +{ + get(Settings::findIndexStrict(setting_name), min_value, max_value, read_only); +} + +void SettingsConstraints::get(size_t setting_index, Field & min_value, Field & max_value, bool & read_only) const { - size_t setting_index = Settings::findIndexStrict(name); const auto * ptr = tryGetConstraint(setting_index); if (ptr) { @@ -112,7 +139,6 @@ void SettingsConstraints::get(const StringRef & name, Field & min_value, Field & } } - void SettingsConstraints::merge(const SettingsConstraints & other) { for (const auto & [setting_index, other_constraint] : other.constraints_by_index) diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index 074dc66d123..65537250957 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -1,7 +1,10 @@ #pragma once +#include #include -#include +#include +#include + namespace Poco { @@ -60,17 +63,25 @@ public: void clear(); bool empty() const { return constraints_by_index.empty(); } - void setMinValue(const StringRef & name, const Field & min_value); - Field getMinValue(const StringRef & name) const; + void setMinValue(const StringRef & setting_name, const Field & min_value); + void setMinValue(size_t setting_index, const Field & min_value); + Field getMinValue(const StringRef & setting_name) const; + Field getMinValue(size_t setting_index) const; - void setMaxValue(const StringRef & name, const Field & max_value); - Field getMaxValue(const StringRef & name) const; + void setMaxValue(const StringRef & setting_name, const Field & max_value); + void setMaxValue(size_t setting_index, const Field & max_value); + Field getMaxValue(const StringRef & setting_name) const; + Field getMaxValue(size_t setting_index) const; - void setReadOnly(const StringRef & name, bool read_only); - bool isReadOnly(const StringRef & name) const; + void setReadOnly(const StringRef & setting_name, bool read_only); + void setReadOnly(size_t setting_index, bool read_only); + bool isReadOnly(const StringRef & setting_name) const; + bool isReadOnly(size_t setting_index) const; - void set(const StringRef & name, const Field & min_value, const Field & max_value, bool read_only); - void get(const StringRef & name, Field & min_value, Field & max_value, bool & read_only) const; + void set(const StringRef & setting_name, const Field & min_value, const Field & max_value, bool read_only); + void set(size_t setting_index, const Field & min_value, const Field & max_value, bool read_only); + void get(const StringRef & setting_name, Field & min_value, Field & max_value, bool & read_only) const; + void get(size_t setting_index, Field & min_value, Field & max_value, bool & read_only) const; void merge(const SettingsConstraints & other); diff --git a/src/Access/SettingsProfile.cpp b/src/Access/SettingsProfile.cpp index c2f868502c0..64fb91eb66b 100644 --- a/src/Access/SettingsProfile.cpp +++ b/src/Access/SettingsProfile.cpp @@ -3,6 +3,7 @@ namespace DB { + bool SettingsProfile::equal(const IAccessEntity & other) const { if (!IAccessEntity::equal(other)) @@ -10,4 +11,5 @@ bool SettingsProfile::equal(const IAccessEntity & other) const const auto & other_profile = typeid_cast(other); return (elements == other_profile.elements) && (to_roles == other_profile.to_roles); } + } diff --git a/src/Access/SettingsProfile.h b/src/Access/SettingsProfile.h index b73b45d57cf..9589b5b3eb5 100644 --- a/src/Access/SettingsProfile.h +++ b/src/Access/SettingsProfile.h @@ -18,6 +18,8 @@ struct SettingsProfile : public IAccessEntity bool equal(const IAccessEntity & other) const override; std::shared_ptr clone() const override { return cloneImpl(); } + static constexpr const Type TYPE = Type::SETTINGS_PROFILE; + Type getType() const override { return TYPE; } }; using SettingsProfilePtr = std::shared_ptr; diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index b052f8b5e75..d4f6ff5d0f2 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -33,21 +33,20 @@ void SettingsProfileElement::init(const ASTSettingsProfileElement & ast, const A if (!ast.parent_profile.empty()) parent_profile = name_to_id(ast.parent_profile); - if (!ast.name.empty()) + if (!ast.setting_name.empty()) { - name = ast.name; + setting_index = Settings::findIndexStrict(ast.setting_name); value = ast.value; min_value = ast.min_value; max_value = ast.max_value; readonly = ast.readonly; - size_t index = Settings::findIndexStrict(name); if (!value.isNull()) - value = Settings::valueToCorrespondingType(index, value); + value = Settings::valueToCorrespondingType(setting_index, value); if (!min_value.isNull()) - min_value = Settings::valueToCorrespondingType(index, min_value); + min_value = Settings::valueToCorrespondingType(setting_index, min_value); if (!max_value.isNull()) - max_value = Settings::valueToCorrespondingType(index, max_value); + max_value = Settings::valueToCorrespondingType(setting_index, max_value); } } @@ -60,7 +59,9 @@ std::shared_ptr SettingsProfileElement::toAST() const if (parent_profile) ast->parent_profile = ::DB::toString(*parent_profile); - ast->name = name; + if (setting_index != static_cast(-1)) + ast->setting_name = Settings::getName(setting_index).toString(); + ast->value = value; ast->min_value = min_value; ast->max_value = max_value; @@ -81,7 +82,9 @@ std::shared_ptr SettingsProfileElement::toASTWithName ast->parent_profile = *parent_profile_name; } - ast->name = name; + if (setting_index != static_cast(-1)) + ast->setting_name = Settings::getName(setting_index).toString(); + ast->value = value; ast->min_value = min_value; ast->max_value = max_value; @@ -132,8 +135,8 @@ Settings SettingsProfileElements::toSettings() const Settings res; for (const auto & elem : *this) { - if (!elem.name.empty() && !elem.value.isNull()) - res.set(elem.name, elem.value); + if ((elem.setting_index != static_cast(-1)) && !elem.value.isNull()) + res.set(elem.setting_index, elem.value); } return res; } @@ -143,8 +146,8 @@ SettingsChanges SettingsProfileElements::toSettingsChanges() const SettingsChanges res; for (const auto & elem : *this) { - if (!elem.name.empty() && !elem.value.isNull()) - res.push_back({elem.name, elem.value}); + if ((elem.setting_index != static_cast(-1)) && !elem.value.isNull()) + res.push_back({Settings::getName(elem.setting_index).toString(), elem.value}); } return res; } @@ -154,14 +157,14 @@ SettingsConstraints SettingsProfileElements::toSettingsConstraints() const SettingsConstraints res; for (const auto & elem : *this) { - if (!elem.name.empty()) + if (elem.setting_index != static_cast(-1)) { if (!elem.min_value.isNull()) - res.setMinValue(elem.name, elem.min_value); + res.setMinValue(elem.setting_index, elem.min_value); if (!elem.max_value.isNull()) - res.setMaxValue(elem.name, elem.max_value); + res.setMaxValue(elem.setting_index, elem.max_value); if (elem.readonly) - res.setReadOnly(elem.name, *elem.readonly); + res.setReadOnly(elem.setting_index, *elem.readonly); } } return res; diff --git a/src/Access/SettingsProfileElement.h b/src/Access/SettingsProfileElement.h index abcac2567c8..f64317344b8 100644 --- a/src/Access/SettingsProfileElement.h +++ b/src/Access/SettingsProfileElement.h @@ -20,13 +20,13 @@ class AccessControlManager; struct SettingsProfileElement { std::optional parent_profile; - String name; + size_t setting_index = static_cast(-1); Field value; Field min_value; Field max_value; std::optional readonly; - auto toTuple() const { return std::tie(parent_profile, name, value, min_value, max_value, readonly); } + auto toTuple() const { return std::tie(parent_profile, setting_index, value, min_value, max_value, readonly); } friend bool operator==(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() == rhs.toTuple(); } friend bool operator!=(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return !(lhs == rhs); } friend bool operator <(const SettingsProfileElement & lhs, const SettingsProfileElement & rhs) { return lhs.toTuple() < rhs.toTuple(); } diff --git a/src/Access/SettingsProfilesCache.cpp b/src/Access/SettingsProfilesCache.cpp index f283715e129..95074ff7d0b 100644 --- a/src/Access/SettingsProfilesCache.cpp +++ b/src/Access/SettingsProfilesCache.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -149,32 +150,34 @@ void SettingsProfilesCache::mergeSettingsAndConstraintsFor(EnabledSettings & ena void SettingsProfilesCache::substituteProfiles(SettingsProfileElements & elements) const { - bool stop_substituting = false; boost::container::flat_set already_substituted; - while (!stop_substituting) + for (size_t i = 0; i != elements.size();) { - stop_substituting = true; - for (size_t i = 0; i != elements.size(); ++i) + auto & element = elements[i]; + if (!element.parent_profile) { - auto & element = elements[i]; - if (!element.parent_profile) - continue; - - auto parent_profile_id = *element.parent_profile; - element.parent_profile.reset(); - if (already_substituted.count(parent_profile_id)) - continue; - - already_substituted.insert(parent_profile_id); - auto parent_profile = all_profiles.find(parent_profile_id); - if (parent_profile == all_profiles.end()) - continue; - - const auto & parent_profile_elements = parent_profile->second->elements; - elements.insert(elements.begin() + i + 1, parent_profile_elements.begin(), parent_profile_elements.end()); - i += parent_profile_elements.size(); - stop_substituting = false; + ++i; + continue; } + + auto parent_profile_id = *element.parent_profile; + element.parent_profile.reset(); + if (already_substituted.count(parent_profile_id)) + { + ++i; + continue; + } + + already_substituted.insert(parent_profile_id); + auto parent_profile = all_profiles.find(parent_profile_id); + if (parent_profile == all_profiles.end()) + { + ++i; + continue; + } + + const auto & parent_profile_elements = parent_profile->second->elements; + elements.insert(elements.begin() + i, parent_profile_elements.begin(), parent_profile_elements.end()); } } @@ -182,7 +185,7 @@ void SettingsProfilesCache::substituteProfiles(SettingsProfileElements & element std::shared_ptr SettingsProfilesCache::getEnabledSettings( const UUID & user_id, const SettingsProfileElements & settings_from_user, - const std::vector & enabled_roles, + const boost::container::flat_set & enabled_roles, const SettingsProfileElements & settings_from_enabled_roles) { std::lock_guard lock{mutex}; diff --git a/src/Access/SettingsProfilesCache.h b/src/Access/SettingsProfilesCache.h index 656ffc6fce6..42dd05df351 100644 --- a/src/Access/SettingsProfilesCache.h +++ b/src/Access/SettingsProfilesCache.h @@ -29,7 +29,7 @@ public: std::shared_ptr getEnabledSettings( const UUID & user_id, const SettingsProfileElements & settings_from_user_, - const std::vector & enabled_roles, + const boost::container::flat_set & enabled_roles, const SettingsProfileElements & settings_from_enabled_roles_); std::shared_ptr getProfileSettings(const String & profile_name); diff --git a/src/Access/User.cpp b/src/Access/User.cpp index 4a751c31e25..f57ec7c1359 100644 --- a/src/Access/User.cpp +++ b/src/Access/User.cpp @@ -10,9 +10,8 @@ bool User::equal(const IAccessEntity & other) const return false; const auto & other_user = typeid_cast(other); return (authentication == other_user.authentication) && (allowed_client_hosts == other_user.allowed_client_hosts) - && (access == other_user.access) && (access_with_grant_option == other_user.access_with_grant_option) - && (granted_roles == other_user.granted_roles) && (granted_roles_with_admin_option == other_user.granted_roles_with_admin_option) - && (default_roles == other_user.default_roles) && (settings == other_user.settings); + && (access == other_user.access) && (granted_roles == other_user.granted_roles) && (default_roles == other_user.default_roles) + && (settings == other_user.settings); } } diff --git a/src/Access/User.h b/src/Access/User.h index 6df3b3e4d3c..da2fb14e131 100644 --- a/src/Access/User.h +++ b/src/Access/User.h @@ -3,11 +3,10 @@ #include #include #include -#include +#include +#include #include #include -#include -#include namespace DB @@ -18,15 +17,15 @@ struct User : public IAccessEntity { Authentication authentication; AllowedClientHosts allowed_client_hosts = AllowedClientHosts::AnyHostTag{}; - AccessRights access; - AccessRights access_with_grant_option; - boost::container::flat_set granted_roles; - boost::container::flat_set granted_roles_with_admin_option; + GrantedAccess access; + GrantedRoles granted_roles; ExtendedRoleSet default_roles = ExtendedRoleSet::AllTag{}; SettingsProfileElements settings; bool equal(const IAccessEntity & other) const override; std::shared_ptr clone() const override { return cloneImpl(); } + static constexpr const Type TYPE = Type::USER; + Type getType() const override { return TYPE; } }; using UserPtr = std::shared_ptr; diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 0842839dec8..ce33383548f 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -26,34 +27,23 @@ namespace ErrorCodes namespace { - char getTypeChar(std::type_index type) - { - if (type == typeid(User)) - return 'U'; - if (type == typeid(Quota)) - return 'Q'; - if (type == typeid(RowPolicy)) - return 'P'; - if (type == typeid(SettingsProfile)) - return 'S'; - return 0; - } + using EntityType = IAccessStorage::EntityType; + using EntityTypeInfo = IAccessStorage::EntityTypeInfo; - - UUID generateID(std::type_index type, const String & name) + UUID generateID(EntityType type, const String & name) { Poco::MD5Engine md5; md5.update(name); char type_storage_chars[] = " USRSXML"; - type_storage_chars[0] = getTypeChar(type); + type_storage_chars[0] = EntityTypeInfo::get(type).unique_char; md5.update(type_storage_chars, strlen(type_storage_chars)); UUID result; memcpy(&result, md5.digest().data(), md5.digestLength()); return result; } + UUID generateID(const IAccessEntity & entity) { return generateID(entity.getType(), entity.getName()); } - UUID generateID(const IAccessEntity & entity) { return generateID(entity.getType(), entity.getFullName()); } UserPtr parseUser(const Poco::Util::AbstractConfiguration & config, const String & user_name) { @@ -94,7 +84,7 @@ namespace { auto profile_name = config.getString(profile_name_config); SettingsProfileElement profile_element; - profile_element.parent_profile = generateID(typeid(SettingsProfile), profile_name); + profile_element.parent_profile = generateID(EntityType::SETTINGS_PROFILE, profile_name); user->settings.push_back(std::move(profile_element)); } @@ -151,30 +141,30 @@ namespace } } - user->access.grant(AccessType::ALL); /// By default all databases are accessible. + /// By default all databases are accessible + /// and the user can grant everything he has. + user->access.grantWithGrantOption(AccessType::ALL); if (databases) { user->access.revoke(AccessFlags::allFlags() - AccessFlags::allGlobalFlags()); - user->access.grant(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG); + user->access.grantWithGrantOption(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG); for (const String & database : *databases) - user->access.grant(AccessFlags::allFlags(), database); + user->access.grantWithGrantOption(AccessFlags::allFlags(), database); } if (dictionaries) { user->access.revoke(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG); for (const String & dictionary : *dictionaries) - user->access.grant(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG, dictionary); + user->access.grantWithGrantOption(AccessFlags::allDictionaryFlags(), IDictionary::NO_DATABASE_TAG, dictionary); } - user->access_with_grant_option = user->access; /// By default the user can grant everything he has. - bool access_management = config.getBool(user_config + ".access_management", false); if (!access_management) { user->access.revoke(AccessType::ACCESS_MANAGEMENT); - user->access_with_grant_option.clear(); + user->access.revokeGrantOption(AccessType::ALL); } return user; @@ -235,14 +225,13 @@ namespace limits.duration = duration; limits.randomize_interval = config.getBool(interval_config + ".randomize", false); - using ResourceType = Quota::ResourceType; - limits.max[ResourceType::QUERIES] = config.getUInt64(interval_config + ".queries", Quota::UNLIMITED); - limits.max[ResourceType::ERRORS] = config.getUInt64(interval_config + ".errors", Quota::UNLIMITED); - limits.max[ResourceType::RESULT_ROWS] = config.getUInt64(interval_config + ".result_rows", Quota::UNLIMITED); - limits.max[ResourceType::RESULT_BYTES] = config.getUInt64(interval_config + ".result_bytes", Quota::UNLIMITED); - limits.max[ResourceType::READ_ROWS] = config.getUInt64(interval_config + ".read_rows", Quota::UNLIMITED); - limits.max[ResourceType::READ_BYTES] = config.getUInt64(interval_config + ".read_bytes", Quota::UNLIMITED); - limits.max[ResourceType::EXECUTION_TIME] = Quota::secondsToExecutionTime(config.getUInt64(interval_config + ".execution_time", Quota::UNLIMITED)); + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) + { + const auto & type_info = Quota::ResourceTypeInfo::get(resource_type); + auto value = config.getString(interval_config + "." + type_info.name, "0"); + if (value != "0") + limits.max[resource_type] = type_info.amountFromString(value); + } } quota->to_roles.add(user_ids); @@ -259,7 +248,7 @@ namespace for (const auto & user_name : user_names) { if (config.has("users." + user_name + ".quota")) - quota_to_user_ids[config.getString("users." + user_name + ".quota")].push_back(generateID(typeid(User), user_name)); + quota_to_user_ids[config.getString("users." + user_name + ".quota")].push_back(generateID(EntityType::USER, user_name)); } Poco::Util::AbstractConfiguration::Keys quota_names; @@ -343,9 +332,9 @@ namespace String filter = (it != user_to_filters.end()) ? it->second : "1"; auto policy = std::make_shared(); - policy->setFullName(database, table_name, user_name); + policy->setNameParts(user_name, database, table_name); policy->conditions[RowPolicy::SELECT_FILTER] = filter; - policy->to_roles.add(generateID(typeid(User), user_name)); + policy->to_roles.add(generateID(EntityType::USER, user_name)); policies.push_back(policy); } } @@ -362,7 +351,7 @@ namespace for (const String & name : names) { SettingsProfileElement profile_element; - profile_element.name = name; + profile_element.setting_index = Settings::findIndexStrict(name); Poco::Util::AbstractConfiguration::Keys constraint_types; String path_to_name = path_to_constraints + "." + name; config.keys(path_to_name, constraint_types); @@ -399,7 +388,7 @@ namespace { String parent_profile_name = config.getString(profile_config + "." + key); SettingsProfileElement profile_element; - profile_element.parent_profile = generateID(typeid(SettingsProfile), parent_profile_name); + profile_element.parent_profile = generateID(EntityType::SETTINGS_PROFILE, parent_profile_name); profile->elements.emplace_back(std::move(profile_element)); continue; } @@ -411,7 +400,7 @@ namespace } SettingsProfileElement profile_element; - profile_element.name = key; + profile_element.setting_index = Settings::findIndexStrict(key); profile_element.value = config.getString(profile_config + "." + key); profile->elements.emplace_back(std::move(profile_element)); } @@ -461,13 +450,13 @@ void UsersConfigAccessStorage::setConfiguration(const Poco::Util::AbstractConfig } -std::optional UsersConfigAccessStorage::findImpl(std::type_index type, const String & name) const +std::optional UsersConfigAccessStorage::findImpl(EntityType type, const String & name) const { return memory_storage.find(type, name); } -std::vector UsersConfigAccessStorage::findAllImpl(std::type_index type) const +std::vector UsersConfigAccessStorage::findAllImpl(EntityType type) const { return memory_storage.findAll(type); } @@ -493,21 +482,21 @@ String UsersConfigAccessStorage::readNameImpl(const UUID & id) const UUID UsersConfigAccessStorage::insertImpl(const AccessEntityPtr & entity, bool) { - throwReadonlyCannotInsert(entity->getType(), entity->getFullName()); + throwReadonlyCannotInsert(entity->getType(), entity->getName()); } void UsersConfigAccessStorage::removeImpl(const UUID & id) { auto entity = read(id); - throwReadonlyCannotRemove(entity->getType(), entity->getFullName()); + throwReadonlyCannotRemove(entity->getType(), entity->getName()); } void UsersConfigAccessStorage::updateImpl(const UUID & id, const UpdateFunc &) { auto entity = read(id); - throwReadonlyCannotUpdate(entity->getType(), entity->getFullName()); + throwReadonlyCannotUpdate(entity->getType(), entity->getName()); } @@ -517,7 +506,7 @@ ext::scope_guard UsersConfigAccessStorage::subscribeForChangesImpl(const UUID & } -ext::scope_guard UsersConfigAccessStorage::subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const +ext::scope_guard UsersConfigAccessStorage::subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const { return memory_storage.subscribeForChanges(type, handler); } @@ -529,7 +518,7 @@ bool UsersConfigAccessStorage::hasSubscriptionImpl(const UUID & id) const } -bool UsersConfigAccessStorage::hasSubscriptionImpl(std::type_index type) const +bool UsersConfigAccessStorage::hasSubscriptionImpl(EntityType type) const { return memory_storage.hasSubscription(type); } diff --git a/src/Access/UsersConfigAccessStorage.h b/src/Access/UsersConfigAccessStorage.h index 773d8caa570..d7012cda4ff 100644 --- a/src/Access/UsersConfigAccessStorage.h +++ b/src/Access/UsersConfigAccessStorage.h @@ -23,8 +23,8 @@ public: void setConfiguration(const Poco::Util::AbstractConfiguration & config); private: - std::optional findImpl(std::type_index type, const String & name) const override; - std::vector findAllImpl(std::type_index type) const override; + std::optional findImpl(EntityType type, const String & name) const override; + std::vector findAllImpl(EntityType type) const override; bool existsImpl(const UUID & id) const override; AccessEntityPtr readImpl(const UUID & id) const override; String readNameImpl(const UUID & id) const override; @@ -33,9 +33,9 @@ private: void removeImpl(const UUID & id) override; void updateImpl(const UUID & id, const UpdateFunc & update_func) override; ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; - ext::scope_guard subscribeForChangesImpl(std::type_index type, const OnChangedHandler & handler) const override; + ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; bool hasSubscriptionImpl(const UUID & id) const override; - bool hasSubscriptionImpl(std::type_index type) const override; + bool hasSubscriptionImpl(EntityType type) const override; MemoryAccessStorage memory_storage; }; diff --git a/src/Access/ya.make b/src/Access/ya.make index fb2e23e0684..970c0714a93 100644 --- a/src/Access/ya.make +++ b/src/Access/ya.make @@ -18,13 +18,15 @@ SRCS( EnabledRowPolicies.cpp EnabledSettings.cpp ExtendedRoleSet.cpp + GrantedAccess.cpp + GrantedRoles.cpp IAccessEntity.cpp IAccessStorage.cpp MemoryAccessStorage.cpp MultipleAccessStorage.cpp Quota.cpp QuotaCache.cpp - QuotaUsageInfo.cpp + QuotaUsage.cpp Role.cpp RoleCache.cpp RowPolicy.cpp diff --git a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h index 72c2ce014e4..96c07cc3d41 100644 --- a/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h +++ b/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h @@ -80,20 +80,28 @@ struct VarMoments readPODBinary(*this, buf); } - T NO_SANITIZE_UNDEFINED getPopulation() const - { - return (m[2] - m[1] * m[1] / m[0]) / m[0]; - } - - T NO_SANITIZE_UNDEFINED getSample() const + T getPopulation() const { if (m[0] == 0) return std::numeric_limits::quiet_NaN(); - return (m[2] - m[1] * m[1] / m[0]) / (m[0] - 1); + + /// Due to numerical errors, the result can be slightly less than zero, + /// but it should be impossible. Trim to zero. + + return std::max(T{}, (m[2] - m[1] * m[1] / m[0]) / m[0]); } - T NO_SANITIZE_UNDEFINED getMoment3() const + T getSample() const { + if (m[0] <= 1) + return std::numeric_limits::quiet_NaN(); + return std::max(T{}, (m[2] - m[1] * m[1] / m[0]) / (m[0] - 1)); + } + + T getMoment3() const + { + if (m[0] == 0) + return std::numeric_limits::quiet_NaN(); // to avoid accuracy problem if (m[0] == 1) return 0; @@ -104,8 +112,10 @@ struct VarMoments ) / m[0]; } - T NO_SANITIZE_UNDEFINED getMoment4() const + T getMoment4() const { + if (m[0] == 0) + return std::numeric_limits::quiet_NaN(); // to avoid accuracy problem if (m[0] == 1) return 0; @@ -180,7 +190,7 @@ struct VarMomentsDecimal if (common::mulOverflow(getM(1), getM(1), tmp) || common::subOverflow(getM(2), NativeType(tmp / m0), tmp)) throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); - return convertFromDecimal, DataTypeNumber>(tmp / m0, scale); + return std::max(Float64{}, convertFromDecimal, DataTypeNumber>(tmp / m0, scale)); } Float64 getSample(UInt32 scale) const @@ -194,7 +204,7 @@ struct VarMomentsDecimal if (common::mulOverflow(getM(1), getM(1), tmp) || common::subOverflow(getM(2), NativeType(tmp / m0), tmp)) throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); - return convertFromDecimal, DataTypeNumber>(tmp / (m0 - 1), scale); + return std::max(Float64{}, convertFromDecimal, DataTypeNumber>(tmp / (m0 - 1), scale)); } Float64 getMoment3(UInt32 scale) const diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5ea862b7db4..222a3e486f9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -322,6 +322,11 @@ if (OPENSSL_CRYPTO_LIBRARY) target_link_libraries (clickhouse_common_io PRIVATE ${OPENSSL_CRYPTO_LIBRARY}) endif () +if (USE_LDAP) + dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${OPENLDAP_INCLUDE_DIR}) + dbms_target_link_libraries (PRIVATE ${OPENLDAP_LIBRARIES}) +endif () + dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR}) dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) @@ -365,7 +370,7 @@ if (ENABLE_TESTS AND USE_GTEST) endmacro() # attach all dbms gtest sources - grep_gtest_sources(${ClickHouse_SOURCE_DIR}/dbms dbms_gtest_sources) + grep_gtest_sources(${ClickHouse_SOURCE_DIR}/src dbms_gtest_sources) add_executable(unit_tests_dbms ${dbms_gtest_sources}) # gtest framework has substandard code diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index 62e414a676b..1f56b7c4242 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -113,7 +113,7 @@ public: Field operator[](size_t n) const override { return DecimalField(data[n], scale); } - StringRef getRawData() const override { return StringRef(reinterpret_cast(data.data()), data.size()); } + StringRef getRawData() const override { return StringRef(reinterpret_cast(data.data()), byteSize()); } StringRef getDataAt(size_t n) const override { return StringRef(reinterpret_cast(&data[n]), sizeof(data[n])); } void get(size_t n, Field & res) const override { res = (*this)[n]; } bool getBool(size_t n) const override { return bool(data[n]); } diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index e87b3b4cbf6..d6f0df1d53a 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -190,7 +190,7 @@ void ColumnLowCardinality::insertRangeFrom(const IColumn & src, size_t start, si /// TODO: Support native insertion from other unique column. It will help to avoid null map creation. - auto sub_idx = (*low_cardinality_src->getIndexes().cut(start, length)).mutate(); + auto sub_idx = IColumn::mutate(low_cardinality_src->getIndexes().cut(start, length)); auto idx_map = mapUniqueIndex(*sub_idx); auto src_nested = low_cardinality_src->getDictionary().getNestedColumn(); @@ -268,7 +268,7 @@ MutableColumnPtr ColumnLowCardinality::cloneResized(size_t size) const if (size == 0) unique_ptr = unique_ptr->cloneEmpty(); - return ColumnLowCardinality::create((*std::move(unique_ptr)).mutate(), getIndexes().cloneResized(size)); + return ColumnLowCardinality::create(IColumn::mutate(std::move(unique_ptr)), getIndexes().cloneResized(size)); } int ColumnLowCardinality::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const @@ -320,7 +320,7 @@ std::vector ColumnLowCardinality::scatter(ColumnIndex num_colu for (auto & column : columns) { auto unique_ptr = dictionary.getColumnUniquePtr(); - column = ColumnLowCardinality::create((*std::move(unique_ptr)).mutate(), std::move(column)); + column = ColumnLowCardinality::create(IColumn::mutate(std::move(unique_ptr)), std::move(column)); } return columns; @@ -337,7 +337,7 @@ void ColumnLowCardinality::setSharedDictionary(const ColumnPtr & column_unique) ColumnLowCardinality::MutablePtr ColumnLowCardinality::cutAndCompact(size_t start, size_t length) const { - auto sub_positions = (*idx.getPositions()->cut(start, length)).mutate(); + auto sub_positions = IColumn::mutate(idx.getPositions()->cut(start, length)); /// Create column with new indexes and old dictionary. /// Dictionary is shared, but will be recreated after compactInplace call. auto column = ColumnLowCardinality::create(getDictionary().assumeMutable(), std::move(sub_positions)); @@ -364,7 +364,7 @@ void ColumnLowCardinality::compactIfSharedDictionary() ColumnLowCardinality::DictionaryEncodedColumn ColumnLowCardinality::getMinimalDictionaryEncodedColumn(UInt64 offset, UInt64 limit) const { - MutableColumnPtr sub_indexes = (*std::move(idx.getPositions()->cut(offset, limit))).mutate(); + MutableColumnPtr sub_indexes = IColumn::mutate(idx.getPositions()->cut(offset, limit)); auto indexes_map = mapUniqueIndex(*sub_indexes); auto sub_keys = getDictionary().getNestedColumn()->index(*indexes_map, 0); @@ -710,7 +710,7 @@ void ColumnLowCardinality::Dictionary::compact(ColumnPtr & positions) auto sub_keys = unique.getNestedColumn()->index(*indexes, 0); auto new_indexes = new_unique.uniqueInsertRangeFrom(*sub_keys, 0, sub_keys->size()); - positions = (*new_indexes->index(*positions, 0)).mutate(); + positions = IColumn::mutate(new_indexes->index(*positions, 0)); column_unique = std::move(new_column_unique); shared = false; diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 3551efe890c..2fd177625cc 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -264,7 +264,7 @@ public: bool isFixedAndContiguous() const override { return true; } size_t sizeOfValueIfFixed() const override { return sizeof(T); } - StringRef getRawData() const override { return StringRef(reinterpret_cast(data.data()), data.size()); } + StringRef getRawData() const override { return StringRef(reinterpret_cast(data.data()), byteSize()); } bool structureEquals(const IColumn & rhs) const override diff --git a/src/Columns/FilterDescription.cpp b/src/Columns/FilterDescription.cpp index 27336d3db58..d216094eaab 100644 --- a/src/Columns/FilterDescription.cpp +++ b/src/Columns/FilterDescription.cpp @@ -64,7 +64,7 @@ FilterDescription::FilterDescription(const IColumn & column_) if (const auto * nullable_column = checkAndGetColumn(column)) { ColumnPtr nested_column = nullable_column->getNestedColumnPtr(); - MutableColumnPtr mutable_holder = (*std::move(nested_column)).mutate(); + MutableColumnPtr mutable_holder = IColumn::mutate(std::move(nested_column)); ColumnUInt8 * concrete_column = typeid_cast(mutable_holder.get()); if (!concrete_column) diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 4af593bb658..11ade9b3b84 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -304,10 +304,10 @@ public: } - MutablePtr mutate() const && + static MutablePtr mutate(Ptr ptr) { - MutablePtr res = shallowMutate(); - res->forEachSubcolumn([](WrappedPtr & subcolumn) { subcolumn = std::move(*subcolumn).mutate(); }); + MutablePtr res = ptr->shallowMutate(); + res->forEachSubcolumn([](WrappedPtr & subcolumn) { subcolumn = IColumn::mutate(std::move(subcolumn)); }); return res; } diff --git a/src/Common/COW.h b/src/Common/COW.h index d8b50c54368..3c2171436bf 100644 --- a/src/Common/COW.h +++ b/src/Common/COW.h @@ -50,7 +50,7 @@ /// Change value of x. { /// Creating mutable ptr. It can clone an object under the hood if it was shared. - Column::MutablePtr mutate_x = std::move(*x).mutate(); + Column::MutablePtr mutate_x = IColumn::mutate(std::move(x)); /// Using non-const methods of an object. mutate_x->set(2); /// Assigning pointer 'x' to mutated object. @@ -79,22 +79,12 @@ private: Derived * derived() { return static_cast(this); } const Derived * derived() const { return static_cast(this); } - template - class IntrusivePtr : public boost::intrusive_ptr - { - public: - using boost::intrusive_ptr::intrusive_ptr; - - T & operator*() const & { return boost::intrusive_ptr::operator*(); } - T && operator*() const && { return const_cast::type &&>(*boost::intrusive_ptr::get()); } - }; - protected: template - class mutable_ptr : public IntrusivePtr + class mutable_ptr : public boost::intrusive_ptr { private: - using Base = IntrusivePtr; + using Base = boost::intrusive_ptr; template friend class COW; template friend class COWHelper; @@ -123,10 +113,10 @@ public: protected: template - class immutable_ptr : public IntrusivePtr + class immutable_ptr : public boost::intrusive_ptr { private: - using Base = IntrusivePtr; + using Base = boost::intrusive_ptr; template friend class COW; template friend class COWHelper; @@ -185,9 +175,9 @@ protected: } public: - MutablePtr mutate() const && + static MutablePtr mutate(Ptr ptr) { - return shallowMutate(); + return ptr->shallowMutate(); } MutablePtr assumeMutable() const diff --git a/src/Common/Dwarf.cpp b/src/Common/Dwarf.cpp index 6d3d8fdf0ee..7a697a2c9ef 100644 --- a/src/Common/Dwarf.cpp +++ b/src/Common/Dwarf.cpp @@ -104,7 +104,7 @@ namespace // Read (bitwise) one object of type T template -std::enable_if_t, T> read(std::string_view & sp) +std::enable_if_t && std::is_standard_layout_v, T> read(std::string_view & sp) { SAFE_CHECK(sp.size() >= sizeof(T), "underflow"); T x; diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 75f22ddc5eb..374315005de 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -494,6 +494,7 @@ namespace ErrorCodes extern const int CANNOT_DETACH_DICTIONARY_AS_TABLE = 520; extern const int ATOMIC_RENAME_FAIL = 521; extern const int OPENCL_ERROR = 522; + extern const int UNKNOWN_ROW_POLICY = 523; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/src/Common/tests/cow_columns.cpp b/src/Common/tests/cow_columns.cpp index b4c3637be5a..fa84fc9ebc2 100644 --- a/src/Common/tests/cow_columns.cpp +++ b/src/Common/tests/cow_columns.cpp @@ -53,7 +53,7 @@ int main(int, char **) std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n"; { - MutableColumnPtr mut = std::move(*y).mutate(); + MutableColumnPtr mut = IColumn::mutate(std::move(y)); mut->set(2); std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << ", " << mut->use_count() << "\n"; @@ -72,7 +72,7 @@ int main(int, char **) std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n"; { - MutableColumnPtr mut = std::move(*y).mutate(); + MutableColumnPtr mut = IColumn::mutate(std::move(y)); mut->set(3); std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << ", " << mut->use_count() << "\n"; diff --git a/src/Common/tests/cow_compositions.cpp b/src/Common/tests/cow_compositions.cpp index 0335693d1bd..be33f392497 100644 --- a/src/Common/tests/cow_compositions.cpp +++ b/src/Common/tests/cow_compositions.cpp @@ -18,7 +18,7 @@ public: virtual int get() const = 0; virtual void set(int value) = 0; - MutablePtr mutate() const && { return deepMutate(); } + static MutablePtr mutate(Ptr ptr) { return ptr->deepMutate(); } }; using ColumnPtr = IColumn::Ptr; @@ -52,7 +52,7 @@ private: { std::cerr << "Mutating\n"; auto res = shallowMutate(); - res->wrapped = std::move(*wrapped).mutate(); + res->wrapped = IColumn::mutate(std::move(wrapped)); return res; } @@ -72,7 +72,7 @@ int main(int, char **) std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n"; { - MutableColumnPtr mut = std::move(*y).mutate(); + MutableColumnPtr mut = IColumn::mutate(std::move(y)); mut->set(2); std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << ", " << mut->use_count() << "\n"; @@ -91,7 +91,7 @@ int main(int, char **) std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n"; { - MutableColumnPtr mut = std::move(*y).mutate(); + MutableColumnPtr mut = IColumn::mutate(std::move(y)); mut->set(3); std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << ", " << mut->use_count() << "\n"; diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 75b50377a64..da8bfa5451b 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -335,7 +335,7 @@ MutableColumns Block::mutateColumns() size_t num_columns = data.size(); MutableColumns columns(num_columns); for (size_t i = 0; i < num_columns; ++i) - columns[i] = data[i].column ? (*std::move(data[i].column)).mutate() : data[i].type->createColumn(); + columns[i] = data[i].column ? IColumn::mutate(std::move(data[i].column)) : data[i].type->createColumn(); return columns; } diff --git a/src/Core/Defines.h b/src/Core/Defines.h index 5552de3b045..13070c565b4 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -58,7 +58,7 @@ /// Minimum revision with exactly the same set of aggregation methods and rules to select them. /// Two-level (bucketed) aggregation is incompatible if servers are inconsistent in these rules /// (keys will be placed in different buckets and result will not be fully aggregated). -#define DBMS_MIN_REVISION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD 54408 +#define DBMS_MIN_REVISION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD 54431 #define DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA 54410 #define DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE 54405 diff --git a/src/Core/Names.h b/src/Core/Names.h index 5489a233b6e..3281daa560e 100644 --- a/src/Core/Names.h +++ b/src/Core/Names.h @@ -15,6 +15,7 @@ using NameSet = std::unordered_set; using NameOrderedSet = std::set; using NameToNameMap = std::unordered_map; using NameToNameSetMap = std::unordered_map; +using NameToNameVector = std::vector>; using NameWithAlias = std::pair; using NamesWithAliases = std::vector; diff --git a/src/Core/Settings.h b/src/Core/Settings.h index ff151e24a99..34d05900f77 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -189,7 +189,7 @@ struct Settings : public SettingsCollection M(SettingUInt64, max_http_get_redirects, 0, "Max number of http GET redirects hops allowed. Make sure additional security measures are in place to prevent a malicious server to redirect your requests to unexpected services.", 0) \ \ M(SettingBool, input_format_skip_unknown_fields, false, "Skip columns with unknown names from input data (it works for JSONEachRow, CSVWithNames, TSVWithNames and TSKV formats).", 0) \ - M(SettingBool, input_format_with_names_use_header, false, "For TSVWithNames and CSVWithNames input formats this controls whether format parser is to assume that column data appear in the input exactly as they are specified in the header.", 0) \ + M(SettingBool, input_format_with_names_use_header, true, "For TSVWithNames and CSVWithNames input formats this controls whether format parser is to assume that column data appear in the input exactly as they are specified in the header.", 0) \ M(SettingBool, input_format_import_nested_json, false, "Map nested JSON data to nested tables (it works for JSONEachRow format).", 0) \ M(SettingBool, input_format_defaults_for_omitted_fields, true, "For input data calculate default expressions for omitted fields (it works for JSONEachRow, CSV and TSV formats).", IMPORTANT) \ M(SettingBool, input_format_tsv_empty_as_default, false, "Treat empty fields in TSV input as default values.", 0) \ @@ -385,8 +385,6 @@ struct Settings : public SettingsCollection M(SettingBool, cancel_http_readonly_queries_on_client_close, false, "Cancel HTTP readonly queries when a client closes the connection without waiting for response.", 0) \ M(SettingBool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql' and 'odbc' table functions.", 0) \ \ - M(SettingBool, experimental_use_processors, true, "Use processors pipeline.", 0) \ - \ M(SettingBool, allow_hyperscan, true, "Allow functions that use Hyperscan library. Disable to avoid potentially long compilation times and excessive resource usage.", 0) \ M(SettingBool, allow_simdjson, true, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.", 0) \ M(SettingBool, allow_introspection_functions, false, "Allow functions for introspection of ELF and DWARF for query profiling. These functions are slow and may impose security considerations.", 0) \ diff --git a/src/DataStreams/AddingDefaultsBlockInputStream.cpp b/src/DataStreams/AddingDefaultsBlockInputStream.cpp index d2df3dbc496..4caf396eb49 100644 --- a/src/DataStreams/AddingDefaultsBlockInputStream.cpp +++ b/src/DataStreams/AddingDefaultsBlockInputStream.cpp @@ -191,7 +191,7 @@ Block AddingDefaultsBlockInputStream::readImpl() /// TODO: FixedString if (isColumnedAsNumber(column_read.type) || isDecimal(column_read.type)) { - MutableColumnPtr column_mixed = (*std::move(column_read.column)).mutate(); + MutableColumnPtr column_mixed = IColumn::mutate(std::move(column_read.column)); mixNumberColumns(column_read.type->getTypeId(), column_mixed, column_def.column, defaults_mask); column_read.column = std::move(column_mixed); } diff --git a/src/DataStreams/BlockStreamProfileInfo.cpp b/src/DataStreams/BlockStreamProfileInfo.cpp index 81e2be8d4a0..09ad8a8e4ac 100644 --- a/src/DataStreams/BlockStreamProfileInfo.cpp +++ b/src/DataStreams/BlockStreamProfileInfo.cpp @@ -62,10 +62,15 @@ bool BlockStreamProfileInfo::hasAppliedLimit() const void BlockStreamProfileInfo::update(Block & block) +{ + update(block.rows(), block.bytes()); +} + +void BlockStreamProfileInfo::update(size_t num_rows, size_t num_bytes) { ++blocks; - rows += block.rows(); - bytes += block.bytes(); + rows += num_rows; + bytes += num_bytes; } diff --git a/src/DataStreams/BlockStreamProfileInfo.h b/src/DataStreams/BlockStreamProfileInfo.h index 6b60fa95e24..5f75cf9ddea 100644 --- a/src/DataStreams/BlockStreamProfileInfo.h +++ b/src/DataStreams/BlockStreamProfileInfo.h @@ -40,6 +40,7 @@ struct BlockStreamProfileInfo bool hasAppliedLimit() const; void update(Block & block); + void update(size_t num_rows, size_t num_bytes); /// Binary serialization and deserialization of main fields. /// Writes only main fields i.e. fields that required by internal transmission protocol. diff --git a/src/DataStreams/MergingSortedBlockInputStream.cpp b/src/DataStreams/MergingSortedBlockInputStream.cpp index 1ed7c6dff22..ae10617d45a 100644 --- a/src/DataStreams/MergingSortedBlockInputStream.cpp +++ b/src/DataStreams/MergingSortedBlockInputStream.cpp @@ -185,7 +185,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, TSort throw Exception("Logical error in MergingSortedBlockInputStream", ErrorCodes::LOGICAL_ERROR); for (size_t i = 0; i < num_columns; ++i) - merged_columns[i] = (*std::move(source_blocks[source_num].getByPosition(i).column)).mutate(); + merged_columns[i] = IColumn::mutate(std::move(source_blocks[source_num].getByPosition(i).column)); // std::cerr << "copied columns\n"; @@ -198,7 +198,7 @@ void MergingSortedBlockInputStream::merge(MutableColumns & merged_columns, TSort for (size_t i = 0; i < num_columns; ++i) { auto & column = merged_columns[i]; - column = (*column->cut(0, merged_rows)).mutate(); + column = IColumn::mutate(column->cut(0, merged_rows)); } cancel(false); diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/src/DataStreams/PushingToViewsBlockOutputStream.cpp index a79cc61bd2d..ce0922bf282 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -90,7 +90,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( else out = std::make_shared(dependent_table, *views_context, ASTPtr()); - views.emplace_back(ViewInfo{std::move(query), database_table, std::move(out)}); + views.emplace_back(ViewInfo{std::move(query), database_table, std::move(out), nullptr}); } /// Do not push to destination table if the flag is set @@ -162,7 +162,12 @@ void PushingToViewsBlockOutputStream::write(const Block & block) { // Process sequentially for (size_t view_num = 0; view_num < views.size(); ++view_num) + { process(block, view_num); + + if (views[view_num].exception) + std::rethrow_exception(views[view_num].exception); + } } } @@ -190,8 +195,18 @@ void PushingToViewsBlockOutputStream::writeSuffix() if (output) output->writeSuffix(); + std::exception_ptr first_exception; + for (auto & view : views) { + if (view.exception) + { + if (!first_exception) + first_exception = view.exception; + + continue; + } + try { view.out->writeSuffix(); @@ -202,6 +217,9 @@ void PushingToViewsBlockOutputStream::writeSuffix() throw; } } + + if (first_exception) + std::rethrow_exception(first_exception); } void PushingToViewsBlockOutputStream::flush() @@ -270,7 +288,11 @@ void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_n catch (Exception & ex) { ex.addMessage("while pushing to view " + view.table_id.getNameForLogs()); - throw; + view.exception = std::current_exception(); + } + catch (...) + { + view.exception = std::current_exception(); } } diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.h b/src/DataStreams/PushingToViewsBlockOutputStream.h index 162c2e1b447..a2a1ca5caf5 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.h +++ b/src/DataStreams/PushingToViewsBlockOutputStream.h @@ -40,6 +40,7 @@ private: ASTPtr query; StorageID table_id; BlockOutputStreamPtr out; + std::exception_ptr exception; }; std::vector views; diff --git a/src/DataStreams/SquashingTransform.cpp b/src/DataStreams/SquashingTransform.cpp index 8cbbebb75ab..c57e2351230 100644 --- a/src/DataStreams/SquashingTransform.cpp +++ b/src/DataStreams/SquashingTransform.cpp @@ -95,8 +95,7 @@ void SquashingTransform::append(ReferenceType input_block) { const auto source_column = input_block.getByPosition(i).column; - auto mutable_column = (*std::move( - accumulated_block.getByPosition(i).column)).mutate(); + auto mutable_column = IColumn::mutate(std::move(accumulated_block.getByPosition(i).column)); if (reserve_memory) { diff --git a/src/DataStreams/finalizeBlock.cpp b/src/DataStreams/finalizeBlock.cpp index 144f1a28129..56068edcc29 100644 --- a/src/DataStreams/finalizeBlock.cpp +++ b/src/DataStreams/finalizeBlock.cpp @@ -18,7 +18,7 @@ namespace DB current.type = unfinalized_type->getReturnType(); if (current.column) { - auto mut_column = (*std::move(current.column)).mutate(); + auto mut_column = IColumn::mutate(std::move(current.column)); current.column = ColumnAggregateFunction::convertToValues(std::move(mut_column)); } } diff --git a/src/DataStreams/tests/finish_sorting_stream.cpp b/src/DataStreams/tests/finish_sorting_stream.cpp index cfc9ba217b3..d10ae3fbb63 100644 --- a/src/DataStreams/tests/finish_sorting_stream.cpp +++ b/src/DataStreams/tests/finish_sorting_stream.cpp @@ -80,7 +80,7 @@ int main(int argc, char ** argv) { for (size_t i = 0; i < block.columns(); ++i) { - MutableColumnPtr ptr = (*std::move(res_block.getByPosition(i).column)).mutate(); + MutableColumnPtr ptr = IColumn::mutate(std::move(res_block.getByPosition(i).column)); ptr->insertRangeFrom(*block.getByPosition(i).column.get(), 0, block.rows()); } } diff --git a/src/DataTypes/DataTypeLowCardinality.cpp b/src/DataTypes/DataTypeLowCardinality.cpp index 95ef60bed6e..b4f28a8853f 100644 --- a/src/DataTypes/DataTypeLowCardinality.cpp +++ b/src/DataTypes/DataTypeLowCardinality.cpp @@ -672,7 +672,7 @@ void DataTypeLowCardinality::deserializeBinaryBulkWithMultipleStreams( ColumnLowCardinality::Index(indexes_column->getPtr()).check( maps.dictionary_map->size() + maps.additional_keys_map->size()); - auto used_keys = (*std::move(global_dictionary->getNestedColumn()->index(*maps.dictionary_map, 0))).mutate(); + auto used_keys = IColumn::mutate(global_dictionary->getNestedColumn()->index(*maps.dictionary_map, 0)); if (!maps.additional_keys_map->empty()) { diff --git a/src/DataTypes/DataTypesDecimal.cpp b/src/DataTypes/DataTypesDecimal.cpp index cbc38429183..0d48845b4fe 100644 --- a/src/DataTypes/DataTypesDecimal.cpp +++ b/src/DataTypes/DataTypesDecimal.cpp @@ -22,9 +22,9 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int DECIMAL_OVERFLOW; } -// template std::string DataTypeDecimal::doGetName() const @@ -61,10 +61,13 @@ template bool DataTypeDecimal::tryReadText(T & x, ReadBuffer & istr, UInt32 precision, UInt32 scale) { UInt32 unread_scale = scale; - bool done = tryReadDecimalText(istr, x, precision, unread_scale); + if (!tryReadDecimalText(istr, x, precision, unread_scale)) + return false; - x *= T::getScaleMultiplier(unread_scale); - return done; + if (common::mulOverflow(x.value, T::getScaleMultiplier(unread_scale), x.value)) + return false; + + return true; } template @@ -75,7 +78,9 @@ void DataTypeDecimal::readText(T & x, ReadBuffer & istr, UInt32 precision, UI readCSVDecimalText(istr, x, precision, unread_scale); else readDecimalText(istr, x, precision, unread_scale); - x *= T::getScaleMultiplier(unread_scale); + + if (common::mulOverflow(x.value, T::getScaleMultiplier(unread_scale), x.value)) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); } template @@ -101,7 +106,9 @@ T DataTypeDecimal::parseFromString(const String & str) const T x; UInt32 unread_scale = this->scale; readDecimalText(buf, x, this->precision, unread_scale, true); - x *= T::getScaleMultiplier(unread_scale); + + if (common::mulOverflow(x.value, T::getScaleMultiplier(unread_scale), x.value)) + throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); return x; } diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index bafa65b10ba..09ff1b8944e 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -227,7 +228,7 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora void DatabaseAtomic::commitAlterTable(const StorageID & table_id, const String & table_metadata_tmp_path, const String & table_metadata_path) { - SCOPE_EXIT({ Poco::File(table_metadata_tmp_path).remove(); }); + SCOPE_EXIT({ std::error_code code; std::filesystem::remove(table_metadata_tmp_path, code); }); std::unique_lock lock{mutex}; auto actual_table_id = getTableUnlocked(table_id.table_name, lock)->getStorageID(); diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 98ed7985acb..8c736bc0e56 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -77,9 +77,6 @@ ClickHouseDictionarySource::ClickHouseDictionarySource( context.setUser(user, password, Poco::Net::SocketAddress("127.0.0.1", 0), {}); context = copyContextAndApplySettings(path_to_settings, context, config); - /// Processors are not supported here yet. - context.setSetting("experimental_use_processors", false); - /// Query context is needed because some code in executeQuery function may assume it exists. /// Current example is Context::getSampleBlockCache from InterpreterSelectWithUnionQuery::getSampleBlock. context.makeQueryContext(); @@ -134,7 +131,7 @@ BlockInputStreamPtr ClickHouseDictionarySource::loadAll() */ if (is_local) { - BlockIO res = executeQuery(load_all_query, context, true); + BlockIO res = executeQuery(load_all_query, context, true, QueryProcessingStage::Complete, false, false); /// FIXME res.in may implicitly use some objects owned be res, but them will be destructed after return res.in = std::make_shared(res.in, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); return res.in; @@ -147,7 +144,7 @@ BlockInputStreamPtr ClickHouseDictionarySource::loadUpdatedAll() std::string load_update_query = getUpdateFieldAndDate(); if (is_local) { - auto res = executeQuery(load_update_query, context, true); + auto res = executeQuery(load_update_query, context, true, QueryProcessingStage::Complete, false, false); res.in = std::make_shared(res.in, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); return res.in; } @@ -194,7 +191,7 @@ BlockInputStreamPtr ClickHouseDictionarySource::createStreamForSelectiveLoad(con { if (is_local) { - auto res = executeQuery(query, context, true); + auto res = executeQuery(query, context, true, QueryProcessingStage::Complete, false, false); res.in = std::make_shared( res.in, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); return res.in; @@ -209,7 +206,8 @@ std::string ClickHouseDictionarySource::doInvalidateQuery(const std::string & re if (is_local) { Context query_context = context; - auto input_block = executeQuery(request, query_context, true).in; + auto input_block = executeQuery(request, query_context, true, + QueryProcessingStage::Complete, false, false).in; return readInvalidateQuery(*input_block); } else diff --git a/src/Dictionaries/PolygonDictionary.cpp b/src/Dictionaries/PolygonDictionary.cpp index dd231b3f2d7..01c9522e952 100644 --- a/src/Dictionaries/PolygonDictionary.cpp +++ b/src/Dictionaries/PolygonDictionary.cpp @@ -200,7 +200,7 @@ void IPolygonDictionary::blockToAttributes(const DB::Block &block) const auto & column = block.safeGetByPosition(i + 1); if (attributes[i]) { - MutableColumnPtr mutated = std::move(*attributes[i]).mutate(); + MutableColumnPtr mutated = IColumn::mutate(std::move(attributes[i])); mutated->insertRangeFrom(*column.column, 0, column.column->size()); attributes[i] = std::move(mutated); } diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index 4f808c4766d..8e538bf780b 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -55,6 +55,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } + /** Functions that use plug-ins (external) dictionaries_loader. * * Get the value of the attribute of the specified type. @@ -70,6 +71,51 @@ namespace ErrorCodes */ +class FunctionDictHelper +{ +public: + FunctionDictHelper(const Context & context_) : context(context_), external_loader(context.getExternalDictionariesLoader()) {} + + std::shared_ptr getDictionary(const String & dictionary_name) + { + auto dict = std::atomic_load(&dictionary); + if (dict) + return dict; + dict = external_loader.getDictionary(dictionary_name); + context.checkAccess(AccessType::dictGet, dict->getDatabaseOrNoDatabaseTag(), dict->getName()); + std::atomic_store(&dictionary, dict); + return dict; + } + + std::shared_ptr getDictionary(const ColumnWithTypeAndName & column) + { + const auto dict_name_col = checkAndGetColumnConst(column.column.get()); + return getDictionary(dict_name_col->getValue()); + } + + bool isDictGetFunctionInjective(const Block & sample_block) + { + if (sample_block.columns() != 3 && sample_block.columns() != 4) + throw Exception{"Function dictGet... takes 3 or 4 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; + + const auto dict_name_col = checkAndGetColumnConst(sample_block.getByPosition(0).column.get()); + if (!dict_name_col) + throw Exception{"First argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; + + const auto attr_name_col = checkAndGetColumnConst(sample_block.getByPosition(1).column.get()); + if (!attr_name_col) + throw Exception{"Second argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; + + return getDictionary(dict_name_col->getValue())->isInjective(attr_name_col->getValue()); + } + +private: + const Context & context; + const ExternalDictionariesLoader & external_loader; + mutable std::shared_ptr dictionary; +}; + + class FunctionDictHas final : public IFunction { public: @@ -77,12 +123,10 @@ public: static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictHas(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) - : dictionaries_loader(dictionaries_loader_) - , context(context_) {} + FunctionDictHas(const Context & context_) : helper(context_) {} String getName() const override { return name; } @@ -110,10 +154,6 @@ private: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - /** Do not require existence of the dictionary if the function is called for empty block. * This is needed to allow successful query analysis on a server, * that is the initiator of a distributed query, @@ -128,29 +168,27 @@ private: return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatchSimple(block, arguments, result, dict_ptr) && - !executeDispatchSimple(block, arguments, result, dict_ptr) && - !executeDispatchSimple(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && + if (!executeDispatchSimple(block, arguments, result, dict) && + !executeDispatchSimple(block, arguments, result, dict) && + !executeDispatchSimple(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && #if !defined(ARCADIA_BUILD) - !executeDispatchComplex(block, arguments, result, dict_ptr) && + !executeDispatchComplex(block, arguments, result, dict) && #endif - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchSimple(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchSimple(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template bool executeDispatchSimple( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -171,9 +209,9 @@ private: template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -195,28 +233,11 @@ private: return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; +private: + mutable FunctionDictHelper helper; }; -static bool isDictGetFunctionInjective(const ExternalDictionariesLoader & dictionaries_loader, const Block & sample_block) -{ - if (sample_block.columns() != 3 && sample_block.columns() != 4) - throw Exception{"Function dictGet... takes 3 or 4 arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - const auto dict_name_col = checkAndGetColumnConst(sample_block.getByPosition(0).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - const auto attr_name_col = checkAndGetColumnConst(sample_block.getByPosition(1).column.get()); - if (!attr_name_col) - throw Exception{"Second argument of function dictGet... must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - - return dictionaries_loader.getDictionary(dict_name_col->getValue())->isInjective(attr_name_col->getValue()); -} - - /** For ColumnVector. Either returns a reference to internal data, * or convert it to T type, stores the result in backup_storage and returns a reference to it. */ @@ -231,12 +252,10 @@ public: static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictGetString(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) - : dictionaries_loader(dictionaries_loader_) - , context(context_) {} + FunctionDictGetString(const Context & context_) : helper(context_) {} String getName() const override { return name; } @@ -249,7 +268,7 @@ private: bool isInjective(const Block & sample_block) const override { - return isDictGetFunctionInjective(dictionaries_loader, sample_block); + return helper.isDictGetFunctionInjective(sample_block); } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override @@ -292,10 +311,6 @@ private: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -303,30 +318,28 @@ private: return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && + if (!executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && #if !defined(ARCADIA_BUILD) - !executeDispatchComplex(block, arguments, result, dict_ptr) && + !executeDispatchComplex(block, arguments, result, dict) && #endif - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchRange(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchRange(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template bool executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -355,9 +368,9 @@ private: template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -392,9 +405,9 @@ private: template bool executeDispatchRange( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -423,8 +436,8 @@ private: return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; +private: + mutable FunctionDictHelper helper; }; @@ -435,12 +448,10 @@ public: static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictGetStringOrDefault(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) - : dictionaries_loader(dictionaries_loader_) - , context(context_) {} + FunctionDictGetStringOrDefault(const Context & context_) : helper(context_) {} String getName() const override { return name; } @@ -478,10 +489,6 @@ private: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -489,29 +496,27 @@ private: return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && + if (!executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && #if !defined(ARCADIA_BUILD) - !executeDispatchComplex(block, arguments, result, dict_ptr) && + !executeDispatchComplex(block, arguments, result, dict) && #endif - !executeDispatchComplex(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + !executeDispatchComplex(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template bool executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -534,7 +539,7 @@ private: template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dictionary, + Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, const std::string & attr_name, const ColumnUInt64 * id_col) { const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); @@ -544,7 +549,7 @@ private: /// vector ids, vector defaults auto out = ColumnString::create(); const auto & ids = id_col->getData(); - dictionary->getString(attr_name, ids, default_col, out.get()); + dict->getString(attr_name, ids, default_col, out.get()); block.getByPosition(result).column = std::move(out); } else if (const auto default_col_const = checkAndGetColumnConstStringOrFixedString(default_col_untyped)) @@ -553,7 +558,7 @@ private: auto out = ColumnString::create(); const auto & ids = id_col->getData(); String def = default_col_const->getValue(); - dictionary->getString(attr_name, ids, def, out.get()); + dict->getString(attr_name, ids, def, out.get()); block.getByPosition(result).column = std::move(out); } else @@ -562,7 +567,7 @@ private: template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dictionary, + Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, const std::string & attr_name, const ColumnConst * id_col) { const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); @@ -572,11 +577,11 @@ private: /// const ids, vector defaults const PaddedPODArray ids(1, id_col->getValue()); PaddedPODArray flags(1); - dictionary->has(ids, flags); + dict->has(ids, flags); if (flags.front()) { auto out = ColumnString::create(); - dictionary->getString(attr_name, ids, String(), out.get()); + dict->getString(attr_name, ids, String(), out.get()); block.getByPosition(result).column = DataTypeString().createColumnConst(id_col->size(), out->getDataAt(0).toString()); } else @@ -588,7 +593,7 @@ private: const PaddedPODArray ids(1, id_col->getValue()); auto out = ColumnString::create(); String def = default_col_const->getValue(); - dictionary->getString(attr_name, ids, def, out.get()); + dict->getString(attr_name, ids, def, out.get()); block.getByPosition(result).column = DataTypeString().createColumnConst(id_col->size(), out->getDataAt(0).toString()); } else @@ -597,9 +602,9 @@ private: template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -635,8 +640,7 @@ private: return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; + mutable FunctionDictHelper helper; }; @@ -759,12 +763,11 @@ public: static FunctionPtr create(const Context & context, UInt32 dec_scale = 0) { - return std::make_shared(context.getExternalDictionariesLoader(), context, dec_scale); + return std::make_shared(context, dec_scale); } - FunctionDictGet(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_, UInt32 dec_scale = 0) - : dictionaries_loader(dictionaries_loader_) - , context(context_) + FunctionDictGet(const Context & context_, UInt32 dec_scale = 0) + : helper(context_) , decimal_scale(dec_scale) {} @@ -779,7 +782,7 @@ private: bool isInjective(const Block & sample_block) const override { - return isDictGetFunctionInjective(dictionaries_loader, sample_block); + return helper.isDictGetFunctionInjective(sample_block); } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override @@ -820,10 +823,6 @@ private: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -831,30 +830,27 @@ private: return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && + if (!executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && #if !defined(ARCADIA_BUILD) - !executeDispatchComplex(block, arguments, result, dict_ptr) && + !executeDispatchComplex(block, arguments, result, dict) && #endif - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchRange(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchRange(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template - bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, - const IDictionaryBase * dictionary) + bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -908,9 +904,9 @@ private: template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -951,9 +947,9 @@ private: template bool executeDispatchRange( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -987,8 +983,7 @@ private: return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; + mutable FunctionDictHelper helper; UInt32 decimal_scale; }; @@ -1038,12 +1033,11 @@ public: static FunctionPtr create(const Context & context, UInt32 dec_scale = 0) { - return std::make_shared(context.getExternalDictionariesLoader(), context, dec_scale); + return std::make_shared(context, dec_scale); } - FunctionDictGetOrDefault(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_, UInt32 dec_scale = 0) - : dictionaries_loader(dictionaries_loader_) - , context(context_) + FunctionDictGetOrDefault(const Context & context_, UInt32 dec_scale = 0) + : helper(context_) , decimal_scale(dec_scale) {} @@ -1084,10 +1078,6 @@ private: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -1095,29 +1085,26 @@ private: return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && - !executeDispatchComplex(block, arguments, result, dict_ptr) && + if (!executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && + !executeDispatchComplex(block, arguments, result, dict) && #if !defined(ARCADIA_BUILD) - !executeDispatchComplex(block, arguments, result, dict_ptr) && + !executeDispatchComplex(block, arguments, result, dict) && #endif - !executeDispatchComplex(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + !executeDispatchComplex(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template - bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, - const IDictionaryBase * dictionary) + bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -1140,7 +1127,7 @@ private: template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dictionary, + Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, const std::string & attr_name, const ColumnUInt64 * id_col) { const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); @@ -1156,7 +1143,7 @@ private: const auto & ids = id_col->getData(); auto & data = out->getData(); const auto & defs = default_col->getData(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, defs, data); + DictGetTraits::getOrDefault(dict, attr_name, ids, defs, data); block.getByPosition(result).column = std::move(out); } else if (const auto default_col_const = checkAndGetColumnConst(default_col_untyped)) @@ -1170,7 +1157,7 @@ private: const auto & ids = id_col->getData(); auto & data = out->getData(); const auto def = default_col_const->template getValue(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, def, data); + DictGetTraits::getOrDefault(dict, attr_name, ids, def, data); block.getByPosition(result).column = std::move(out); } else @@ -1179,7 +1166,7 @@ private: template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dictionary, + Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, const std::string & attr_name, const ColumnConst * id_col) { const auto default_col_untyped = block.getByPosition(arguments[3]).column.get(); @@ -1189,13 +1176,13 @@ private: /// const ids, vector defaults const PaddedPODArray ids(1, id_col->getValue()); PaddedPODArray flags(1); - dictionary->has(ids, flags); + dict->has(ids, flags); if (flags.front()) { if constexpr (IsDataTypeDecimal) { DecimalPaddedPODArray data(1, decimal_scale); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, Type(), data); + DictGetTraits::getOrDefault(dict, attr_name, ids, Type(), data); block.getByPosition(result).column = DataType(DataType::maxPrecision(), decimal_scale).createColumnConst( id_col->size(), toField(data.front(), decimal_scale)); @@ -1203,7 +1190,7 @@ private: else { PaddedPODArray data(1); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, Type(), data); + DictGetTraits::getOrDefault(dict, attr_name, ids, Type(), data); block.getByPosition(result).column = DataType().createColumnConst(id_col->size(), toField(data.front())); } } @@ -1219,7 +1206,7 @@ private: { DecimalPaddedPODArray data(1, decimal_scale); const auto & def = default_col_const->template getValue(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, def, data); + DictGetTraits::getOrDefault(dict, attr_name, ids, def, data); block.getByPosition(result).column = DataType(DataType::maxPrecision(), decimal_scale).createColumnConst( id_col->size(), toField(data.front(), decimal_scale)); @@ -1228,7 +1215,7 @@ private: { PaddedPODArray data(1); const auto & def = default_col_const->template getValue(); - DictGetTraits::getOrDefault(dictionary, attr_name, ids, def, data); + DictGetTraits::getOrDefault(dict, attr_name, ids, def, data); block.getByPosition(result).column = DataType().createColumnConst(id_col->size(), toField(data.front())); } } @@ -1238,9 +1225,9 @@ private: template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const IDictionaryBase * dictionary) + Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -1288,8 +1275,7 @@ private: return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; + mutable FunctionDictHelper helper; UInt32 decimal_scale; }; @@ -1336,10 +1322,10 @@ public: static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictGetNoType(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) : dictionaries_loader(dictionaries_loader_), context(context_) {} + FunctionDictGetNoType(const Context & context_) : context(context_), helper(context_) {} String getName() const override { return name; } @@ -1352,7 +1338,7 @@ private: bool isInjective(const Block & sample_block) const override { - return isDictGetFunctionInjective(dictionaries_loader, sample_block); + return helper.isDictGetFunctionInjective(sample_block); } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override @@ -1392,7 +1378,7 @@ private: + ", must be convertible to " + TypeName::get() + ".", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; } - auto dict = dictionaries_loader.getDictionary(dict_name); + auto dict = helper.getDictionary(dict_name); const DictionaryStructure & structure = dict->getStructure(); for (const auto idx : ext::range(0, structure.attributes.size())) @@ -1472,8 +1458,8 @@ private: } private: - const ExternalDictionariesLoader & dictionaries_loader; const Context & context; + mutable FunctionDictHelper helper; mutable FunctionPtr impl; // underlying function used by dictGet function without explicit type info }; @@ -1485,10 +1471,10 @@ public: static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictGetNoTypeOrDefault(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) : dictionaries_loader(dictionaries_loader_), context(context_) {} + FunctionDictGetNoTypeOrDefault(const Context & context_) : context(context_), helper(context_) {} String getName() const override { return name; } @@ -1500,7 +1486,7 @@ private: bool isInjective(const Block & sample_block) const override { - return isDictGetFunctionInjective(dictionaries_loader, sample_block); + return helper.isDictGetFunctionInjective(sample_block); } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override @@ -1528,7 +1514,7 @@ private: throw Exception{"Illegal type " + arguments[2].type->getName() + " of third argument of function " + getName() + ", must be UInt64 or tuple(...).", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - auto dict = dictionaries_loader.getDictionary(dict_name); + auto dict = helper.getDictionary(dict_name); const DictionaryStructure & structure = dict->getStructure(); for (const auto idx : ext::range(0, structure.attributes.size())) @@ -1614,8 +1600,8 @@ private: } private: - const ExternalDictionariesLoader & dictionaries_loader; const Context & context; + mutable FunctionDictHelper helper; mutable FunctionPtr impl; // underlying function used by dictGet function without explicit type info }; @@ -1628,12 +1614,10 @@ public: static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictGetHierarchy(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) - : dictionaries_loader(dictionaries_loader_) - , context(context_) {} + FunctionDictGetHierarchy(const Context & context_) : helper(context_) {} String getName() const override { return name; } @@ -1661,10 +1645,6 @@ private: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -1672,22 +1652,20 @@ private: return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr) && - !executeDispatch(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + if (!executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict) && + !executeDispatch(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, - const IDictionaryBase * dictionary) + const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -1778,8 +1756,7 @@ private: return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; + mutable FunctionDictHelper helper; }; @@ -1790,12 +1767,11 @@ public: static FunctionPtr create(const Context & context) { - return std::make_shared(context.getExternalDictionariesLoader(), context); + return std::make_shared(context); } - FunctionDictIsIn(const ExternalDictionariesLoader & dictionaries_loader_, const Context & context_) - : dictionaries_loader(dictionaries_loader_) - , context(context_) {} + FunctionDictIsIn(const Context & context_) + : helper(context_) {} String getName() const override { return name; } @@ -1826,10 +1802,6 @@ private: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - const auto dict_name_col = checkAndGetColumnConst(block.getByPosition(arguments[0]).column.get()); - if (!dict_name_col) - throw Exception{"First argument of function " + getName() + " must be a constant string", ErrorCodes::ILLEGAL_COLUMN}; - if (input_rows_count == 0) { auto & elem = block.getByPosition(result); @@ -1837,22 +1809,20 @@ private: return; } - auto dict = dictionaries_loader.getDictionary(dict_name_col->getValue()); - const auto dict_ptr = dict.get(); - context.checkAccess(AccessType::dictGet, dict_ptr->getDatabaseOrNoDatabaseTag(), dict_ptr->getName()); + auto dict = helper.getDictionary(block.getByPosition(arguments[0])); - if (!executeDispatch(block, arguments, result, dict_ptr) - && !executeDispatch(block, arguments, result, dict_ptr) - && !executeDispatch(block, arguments, result, dict_ptr) - && !executeDispatch(block, arguments, result, dict_ptr)) - throw Exception{"Unsupported dictionary type " + dict_ptr->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; + if (!executeDispatch(block, arguments, result, dict) + && !executeDispatch(block, arguments, result, dict) + && !executeDispatch(block, arguments, result, dict) + && !executeDispatch(block, arguments, result, dict)) + throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; } template bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, - const IDictionaryBase * dictionary) + const std::shared_ptr & dict_ptr) { - const auto dict = typeid_cast(dictionary); + const auto dict = typeid_cast(dict_ptr.get()); if (!dict) return false; @@ -1874,7 +1844,7 @@ private: } template - bool execute(Block & block, const size_t result, const DictionaryType * dictionary, + bool execute(Block & block, const size_t result, const DictionaryType * dict, const ColumnUInt64 * child_id_col, const IColumn * ancestor_id_col_untyped) { if (const auto ancestor_id_col = checkAndGetColumn(ancestor_id_col_untyped)) @@ -1887,7 +1857,7 @@ private: const auto size = child_id_col->size(); data.resize(size); - dictionary->isInVectorVector(child_ids, ancestor_ids, data); + dict->isInVectorVector(child_ids, ancestor_ids, data); block.getByPosition(result).column = std::move(out); } else if (const auto ancestor_id_col_const = checkAndGetColumnConst>(ancestor_id_col_untyped)) @@ -1900,7 +1870,7 @@ private: const auto size = child_id_col->size(); data.resize(size); - dictionary->isInVectorConstant(child_ids, ancestor_id, data); + dict->isInVectorConstant(child_ids, ancestor_id, data); block.getByPosition(result).column = std::move(out); } else @@ -1913,7 +1883,7 @@ private: } template - bool execute(Block & block, const size_t result, const DictionaryType * dictionary, + bool execute(Block & block, const size_t result, const DictionaryType * dict, const ColumnConst * child_id_col, const IColumn * ancestor_id_col_untyped) { if (const auto ancestor_id_col = checkAndGetColumn(ancestor_id_col_untyped)) @@ -1926,7 +1896,7 @@ private: const auto size = child_id_col->size(); data.resize(size); - dictionary->isInConstantVector(child_id, ancestor_ids, data); + dict->isInConstantVector(child_id, ancestor_ids, data); block.getByPosition(result).column = std::move(out); } else if (const auto ancestor_id_col_const = checkAndGetColumnConst>(ancestor_id_col_untyped)) @@ -1935,7 +1905,7 @@ private: const auto ancestor_id = ancestor_id_col_const->getValue(); UInt8 res = 0; - dictionary->isInConstantConstant(child_id, ancestor_id, res); + dict->isInConstantConstant(child_id, ancestor_id, res); block.getByPosition(result).column = DataTypeUInt8().createColumnConst(child_id_col->size(), res); } else @@ -1945,8 +1915,7 @@ private: return true; } - const ExternalDictionariesLoader & dictionaries_loader; - const Context & context; + mutable FunctionDictHelper helper; }; diff --git a/src/Functions/FunctionsExternalModels.cpp b/src/Functions/FunctionsExternalModels.cpp index a7ec5947c4f..f570fb661dd 100644 --- a/src/Functions/FunctionsExternalModels.cpp +++ b/src/Functions/FunctionsExternalModels.cpp @@ -100,7 +100,7 @@ void FunctionModelEvaluate::executeImpl(Block & block, const ColumnNumbers & arg null_map = col_nullable->getNullMapColumnPtr(); else { - auto mut_null_map = (*std::move(null_map)).mutate(); + auto mut_null_map = IColumn::mutate(std::move(null_map)); NullMap & result_null_map = assert_cast(*mut_null_map).getData(); const NullMap & src_null_map = col_nullable->getNullMapColumn().getData(); diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp index 618caab232e..4bb702015df 100644 --- a/src/Functions/IFunction.cpp +++ b/src/Functions/IFunction.cpp @@ -147,7 +147,7 @@ ColumnPtr wrapInNullable(const ColumnPtr & src, const Block & block, const Colum } else { - MutableColumnPtr mutable_result_null_map_column = (*std::move(result_null_map_column)).mutate(); + MutableColumnPtr mutable_result_null_map_column = IColumn::mutate(std::move(result_null_map_column)); NullMap & result_null_map = assert_cast(*mutable_result_null_map_column).getData(); const NullMap & src_null_map = assert_cast(*null_map_column).getData(); diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index 3313f2a8e6d..ee4c5c083d8 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -222,7 +222,7 @@ public: } /// Put all the necessary columns multiplied by the sizes of arrays into the block. - auto replicated_column_function_ptr = (*column_function->replicate(column_first_array->getOffsets())).mutate(); + auto replicated_column_function_ptr = IColumn::mutate(column_function->replicate(column_first_array->getOffsets())); auto * replicated_column_function = typeid_cast(replicated_column_function_ptr.get()); replicated_column_function->appendArguments(arrays); diff --git a/src/Functions/array/arrayUniq.cpp b/src/Functions/array/arrayUniq.cpp index 4b09e725f25..02129781c13 100644 --- a/src/Functions/array/arrayUniq.cpp +++ b/src/Functions/array/arrayUniq.cpp @@ -119,7 +119,8 @@ private: void FunctionArrayUniq::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) { const ColumnArray::Offsets * offsets = nullptr; - size_t num_arguments = arguments.size(); + const size_t num_arguments = arguments.size(); + assert(num_arguments > 0); ColumnRawPtrs data_columns(num_arguments); Columns array_holders; diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index d83fbed30e9..5bdc070462c 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -116,14 +116,17 @@ private: void executeFormatImpl(Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { + const size_t num_arguments = arguments.size(); + assert(num_arguments >= 2); + auto c_res = ColumnString::create(); - std::vector data(arguments.size()); - std::vector offsets(arguments.size()); - std::vector fixed_string_sizes(arguments.size()); - std::vector constant_strings(arguments.size()); + std::vector data(num_arguments); + std::vector offsets(num_arguments); + std::vector fixed_string_sizes(num_arguments); + std::vector constant_strings(num_arguments); bool has_column_string = false; bool has_column_fixed_string = false; - for (size_t i = 0; i < arguments.size(); ++i) + for (size_t i = 0; i < num_arguments; ++i) { const ColumnPtr & column = block.getByPosition(arguments[i]).column; if (const ColumnString * col = checkAndGetColumn(column.get())) @@ -148,9 +151,9 @@ private: } String pattern; - pattern.reserve(2 * arguments.size()); + pattern.reserve(2 * num_arguments); - for (size_t i = 0; i < arguments.size(); ++i) + for (size_t i = 0; i < num_arguments; ++i) pattern += "{}"; FormatImpl::formatExecute( diff --git a/src/Functions/currentQuota.cpp b/src/Functions/currentQuota.cpp deleted file mode 100644 index b16a8a7c1ec..00000000000 --- a/src/Functions/currentQuota.cpp +++ /dev/null @@ -1,135 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -class FunctionCurrentQuota : public IFunction -{ - const String quota_name; - -public: - static constexpr auto name = "currentQuota"; - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getQuota()->getUsageInfo().quota_name); - } - - explicit FunctionCurrentQuota(const String & quota_name_) : quota_name{quota_name_} - { - } - - String getName() const override - { - return name; - } - size_t getNumberOfArguments() const override - { - return 0; - } - - DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override - { - return std::make_shared(); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) override - { - block.getByPosition(result).column = DataTypeString().createColumnConst(input_rows_count, quota_name); - } -}; - - -class FunctionCurrentQuotaId : public IFunction -{ - const UUID quota_id; - -public: - static constexpr auto name = "currentQuotaID"; - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getQuota()->getUsageInfo().quota_id); - } - - explicit FunctionCurrentQuotaId(const UUID quota_id_) : quota_id{quota_id_} - { - } - - String getName() const override - { - return name; - } - size_t getNumberOfArguments() const override - { - return 0; - } - - DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override - { - return std::make_shared(); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) override - { - block.getByPosition(result).column = DataTypeUUID().createColumnConst(input_rows_count, quota_id); - } -}; - - -class FunctionCurrentQuotaKey : public IFunction -{ - const String quota_key; - -public: - static constexpr auto name = "currentQuotaKey"; - static FunctionPtr create(const Context & context) - { - return std::make_shared(context.getQuota()->getUsageInfo().quota_key); - } - - explicit FunctionCurrentQuotaKey(const String & quota_key_) : quota_key{quota_key_} - { - } - - String getName() const override - { - return name; - } - size_t getNumberOfArguments() const override - { - return 0; - } - - DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override - { - return std::make_shared(); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) override - { - block.getByPosition(result).column = DataTypeString().createColumnConst(input_rows_count, quota_key); - } -}; - - -void registerFunctionCurrentQuota(FunctionFactory & factory) -{ - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); -} - -} diff --git a/src/Functions/currentRowPolicies.cpp b/src/Functions/currentRowPolicies.cpp deleted file mode 100644 index 0248f77c9b5..00000000000 --- a/src/Functions/currentRowPolicies.cpp +++ /dev/null @@ -1,237 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; -} - - -/// The currentRowPolicies() function can be called with 0..2 arguments: -/// currentRowPolicies() returns array of tuples (database, table_name, row_policy_name) for all the row policies applied for the current user; -/// currentRowPolicies(table_name) is equivalent to currentRowPolicies(currentDatabase(), table_name); -/// currentRowPolicies(database, table_name) returns array of names of the row policies applied to a specific table and for the current user. -class FunctionCurrentRowPolicies : public IFunction -{ -public: - static constexpr auto name = "currentRowPolicies"; - - static FunctionPtr create(const Context & context_) { return std::make_shared(context_); } - explicit FunctionCurrentRowPolicies(const Context & context_) : context(context_) {} - - String getName() const override { return name; } - size_t getNumberOfArguments() const override { return 0; } - bool isVariadic() const override { return true; } - - void checkNumberOfArgumentsIfVariadic(size_t number_of_arguments) const override - { - if (number_of_arguments > 2) - throw Exception("Number of arguments for function " + String(name) + " doesn't match: passed " - + toString(number_of_arguments) + ", should be 0..2", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (arguments.empty()) - return std::make_shared(std::make_shared( - DataTypes{std::make_shared(), std::make_shared(), std::make_shared()})); - else - return std::make_shared(std::make_shared()); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result_pos, size_t input_rows_count) override - { - if (arguments.empty()) - { - auto database_column = ColumnString::create(); - auto table_name_column = ColumnString::create(); - auto policy_name_column = ColumnString::create(); - if (auto policies = context.getRowPolicies()) - { - for (const auto & policy_id : policies->getCurrentPolicyIDs()) - { - const auto policy = context.getAccessControlManager().tryRead(policy_id); - if (policy) - { - const String database = policy->getDatabase(); - const String table_name = policy->getTableName(); - const String policy_name = policy->getName(); - database_column->insertData(database.data(), database.length()); - table_name_column->insertData(table_name.data(), table_name.length()); - policy_name_column->insertData(policy_name.data(), policy_name.length()); - } - } - } - auto offset_column = ColumnArray::ColumnOffsets::create(); - offset_column->insertValue(policy_name_column->size()); - block.getByPosition(result_pos).column = ColumnConst::create( - ColumnArray::create( - ColumnTuple::create(Columns{std::move(database_column), std::move(table_name_column), std::move(policy_name_column)}), - std::move(offset_column)), - input_rows_count); - return; - } - - const IColumn * database_column = nullptr; - if (arguments.size() == 2) - { - const auto & database_column_with_type = block.getByPosition(arguments[0]); - if (!isStringOrFixedString(database_column_with_type.type)) - throw Exception{"The first argument of function " + String(name) - + " should be a string containing database name, illegal type: " - + database_column_with_type.type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - database_column = database_column_with_type.column.get(); - } - - const auto & table_name_column_with_type = block.getByPosition(arguments[arguments.size() - 1]); - if (!isStringOrFixedString(table_name_column_with_type.type)) - throw Exception{"The" + String(database_column ? " last" : "") + " argument of function " + String(name) - + " should be a string containing table name, illegal type: " + table_name_column_with_type.type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - const IColumn * table_name_column = table_name_column_with_type.column.get(); - - auto policy_name_column = ColumnString::create(); - auto offset_column = ColumnArray::ColumnOffsets::create(); - for (const auto i : ext::range(0, input_rows_count)) - { - String database = database_column ? database_column->getDataAt(i).toString() : context.getCurrentDatabase(); - String table_name = table_name_column->getDataAt(i).toString(); - if (auto policies = context.getRowPolicies()) - { - for (const auto & policy_id : policies->getCurrentPolicyIDs(database, table_name)) - { - const auto policy = context.getAccessControlManager().tryRead(policy_id); - if (policy) - { - const String policy_name = policy->getName(); - policy_name_column->insertData(policy_name.data(), policy_name.length()); - } - } - } - offset_column->insertValue(policy_name_column->size()); - } - - block.getByPosition(result_pos).column = ColumnArray::create(std::move(policy_name_column), std::move(offset_column)); - } - -private: - const Context & context; -}; - - -/// The currentRowPolicyIDs() function can be called with 0..2 arguments: -/// currentRowPolicyIDs() returns array of IDs of all the row policies applied for the current user; -/// currentRowPolicyIDs(table_name) is equivalent to currentRowPolicyIDs(currentDatabase(), table_name); -/// currentRowPolicyIDs(database, table_name) returns array of IDs of the row policies applied to a specific table and for the current user. -class FunctionCurrentRowPolicyIDs : public IFunction -{ -public: - static constexpr auto name = "currentRowPolicyIDs"; - - static FunctionPtr create(const Context & context_) { return std::make_shared(context_); } - explicit FunctionCurrentRowPolicyIDs(const Context & context_) : context(context_) {} - - String getName() const override { return name; } - size_t getNumberOfArguments() const override { return 0; } - bool isVariadic() const override { return true; } - - void checkNumberOfArgumentsIfVariadic(size_t number_of_arguments) const override - { - if (number_of_arguments > 2) - throw Exception("Number of arguments for function " + String(name) + " doesn't match: passed " - + toString(number_of_arguments) + ", should be 0..2", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - } - - DataTypePtr getReturnTypeImpl(const DataTypes & /* arguments */) const override - { - return std::make_shared(std::make_shared()); - } - - bool isDeterministic() const override { return false; } - - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result_pos, size_t input_rows_count) override - { - if (arguments.empty()) - { - auto policy_id_column = ColumnVector::create(); - if (auto policies = context.getRowPolicies()) - { - for (const auto & policy_id : policies->getCurrentPolicyIDs()) - policy_id_column->insertValue(policy_id); - } - auto offset_column = ColumnArray::ColumnOffsets::create(); - offset_column->insertValue(policy_id_column->size()); - block.getByPosition(result_pos).column - = ColumnConst::create(ColumnArray::create(std::move(policy_id_column), std::move(offset_column)), input_rows_count); - return; - } - - const IColumn * database_column = nullptr; - if (arguments.size() == 2) - { - const auto & database_column_with_type = block.getByPosition(arguments[0]); - if (!isStringOrFixedString(database_column_with_type.type)) - throw Exception{"The first argument of function " + String(name) - + " should be a string containing database name, illegal type: " - + database_column_with_type.type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - database_column = database_column_with_type.column.get(); - } - - const auto & table_name_column_with_type = block.getByPosition(arguments[arguments.size() - 1]); - if (!isStringOrFixedString(table_name_column_with_type.type)) - throw Exception{"The" + String(database_column ? " last" : "") + " argument of function " + String(name) - + " should be a string containing table name, illegal type: " + table_name_column_with_type.type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - const IColumn * table_name_column = table_name_column_with_type.column.get(); - - auto policy_id_column = ColumnVector::create(); - auto offset_column = ColumnArray::ColumnOffsets::create(); - for (const auto i : ext::range(0, input_rows_count)) - { - String database = database_column ? database_column->getDataAt(i).toString() : context.getCurrentDatabase(); - String table_name = table_name_column->getDataAt(i).toString(); - if (auto policies = context.getRowPolicies()) - { - for (const auto & policy_id : policies->getCurrentPolicyIDs(database, table_name)) - policy_id_column->insertValue(policy_id); - } - offset_column->insertValue(policy_id_column->size()); - } - - block.getByPosition(result_pos).column = ColumnArray::create(std::move(policy_id_column), std::move(offset_column)); - } - -private: - const Context & context; -}; - - -void registerFunctionCurrentRowPolicies(FunctionFactory & factory) -{ - factory.registerFunction(); - factory.registerFunction(); -} - -} diff --git a/src/Functions/finalizeAggregation.cpp b/src/Functions/finalizeAggregation.cpp index 66c6268841b..522a645b8e7 100644 --- a/src/Functions/finalizeAggregation.cpp +++ b/src/Functions/finalizeAggregation.cpp @@ -65,7 +65,7 @@ public: ErrorCodes::ILLEGAL_COLUMN); /// Column is copied here, because there is no guarantee that we own it. - auto mut_column = (*std::move(column)).mutate(); + auto mut_column = IColumn::mutate(std::move(column)); block.getByPosition(result).column = ColumnAggregateFunction::convertToValues(std::move(mut_column)); } }; diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp index 58dbf74af5e..02c3d938d2b 100644 --- a/src/Functions/if.cpp +++ b/src/Functions/if.cpp @@ -816,7 +816,7 @@ private: if (isColumnNullable(*arg_else.column)) { auto arg_else_column = arg_else.column; - auto result_column = (*std::move(arg_else_column)).mutate(); + auto result_column = IColumn::mutate(std::move(arg_else_column)); assert_cast(*result_column).applyNullMap(assert_cast(*arg_cond.column)); block.getByPosition(result).column = std::move(result_column); } @@ -858,7 +858,7 @@ private: if (isColumnNullable(*arg_then.column)) { auto arg_then_column = arg_then.column; - auto result_column = (*std::move(arg_then_column)).mutate(); + auto result_column = IColumn::mutate(std::move(arg_then_column)); assert_cast(*result_column).applyNegatedNullMap(assert_cast(*arg_cond.column)); block.getByPosition(result).column = std::move(result_column); } diff --git a/src/Functions/ignoreExceptNull.cpp b/src/Functions/ignoreExceptNull.cpp deleted file mode 100644 index ff009533e37..00000000000 --- a/src/Functions/ignoreExceptNull.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include -#include -#include -#include - - -namespace DB -{ - -/** ignoreExceptNull(...) is a function that takes any arguments, and always returns 0 except Null. - */ - class FunctionIgnoreExceptNull : public IFunction - { - public: - static constexpr auto name = "ignoreExceptNull"; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } - - bool isVariadic() const override - { - return true; - } - size_t getNumberOfArguments() const override - { - return 0; - } - - String getName() const override - { - return name; - } - - DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override - { - return std::make_shared(); - } - - void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) override - { - /// This function is mainly used in query analysis instead of "in" functions - /// in the case when only header is needed and set for in is not calculated. - /// Because of that function must return the same column type as "in" function, which is ColumnUInt8. - auto res = ColumnUInt8::create(input_rows_count, 0); - block.getByPosition(result).column = std::move(res); - } - }; - - - void registerFunctionIgnoreExceptNull(FunctionFactory & factory) - { - factory.registerFunction(); - } - -} diff --git a/src/Functions/in.cpp b/src/Functions/in.cpp index a89535c675a..81b75f67764 100644 --- a/src/Functions/in.cpp +++ b/src/Functions/in.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -21,62 +22,34 @@ namespace ErrorCodes * notIn(x, set) - and NOT IN. */ -template +template struct FunctionInName; -template <> -struct FunctionInName -{ - static constexpr auto name = "in"; -}; +template <> struct FunctionInName { static constexpr auto name = "in"; }; +template <> struct FunctionInName { static constexpr auto name = "globalIn"; }; +template <> struct FunctionInName { static constexpr auto name = "notIn"; }; +template <> struct FunctionInName { static constexpr auto name = "globalNotIn"; }; +template <> struct FunctionInName { static constexpr auto name = "nullIn"; }; +template <> struct FunctionInName { static constexpr auto name = "globalNullIn"; }; +template <> struct FunctionInName { static constexpr auto name = "notNullIn"; }; +template <> struct FunctionInName { static constexpr auto name = "globalNotNullIn"; }; +template <> struct FunctionInName { static constexpr auto name = "inIgnoreSet"; }; +template <> struct FunctionInName { static constexpr auto name = "globalInIgnoreSet"; }; +template <> struct FunctionInName { static constexpr auto name = "notInIgnoreSet"; }; +template <> struct FunctionInName { static constexpr auto name = "globalNotInIgnoreSet"; }; +template <> struct FunctionInName { static constexpr auto name = "nullInIgnoreSet"; }; +template <> struct FunctionInName { static constexpr auto name = "globalNullInIgnoreSet"; }; +template <> struct FunctionInName { static constexpr auto name = "notNullInIgnoreSet"; }; +template <> struct FunctionInName { static constexpr auto name = "globalNotNullInIgnoreSet"; }; -template <> -struct FunctionInName -{ - static constexpr auto name = "globalIn"; -}; - -template <> -struct FunctionInName -{ - static constexpr auto name = "notIn"; -}; - -template <> -struct FunctionInName -{ - static constexpr auto name = "globalNotIn"; -}; - -template <> -struct FunctionInName -{ - static constexpr auto name = "nullIn"; -}; - -template <> -struct FunctionInName -{ - static constexpr auto name = "globalNullIn"; -}; - -template <> -struct FunctionInName -{ - static constexpr auto name = "notNullIn"; -}; - -template <> -struct FunctionInName -{ - static constexpr auto name = "globalNotNullIn"; -}; - -template +template class FunctionIn : public IFunction { public: - static constexpr auto name = FunctionInName::name; + /// ignore_set flag means that we don't use set from the second argument, just return zero column. + /// It is needed to perform type analysis without creation of set. + static constexpr auto name = FunctionInName::name; + static FunctionPtr create(const Context &) { return std::make_shared(); @@ -97,13 +70,21 @@ public: return std::make_shared(); } - bool useDefaultImplementationForConstants() const override { return true; } + bool useDefaultImplementationForConstants() const override + { + /// Never return constant for -IgnoreSet functions to avoid constant folding. + return !ignore_set; + } bool useDefaultImplementationForNulls() const override { return null_is_skipped; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, [[maybe_unused]] size_t input_rows_count) override { - /// NOTE: after updating this code, check that FunctionIgnoreExceptNull returns the same type of column. + if constexpr (ignore_set) + { + block.getByPosition(result).column = ColumnUInt8::create(input_rows_count, 0u); + return; + } /// Second argument must be ColumnSet. ColumnPtr column_set_ptr = block.getByPosition(arguments[1]).column; @@ -146,17 +127,23 @@ public: } }; +template +static void registerFunctionsInImpl(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); +} void registerFunctionsIn(FunctionFactory & factory) { - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); + registerFunctionsInImpl(factory); + registerFunctionsInImpl(factory); } } diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 221e14fcce1..3e6099ee25a 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -9,8 +9,6 @@ class FunctionFactory; void registerFunctionCurrentDatabase(FunctionFactory &); void registerFunctionCurrentUser(FunctionFactory &); -void registerFunctionCurrentQuota(FunctionFactory &); -void registerFunctionCurrentRowPolicies(FunctionFactory &); void registerFunctionHostName(FunctionFactory &); void registerFunctionFQDN(FunctionFactory &); void registerFunctionVisibleWidth(FunctionFactory &); @@ -29,7 +27,6 @@ void registerFunctionSleep(FunctionFactory &); void registerFunctionSleepEachRow(FunctionFactory &); void registerFunctionMaterialize(FunctionFactory &); void registerFunctionIgnore(FunctionFactory &); -void registerFunctionIgnoreExceptNull(FunctionFactory &); void registerFunctionIdentity(FunctionFactory &); void registerFunctionArrayJoin(FunctionFactory &); void registerFunctionReplicate(FunctionFactory &); @@ -68,8 +65,6 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) { registerFunctionCurrentDatabase(factory); registerFunctionCurrentUser(factory); - registerFunctionCurrentQuota(factory); - registerFunctionCurrentRowPolicies(factory); registerFunctionHostName(factory); registerFunctionFQDN(factory); registerFunctionVisibleWidth(factory); @@ -88,7 +83,6 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionSleepEachRow(factory); registerFunctionMaterialize(factory); registerFunctionIgnore(factory); - registerFunctionIgnoreExceptNull(factory); registerFunctionIdentity(factory); registerFunctionArrayJoin(factory); registerFunctionReplicate(factory); diff --git a/src/Functions/ya.make b/src/Functions/ya.make index afce90f0dee..999a63e8fa3 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -127,8 +127,6 @@ SRCS( cos.cpp CRC.cpp currentDatabase.cpp - currentQuota.cpp - currentRowPolicies.cpp currentUser.cpp dateDiff.cpp defaultValueOfArgumentType.cpp @@ -207,7 +205,6 @@ SRCS( ifNull.cpp IFunction.cpp ignore.cpp - ignoreExceptNull.cpp in.cpp intDiv.cpp intDivOrZero.cpp diff --git a/src/IO/WriteBufferFromFileDescriptor.cpp b/src/IO/WriteBufferFromFileDescriptor.cpp index fe528bdf810..9710c5a979b 100644 --- a/src/IO/WriteBufferFromFileDescriptor.cpp +++ b/src/IO/WriteBufferFromFileDescriptor.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -90,6 +91,8 @@ WriteBufferFromFileDescriptor::~WriteBufferFromFileDescriptor() { if (fd >= 0) next(); + else + assert(!offset() && "attempt to write after close"); } catch (...) { diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 4d032e552ca..81f19d0b3dc 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -381,11 +381,13 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & if (!data.only_consts) { /// We are in the part of the tree that we are not going to compute. You just need to define types. - /// Do not subquery and create sets. We treat "IN" as "ignoreExceptNull" function. + /// Do not subquery and create sets. We replace "in*" function to "in*IgnoreSet". + + auto argument_name = node.arguments->children.at(0)->getColumnName(); data.addAction(ExpressionAction::applyFunction( - FunctionFactory::instance().get("ignoreExceptNull", data.context), - { node.arguments->children.at(0)->getColumnName() }, + FunctionFactory::instance().get(node.name + "IgnoreSet", data.context), + { argument_name, argument_name }, column_name.get(ast))); } return; diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index 151dfc5c9bb..7fad9af8960 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -328,8 +328,8 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, const Setting /// In case of internal_replication we will be appending names to dir_name_for_internal_replication std::string dir_name_for_internal_replication; + std::string dir_name_for_internal_replication_with_local; - auto first = true; for (const auto & replica_key : replica_keys) { if (startsWith(replica_key, "weight") || startsWith(replica_key, "internal_replication")) @@ -340,18 +340,20 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, const Setting replica_addresses.emplace_back(config, partial_prefix + replica_key, current_shard_num, current_replica_num); ++current_replica_num; - if (!replica_addresses.back().is_local) + if (internal_replication) { - if (internal_replication) + auto dir_name = replica_addresses.back().toFullString(settings.use_compact_format_in_distributed_parts_names); + if (!replica_addresses.back().is_local) { - auto dir_name = replica_addresses.back().toFullString(settings.use_compact_format_in_distributed_parts_names); - if (first) + if (dir_name_for_internal_replication.empty()) dir_name_for_internal_replication = dir_name; else dir_name_for_internal_replication += "," + dir_name; } - - if (first) first = false; + if (dir_name_for_internal_replication_with_local.empty()) + dir_name_for_internal_replication_with_local = dir_name; + else + dir_name_for_internal_replication_with_local += "," + dir_name; } } else @@ -383,8 +385,16 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, const Setting if (weight) slot_to_shard.insert(std::end(slot_to_shard), weight, shards_info.size()); - shards_info.push_back({std::move(dir_name_for_internal_replication), current_shard_num, weight, - std::move(shard_local_addresses), std::move(shard_pool), std::move(all_replicas_pools), internal_replication}); + shards_info.push_back({ + std::move(dir_name_for_internal_replication), + std::move(dir_name_for_internal_replication_with_local), + current_shard_num, + weight, + std::move(shard_local_addresses), + std::move(shard_pool), + std::move(all_replicas_pools), + internal_replication + }); } else throw Exception("Unknown element in config: " + key, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); @@ -433,8 +443,16 @@ Cluster::Cluster(const Settings & settings, const std::vector #include #include +#include #include +#include #include #include #include @@ -681,14 +683,12 @@ void Context::setUser(const String & name, const String & password, const Poco:: std::shared_ptr Context::getUser() const { - auto lock = getLock(); - return access->getUser(); + return getAccess()->getUser(); } String Context::getUserName() const { - auto lock = getLock(); - return access->getUserName(); + return getAccess()->getUserName(); } std::optional Context::getUserID() const @@ -698,7 +698,7 @@ std::optional Context::getUserID() const } -void Context::setCurrentRoles(const std::vector & current_roles_) +void Context::setCurrentRoles(const boost::container::flat_set & current_roles_) { auto lock = getLock(); if (current_roles == current_roles_ && !use_default_roles) @@ -718,24 +718,19 @@ void Context::setCurrentRolesDefault() calculateAccessRights(); } -std::vector Context::getCurrentRoles() const +boost::container::flat_set Context::getCurrentRoles() const { - return getAccess()->getCurrentRoles(); + return getRolesInfo()->current_roles; } -Strings Context::getCurrentRolesNames() const +boost::container::flat_set Context::getEnabledRoles() const { - return getAccess()->getCurrentRolesNames(); + return getRolesInfo()->enabled_roles; } -std::vector Context::getEnabledRoles() const +std::shared_ptr Context::getRolesInfo() const { - return getAccess()->getEnabledRoles(); -} - -Strings Context::getEnabledRolesNames() const -{ - return getAccess()->getEnabledRolesNames(); + return getAccess()->getRolesInfo(); } @@ -780,11 +775,6 @@ ASTPtr Context::getRowPolicyCondition(const String & database, const String & ta return getAccess()->getRowPolicyCondition(database, table_name, type, initial_condition); } -std::shared_ptr Context::getRowPolicies() const -{ - return getAccess()->getRowPolicies(); -} - void Context::setInitialRowPolicy() { auto lock = getLock(); @@ -801,6 +791,12 @@ std::shared_ptr Context::getQuota() const } +std::optional Context::getQuotaUsage() const +{ + return getAccess()->getQuotaUsage(); +} + + void Context::setProfile(const String & profile_name) { applySettingsChanges(*getAccessControlManager().getProfileSettings(profile_name)); @@ -1011,8 +1007,7 @@ void Context::clampToSettingsConstraints(SettingsChanges & changes) const std::shared_ptr Context::getSettingsConstraints() const { - auto lock = getLock(); - return access->getSettingsConstraints(); + return getAccess()->getSettingsConstraints(); } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 0d2b3cdb5af..31d7b1f1e13 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -51,8 +51,10 @@ class Context; class ContextAccess; struct User; using UserPtr = std::shared_ptr; +struct EnabledRolesInfo; class EnabledRowPolicies; class EnabledQuota; +struct QuotaUsage; class AccessFlags; struct AccessRightsElement; class AccessRightsElements; @@ -166,7 +168,7 @@ private: InputBlocksReader input_blocks_reader; std::optional user_id; - std::vector current_roles; + boost::container::flat_set current_roles; bool use_default_roles = false; std::shared_ptr access; std::shared_ptr initial_row_policy; @@ -254,12 +256,11 @@ public: String getUserName() const; std::optional getUserID() const; - void setCurrentRoles(const std::vector & current_roles_); + void setCurrentRoles(const boost::container::flat_set & current_roles_); void setCurrentRolesDefault(); - std::vector getCurrentRoles() const; - Strings getCurrentRolesNames() const; - std::vector getEnabledRoles() const; - Strings getEnabledRolesNames() const; + boost::container::flat_set getCurrentRoles() const; + boost::container::flat_set getEnabledRoles() const; + std::shared_ptr getRolesInfo() const; /// Checks access rights. /// Empty database means the current database. @@ -278,7 +279,6 @@ public: std::shared_ptr getAccess() const; - std::shared_ptr getRowPolicies() const; ASTPtr getRowPolicyCondition(const String & database, const String & table_name, RowPolicy::ConditionType type) const; /// Sets an extra row policy based on `client_info.initial_user`, if it exists. @@ -287,6 +287,7 @@ public: void setInitialRowPolicy(); std::shared_ptr getQuota() const; + std::optional getQuotaUsage() const; /// We have to copy external tables inside executeQuery() to track limits. Therefore, set callback for it. Must set once. void setExternalTablesInitializer(ExternalTablesInitializer && initializer); diff --git a/src/Interpreters/DictionaryReader.cpp b/src/Interpreters/DictionaryReader.cpp index e996be65095..61d40748617 100644 --- a/src/Interpreters/DictionaryReader.cpp +++ b/src/Interpreters/DictionaryReader.cpp @@ -119,7 +119,7 @@ void DictionaryReader::readKeys(const IColumn & keys, Block & out_block, ColumnV /// calculate and extract dictHas() function_has->execute(working_block, size); ColumnWithTypeAndName & has_column = working_block.getByPosition(has_position); - auto mutable_has = (*std::move(has_column.column)).mutate(); + auto mutable_has = IColumn::mutate(std::move(has_column.column)); found.swap(typeid_cast &>(*mutable_has).getData()); has_column.column = nullptr; diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index e5b3f9e55f3..0e463a84dc4 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -108,7 +108,7 @@ static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column, JoinCommon::convertColumnToNullable(column); if (column.type->isNullable() && !negative_null_map.empty()) { - MutableColumnPtr mutable_column = (*std::move(column.column)).mutate(); + MutableColumnPtr mutable_column = IColumn::mutate(std::move(column.column)); assert_cast(*mutable_column).applyNegatedNullMap(negative_null_map); column.column = std::move(mutable_column); } @@ -127,7 +127,7 @@ static void changeNullability(MutableColumnPtr & mutable_column) else column = makeNullable(column); - mutable_column = (*std::move(column)).mutate(); + mutable_column = IColumn::mutate(std::move(column)); } static ColumnPtr emptyNotNullableClone(const ColumnPtr & column) diff --git a/src/Interpreters/InterpreterCreateQuotaQuery.cpp b/src/Interpreters/InterpreterCreateQuotaQuery.cpp index 80987993c96..907532c3d89 100644 --- a/src/Interpreters/InterpreterCreateQuotaQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuotaQuery.cpp @@ -56,12 +56,7 @@ void updateQuotaFromQueryImpl(Quota & quota, const ASTCreateQuotaQuery & query, auto & quota_limits = *it; quota_limits.randomize_interval = query_limits.randomize_interval; for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) - { - if (query_limits.max[resource_type]) - quota_limits.max[resource_type] = *query_limits.max[resource_type]; - else - quota_limits.max[resource_type] = Quota::UNLIMITED; - } + quota_limits.max[resource_type] = query_limits.max[resource_type]; } const ExtendedRoleSet * roles = nullptr; diff --git a/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp b/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp index c3de3876c46..778a32019d9 100644 --- a/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp +++ b/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp @@ -11,43 +11,30 @@ namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - namespace { - const String & checkCurrentDatabase(const String & current_database) - { - if (current_database.empty()) - throw Exception("No current database", ErrorCodes::LOGICAL_ERROR); - return current_database; - } - void updateRowPolicyFromQueryImpl( RowPolicy & policy, const ASTCreateRowPolicyQuery & query, - const std::optional & roles_from_query = {}, - const String & current_database = {}) + const std::optional & roles_from_query = {}) { if (query.alter) { - if (!query.new_policy_name.empty()) - policy.setName(query.new_policy_name); + if (!query.new_short_name.empty()) + policy.setShortName(query.new_short_name); } else - { - policy.setDatabase(!query.name_parts.database.empty() ? query.name_parts.database : checkCurrentDatabase(current_database)); - policy.setTableName(query.name_parts.table_name); - policy.setName(query.name_parts.policy_name); - } + policy.setNameParts(query.name_parts); if (query.is_restrictive) policy.setRestrictive(*query.is_restrictive); - for (const auto & [index, condition] : query.conditions) - policy.conditions[index] = condition ? serializeAST(*condition) : String{}; + for (auto condition_type : ext::range(RowPolicy::MAX_CONDITION_TYPE)) + { + const auto & condition = query.conditions[condition_type]; + if (condition) + policy.conditions[condition_type] = *condition ? serializeAST(**condition) : String{}; + } const ExtendedRoleSet * roles = nullptr; std::optional temp_role_set; @@ -78,29 +65,29 @@ BlockIO InterpreterCreateRowPolicyQuery::execute() if (query.roles) roles_from_query = ExtendedRoleSet{*query.roles, access_control, context.getUserID()}; - const String current_database = context.getCurrentDatabase(); + if (query.name_parts.database.empty()) + query.name_parts.database = context.getCurrentDatabase(); if (query.alter) { auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr { auto updated_policy = typeid_cast>(entity->clone()); - updateRowPolicyFromQueryImpl(*updated_policy, query, roles_from_query, current_database); + updateRowPolicyFromQueryImpl(*updated_policy, query, roles_from_query); return updated_policy; }; - String full_name = query.name_parts.getFullName(context); if (query.if_exists) { - if (auto id = access_control.find(full_name)) + if (auto id = access_control.find(query.name_parts.getName())) access_control.tryUpdate(*id, update_func); } else - access_control.update(access_control.getID(full_name), update_func); + access_control.update(access_control.getID(query.name_parts.getName()), update_func); } else { auto new_policy = std::make_shared(); - updateRowPolicyFromQueryImpl(*new_policy, query, roles_from_query, current_database); + updateRowPolicyFromQueryImpl(*new_policy, query, roles_from_query); if (query.if_not_exists) access_control.tryInsert(new_policy); diff --git a/src/Interpreters/InterpreterCreateUserQuery.cpp b/src/Interpreters/InterpreterCreateUserQuery.cpp index e4900e5c518..7c488ddf8e9 100644 --- a/src/Interpreters/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/InterpreterCreateUserQuery.cpp @@ -48,7 +48,7 @@ namespace if (default_roles) { if (!query.alter && !default_roles->all) - boost::range::copy(default_roles->getMatchingIDs(), std::inserter(user.granted_roles, user.granted_roles.end())); + user.granted_roles.grant(default_roles->getMatchingIDs()); InterpreterSetRoleQuery::updateUserSetDefaultRoles(user, *default_roles); } diff --git a/src/Interpreters/InterpreterDropAccessEntityQuery.cpp b/src/Interpreters/InterpreterDropAccessEntityQuery.cpp index e67e0659796..be82147a322 100644 --- a/src/Interpreters/InterpreterDropAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterDropAccessEntityQuery.cpp @@ -9,71 +9,66 @@ #include #include #include -#include namespace DB { -namespace +namespace ErrorCodes { - using Kind = ASTDropAccessEntityQuery::Kind; - - std::type_index getType(Kind kind) - { - switch (kind) - { - case Kind::USER: return typeid(User); - case Kind::ROLE: return typeid(Role); - case Kind::QUOTA: return typeid(Quota); - case Kind::ROW_POLICY: return typeid(RowPolicy); - case Kind::SETTINGS_PROFILE: return typeid(SettingsProfile); - } - __builtin_unreachable(); - } - - AccessType getRequiredAccessType(Kind kind) - { - switch (kind) - { - case Kind::USER: return AccessType::DROP_USER; - case Kind::ROLE: return AccessType::DROP_ROLE; - case Kind::QUOTA: return AccessType::DROP_QUOTA; - case Kind::ROW_POLICY: return AccessType::DROP_ROW_POLICY; - case Kind::SETTINGS_PROFILE: return AccessType::DROP_SETTINGS_PROFILE; - } - __builtin_unreachable(); - } + extern const int NOT_IMPLEMENTED; } +using EntityType = IAccessEntity::Type; + + BlockIO InterpreterDropAccessEntityQuery::execute() { - const auto & query = query_ptr->as(); + auto & query = query_ptr->as(); auto & access_control = context.getAccessControlManager(); - - std::type_index type = getType(query.kind); - context.checkAccess(getRequiredAccessType(query.kind)); + context.checkAccess(getRequiredAccess()); if (!query.cluster.empty()) return executeDDLQueryOnCluster(query_ptr, context); - if (query.kind == Kind::ROW_POLICY) + if (query.type == EntityType::ROW_POLICY) { - Strings full_names; - boost::range::transform( - query.row_policies_names, std::back_inserter(full_names), - [this](const RowPolicy::FullNameParts & row_policy_name) { return row_policy_name.getFullName(context); }); + Strings names; + for (auto & name_parts : query.row_policies_name_parts) + { + if (name_parts.database.empty()) + name_parts.database = context.getCurrentDatabase(); + names.emplace_back(name_parts.getName()); + } if (query.if_exists) - access_control.tryRemove(access_control.find(full_names)); + access_control.tryRemove(access_control.find(names)); else - access_control.remove(access_control.getIDs(full_names)); + access_control.remove(access_control.getIDs(names)); return {}; } if (query.if_exists) - access_control.tryRemove(access_control.find(type, query.names)); + access_control.tryRemove(access_control.find(query.type, query.names)); else - access_control.remove(access_control.getIDs(type, query.names)); + access_control.remove(access_control.getIDs(query.type, query.names)); return {}; } + +AccessRightsElements InterpreterDropAccessEntityQuery::getRequiredAccess() const +{ + const auto & query = query_ptr->as(); + AccessRightsElements res; + switch (query.type) + { + case EntityType::USER: res.emplace_back(AccessType::DROP_USER); return res; + case EntityType::ROLE: res.emplace_back(AccessType::DROP_ROLE); return res; + case EntityType::SETTINGS_PROFILE: res.emplace_back(AccessType::DROP_SETTINGS_PROFILE); return res; + case EntityType::ROW_POLICY: res.emplace_back(AccessType::DROP_ROW_POLICY); return res; + case EntityType::QUOTA: res.emplace_back(AccessType::DROP_QUOTA); return res; + case EntityType::MAX: break; + } + throw Exception( + toString(query.type) + ": type is not supported by DROP query", ErrorCodes::NOT_IMPLEMENTED); +} + } diff --git a/src/Interpreters/InterpreterDropAccessEntityQuery.h b/src/Interpreters/InterpreterDropAccessEntityQuery.h index 2a0e749b265..0db68a0ad78 100644 --- a/src/Interpreters/InterpreterDropAccessEntityQuery.h +++ b/src/Interpreters/InterpreterDropAccessEntityQuery.h @@ -6,6 +6,8 @@ namespace DB { +class AccessRightsElements; + class InterpreterDropAccessEntityQuery : public IInterpreter { public: @@ -14,6 +16,8 @@ public: BlockIO execute() override; private: + AccessRightsElements getRequiredAccess() const; + ASTPtr query_ptr; Context & context; }; diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 0c34d6ed79f..60302848367 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -16,11 +16,11 @@ #include #include #include +#include #include -#include #include -#include -#include +#include +#include #include #include #include @@ -50,12 +50,12 @@ #include #include #include +#include #include +#include +#include #include #include -#include -#include -#include #include #include #include @@ -226,13 +226,13 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, Context & { return std::make_unique(query, context); } - else if (query->as()) + else if (query->as()) { - return std::make_unique(query, context); + return std::make_unique(query, context); } - else if (query->as()) + else if (query->as()) { - return std::make_unique(query, context); + return std::make_unique(query, context); } else throw Exception("Unknown type of query: " + query->getID(), ErrorCodes::UNKNOWN_TYPE_OF_QUERY); diff --git a/src/Interpreters/InterpreterGrantQuery.cpp b/src/Interpreters/InterpreterGrantQuery.cpp index a5f13dbbbfe..c72e48c2019 100644 --- a/src/Interpreters/InterpreterGrantQuery.cpp +++ b/src/Interpreters/InterpreterGrantQuery.cpp @@ -23,14 +23,16 @@ namespace { if (query.kind == Kind::GRANT) { - grantee.access.grant(query.access_rights_elements, current_database); if (query.grant_option) - grantee.access_with_grant_option.grant(query.access_rights_elements, current_database); + grantee.access.grantWithGrantOption(query.access_rights_elements, current_database); + else + grantee.access.grant(query.access_rights_elements, current_database); } else { - grantee.access_with_grant_option.revoke(query.access_rights_elements, current_database); - if (!query.grant_option) + if (query.grant_option) + grantee.access.revokeGrantOption(query.access_rights_elements, current_database); + else grantee.access.revoke(query.access_rights_elements, current_database); } } @@ -39,18 +41,21 @@ namespace { if (query.kind == Kind::GRANT) { - boost::range::copy(roles_from_query, std::inserter(grantee.granted_roles, grantee.granted_roles.end())); if (query.admin_option) - boost::range::copy(roles_from_query, std::inserter(grantee.granted_roles_with_admin_option, grantee.granted_roles_with_admin_option.end())); + grantee.granted_roles.grantWithAdminOption(roles_from_query); + else + grantee.granted_roles.grant(roles_from_query); } else { - for (const UUID & role_from_query : roles_from_query) + if (query.admin_option) + grantee.granted_roles.revokeAdminOption(roles_from_query); + else + grantee.granted_roles.revoke(roles_from_query); + + if constexpr (std::is_same_v) { - grantee.granted_roles_with_admin_option.erase(role_from_query); - if (!query.admin_option) - grantee.granted_roles.erase(role_from_query); - if constexpr (std::is_same_v) + for (const UUID & role_from_query : roles_from_query) grantee.default_roles.ids.erase(role_from_query); } } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index f09b0fad8b7..6c7a17ffd77 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -423,11 +423,14 @@ InterpreterSelectQuery::InterpreterSelectQuery( context->checkAccess(AccessType::SELECT, left_table_id, required_columns); /// Remove limits for some tables in the `system` database. - if (left_table_id.database_name == "system" && - ((left_table_id.table_name == "quotas") || (left_table_id.table_name == "quota_usage") || (left_table_id.table_name == "one"))) + if (left_table_id.database_name == "system") { - options.ignore_quota = true; - options.ignore_limits = true; + static const boost::container::flat_set system_tables_ignoring_quota{"quotas", "quota_limits", "quota_usage", "quotas_usage", "one"}; + if (system_tables_ignoring_quota.count(left_table_id.table_name)) + { + options.ignore_quota = true; + options.ignore_limits = true; + } } /// Blocks used in expression analysis contains size 1 const columns for constant folding and diff --git a/src/Interpreters/InterpreterSetRoleQuery.cpp b/src/Interpreters/InterpreterSetRoleQuery.cpp index 8f085d66c4c..f8e0167d748 100644 --- a/src/Interpreters/InterpreterSetRoleQuery.cpp +++ b/src/Interpreters/InterpreterSetRoleQuery.cpp @@ -39,20 +39,20 @@ void InterpreterSetRoleQuery::setRole(const ASTSetRoleQuery & query) else { ExtendedRoleSet roles_from_query{*query.roles, access_control}; - std::vector new_current_roles; + boost::container::flat_set new_current_roles; if (roles_from_query.all) { - for (const auto & id : user->granted_roles) + for (const auto & id : user->granted_roles.roles) if (roles_from_query.match(id)) - new_current_roles.push_back(id); + new_current_roles.emplace(id); } else { for (const auto & id : roles_from_query.getMatchingIDs()) { - if (!user->granted_roles.count(id)) + if (!user->granted_roles.roles.count(id)) throw Exception("Role should be granted to set current", ErrorCodes::SET_NON_GRANTED_ROLE); - new_current_roles.push_back(id); + new_current_roles.emplace(id); } } session_context.setCurrentRoles(new_current_roles); @@ -85,7 +85,7 @@ void InterpreterSetRoleQuery::updateUserSetDefaultRoles(User & user, const Exten { for (const auto & id : roles_from_query.getMatchingIDs()) { - if (!user.granted_roles.count(id)) + if (!user.granted_roles.roles.count(id)) throw Exception("Role should be granted to set default", ErrorCodes::SET_NON_GRANTED_ROLE); } } diff --git a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp new file mode 100644 index 00000000000..8d54da790f8 --- /dev/null +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp @@ -0,0 +1,121 @@ +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + +using EntityType = IAccessEntity::Type; + + +InterpreterShowAccessEntitiesQuery::InterpreterShowAccessEntitiesQuery(const ASTPtr & query_ptr_, Context & context_) + : query_ptr(query_ptr_), context(context_), ignore_quota(query_ptr->as().type == EntityType::QUOTA) +{ +} + + +BlockIO InterpreterShowAccessEntitiesQuery::execute() +{ + return executeQuery(getRewrittenQuery(), context, true); +} + + +String InterpreterShowAccessEntitiesQuery::getRewrittenQuery() const +{ + const auto & query = query_ptr->as(); + String origin; + String expr = "*"; + String filter, order; + + switch (query.type) + { + case EntityType::ROW_POLICY: + { + origin = "row_policies"; + expr = "name"; + const String & table_name = query.table_name; + if (!table_name.empty()) + { + String database = query.database; + if (database.empty()) + database = context.getCurrentDatabase(); + filter = "database = " + quoteString(database) + " AND table = " + quoteString(table_name); + expr = "short_name"; + } + break; + } + + case EntityType::QUOTA: + { + if (query.current_quota) + { + origin = "quota_usage"; + order = "duration"; + } + else + { + origin = "quotas"; + expr = "name"; + } + break; + } + + case EntityType::SETTINGS_PROFILE: + { + origin = "settings_profiles"; + expr = "name"; + break; + } + + case EntityType::USER: + { + origin = "users"; + expr = "name"; + break; + } + + case EntityType::ROLE: + { + if (query.current_roles) + { + origin = "current_roles"; + order = "role_name"; + } + else if (query.enabled_roles) + { + origin = "enabled_roles"; + order = "role_name"; + } + else + { + origin = "roles"; + expr = "name"; + } + break; + } + + case EntityType::MAX: + break; + } + + if (origin.empty()) + throw Exception(toString(query.type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); + + if (order.empty() && expr != "*") + order = expr; + + return "SELECT " + expr + " from system." + origin + + (filter.empty() ? "" : " WHERE " + filter) + + (order.empty() ? "" : " ORDER BY " + order); +} + +} diff --git a/src/Interpreters/InterpreterShowAccessEntitiesQuery.h b/src/Interpreters/InterpreterShowAccessEntitiesQuery.h new file mode 100644 index 00000000000..5e20bdfa231 --- /dev/null +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +class InterpreterShowAccessEntitiesQuery : public IInterpreter +{ +public: + InterpreterShowAccessEntitiesQuery(const ASTPtr & query_ptr_, Context & context_); + + BlockIO execute() override; + + bool ignoreQuota() const override { return ignore_quota; } + bool ignoreLimits() const override { return ignore_quota; } + +private: + String getRewrittenQuery() const; + + ASTPtr query_ptr; + Context & context; + bool ignore_quota = false; +}; + +} diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index 0d3b88facce..87e1265f793 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -30,9 +30,10 @@ namespace DB { namespace ErrorCodes { - extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; } + namespace { ASTPtr getCreateQueryImpl( @@ -137,8 +138,7 @@ namespace create_query_limits.duration = limits.duration; create_query_limits.randomize_interval = limits.randomize_interval; for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) - if (limits.max[resource_type] != Quota::UNLIMITED) - create_query_limits.max[resource_type] = limits.max[resource_type]; + create_query_limits.max[resource_type] = limits.max[resource_type]; query->all_limits.push_back(create_query_limits); } @@ -160,20 +160,20 @@ namespace bool attach_mode) { auto query = std::make_shared(); - query->name_parts = RowPolicy::FullNameParts{policy.getDatabase(), policy.getTableName(), policy.getName()}; + query->name_parts = policy.getNameParts(); query->attach = attach_mode; if (policy.isRestrictive()) query->is_restrictive = policy.isRestrictive(); - for (auto index : ext::range_with_static_cast(RowPolicy::MAX_CONDITION_TYPE)) + for (auto type : ext::range(RowPolicy::MAX_CONDITION_TYPE)) { - const auto & condition = policy.conditions[index]; + const auto & condition = policy.conditions[static_cast(type)]; if (!condition.empty()) { ParserExpression parser; ASTPtr expr = parseQuery(parser, condition, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); - query->conditions.push_back(std::pair{index, expr}); + query->conditions[static_cast(type)] = expr; } } @@ -203,23 +203,16 @@ namespace return getCreateQueryImpl(*quota, manager, attach_mode); if (const SettingsProfile * profile = typeid_cast(&entity)) return getCreateQueryImpl(*profile, manager, attach_mode); - throw Exception("Unexpected type of access entity: " + entity.getTypeName(), ErrorCodes::LOGICAL_ERROR); + throw Exception(entity.outputTypeAndName() + ": type is not supported by SHOW CREATE query", ErrorCodes::NOT_IMPLEMENTED); } - using Kind = ASTShowCreateAccessEntityQuery::Kind; + using EntityType = IAccessEntity::Type; +} - std::type_index getType(Kind kind) - { - switch (kind) - { - case Kind::USER: return typeid(User); - case Kind::ROLE: return typeid(Role); - case Kind::QUOTA: return typeid(Quota); - case Kind::ROW_POLICY: return typeid(RowPolicy); - case Kind::SETTINGS_PROFILE: return typeid(SettingsProfile); - } - __builtin_unreachable(); - } + +InterpreterShowCreateAccessEntityQuery::InterpreterShowCreateAccessEntityQuery(const ASTPtr & query_ptr_, const Context & context_) + : query_ptr(query_ptr_), context(context_), ignore_quota(query_ptr->as().type == EntityType::QUOTA) +{ } @@ -233,17 +226,20 @@ BlockIO InterpreterShowCreateAccessEntityQuery::execute() BlockInputStreamPtr InterpreterShowCreateAccessEntityQuery::executeImpl() { - const auto & show_query = query_ptr->as(); + auto & show_query = query_ptr->as(); /// Build a create query. ASTPtr create_query = getCreateQuery(show_query); /// Build the result column. - std::stringstream create_query_ss; - formatAST(*create_query, create_query_ss, false, true); - String create_query_str = create_query_ss.str(); MutableColumnPtr column = ColumnString::create(); - column->insert(create_query_str); + if (create_query) + { + std::stringstream create_query_ss; + formatAST(*create_query, create_query_ss, false, true); + String create_query_str = create_query_ss.str(); + column->insert(create_query_str); + } /// Prepare description of the result column. std::stringstream desc_ss; @@ -257,7 +253,7 @@ BlockInputStreamPtr InterpreterShowCreateAccessEntityQuery::executeImpl() } -ASTPtr InterpreterShowCreateAccessEntityQuery::getCreateQuery(const ASTShowCreateAccessEntityQuery & show_query) const +ASTPtr InterpreterShowCreateAccessEntityQuery::getCreateQuery(ASTShowCreateAccessEntityQuery & show_query) const { const auto & access_control = context.getAccessControlManager(); context.checkAccess(getRequiredAccess()); @@ -265,46 +261,52 @@ ASTPtr InterpreterShowCreateAccessEntityQuery::getCreateQuery(const ASTShowCreat if (show_query.current_user) { auto user = context.getUser(); + if (!user) + return nullptr; return getCreateQueryImpl(*user, &access_control, false); } if (show_query.current_quota) { - auto quota = access_control.read(context.getQuota()->getUsageInfo().quota_id); + auto usage = context.getQuotaUsage(); + if (!usage) + return nullptr; + auto quota = access_control.read(usage->quota_id); return getCreateQueryImpl(*quota, &access_control, false); } - auto type = getType(show_query.kind); - if (show_query.kind == Kind::ROW_POLICY) + if (show_query.type == EntityType::ROW_POLICY) { - RowPolicyPtr policy = access_control.read(show_query.row_policy_name.getFullName(context)); + if (show_query.row_policy_name_parts.database.empty()) + show_query.row_policy_name_parts.database = context.getCurrentDatabase(); + RowPolicyPtr policy = access_control.read(show_query.row_policy_name_parts.getName()); return getCreateQueryImpl(*policy, &access_control, false); } - auto entity = access_control.read(access_control.getID(type, show_query.name)); + auto entity = access_control.read(access_control.getID(show_query.type, show_query.name)); return getCreateQueryImpl(*entity, &access_control, false); } -AccessRightsElements InterpreterShowCreateAccessEntityQuery::getRequiredAccess() const -{ - const auto & show_query = query_ptr->as(); - AccessRightsElements res; - switch (show_query.kind) - { - case Kind::USER: res.emplace_back(AccessType::SHOW_USERS); break; - case Kind::ROLE: res.emplace_back(AccessType::SHOW_ROLES); break; - case Kind::ROW_POLICY: res.emplace_back(AccessType::SHOW_ROW_POLICIES); break; - case Kind::SETTINGS_PROFILE: res.emplace_back(AccessType::SHOW_SETTINGS_PROFILES); break; - case Kind::QUOTA: res.emplace_back(AccessType::SHOW_QUOTAS); break; - } - return res; -} - - ASTPtr InterpreterShowCreateAccessEntityQuery::getAttachQuery(const IAccessEntity & entity) { return getCreateQueryImpl(entity, nullptr, true); } + +AccessRightsElements InterpreterShowCreateAccessEntityQuery::getRequiredAccess() const +{ + const auto & show_query = query_ptr->as(); + AccessRightsElements res; + switch (show_query.type) + { + case EntityType::USER: res.emplace_back(AccessType::SHOW_USERS); return res; + case EntityType::ROLE: res.emplace_back(AccessType::SHOW_ROLES); return res; + case EntityType::SETTINGS_PROFILE: res.emplace_back(AccessType::SHOW_SETTINGS_PROFILES); return res; + case EntityType::ROW_POLICY: res.emplace_back(AccessType::SHOW_ROW_POLICIES); return res; + case EntityType::QUOTA: res.emplace_back(AccessType::SHOW_QUOTAS); return res; + case EntityType::MAX: break; + } + throw Exception(toString(show_query.type) + ": type is not supported by SHOW CREATE query", ErrorCodes::NOT_IMPLEMENTED); +} } diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h index 0183c59766f..ee28bfbdc4a 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.h @@ -18,23 +18,23 @@ struct IAccessEntity; class InterpreterShowCreateAccessEntityQuery : public IInterpreter { public: - InterpreterShowCreateAccessEntityQuery(const ASTPtr & query_ptr_, const Context & context_) - : query_ptr(query_ptr_), context(context_) {} + InterpreterShowCreateAccessEntityQuery(const ASTPtr & query_ptr_, const Context & context_); BlockIO execute() override; - bool ignoreQuota() const override { return true; } - bool ignoreLimits() const override { return true; } + bool ignoreQuota() const override { return ignore_quota; } + bool ignoreLimits() const override { return ignore_quota; } static ASTPtr getAttachQuery(const IAccessEntity & entity); private: BlockInputStreamPtr executeImpl(); - ASTPtr getCreateQuery(const ASTShowCreateAccessEntityQuery & show_query) const; + ASTPtr getCreateQuery(ASTShowCreateAccessEntityQuery & show_query) const; AccessRightsElements getRequiredAccess() const; ASTPtr query_ptr; const Context & context; + bool ignore_quota = false; }; diff --git a/src/Interpreters/InterpreterShowGrantsQuery.cpp b/src/Interpreters/InterpreterShowGrantsQuery.cpp index da1d46f0cab..130749526c7 100644 --- a/src/Interpreters/InterpreterShowGrantsQuery.cpp +++ b/src/Interpreters/InterpreterShowGrantsQuery.cpp @@ -10,8 +10,6 @@ #include #include #include -#include -#include namespace DB @@ -23,37 +21,6 @@ namespace ErrorCodes namespace { - std::vector groupByTable(AccessRightsElements && elements) - { - using Key = std::tuple; - std::map grouping_map; - for (auto & element : elements) - { - Key key(element.database, element.any_database, element.table, element.any_table); - grouping_map[key].emplace_back(std::move(element)); - } - std::vector res; - res.reserve(grouping_map.size()); - boost::range::copy(grouping_map | boost::adaptors::map_values, std::back_inserter(res)); - return res; - } - - - struct GroupedGrantsAndPartialRevokes - { - std::vector grants; - std::vector partial_revokes; - }; - - GroupedGrantsAndPartialRevokes groupByTable(AccessRights::Elements && elements) - { - GroupedGrantsAndPartialRevokes res; - res.grants = groupByTable(std::move(elements.grants)); - res.partial_revokes = groupByTable(std::move(elements.partial_revokes)); - return res; - } - - template ASTs getGrantQueriesImpl( const T & grantee, @@ -65,35 +32,51 @@ namespace std::shared_ptr to_roles = std::make_shared(); to_roles->names.push_back(grantee.getName()); - for (bool grant_option : {true, false}) - { - if (!grant_option && (grantee.access == grantee.access_with_grant_option)) - continue; - const auto & access_rights = grant_option ? grantee.access_with_grant_option : grantee.access; - const auto grouped_elements = groupByTable(access_rights.getElements()); + auto grants_and_partial_revokes = grantee.access.getGrantsAndPartialRevokes(); + for (bool grant_option : {false, true}) + { using Kind = ASTGrantQuery::Kind; for (Kind kind : {Kind::GRANT, Kind::REVOKE}) { - for (const auto & elements : (kind == Kind::GRANT ? grouped_elements.grants : grouped_elements.partial_revokes)) + AccessRightsElements * elements = nullptr; + if (grant_option) + elements = (kind == Kind::GRANT) ? &grants_and_partial_revokes.grants_with_grant_option : &grants_and_partial_revokes.revokes_grant_option; + else + elements = (kind == Kind::GRANT) ? &grants_and_partial_revokes.grants : &grants_and_partial_revokes.revokes; + elements->normalize(); + + std::shared_ptr grant_query = nullptr; + for (size_t i = 0; i != elements->size(); ++i) { - auto grant_query = std::make_shared(); - grant_query->kind = kind; - grant_query->attach = attach_mode; - grant_query->grant_option = grant_option; - grant_query->to_roles = to_roles; - grant_query->access_rights_elements = elements; - res.push_back(std::move(grant_query)); + const auto & element = (*elements)[i]; + bool prev_element_on_same_db_and_table = false; + if (grant_query) + { + const auto & prev_element = grant_query->access_rights_elements.back(); + if ((element.database == prev_element.database) && (element.any_database == prev_element.any_database) + && (element.table == prev_element.table) && (element.any_table == prev_element.any_table)) + prev_element_on_same_db_and_table = true; + } + if (!prev_element_on_same_db_and_table) + { + grant_query = std::make_shared(); + grant_query->kind = kind; + grant_query->attach = attach_mode; + grant_query->grant_option = grant_option; + grant_query->to_roles = to_roles; + res.push_back(grant_query); + } + grant_query->access_rights_elements.emplace_back(std::move(element)); } } } - for (bool admin_option : {true, false}) - { - if (!admin_option && (grantee.granted_roles == grantee.granted_roles_with_admin_option)) - continue; + auto grants_roles = grantee.granted_roles.getGrants(); - const auto & roles = admin_option ? grantee.granted_roles_with_admin_option : grantee.granted_roles; + for (bool admin_option : {false, true}) + { + const auto & roles = admin_option ? grants_roles.grants_with_admin_option : grants_roles.grants; if (roles.empty()) continue; @@ -122,7 +105,7 @@ namespace return getGrantQueriesImpl(*user, manager, attach_mode); if (const Role * role = typeid_cast(&entity)) return getGrantQueriesImpl(*role, manager, attach_mode); - throw Exception("Unexpected type of access entity: " + entity.getTypeName(), ErrorCodes::LOGICAL_ERROR); + throw Exception(entity.outputTypeAndName() + " is expected to be user or role", ErrorCodes::LOGICAL_ERROR); } } diff --git a/src/Interpreters/InterpreterShowPrivilegesQuery.cpp b/src/Interpreters/InterpreterShowPrivilegesQuery.cpp new file mode 100644 index 00000000000..e88b97d8671 --- /dev/null +++ b/src/Interpreters/InterpreterShowPrivilegesQuery.cpp @@ -0,0 +1,18 @@ +#include +#include + + +namespace DB +{ +InterpreterShowPrivilegesQuery::InterpreterShowPrivilegesQuery(const ASTPtr & query_ptr_, Context & context_) + : query_ptr(query_ptr_), context(context_) +{ +} + + +BlockIO InterpreterShowPrivilegesQuery::execute() +{ + return executeQuery("SELECT * FROM system.privileges", context, true); +} + +} diff --git a/src/Interpreters/InterpreterShowQuotasQuery.h b/src/Interpreters/InterpreterShowPrivilegesQuery.h similarity index 66% rename from src/Interpreters/InterpreterShowQuotasQuery.h rename to src/Interpreters/InterpreterShowPrivilegesQuery.h index ae608e81ce5..0c1a30107c0 100644 --- a/src/Interpreters/InterpreterShowQuotasQuery.h +++ b/src/Interpreters/InterpreterShowPrivilegesQuery.h @@ -8,10 +8,10 @@ namespace DB { class Context; -class InterpreterShowQuotasQuery : public IInterpreter +class InterpreterShowPrivilegesQuery : public IInterpreter { public: - InterpreterShowQuotasQuery(const ASTPtr & query_ptr_, Context & context_); + InterpreterShowPrivilegesQuery(const ASTPtr & query_ptr_, Context & context_); BlockIO execute() override; @@ -21,8 +21,6 @@ public: private: ASTPtr query_ptr; Context & context; - - String getRewrittenQuery(); }; } diff --git a/src/Interpreters/InterpreterShowQuotasQuery.cpp b/src/Interpreters/InterpreterShowQuotasQuery.cpp deleted file mode 100644 index 73653e26781..00000000000 --- a/src/Interpreters/InterpreterShowQuotasQuery.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -InterpreterShowQuotasQuery::InterpreterShowQuotasQuery(const ASTPtr & query_ptr_, Context & context_) - : query_ptr(query_ptr_), context(context_) -{ -} - - -String InterpreterShowQuotasQuery::getRewrittenQuery() -{ - const auto & query = query_ptr->as(); - - /// Transform the query into some kind of "SELECT from system.quotas" query. - String expr; - String filter; - String table_name; - String order_by; - if (query.usage) - { - expr = "name || ' key=\\'' || key || '\\'' || if(isNull(end_of_interval), '', ' interval=[' || " - "toString(end_of_interval - duration) || ' .. ' || " - "toString(end_of_interval) || ']'"; - for (auto resource_type : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) - { - String column_name = Quota::resourceTypeToColumnName(resource_type); - expr += String{" || ' "} + column_name + "=' || toString(" + column_name + ")"; - expr += String{" || if(max_"} + column_name + "=0, '', '/' || toString(max_" + column_name + "))"; - } - expr += ")"; - - if (query.current) - filter = "(id = currentQuotaID()) AND (key = currentQuotaKey())"; - - table_name = "system.quota_usage"; - order_by = "name, key, duration"; - } - else - { - expr = "name"; - table_name = "system.quotas"; - order_by = "name"; - } - - /// Prepare description of the result column. - std::stringstream ss; - formatAST(query, ss, false, true); - String desc = ss.str(); - String prefix = "SHOW "; - if (startsWith(desc, prefix)) - desc = desc.substr(prefix.length()); /// `desc` always starts with "SHOW ", so we can trim this prefix. - - /// Build a new query. - return "SELECT " + expr + " AS " + backQuote(desc) + " FROM " + table_name + (filter.empty() ? "" : (" WHERE " + filter)) - + (order_by.empty() ? "" : (" ORDER BY " + order_by)); -} - - -BlockIO InterpreterShowQuotasQuery::execute() -{ - return executeQuery(getRewrittenQuery(), context, true); -} - -} diff --git a/src/Interpreters/InterpreterShowRowPoliciesQuery.cpp b/src/Interpreters/InterpreterShowRowPoliciesQuery.cpp deleted file mode 100644 index 3fcae82064d..00000000000 --- a/src/Interpreters/InterpreterShowRowPoliciesQuery.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -InterpreterShowRowPoliciesQuery::InterpreterShowRowPoliciesQuery(const ASTPtr & query_ptr_, Context & context_) - : query_ptr(query_ptr_), context(context_) -{ -} - - -BlockIO InterpreterShowRowPoliciesQuery::execute() -{ - return executeQuery(getRewrittenQuery(), context, true); -} - - -String InterpreterShowRowPoliciesQuery::getRewrittenQuery() const -{ - const auto & query = query_ptr->as(); - - const String & table_name = query.table_name; - String database; - if (!table_name.empty()) - { - database = query.database; - if (database.empty()) - database = context.getCurrentDatabase(); - } - - String filter; - if (query.current) - { - if (table_name.empty()) - filter = "has(currentRowPolicyIDs(), id)"; - else - filter = "has(currentRowPolicyIDs(" + quoteString(database) + ", " + quoteString(table_name) + "), id)"; - } - else - { - if (!table_name.empty()) - filter = "database = " + quoteString(database) + " AND table = " + quoteString(table_name); - } - - String expr = table_name.empty() ? "full_name" : "name"; - - return "SELECT " + expr + " AS " + backQuote(getResultDescription()) + " from system.row_policies" - + (filter.empty() ? "" : " WHERE " + filter) + " ORDER BY " + expr; -} - - -String InterpreterShowRowPoliciesQuery::getResultDescription() const -{ - std::stringstream ss; - formatAST(*query_ptr, ss, false, true); - String desc = ss.str(); - String prefix = "SHOW "; - if (startsWith(desc, prefix)) - desc = desc.substr(prefix.length()); /// `desc` always starts with "SHOW ", so we can trim this prefix. - return desc; -} -} diff --git a/src/Interpreters/InterpreterShowRowPoliciesQuery.h b/src/Interpreters/InterpreterShowRowPoliciesQuery.h deleted file mode 100644 index 84cf6299b84..00000000000 --- a/src/Interpreters/InterpreterShowRowPoliciesQuery.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include -#include - - -namespace DB -{ -class Context; - -class InterpreterShowRowPoliciesQuery : public IInterpreter -{ -public: - InterpreterShowRowPoliciesQuery(const ASTPtr & query_ptr_, Context & context_); - BlockIO execute() override; - -private: - String getRewrittenQuery() const; - String getResultDescription() const; - - ASTPtr query_ptr; - Context & context; -}; - -} diff --git a/src/Interpreters/MetricLog.cpp b/src/Interpreters/MetricLog.cpp index bd898170705..96fe55c26e6 100644 --- a/src/Interpreters/MetricLog.cpp +++ b/src/Interpreters/MetricLog.cpp @@ -50,6 +50,8 @@ void MetricLogElement::appendToBlock(Block & block) const for (size_t i = 0, end = CurrentMetrics::end(); i < end; ++i) columns[column_idx++]->insert(current_metrics[i]); + + block.setColumns(std::move(columns)); } diff --git a/src/Interpreters/NullableUtils.cpp b/src/Interpreters/NullableUtils.cpp index aea69f8ea6d..d1586bec06b 100644 --- a/src/Interpreters/NullableUtils.cpp +++ b/src/Interpreters/NullableUtils.cpp @@ -33,7 +33,7 @@ ColumnPtr extractNestedColumnsAndNullMap(ColumnRawPtrs & key_columns, ConstNullM } else { - MutableColumnPtr mutable_null_map_holder = (*std::move(null_map_holder)).mutate(); + MutableColumnPtr mutable_null_map_holder = IColumn::mutate(std::move(null_map_holder)); PaddedPODArray & mutable_null_map = assert_cast(*mutable_null_map_holder).getData(); const PaddedPODArray & other_null_map = column_nullable->getNullMapData(); diff --git a/src/Interpreters/QueryThreadLog.cpp b/src/Interpreters/QueryThreadLog.cpp index 1a2975bd43b..f539a720449 100644 --- a/src/Interpreters/QueryThreadLog.cpp +++ b/src/Interpreters/QueryThreadLog.cpp @@ -107,6 +107,8 @@ void QueryThreadLogElement::appendToBlock(Block & block) const columns[i++]->insertDefault(); columns[i++]->insertDefault(); } + + block.setColumns(std::move(columns)); } } diff --git a/src/Interpreters/TextLog.cpp b/src/Interpreters/TextLog.cpp index b5f1d987b91..35e027616f6 100644 --- a/src/Interpreters/TextLog.cpp +++ b/src/Interpreters/TextLog.cpp @@ -68,6 +68,8 @@ void TextLogElement::appendToBlock(Block & block) const columns[i++]->insert(source_file); columns[i++]->insert(source_line); + + block.setColumns(std::move(columns)); } TextLog::TextLog(Context & context_, const String & database_name_, diff --git a/src/Interpreters/addMissingDefaults.h b/src/Interpreters/addMissingDefaults.h index d9d8d3d4f22..cc84f413b16 100644 --- a/src/Interpreters/addMissingDefaults.h +++ b/src/Interpreters/addMissingDefaults.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 62ef4152a2e..4d609395c3a 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -317,7 +317,7 @@ static std::tuple executeQueryImpl( context.resetInputCallbacks(); auto interpreter = InterpreterFactory::get(ast, context, stage); - bool use_processors = settings.experimental_use_processors && allow_processors && interpreter->canExecuteWithProcessors(); + bool use_processors = allow_processors && interpreter->canExecuteWithProcessors(); std::shared_ptr quota; if (!interpreter->ignoreQuota()) diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index 701c520c9f8..e3ca9258892 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -43,7 +43,7 @@ void removeColumnNullability(ColumnWithTypeAndName & column) { const auto * nullable_column = checkAndGetColumn(*column.column); ColumnPtr nested_column = nullable_column->getNestedColumnPtr(); - MutableColumnPtr mutable_column = (*std::move(nested_column)).mutate(); + MutableColumnPtr mutable_column = IColumn::mutate(std::move(nested_column)); column.column = std::move(mutable_column); } } diff --git a/src/Interpreters/tests/users.cpp b/src/Interpreters/tests/users.cpp index acd0cfd0519..5c7d66ed7ed 100644 --- a/src/Interpreters/tests/users.cpp +++ b/src/Interpreters/tests/users.cpp @@ -218,7 +218,7 @@ void runOneTest(const TestDescriptor & test_descriptor) try { - res = acl_manager.read(entry.user_name)->access.isGranted(DB::AccessType::ALL, entry.database_name); + res = acl_manager.read(entry.user_name)->access.access.isGranted(DB::AccessType::ALL, entry.database_name); } catch (const Poco::Exception &) { diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index 12b9f12a2e2..b210a1c5b8c 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -83,12 +83,12 @@ SRCS( InterpreterSelectWithUnionQuery.cpp InterpreterSetQuery.cpp InterpreterSetRoleQuery.cpp + InterpreterShowAccessEntitiesQuery.cpp InterpreterShowCreateAccessEntityQuery.cpp InterpreterShowCreateQuery.cpp InterpreterShowGrantsQuery.cpp + InterpreterShowPrivilegesQuery.cpp InterpreterShowProcesslistQuery.cpp - InterpreterShowQuotasQuery.cpp - InterpreterShowRowPoliciesQuery.cpp InterpreterShowTablesQuery.cpp InterpreterSystemQuery.cpp InterpreterUseQuery.cpp diff --git a/src/Parsers/ASTCreateQuotaQuery.cpp b/src/Parsers/ASTCreateQuotaQuery.cpp index edb510d1a2e..bdfd1b32e96 100644 --- a/src/Parsers/ASTCreateQuotaQuery.cpp +++ b/src/Parsers/ASTCreateQuotaQuery.cpp @@ -10,14 +10,16 @@ namespace DB namespace { using KeyType = Quota::KeyType; + using KeyTypeInfo = Quota::KeyTypeInfo; using ResourceType = Quota::ResourceType; + using ResourceTypeInfo = Quota::ResourceTypeInfo; using ResourceAmount = Quota::ResourceAmount; void formatKeyType(const KeyType & key_type, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " KEYED BY " << (settings.hilite ? IAST::hilite_none : "") << "'" - << Quota::getNameOfKeyType(key_type) << "'"; + << KeyTypeInfo::get(key_type).name << "'"; } @@ -35,13 +37,9 @@ namespace else settings.ostr << ","; - settings.ostr << " " << (settings.hilite ? IAST::hilite_keyword : "") << Quota::resourceTypeToKeyword(resource_type) - << (settings.hilite ? IAST::hilite_none : "") << " "; - - if (resource_type == Quota::EXECUTION_TIME) - settings.ostr << Quota::executionTimeToSeconds(max); - else - settings.ostr << max; + const auto & type_info = ResourceTypeInfo::get(resource_type); + settings.ostr << " " << (settings.hilite ? IAST::hilite_keyword : "") << type_info.keyword + << (settings.hilite ? IAST::hilite_none : "") << " " << type_info.amountToString(max); } @@ -67,7 +65,7 @@ namespace else { bool limit_found = false; - for (auto resource_type : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) { if (limits.max[resource_type]) { diff --git a/src/Parsers/ASTCreateQuotaQuery.h b/src/Parsers/ASTCreateQuotaQuery.h index b001ec44a0c..b9994e8ec3a 100644 --- a/src/Parsers/ASTCreateQuotaQuery.h +++ b/src/Parsers/ASTCreateQuotaQuery.h @@ -35,18 +35,16 @@ public: bool if_not_exists = false; bool or_replace = false; + using KeyType = Quota::KeyType; + using ResourceAmount = Quota::ResourceAmount; + String name; String new_name; - using KeyType = Quota::KeyType; std::optional key_type; - using ResourceType = Quota::ResourceType; - using ResourceAmount = Quota::ResourceAmount; - static constexpr size_t MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; - struct Limits { - std::optional max[MAX_RESOURCE_TYPE]; + std::optional max[Quota::MAX_RESOURCE_TYPE]; bool drop = false; std::chrono::seconds duration = std::chrono::seconds::zero(); bool randomize_interval = false; diff --git a/src/Parsers/ASTCreateRowPolicyQuery.cpp b/src/Parsers/ASTCreateRowPolicyQuery.cpp index fc6af51da6f..973f3c6b930 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.cpp +++ b/src/Parsers/ASTCreateRowPolicyQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -11,11 +12,14 @@ namespace DB namespace { using ConditionType = RowPolicy::ConditionType; + using ConditionTypeInfo = RowPolicy::ConditionTypeInfo; + constexpr auto MAX_CONDITION_TYPE = RowPolicy::MAX_CONDITION_TYPE; - void formatRenameTo(const String & new_policy_name, const IAST::FormatSettings & settings) + + void formatRenameTo(const String & new_short_name, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " RENAME TO " << (settings.hilite ? IAST::hilite_none : "") - << backQuote(new_policy_name); + << backQuote(new_short_name); } @@ -28,90 +32,89 @@ namespace void formatConditionalExpression(const ASTPtr & expr, const IAST::FormatSettings & settings) { - if (!expr) - { + if (expr) + expr->format(settings); + else settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NONE" << (settings.hilite ? IAST::hilite_none : ""); - return; - } - expr->format(settings); } - std::vector> - conditionalExpressionsToStrings(const std::vector> & exprs, const IAST::FormatSettings & settings) + void formatCondition(const boost::container::flat_set & commands, const String & filter, const String & check, bool alter, const IAST::FormatSettings & settings) { - std::vector> result; - std::stringstream ss; - IAST::FormatSettings temp_settings(ss, settings); - boost::range::transform(exprs, std::back_inserter(result), [&](const std::pair & in) - { - formatConditionalExpression(in.second, temp_settings); - auto out = std::pair{in.first, ss.str()}; - ss.str(""); - return out; - }); - return result; - } - - - void formatConditions(const char * op, const std::optional & filter, const std::optional & check, bool alter, const IAST::FormatSettings & settings) - { - if (op) - { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " FOR" << (settings.hilite ? IAST::hilite_none : ""); - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << ' ' << op << (settings.hilite ? IAST::hilite_none : ""); - } - - if (filter) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " USING " << (settings.hilite ? IAST::hilite_none : "") << *filter; - - if (check && (alter || (check != filter))) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH CHECK " << (settings.hilite ? IAST::hilite_none : "") << *check; - } - - - void formatMultipleConditions(const std::vector> & conditions, bool alter, const IAST::FormatSettings & settings) - { - std::optional scond[RowPolicy::MAX_CONDITION_TYPE]; - for (const auto & [index, scondition] : conditionalExpressionsToStrings(conditions, settings)) - scond[index] = scondition; - - if ((scond[RowPolicy::SELECT_FILTER] == scond[RowPolicy::UPDATE_FILTER]) - && (scond[RowPolicy::UPDATE_FILTER] == scond[RowPolicy::DELETE_FILTER]) - && (scond[RowPolicy::INSERT_CHECK] == scond[RowPolicy::UPDATE_CHECK]) - && (scond[RowPolicy::SELECT_FILTER] || scond[RowPolicy::INSERT_CHECK])) - { - formatConditions(nullptr, scond[RowPolicy::SELECT_FILTER], scond[RowPolicy::INSERT_CHECK], alter, settings); - return; - } - + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " FOR " << (settings.hilite ? IAST::hilite_none : ""); bool need_comma = false; - if (scond[RowPolicy::SELECT_FILTER]) + for (const auto & command : commands) { if (std::exchange(need_comma, true)) - settings.ostr << ','; - formatConditions("SELECT", scond[RowPolicy::SELECT_FILTER], {}, alter, settings); - } - if (scond[RowPolicy::INSERT_CHECK]) - { - if (std::exchange(need_comma, true)) - settings.ostr << ','; - formatConditions("INSERT", {}, scond[RowPolicy::INSERT_CHECK], alter, settings); - } - if (scond[RowPolicy::UPDATE_FILTER] || scond[RowPolicy::UPDATE_CHECK]) - { - if (std::exchange(need_comma, true)) - settings.ostr << ','; - formatConditions("UPDATE", scond[RowPolicy::UPDATE_FILTER], scond[RowPolicy::UPDATE_CHECK], alter, settings); - } - if (scond[RowPolicy::DELETE_FILTER]) - { - if (std::exchange(need_comma, true)) - settings.ostr << ','; - formatConditions("DELETE", scond[RowPolicy::DELETE_FILTER], {}, alter, settings); + settings.ostr << ", "; + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << command << (settings.hilite ? IAST::hilite_none : ""); } + + if (!filter.empty()) + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " USING " << (settings.hilite ? IAST::hilite_none : "") << filter; + + if (!check.empty() && (alter || (check != filter))) + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH CHECK " << (settings.hilite ? IAST::hilite_none : "") << check; } + + void formatMultipleConditions(const std::array, MAX_CONDITION_TYPE> & conditions, bool alter, const IAST::FormatSettings & settings) + { + std::array conditions_as_strings; + std::stringstream temp_sstream; + IAST::FormatSettings temp_settings(temp_sstream, settings); + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + { + const auto & condition = conditions[condition_type]; + if (condition) + { + formatConditionalExpression(*condition, temp_settings); + conditions_as_strings[condition_type] = temp_sstream.str(); + temp_sstream.str(""); + } + } + + boost::container::flat_set commands; + String filter, check; + + do + { + commands.clear(); + filter.clear(); + check.clear(); + + /// Collect commands using the same filter and check conditions. + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + { + const String & condition = conditions_as_strings[condition_type]; + if (condition.empty()) + continue; + const auto & type_info = ConditionTypeInfo::get(condition_type); + if (type_info.is_check) + { + if (check.empty()) + check = condition; + else if (check != condition) + continue; + } + else + { + if (filter.empty()) + filter = condition; + else if (filter != condition) + continue; + } + commands.emplace(type_info.command); + conditions_as_strings[condition_type].clear(); /// Skip this condition on the next iteration. + } + + if (!filter.empty() || !check.empty()) + formatCondition(commands, filter, check, alter, settings); + } + while (!filter.empty() || !check.empty()); + } + + void formatToRoles(const ASTExtendedRoleSet & roles, const IAST::FormatSettings & settings) { settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " TO " << (settings.hilite ? IAST::hilite_none : ""); @@ -153,14 +156,14 @@ void ASTCreateRowPolicyQuery::formatImpl(const FormatSettings & settings, Format const String & database = name_parts.database; const String & table_name = name_parts.table_name; - const String & policy_name = name_parts.policy_name; - settings.ostr << " " << backQuoteIfNeed(policy_name) << (settings.hilite ? hilite_keyword : "") << " ON " + const String & short_name = name_parts.short_name; + settings.ostr << " " << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : "") << (database.empty() ? String{} : backQuoteIfNeed(database) + ".") << table_name; formatOnCluster(settings); - if (!new_policy_name.empty()) - formatRenameTo(new_policy_name, settings); + if (!new_short_name.empty()) + formatRenameTo(new_short_name, settings); if (is_restrictive) formatAsRestrictiveOrPermissive(*is_restrictive, settings); diff --git a/src/Parsers/ASTCreateRowPolicyQuery.h b/src/Parsers/ASTCreateRowPolicyQuery.h index 2ad64255c04..8aa44b784aa 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.h +++ b/src/Parsers/ASTCreateRowPolicyQuery.h @@ -3,8 +3,8 @@ #include #include #include -#include -#include +#include +#include namespace DB @@ -36,12 +36,11 @@ public: bool if_not_exists = false; bool or_replace = false; - RowPolicy::FullNameParts name_parts; - String new_policy_name; + RowPolicy::NameParts name_parts; + String new_short_name; std::optional is_restrictive; - using ConditionType = RowPolicy::ConditionType; - std::vector> conditions; + std::array, RowPolicy::MAX_CONDITION_TYPE> conditions; /// `nullopt` means "not set", `nullptr` means set to NONE. std::shared_ptr roles; diff --git a/src/Parsers/ASTDropAccessEntityQuery.cpp b/src/Parsers/ASTDropAccessEntityQuery.cpp index 06a820bfbb5..9f7a1d86221 100644 --- a/src/Parsers/ASTDropAccessEntityQuery.cpp +++ b/src/Parsers/ASTDropAccessEntityQuery.cpp @@ -4,34 +4,12 @@ namespace DB { -namespace -{ - using Kind = ASTDropAccessEntityQuery::Kind; - - const char * getKeyword(Kind kind) - { - switch (kind) - { - case Kind::USER: return "USER"; - case Kind::ROLE: return "ROLE"; - case Kind::QUOTA: return "QUOTA"; - case Kind::ROW_POLICY: return "ROW POLICY"; - case Kind::SETTINGS_PROFILE: return "SETTINGS PROFILE"; - } - __builtin_unreachable(); - } -} - - -ASTDropAccessEntityQuery::ASTDropAccessEntityQuery(Kind kind_) - : kind(kind_) -{ -} +using EntityTypeInfo = IAccessEntity::TypeInfo; String ASTDropAccessEntityQuery::getID(char) const { - return String("DROP ") + getKeyword(kind) + " query"; + return String("DROP ") + toString(type) + " query"; } @@ -44,22 +22,22 @@ ASTPtr ASTDropAccessEntityQuery::clone() const void ASTDropAccessEntityQuery::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const { settings.ostr << (settings.hilite ? hilite_keyword : "") - << "DROP " << getKeyword(kind) + << "DROP " << EntityTypeInfo::get(type).name << (if_exists ? " IF EXISTS" : "") << (settings.hilite ? hilite_none : ""); - if (kind == Kind::ROW_POLICY) + if (type == EntityType::ROW_POLICY) { bool need_comma = false; - for (const auto & row_policy_name : row_policies_names) + for (const auto & name_parts : row_policies_name_parts) { if (need_comma) settings.ostr << ','; need_comma = true; - const String & database = row_policy_name.database; - const String & table_name = row_policy_name.table_name; - const String & policy_name = row_policy_name.policy_name; - settings.ostr << ' ' << backQuoteIfNeed(policy_name) << (settings.hilite ? hilite_keyword : "") << " ON " + const String & database = name_parts.database; + const String & table_name = name_parts.table_name; + const String & short_name = name_parts.short_name; + settings.ostr << ' ' << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : "") << (database.empty() ? String{} : backQuoteIfNeed(database) + ".") << backQuoteIfNeed(table_name); } diff --git a/src/Parsers/ASTDropAccessEntityQuery.h b/src/Parsers/ASTDropAccessEntityQuery.h index a3b358dcfb9..160b0c2e212 100644 --- a/src/Parsers/ASTDropAccessEntityQuery.h +++ b/src/Parsers/ASTDropAccessEntityQuery.h @@ -17,21 +17,13 @@ namespace DB class ASTDropAccessEntityQuery : public IAST, public ASTQueryWithOnCluster { public: - enum class Kind - { - USER, - ROLE, - QUOTA, - ROW_POLICY, - SETTINGS_PROFILE, - }; + using EntityType = IAccessEntity::Type; - const Kind kind; + EntityType type; bool if_exists = false; Strings names; - std::vector row_policies_names; + std::vector row_policies_name_parts; - ASTDropAccessEntityQuery(Kind kind_); String getID(char) const override; ASTPtr clone() const override; void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; diff --git a/src/Parsers/ASTGrantQuery.cpp b/src/Parsers/ASTGrantQuery.cpp index e6764fc067a..8114bef0766 100644 --- a/src/Parsers/ASTGrantQuery.cpp +++ b/src/Parsers/ASTGrantQuery.cpp @@ -1,10 +1,6 @@ #include #include #include -#include -#include -#include -#include namespace DB @@ -16,61 +12,13 @@ namespace ErrorCodes namespace { - using KeywordToColumnsMap = std::map /* columns */>; - using TableToAccessMap = std::map; - - TableToAccessMap prepareTableToAccessMap(const AccessRightsElements & elements) + void formatColumnNames(const Strings & columns, const IAST::FormatSettings & settings) { - TableToAccessMap res; - for (const auto & element : elements) - { - String database_and_table_name; - if (element.any_database) - { - if (element.any_table) - database_and_table_name = "*.*"; - else - database_and_table_name = "*." + backQuoteIfNeed(element.table); - } - else if (element.database.empty()) - { - if (element.any_table) - database_and_table_name = "*"; - else - database_and_table_name = backQuoteIfNeed(element.table); - } - else - { - if (element.any_table) - database_and_table_name = backQuoteIfNeed(element.database) + ".*"; - else - database_and_table_name = backQuoteIfNeed(element.database) + "." + backQuoteIfNeed(element.table); - } - - KeywordToColumnsMap & keyword_to_columns = res[database_and_table_name]; - for (const auto & keyword : element.access_flags.toKeywords()) - boost::range::push_back(keyword_to_columns[keyword], element.columns); - } - - for (auto & keyword_to_columns : res | boost::adaptors::map_values) - { - for (auto & columns : keyword_to_columns | boost::adaptors::map_values) - boost::range::sort(columns); - } - return res; - } - - - void formatColumnNames(const std::vector & columns, const IAST::FormatSettings & settings) - { - if (columns.empty()) - return; - settings.ostr << "("; - bool need_comma_after_column_name = false; + bool need_comma = false; for (const auto & column : columns) { - if (std::exchange(need_comma_after_column_name, true)) + if (std::exchange(need_comma, true)) settings.ostr << ", "; settings.ostr << backQuoteIfNeed(column); } @@ -80,20 +28,50 @@ namespace void formatAccessRightsElements(const AccessRightsElements & elements, const IAST::FormatSettings & settings) { - bool need_comma = false; - for (const auto & [database_and_table, keyword_to_columns] : prepareTableToAccessMap(elements)) + bool no_output = true; + for (size_t i = 0; i != elements.size(); ++i) { - for (const auto & [keyword, columns] : keyword_to_columns) + const auto & element = elements[i]; + auto keywords = element.access_flags.toKeywords(); + if (keywords.empty() || (!element.any_column && element.columns.empty())) + continue; + + for (const auto & keyword : keywords) { - if (std::exchange(need_comma, true)) + if (!std::exchange(no_output, false)) settings.ostr << ", "; settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << keyword << (settings.hilite ? IAST::hilite_none : ""); - formatColumnNames(columns, settings); + if (!element.any_column) + formatColumnNames(element.columns, settings); } - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " ON " << (settings.hilite ? IAST::hilite_none : "") << database_and_table; + bool next_element_on_same_db_and_table = false; + if (i != elements.size() - 1) + { + const auto & next_element = elements[i + 1]; + if ((element.database == next_element.database) && (element.any_database == next_element.any_database) + && (element.table == next_element.table) && (element.any_table == next_element.any_table)) + next_element_on_same_db_and_table = true; + } + + if (!next_element_on_same_db_and_table) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " ON " << (settings.hilite ? IAST::hilite_none : ""); + if (element.any_database) + settings.ostr << "*."; + else if (!element.database.empty()) + settings.ostr << backQuoteIfNeed(element.database) + "."; + + if (element.any_table) + settings.ostr << "*"; + else + settings.ostr << backQuoteIfNeed(element.table); + } } + + if (no_output) + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "USAGE ON " << (settings.hilite ? IAST::hilite_none : "") << "*.*"; } @@ -134,7 +112,7 @@ void ASTGrantQuery::formatImpl(const FormatSettings & settings, FormatState &, F settings.ostr << (settings.hilite ? hilite_keyword : "") << " ADMIN OPTION FOR" << (settings.hilite ? hilite_none : ""); } - if ((!!roles + !access_rights_elements.empty()) != 1) + if (roles && !access_rights_elements.empty()) throw Exception("Either roles or access rights elements should be set", ErrorCodes::LOGICAL_ERROR); settings.ostr << " "; diff --git a/src/Parsers/ASTLiteral.cpp b/src/Parsers/ASTLiteral.cpp index 92d57687426..184d361ae2e 100644 --- a/src/Parsers/ASTLiteral.cpp +++ b/src/Parsers/ASTLiteral.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include namespace DB @@ -14,30 +16,59 @@ void ASTLiteral::updateTreeHashImpl(SipHash & hash_state) const applyVisitor(FieldVisitorHash(hash_state), value); } +/// Writes 'tuple' word before tuple literals for backward compatibility reasons. +/// TODO: remove, when versions lower than 20.3 will be rearely used. +class FieldVisitorToColumnName : public StaticVisitor +{ +public: + template + String operator() (const T & x) const { return visitor(x); } + +private: + FieldVisitorToString visitor; +}; + +template<> +String FieldVisitorToColumnName::operator() (const Tuple & x) const +{ + WriteBufferFromOwnString wb; + + wb << "tuple("; + for (auto it = x.begin(); it != x.end(); ++it) + { + if (it != x.begin()) + wb << ", "; + wb << applyVisitor(*this, *it); + } + wb << ')'; + + return wb.str(); +} + void ASTLiteral::appendColumnNameImpl(WriteBuffer & ostr) const { /// 100 - just arbitrary value. constexpr auto min_elements_for_hashing = 100; - /// Special case for very large arrays and tuples. Instead of listing all elements, will use hash of them. + /// Special case for very large arrays. Instead of listing all elements, will use hash of them. /// (Otherwise column name will be too long, that will lead to significant slowdown of expression analysis.) + /// TODO: Also do hashing for large tuples, when versions lower than 20.3 will be rarely used, because it breaks backward compatibility. auto type = value.getType(); - if ((type == Field::Types::Array && value.get().size() > min_elements_for_hashing) - || (type == Field::Types::Tuple && value.get().size() > min_elements_for_hashing)) + if ((type == Field::Types::Array && value.get().size() > min_elements_for_hashing)) { SipHash hash; applyVisitor(FieldVisitorHash(hash), value); UInt64 low, high; hash.get128(low, high); - writeCString(type == Field::Types::Array ? "__array_" : "__tuple_", ostr); + writeCString("__array_", ostr); writeText(low, ostr); ostr.write('_'); writeText(high, ostr); } else { - String column_name = applyVisitor(FieldVisitorToString(), value); + String column_name = applyVisitor(FieldVisitorToColumnName(), value); writeString(column_name, ostr); } } diff --git a/src/Parsers/ASTSettingsProfileElement.cpp b/src/Parsers/ASTSettingsProfileElement.cpp index 24f1aa60813..a7955411347 100644 --- a/src/Parsers/ASTSettingsProfileElement.cpp +++ b/src/Parsers/ASTSettingsProfileElement.cpp @@ -31,7 +31,7 @@ void ASTSettingsProfileElement::formatImpl(const FormatSettings & settings, Form return; } - settings.ostr << name; + settings.ostr << setting_name; if (!value.isNull()) { diff --git a/src/Parsers/ASTSettingsProfileElement.h b/src/Parsers/ASTSettingsProfileElement.h index ee1ee28c383..6a54bca3213 100644 --- a/src/Parsers/ASTSettingsProfileElement.h +++ b/src/Parsers/ASTSettingsProfileElement.h @@ -13,7 +13,7 @@ class ASTSettingsProfileElement : public IAST { public: String parent_profile; - String name; + String setting_name; Field value; Field min_value; Field max_value; @@ -21,7 +21,7 @@ public: bool id_mode = false; /// If true then `parent_profile` keeps UUID, not a name. bool use_inherit_keyword = false; /// If true then this element is a part of ASTCreateSettingsProfileQuery. - bool empty() const { return parent_profile.empty() && name.empty(); } + bool empty() const { return parent_profile.empty() && setting_name.empty(); } String getID(char) const override { return "SettingsProfileElement"; } ASTPtr clone() const override { return std::make_shared(*this); } diff --git a/src/Parsers/ASTShowAccessEntitiesQuery.cpp b/src/Parsers/ASTShowAccessEntitiesQuery.cpp new file mode 100644 index 00000000000..cb1ccff5273 --- /dev/null +++ b/src/Parsers/ASTShowAccessEntitiesQuery.cpp @@ -0,0 +1,53 @@ +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + + +const char * ASTShowAccessEntitiesQuery::getKeyword() const +{ + switch (type) + { + case EntityType::ROW_POLICY: + return "SHOW ROW POLICIES"; + case EntityType::QUOTA: + return current_quota ? "SHOW CURRENT QUOTA" : "SHOW QUOTAS"; + case EntityType::SETTINGS_PROFILE: + return "SHOW SETTINGS PROFILES"; + case EntityType::USER: + return "SHOW USERS"; + case EntityType::ROLE: + return current_roles ? "SHOW CURRENT ROLES" : (enabled_roles ? "SHOW ENABLED ROLES" : "SHOW ROLES"); + case EntityType::MAX: + break; + } + throw Exception(toString(type) + ": type is not supported by SHOW query", ErrorCodes::NOT_IMPLEMENTED); +} + + +String ASTShowAccessEntitiesQuery::getID(char) const +{ + return String(getKeyword()) + " query"; +} + +void ASTShowAccessEntitiesQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const +{ + const char * keyword = getKeyword(); + settings.ostr << (settings.hilite ? hilite_keyword : "") << keyword << (settings.hilite ? hilite_none : ""); + + if ((type == EntityType::ROW_POLICY) && !table_name.empty()) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); + if (!database.empty()) + settings.ostr << backQuoteIfNeed(database) << "."; + settings.ostr << backQuoteIfNeed(table_name); + } +} + +} diff --git a/src/Parsers/ASTShowAccessEntitiesQuery.h b/src/Parsers/ASTShowAccessEntitiesQuery.h new file mode 100644 index 00000000000..3bf16ad6abd --- /dev/null +++ b/src/Parsers/ASTShowAccessEntitiesQuery.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +/// SHOW [ROW] POLICIES [ON [database.]table] +/// SHOW QUOTAS +/// SHOW [CURRENT] QUOTA +/// SHOW [SETTINGS] PROFILES +/// SHOW USERS +/// SHOW [CURRENT|ENABLED] ROLES +class ASTShowAccessEntitiesQuery : public ASTQueryWithOutput +{ +public: + using EntityType = IAccessEntity::Type; + + EntityType type; + String database; + String table_name; + bool current_quota = false; + bool current_roles = false; + bool enabled_roles = false; + + String getID(char) const override; + ASTPtr clone() const override { return std::make_shared(*this); } + +protected: + void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; + +private: + const char * getKeyword() const; +}; + +} diff --git a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp index 9e562043f09..f81b30428fb 100644 --- a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp +++ b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp @@ -4,34 +4,12 @@ namespace DB { -namespace -{ - using Kind = ASTShowCreateAccessEntityQuery::Kind; - - const char * getKeyword(Kind kind) - { - switch (kind) - { - case Kind::USER: return "USER"; - case Kind::ROLE: return "ROLE"; - case Kind::QUOTA: return "QUOTA"; - case Kind::ROW_POLICY: return "ROW POLICY"; - case Kind::SETTINGS_PROFILE: return "SETTINGS PROFILE"; - } - __builtin_unreachable(); - } -} - - -ASTShowCreateAccessEntityQuery::ASTShowCreateAccessEntityQuery(Kind kind_) - : kind(kind_) -{ -} +using EntityTypeInfo = IAccessEntity::TypeInfo; String ASTShowCreateAccessEntityQuery::getID(char) const { - return String("SHOW CREATE ") + getKeyword(kind) + " query"; + return String("SHOW CREATE ") + toString(type) + " query"; } @@ -44,20 +22,18 @@ ASTPtr ASTShowCreateAccessEntityQuery::clone() const void ASTShowCreateAccessEntityQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const { settings.ostr << (settings.hilite ? hilite_keyword : "") - << "SHOW CREATE " << getKeyword(kind) + << "SHOW CREATE " << EntityTypeInfo::get(type).name << (settings.hilite ? hilite_none : ""); - if (current_user) + if (current_user || current_quota) { } - else if (current_quota) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " CURRENT" << (settings.hilite ? hilite_none : ""); - else if (kind == Kind::ROW_POLICY) + else if (type == EntityType::ROW_POLICY) { - const String & database = row_policy_name.database; - const String & table_name = row_policy_name.table_name; - const String & policy_name = row_policy_name.policy_name; - settings.ostr << ' ' << backQuoteIfNeed(policy_name) << (settings.hilite ? hilite_keyword : "") << " ON " + const String & database = row_policy_name_parts.database; + const String & table_name = row_policy_name_parts.table_name; + const String & short_name = row_policy_name_parts.short_name; + settings.ostr << ' ' << backQuoteIfNeed(short_name) << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : "") << (database.empty() ? String{} : backQuoteIfNeed(database) + ".") << backQuoteIfNeed(table_name); } diff --git a/src/Parsers/ASTShowCreateAccessEntityQuery.h b/src/Parsers/ASTShowCreateAccessEntityQuery.h index e76a9177979..df7be2e257c 100644 --- a/src/Parsers/ASTShowCreateAccessEntityQuery.h +++ b/src/Parsers/ASTShowCreateAccessEntityQuery.h @@ -15,22 +15,14 @@ namespace DB class ASTShowCreateAccessEntityQuery : public ASTQueryWithOutput { public: - enum class Kind - { - USER, - ROLE, - QUOTA, - ROW_POLICY, - SETTINGS_PROFILE, - }; + using EntityType = IAccessEntity::Type; - const Kind kind; + EntityType type; String name; bool current_quota = false; bool current_user = false; - RowPolicy::FullNameParts row_policy_name; + RowPolicy::NameParts row_policy_name_parts; - ASTShowCreateAccessEntityQuery(Kind kind_); String getID(char) const override; ASTPtr clone() const override; diff --git a/src/Parsers/ASTShowPrivilegesQuery.cpp b/src/Parsers/ASTShowPrivilegesQuery.cpp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/Parsers/ASTShowPrivilegesQuery.h b/src/Parsers/ASTShowPrivilegesQuery.h new file mode 100644 index 00000000000..8cc4ed16f96 --- /dev/null +++ b/src/Parsers/ASTShowPrivilegesQuery.h @@ -0,0 +1,17 @@ +#pragma once + +#include + + +namespace DB +{ + +struct ASTShowPrivilegesIDAndQueryName +{ + static constexpr auto ID = "ShowPrivilegesQuery"; + static constexpr auto Query = "SHOW PRIVILEGES"; +}; + +using ASTShowPrivilegesQuery = ASTQueryWithOutputImpl; + +} diff --git a/src/Parsers/ASTShowQuotasQuery.cpp b/src/Parsers/ASTShowQuotasQuery.cpp deleted file mode 100644 index ca7bd5e853f..00000000000 --- a/src/Parsers/ASTShowQuotasQuery.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include -#include - - -namespace DB -{ -String ASTShowQuotasQuery::getID(char) const -{ - if (usage) - return "SHOW QUOTA USAGE query"; - else - return "SHOW QUOTAS query"; -} - - -ASTPtr ASTShowQuotasQuery::clone() const -{ - return std::make_shared(*this); -} - - -void ASTShowQuotasQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const -{ - settings.ostr << (settings.hilite ? hilite_keyword : ""); - - if (usage && current) - settings.ostr << "SHOW QUOTA USAGE"; - else if (usage) - settings.ostr << "SHOW QUOTA USAGE ALL"; - else - settings.ostr << "SHOW QUOTAS"; - - settings.ostr << (settings.hilite ? hilite_none : ""); -} -} diff --git a/src/Parsers/ASTShowQuotasQuery.h b/src/Parsers/ASTShowQuotasQuery.h deleted file mode 100644 index 27a08a99a54..00000000000 --- a/src/Parsers/ASTShowQuotasQuery.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include - - -namespace DB -{ -/** SHOW QUOTAS - * SHOW QUOTA USAGE [CURRENT | ALL] - */ -class ASTShowQuotasQuery : public ASTQueryWithOutput -{ -public: - bool usage = false; - bool current = false; - - String getID(char) const override; - ASTPtr clone() const override; - -protected: - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; -}; - -} diff --git a/src/Parsers/ASTShowRowPoliciesQuery.cpp b/src/Parsers/ASTShowRowPoliciesQuery.cpp deleted file mode 100644 index 15e0e81f218..00000000000 --- a/src/Parsers/ASTShowRowPoliciesQuery.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include -#include - - -namespace DB -{ -void ASTShowRowPoliciesQuery::formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const -{ - settings.ostr << (settings.hilite ? hilite_keyword : "") << "SHOW POLICIES" << (settings.hilite ? hilite_none : ""); - - if (current) - settings.ostr << (settings.hilite ? hilite_keyword : "") << " CURRENT" << (settings.hilite ? hilite_none : ""); - - if (!table_name.empty()) - { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : ""); - if (!database.empty()) - settings.ostr << backQuoteIfNeed(database) << "."; - settings.ostr << backQuoteIfNeed(table_name); - } -} -} diff --git a/src/Parsers/ASTShowRowPoliciesQuery.h b/src/Parsers/ASTShowRowPoliciesQuery.h deleted file mode 100644 index ce82902e96d..00000000000 --- a/src/Parsers/ASTShowRowPoliciesQuery.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include - - -namespace DB -{ -/// SHOW [ROW] POLICIES [CURRENT] [ON [database.]table] -class ASTShowRowPoliciesQuery : public ASTQueryWithOutput -{ -public: - bool current = false; - String database; - String table_name; - - String getID(char) const override { return "SHOW POLICIES query"; } - ASTPtr clone() const override { return std::make_shared(*this); } - -protected: - void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; -}; - -} diff --git a/src/Parsers/ParserCreateQuotaQuery.cpp b/src/Parsers/ParserCreateQuotaQuery.cpp index 6007d6206ec..b505fd25a95 100644 --- a/src/Parsers/ParserCreateQuotaQuery.cpp +++ b/src/Parsers/ParserCreateQuotaQuery.cpp @@ -22,9 +22,12 @@ namespace ErrorCodes namespace { using KeyType = Quota::KeyType; + using KeyTypeInfo = Quota::KeyTypeInfo; using ResourceType = Quota::ResourceType; + using ResourceTypeInfo = Quota::ResourceTypeInfo; using ResourceAmount = Quota::ResourceAmount; + bool parseRenameTo(IParserBase::Pos & pos, Expected & expected, String & new_name) { return IParserBase::wrapParseImpl(pos, [&] @@ -48,16 +51,16 @@ namespace return false; const String & key_type_str = key_type_ast->as().value.safeGet(); - for (auto kt : ext::range_with_static_cast(Quota::MAX_KEY_TYPE)) - if (boost::iequals(Quota::getNameOfKeyType(kt), key_type_str)) + for (auto kt : ext::range(Quota::KeyType::MAX)) + if (boost::iequals(KeyTypeInfo::get(kt).name, key_type_str)) { key_type = kt; return true; } String all_key_types_str; - for (auto kt : ext::range_with_static_cast(Quota::MAX_KEY_TYPE)) - all_key_types_str += String(all_key_types_str.empty() ? "" : ", ") + "'" + Quota::getNameOfKeyType(kt) + "'"; + for (auto kt : ext::range(Quota::KeyType::MAX)) + all_key_types_str += String(all_key_types_str.empty() ? "" : ", ") + "'" + KeyTypeInfo::get(kt).name + "'"; String msg = "Quota cannot be keyed by '" + key_type_str + "'. Expected one of these literals: " + all_key_types_str; throw Exception(msg, ErrorCodes::SYNTAX_ERROR); }); @@ -80,31 +83,35 @@ namespace ParserKeyword{"MAX"}.ignore(pos, expected); } - bool resource_type_set = false; - for (auto rt : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) + std::optional res_resource_type; + for (auto rt : ext::range(Quota::MAX_RESOURCE_TYPE)) { - if (ParserKeyword{Quota::resourceTypeToKeyword(rt)}.ignore(pos, expected)) + if (ParserKeyword{ResourceTypeInfo::get(rt).keyword.c_str()}.ignore(pos, expected)) { - resource_type = rt; - resource_type_set = true; + res_resource_type = rt; break; } } - if (!resource_type_set) + if (!res_resource_type) return false; + ResourceAmount res_max; ASTPtr max_ast; if (ParserNumber{}.parse(pos, max_ast, expected)) { const Field & max_field = max_ast->as().value; - if (resource_type == Quota::EXECUTION_TIME) - max = Quota::secondsToExecutionTime(applyVisitor(FieldVisitorConvertToNumber(), max_field)); + const auto & type_info = ResourceTypeInfo::get(*res_resource_type); + if (type_info.output_denominator == 1) + res_max = applyVisitor(FieldVisitorConvertToNumber(), max_field); else - max = applyVisitor(FieldVisitorConvertToNumber(), max_field); + res_max = static_cast( + applyVisitor(FieldVisitorConvertToNumber(), max_field) * type_info.output_denominator); } else return false; + resource_type = *res_resource_type; + max = res_max; return true; }); } diff --git a/src/Parsers/ParserCreateRowPolicyQuery.cpp b/src/Parsers/ParserCreateRowPolicyQuery.cpp index b6840f0ed6a..2456cb80368 100644 --- a/src/Parsers/ParserCreateRowPolicyQuery.cpp +++ b/src/Parsers/ParserCreateRowPolicyQuery.cpp @@ -8,27 +8,26 @@ #include #include #include +#include namespace DB { -namespace ErrorCodes -{ -} - - namespace { using ConditionType = RowPolicy::ConditionType; + using ConditionTypeInfo = RowPolicy::ConditionTypeInfo; + constexpr auto MAX_CONDITION_TYPE = RowPolicy::MAX_CONDITION_TYPE; - bool parseRenameTo(IParserBase::Pos & pos, Expected & expected, String & new_policy_name) + + bool parseRenameTo(IParserBase::Pos & pos, Expected & expected, String & new_short_name) { return IParserBase::wrapParseImpl(pos, [&] { if (!ParserKeyword{"RENAME TO"}.ignore(pos, expected)) return false; - return parseIdentifierOrStringLiteral(pos, expected, new_policy_name); + return parseIdentifierOrStringLiteral(pos, expected, new_short_name); }); } @@ -73,111 +72,93 @@ namespace }); } - bool parseConditions(IParserBase::Pos & pos, Expected & expected, bool alter, std::vector> & conditions) + bool parseConditions( + IParserBase::Pos & pos, Expected & expected, bool alter, std::array, MAX_CONDITION_TYPE> & conditions) { return IParserBase::wrapParseImpl(pos, [&] { - static constexpr char select_op[] = "SELECT"; - static constexpr char insert_op[] = "INSERT"; - static constexpr char update_op[] = "UPDATE"; - static constexpr char delete_op[] = "DELETE"; - std::vector ops; + boost::container::flat_set commands; + + auto add_all_commands = [&] + { + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + { + const std::string_view & command = ConditionTypeInfo::get(condition_type).command; + commands.emplace(command); + } + }; if (ParserKeyword{"FOR"}.ignore(pos, expected)) { do { - if (ParserKeyword{"SELECT"}.ignore(pos, expected)) - ops.push_back(select_op); -#if 0 /// INSERT, UPDATE, DELETE are not supported yet - else if (ParserKeyword{"INSERT"}.ignore(pos, expected)) - ops.push_back(insert_op); - else if (ParserKeyword{"UPDATE"}.ignore(pos, expected)) - ops.push_back(update_op); - else if (ParserKeyword{"DELETE"}.ignore(pos, expected)) - ops.push_back(delete_op); - else if (ParserKeyword{"ALL"}.ignore(pos, expected)) + size_t old_size = commands.size(); + if (ParserKeyword{"ALL"}.ignore(pos, expected)) { + add_all_commands(); } -#endif else + { + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + { + const std::string_view & command = ConditionTypeInfo::get(condition_type).command; + if (ParserKeyword{command.data()}.ignore(pos, expected)) + { + commands.emplace(command); + break; + } + } + } + if (commands.size() == old_size) return false; } while (ParserToken{TokenType::Comma}.ignore(pos, expected)); } - if (ops.empty()) - { - ops.push_back(select_op); -#if 0 /// INSERT, UPDATE, DELETE are not supported yet - ops.push_back(insert_op); - ops.push_back(update_op); - ops.push_back(delete_op); -#endif - } - std::optional filter; std::optional check; - bool keyword_using = false, keyword_with_check = false; if (ParserKeyword{"USING"}.ignore(pos, expected)) { - keyword_using = true; if (!parseConditionalExpression(pos, expected, filter)) return false; } -#if 0 /// INSERT, UPDATE, DELETE are not supported yet if (ParserKeyword{"WITH CHECK"}.ignore(pos, expected)) { - keyword_with_check = true; if (!parseConditionalExpression(pos, expected, check)) return false; } -#endif - if (!keyword_using && !keyword_with_check) + + if (!filter && !check) return false; - if (filter && !check && !alter) + if (commands.empty()) + add_all_commands(); + + if (!check && !alter) check = filter; - auto set_condition = [&](ConditionType index, const ASTPtr & condition) + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) { - auto it = std::find_if(conditions.begin(), conditions.end(), [index](const std::pair & element) + const auto & type_info = ConditionTypeInfo::get(condition_type); + if (commands.count(type_info.command)) { - return element.first == index; - }); - if (it == conditions.end()) - it = conditions.insert(conditions.end(), std::pair{index, nullptr}); - it->second = condition; - }; - - for (const auto & op : ops) - { - if ((op == select_op) && filter) - set_condition(RowPolicy::SELECT_FILTER, *filter); - else if ((op == insert_op) && check) - set_condition(RowPolicy::INSERT_CHECK, *check); - else if (op == update_op) - { - if (filter) - set_condition(RowPolicy::UPDATE_FILTER, *filter); - if (check) - set_condition(RowPolicy::UPDATE_CHECK, *check); + if (type_info.is_check && check) + conditions[condition_type] = check; + else if (filter) + conditions[condition_type] = filter; } - else if ((op == delete_op) && filter) - set_condition(RowPolicy::DELETE_FILTER, *filter); - else - __builtin_unreachable(); } return true; }); } - bool parseMultipleConditions(IParserBase::Pos & pos, Expected & expected, bool alter, std::vector> & conditions) + bool parseMultipleConditions( + IParserBase::Pos & pos, Expected & expected, bool alter, std::array, MAX_CONDITION_TYPE> & conditions) { return IParserBase::wrapParseImpl(pos, [&] { - std::vector> res_conditions; + std::array, MAX_CONDITION_TYPE> res_conditions; do { if (!parseConditions(pos, expected, alter, res_conditions)) @@ -246,22 +227,22 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & or_replace = true; } - RowPolicy::FullNameParts name_parts; + RowPolicy::NameParts name_parts; String & database = name_parts.database; String & table_name = name_parts.table_name; - String & policy_name = name_parts.policy_name; - if (!parseIdentifierOrStringLiteral(pos, expected, policy_name) || !ParserKeyword{"ON"}.ignore(pos, expected) + String & short_name = name_parts.short_name; + if (!parseIdentifierOrStringLiteral(pos, expected, short_name) || !ParserKeyword{"ON"}.ignore(pos, expected) || !parseDatabaseAndTableName(pos, expected, database, table_name)) return false; - String new_policy_name; + String new_short_name; std::optional is_restrictive; - std::vector> conditions; + std::array, MAX_CONDITION_TYPE> conditions; String cluster; while (true) { - if (alter && new_policy_name.empty() && parseRenameTo(pos, expected, new_policy_name)) + if (alter && new_short_name.empty() && parseRenameTo(pos, expected, new_short_name)) continue; if (!is_restrictive && parseAsRestrictiveOrPermissive(pos, expected, is_restrictive)) @@ -292,7 +273,7 @@ bool ParserCreateRowPolicyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & query->or_replace = or_replace; query->cluster = std::move(cluster); query->name_parts = std::move(name_parts); - query->new_policy_name = std::move(new_policy_name); + query->new_short_name = std::move(new_short_name); query->is_restrictive = is_restrictive; query->conditions = std::move(conditions); query->roles = std::move(roles); diff --git a/src/Parsers/ParserDropAccessEntityQuery.cpp b/src/Parsers/ParserDropAccessEntityQuery.cpp index ecda1691240..15f8bbf0a62 100644 --- a/src/Parsers/ParserDropAccessEntityQuery.cpp +++ b/src/Parsers/ParserDropAccessEntityQuery.cpp @@ -4,12 +4,16 @@ #include #include #include +#include namespace DB { namespace { + using EntityType = IAccessEntity::Type; + using EntityTypeInfo = IAccessEntity::TypeInfo; + bool parseNames(IParserBase::Pos & pos, Expected & expected, Strings & names) { return IParserBase::wrapParseImpl(pos, [&] @@ -30,25 +34,25 @@ namespace }); } - bool parseRowPolicyNames(IParserBase::Pos & pos, Expected & expected, std::vector & names) + bool parseRowPolicyNames(IParserBase::Pos & pos, Expected & expected, std::vector & name_parts) { return IParserBase::wrapParseImpl(pos, [&] { - std::vector res_names; + std::vector res_name_parts; do { - Strings policy_names; - if (!parseNames(pos, expected, policy_names)) + Strings short_names; + if (!parseNames(pos, expected, short_names)) return false; String database, table_name; if (!ParserKeyword{"ON"}.ignore(pos, expected) || !parseDatabaseAndTableName(pos, expected, database, table_name)) return false; - for (const String & policy_name : policy_names) - res_names.push_back({database, table_name, policy_name}); + for (String & short_name : short_names) + res_name_parts.push_back({std::move(short_name), database, table_name}); } while (ParserToken{TokenType::Comma}.ignore(pos, expected)); - names = std::move(res_names); + name_parts = std::move(res_name_parts); return true; }); } @@ -79,19 +83,17 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & if (!ParserKeyword{"DROP"}.ignore(pos, expected)) return false; - using Kind = ASTDropAccessEntityQuery::Kind; - Kind kind; - if (ParserKeyword{"USER"}.ignore(pos, expected)) - kind = Kind::USER; - else if (ParserKeyword{"ROLE"}.ignore(pos, expected)) - kind = Kind::ROLE; - else if (ParserKeyword{"QUOTA"}.ignore(pos, expected)) - kind = Kind::QUOTA; - else if (ParserKeyword{"POLICY"}.ignore(pos, expected) || ParserKeyword{"ROW POLICY"}.ignore(pos, expected)) - kind = Kind::ROW_POLICY; - else if (ParserKeyword{"SETTINGS PROFILE"}.ignore(pos, expected) || ParserKeyword{"PROFILE"}.ignore(pos, expected)) - kind = Kind::SETTINGS_PROFILE; - else + std::optional type; + for (auto type_i : ext::range(EntityType::MAX)) + { + const auto & type_info = EntityTypeInfo::get(type_i); + if (ParserKeyword{type_info.name.c_str()}.ignore(pos, expected) + || (!type_info.alias.empty() && ParserKeyword{type_info.alias.c_str()}.ignore(pos, expected))) + { + type = type_i; + } + } + if (!type) return false; bool if_exists = false; @@ -99,16 +101,16 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & if_exists = true; Strings names; - std::vector row_policies_names; + std::vector row_policies_name_parts; - if ((kind == Kind::USER) || (kind == Kind::ROLE)) + if ((type == EntityType::USER) || (type == EntityType::ROLE)) { if (!parseUserNames(pos, expected, names)) return false; } - else if (kind == Kind::ROW_POLICY) + else if (type == EntityType::ROW_POLICY) { - if (!parseRowPolicyNames(pos, expected, row_policies_names)) + if (!parseRowPolicyNames(pos, expected, row_policies_name_parts)) return false; } else @@ -124,13 +126,14 @@ bool ParserDropAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & return false; } - auto query = std::make_shared(kind); + auto query = std::make_shared(); node = query; + query->type = *type; query->if_exists = if_exists; query->cluster = std::move(cluster); query->names = std::move(names); - query->row_policies_names = std::move(row_policies_names); + query->row_policies_name_parts = std::move(row_policies_name_parts); return true; } diff --git a/src/Parsers/ParserQueryWithOutput.cpp b/src/Parsers/ParserQueryWithOutput.cpp index f818e72243f..a81305c0557 100644 --- a/src/Parsers/ParserQueryWithOutput.cpp +++ b/src/Parsers/ParserQueryWithOutput.cpp @@ -14,10 +14,10 @@ #include #include #include -#include +#include #include -#include -#include +#include +#include namespace DB @@ -38,10 +38,10 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec ParserOptimizeQuery optimize_p; ParserKillQueryQuery kill_query_p; ParserWatchQuery watch_p; + ParserShowAccessEntitiesQuery show_access_entities_p; ParserShowCreateAccessEntityQuery show_create_access_entity_p; ParserShowGrantsQuery show_grants_p; - ParserShowQuotasQuery show_quotas_p; - ParserShowRowPoliciesQuery show_row_policies_p; + ParserShowPrivilegesQuery show_privileges_p; ASTPtr query; @@ -70,9 +70,9 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec || kill_query_p.parse(pos, query, expected) || optimize_p.parse(pos, query, expected) || watch_p.parse(pos, query, expected) + || show_access_entities_p.parse(pos, query, expected) || show_grants_p.parse(pos, query, expected) - || show_quotas_p.parse(pos, query, expected) - || show_row_policies_p.parse(pos, query, expected); + || show_privileges_p.parse(pos, query, expected); if (!parsed) return false; diff --git a/src/Parsers/ParserSettingsProfileElement.cpp b/src/Parsers/ParserSettingsProfileElement.cpp index 31bc339f544..37044e8ccbe 100644 --- a/src/Parsers/ParserSettingsProfileElement.cpp +++ b/src/Parsers/ParserSettingsProfileElement.cpp @@ -102,7 +102,7 @@ namespace bool ParserSettingsProfileElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { String parent_profile; - String name; + String setting_name; Field value; Field min_value; Field max_value; @@ -119,7 +119,7 @@ bool ParserSettingsProfileElement::parseImpl(Pos & pos, ASTPtr & node, Expected ASTPtr name_ast; if (!ParserIdentifier{}.parse(pos, name_ast, expected)) return false; - name = getIdentifierName(name_ast); + setting_name = getIdentifierName(name_ast); bool has_value_or_constraint = false; while (parseValue(pos, expected, value) || parseMinMaxValue(pos, expected, min_value, max_value) @@ -134,7 +134,7 @@ bool ParserSettingsProfileElement::parseImpl(Pos & pos, ASTPtr & node, Expected auto result = std::make_shared(); result->parent_profile = std::move(parent_profile); - result->name = std::move(name); + result->setting_name = std::move(setting_name); result->value = std::move(value); result->min_value = std::move(min_value); result->max_value = std::move(max_value); diff --git a/src/Parsers/ParserShowAccessEntitiesQuery.cpp b/src/Parsers/ParserShowAccessEntitiesQuery.cpp new file mode 100644 index 00000000000..c50bd5b402c --- /dev/null +++ b/src/Parsers/ParserShowAccessEntitiesQuery.cpp @@ -0,0 +1,89 @@ +#include +#include +#include +#include + + +namespace DB +{ +namespace +{ + using EntityType = IAccessEntity::Type; + + bool parseONDatabaseAndTableName(IParserBase::Pos & pos, Expected & expected, String & database, String & table_name) + { + return IParserBase::wrapParseImpl(pos, [&] + { + database.clear(); + table_name.clear(); + return ParserKeyword{"ON"}.ignore(pos, expected) && parseDatabaseAndTableName(pos, expected, database, table_name); + }); + } +} + + +bool ParserShowAccessEntitiesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + if (!ParserKeyword{"SHOW"}.ignore(pos, expected)) + return false; + + std::optional type; + bool current_quota = false; + bool current_roles = false; + bool enabled_roles = false; + + if (ParserKeyword{"USERS"}.ignore(pos, expected)) + { + type = EntityType::USER; + } + else if (ParserKeyword{"ROLES"}.ignore(pos, expected)) + { + type = EntityType::ROLE; + } + else if (ParserKeyword{"CURRENT ROLES"}.ignore(pos, expected)) + { + type = EntityType::ROLE; + current_roles = true; + } + else if (ParserKeyword{"ENABLED ROLES"}.ignore(pos, expected)) + { + type = EntityType::ROLE; + enabled_roles = true; + } + else if (ParserKeyword{"POLICIES"}.ignore(pos, expected) || ParserKeyword{"ROW POLICIES"}.ignore(pos, expected)) + { + type = EntityType::ROW_POLICY; + } + else if (ParserKeyword{"QUOTAS"}.ignore(pos, expected)) + { + type = EntityType::QUOTA; + } + else if (ParserKeyword{"QUOTA"}.ignore(pos, expected) || ParserKeyword{"CURRENT QUOTA"}.ignore(pos, expected)) + { + type = EntityType::QUOTA; + current_quota = true; + } + else if (ParserKeyword{"PROFILES"}.ignore(pos, expected) || ParserKeyword{"SETTINGS PROFILES"}.ignore(pos, expected)) + { + type = EntityType::SETTINGS_PROFILE; + } + else + return false; + + String database, table_name; + if (type == EntityType::ROW_POLICY) + parseONDatabaseAndTableName(pos, expected, database, table_name); + + auto query = std::make_shared(); + node = query; + + query->type = *type; + query->current_quota = current_quota; + query->current_roles = current_roles; + query->enabled_roles = enabled_roles; + query->database = std::move(database); + query->table_name = std::move(table_name); + + return true; +} +} diff --git a/src/Parsers/ParserShowAccessEntitiesQuery.h b/src/Parsers/ParserShowAccessEntitiesQuery.h new file mode 100644 index 00000000000..bb8b37f40e8 --- /dev/null +++ b/src/Parsers/ParserShowAccessEntitiesQuery.h @@ -0,0 +1,20 @@ +#pragma once + +#include + + +namespace DB +{ +/** Parses queries like + * SHOW [ROW] POLICIES [ON [database.]table] + SHOW QUOTAS + SHOW [CURRENT] QUOTA + SHOW [SETTINGS] PROFILES + */ +class ParserShowAccessEntitiesQuery : public IParserBase +{ +protected: + const char * getName() const override { return "ShowAccessEntitiesQuery"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; +} diff --git a/src/Parsers/ParserShowCreateAccessEntityQuery.cpp b/src/Parsers/ParserShowCreateAccessEntityQuery.cpp index faf9a0a1554..ca520f4df6f 100644 --- a/src/Parsers/ParserShowCreateAccessEntityQuery.cpp +++ b/src/Parsers/ParserShowCreateAccessEntityQuery.cpp @@ -4,85 +4,80 @@ #include #include #include +#include #include namespace DB { +using EntityType = IAccessEntity::Type; +using EntityTypeInfo = IAccessEntity::TypeInfo; + + bool ParserShowCreateAccessEntityQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { if (!ParserKeyword{"SHOW CREATE"}.ignore(pos, expected)) return false; - using Kind = ASTShowCreateAccessEntityQuery::Kind; - Kind kind; - if (ParserKeyword{"USER"}.ignore(pos, expected)) - kind = Kind::USER; - else if (ParserKeyword{"QUOTA"}.ignore(pos, expected)) - kind = Kind::QUOTA; - else if (ParserKeyword{"POLICY"}.ignore(pos, expected) || ParserKeyword{"ROW POLICY"}.ignore(pos, expected)) - kind = Kind::ROW_POLICY; - else if (ParserKeyword{"ROLE"}.ignore(pos, expected)) - kind = Kind::ROLE; - else if (ParserKeyword{"SETTINGS PROFILE"}.ignore(pos, expected) || ParserKeyword{"PROFILE"}.ignore(pos, expected)) - kind = Kind::SETTINGS_PROFILE; - else + std::optional type; + for (auto type_i : ext::range(EntityType::MAX)) + { + const auto & type_info = EntityTypeInfo::get(type_i); + if (ParserKeyword{type_info.name.c_str()}.ignore(pos, expected) + || (!type_info.alias.empty() && ParserKeyword{type_info.alias.c_str()}.ignore(pos, expected))) + { + type = type_i; + } + } + if (!type) return false; String name; bool current_quota = false; bool current_user = false; - RowPolicy::FullNameParts row_policy_name; + RowPolicy::NameParts row_policy_name_parts; - if (kind == Kind::USER) + if (type == EntityType::USER) { if (!parseUserNameOrCurrentUserTag(pos, expected, name, current_user)) current_user = true; } - else if (kind == Kind::ROLE) + else if (type == EntityType::ROLE) { if (!parseRoleName(pos, expected, name)) return false; } - else if (kind == Kind::ROW_POLICY) + else if (type == EntityType::ROW_POLICY) { - String & database = row_policy_name.database; - String & table_name = row_policy_name.table_name; - String & policy_name = row_policy_name.policy_name; - if (!parseIdentifierOrStringLiteral(pos, expected, policy_name) || !ParserKeyword{"ON"}.ignore(pos, expected) + String & database = row_policy_name_parts.database; + String & table_name = row_policy_name_parts.table_name; + String & short_name = row_policy_name_parts.short_name; + if (!parseIdentifierOrStringLiteral(pos, expected, short_name) || !ParserKeyword{"ON"}.ignore(pos, expected) || !parseDatabaseAndTableName(pos, expected, database, table_name)) return false; } - else if (kind == Kind::QUOTA) + else if (type == EntityType::QUOTA) { - if (ParserKeyword{"CURRENT"}.ignore(pos, expected)) - { - /// SHOW CREATE QUOTA CURRENT - current_quota = true; - } - else if (parseIdentifierOrStringLiteral(pos, expected, name)) - { - /// SHOW CREATE QUOTA name - } - else + if (!parseIdentifierOrStringLiteral(pos, expected, name)) { /// SHOW CREATE QUOTA current_quota = true; } } - else if (kind == Kind::SETTINGS_PROFILE) + else if (type == EntityType::SETTINGS_PROFILE) { if (!parseIdentifierOrStringLiteral(pos, expected, name)) return false; } - auto query = std::make_shared(kind); + auto query = std::make_shared(); node = query; + query->type = *type; query->name = std::move(name); query->current_quota = current_quota; query->current_user = current_user; - query->row_policy_name = std::move(row_policy_name); + query->row_policy_name_parts = std::move(row_policy_name_parts); return true; } diff --git a/src/Parsers/ParserShowCreateAccessEntityQuery.h b/src/Parsers/ParserShowCreateAccessEntityQuery.h index 4572b54de27..025949d7fca 100644 --- a/src/Parsers/ParserShowCreateAccessEntityQuery.h +++ b/src/Parsers/ParserShowCreateAccessEntityQuery.h @@ -6,7 +6,7 @@ namespace DB { /** Parses queries like - * SHOW CREATE QUOTA [name | CURRENT] + * SHOW CREATE QUOTA [name] */ class ParserShowCreateAccessEntityQuery : public IParserBase { diff --git a/src/Parsers/ParserShowPrivilegesQuery.cpp b/src/Parsers/ParserShowPrivilegesQuery.cpp new file mode 100644 index 00000000000..56b4327dccf --- /dev/null +++ b/src/Parsers/ParserShowPrivilegesQuery.cpp @@ -0,0 +1,21 @@ +#include +#include +#include + + +namespace DB +{ + +bool ParserShowPrivilegesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + auto query = std::make_shared(); + + if (!ParserKeyword("SHOW PRIVILEGES").ignore(pos, expected)) + return false; + + node = query; + + return true; +} + +} diff --git a/src/Parsers/ParserShowPrivilegesQuery.h b/src/Parsers/ParserShowPrivilegesQuery.h new file mode 100644 index 00000000000..2604e7f28c1 --- /dev/null +++ b/src/Parsers/ParserShowPrivilegesQuery.h @@ -0,0 +1,18 @@ +#pragma once + +#include + + +namespace DB +{ + +/** Query SHOW PRIVILEGES + */ +class ParserShowPrivilegesQuery : public IParserBase +{ +protected: + const char * getName() const override { return "SHOW PRIVILEGES query"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +} diff --git a/src/Parsers/ParserShowQuotasQuery.cpp b/src/Parsers/ParserShowQuotasQuery.cpp deleted file mode 100644 index 69cbd352969..00000000000 --- a/src/Parsers/ParserShowQuotasQuery.cpp +++ /dev/null @@ -1,42 +0,0 @@ -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -bool ParserShowQuotasQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) -{ - bool usage; - bool current; - if (ParserKeyword{"SHOW QUOTAS"}.ignore(pos, expected)) - { - usage = false; - current = false; - } - else if (ParserKeyword{"SHOW QUOTA USAGE"}.ignore(pos, expected)) - { - usage = true; - if (ParserKeyword{"ALL"}.ignore(pos, expected)) - { - current = false; - } - else - { - ParserKeyword{"CURRENT"}.ignore(pos, expected); - current = true; - } - } - else - return false; - - auto query = std::make_shared(); - query->usage = usage; - query->current = current; - node = query; - return true; -} -} diff --git a/src/Parsers/ParserShowQuotasQuery.h b/src/Parsers/ParserShowQuotasQuery.h deleted file mode 100644 index 5b00b525f98..00000000000 --- a/src/Parsers/ParserShowQuotasQuery.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once - -#include - - -namespace DB -{ -/** Parses queries like - * SHOW QUOTAS - * SHOW QUOTA USAGE [CURRENT | ALL] - */ -class ParserShowQuotasQuery : public IParserBase -{ -protected: - const char * getName() const override { return "SHOW QUOTA query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; -}; -} diff --git a/src/Parsers/ParserShowRowPoliciesQuery.cpp b/src/Parsers/ParserShowRowPoliciesQuery.cpp deleted file mode 100644 index b07e7a386ba..00000000000 --- a/src/Parsers/ParserShowRowPoliciesQuery.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include -#include -#include -#include - - -namespace DB -{ -namespace -{ - bool parseONDatabaseAndTableName(IParserBase::Pos & pos, Expected & expected, String & database, String & table_name) - { - return IParserBase::wrapParseImpl(pos, [&] - { - database.clear(); - table_name.clear(); - return ParserKeyword{"ON"}.ignore(pos, expected) && parseDatabaseAndTableName(pos, expected, database, table_name); - }); - } -} - - -bool ParserShowRowPoliciesQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) -{ - if (!ParserKeyword{"SHOW POLICIES"}.ignore(pos, expected) && !ParserKeyword{"SHOW ROW POLICIES"}.ignore(pos, expected)) - return false; - - bool current = ParserKeyword{"CURRENT"}.ignore(pos, expected); - - String database, table_name; - parseONDatabaseAndTableName(pos, expected, database, table_name); - - auto query = std::make_shared(); - query->current = current; - query->database = std::move(database); - query->table_name = std::move(table_name); - node = query; - return true; -} -} diff --git a/src/Parsers/ParserShowRowPoliciesQuery.h b/src/Parsers/ParserShowRowPoliciesQuery.h deleted file mode 100644 index df7413fb604..00000000000 --- a/src/Parsers/ParserShowRowPoliciesQuery.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include - - -namespace DB -{ -/** Parses queries like - * SHOW [ROW] POLICIES [CURRENT] [ON [database.]table] - */ -class ParserShowRowPoliciesQuery : public IParserBase -{ -protected: - const char * getName() const override { return "SHOW POLICIES query"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; -}; -} diff --git a/src/Parsers/ya.make b/src/Parsers/ya.make index 942124c2c7a..8c7e4ff68af 100644 --- a/src/Parsers/ya.make +++ b/src/Parsers/ya.make @@ -42,10 +42,10 @@ SRCS( ASTSelectWithUnionQuery.cpp ASTSetRoleQuery.cpp ASTSettingsProfileElement.cpp + ASTShowAccessEntitiesQuery.cpp ASTShowCreateAccessEntityQuery.cpp ASTShowGrantsQuery.cpp - ASTShowQuotasQuery.cpp - ASTShowRowPoliciesQuery.cpp + ASTShowPrivilegesQuery.cpp ASTShowTablesQuery.cpp ASTSubquery.cpp ASTSystemQuery.cpp @@ -94,10 +94,10 @@ SRCS( ParserSetQuery.cpp ParserSetRoleQuery.cpp ParserSettingsProfileElement.cpp + ParserShowAccessEntitiesQuery.cpp ParserShowCreateAccessEntityQuery.cpp ParserShowGrantsQuery.cpp - ParserShowQuotasQuery.cpp - ParserShowRowPoliciesQuery.cpp + ParserShowPrivilegesQuery.cpp ParserShowTablesQuery.cpp ParserSystemQuery.cpp ParserTablePropertiesQuery.cpp diff --git a/src/Processors/Chunk.cpp b/src/Processors/Chunk.cpp index d68c2bea5ae..1f73c9f276a 100644 --- a/src/Processors/Chunk.cpp +++ b/src/Processors/Chunk.cpp @@ -76,7 +76,7 @@ MutableColumns Chunk::mutateColumns() size_t num_columns = columns.size(); MutableColumns mut_columns(num_columns); for (size_t i = 0; i < num_columns; ++i) - mut_columns[i] = (*std::move(columns[i])).mutate(); + mut_columns[i] = IColumn::mutate(std::move(columns[i])); columns.clear(); num_rows = 0; diff --git a/src/Processors/Executors/ParallelPipelineExecutor.cpp b/src/Processors/Executors/ParallelPipelineExecutor.cpp deleted file mode 100644 index cea020300b6..00000000000 --- a/src/Processors/Executors/ParallelPipelineExecutor.cpp +++ /dev/null @@ -1,105 +0,0 @@ -#include -#include -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} -// -//ParallelPipelineExecutor::ParallelPipelineExecutor(const std::vector & processors, ThreadPool & pool) -// : processors(processors), pool(pool) -//{ -//} -// -// -//ParallelPipelineExecutor::Status ParallelPipelineExecutor::prepare() -//{ -// current_processor = nullptr; -// -// bool has_someone_to_wait = false; -// -// for (auto & element : processors) -// { -// traverse(*element, -// [&] (IProcessor & processor) -// { -// { -// std::lock_guard lock(mutex); -// if (active_processors.count(&processor)) -// { -// has_someone_to_wait = true; -// return Status::Wait; -// } -// } -// -// Status status = processor.prepare(); -// -// if (status == Status::Wait) -// has_someone_to_wait = true; -// -// if (status == Status::Ready || status == Status::Async) -// { -// current_processor = &processor; -// current_status = status; -// } -// -// return status; -// }); -// -// if (current_processor) -// break; -// } -// -// if (current_processor) -// return Status::Async; -// -// if (has_someone_to_wait) -// return Status::Wait; -// -// for (auto & element : processors) -// { -// if (element->prepare() == Status::NeedData) -// throw Exception("Pipeline stuck: " + element->getName() + " processor needs input data but no one is going to generate it", ErrorCodes::LOGICAL_ERROR); -// if (element->prepare() == Status::PortFull) -// throw Exception("Pipeline stuck: " + element->getName() + " processor has data in output port but no one is going to consume it", ErrorCodes::LOGICAL_ERROR); -// } -// -// return Status::Finished; -//} -// -// -//void ParallelPipelineExecutor::schedule(EventCounter & watch) -//{ -// if (!current_processor) -// throw Exception("Bad pipeline", ErrorCodes::LOGICAL_ERROR); -// -// if (current_status == Status::Async) -// { -// current_processor->schedule(watch); -// } -// else -// { -// { -// std::lock_guard lock(mutex); -// active_processors.insert(current_processor); -// } -// -// pool.scheduleOrThrowOnError([processor = current_processor, &watch, this] -// { -// processor->work(); -// { -// std::lock_guard lock(mutex); -// active_processors.erase(processor); -// } -// watch.notify(); -// }); -// } -//} - -} - diff --git a/src/Processors/Executors/ParallelPipelineExecutor.h b/src/Processors/Executors/ParallelPipelineExecutor.h deleted file mode 100644 index 4997f73f699..00000000000 --- a/src/Processors/Executors/ParallelPipelineExecutor.h +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -template -class ThreadPoolImpl; -class ThreadFromGlobalPool; -using ThreadPool = ThreadPoolImpl; - -namespace DB -{ - -/** Wraps pipeline in a single processor. - * This processor has no inputs and outputs and just executes the pipeline, - * performing all synchronous work within a threadpool. - */ -//class ParallelPipelineExecutor : public IProcessor -//{ -//private: -// Processors processors; -// ThreadPool & pool; -// -// std::set active_processors; -// std::mutex mutex; -// -// IProcessor * current_processor = nullptr; -// Status current_status; -// -//public: -// ParallelPipelineExecutor(const Processors & processors, ThreadPool & pool); -// -// String getName() const override { return "ParallelPipelineExecutor"; } -// -// Status prepare() override; -// void schedule(EventCounter & watch) override; -//}; - -} diff --git a/src/Processors/Executors/PullingPipelineExecutor.cpp b/src/Processors/Executors/PullingPipelineExecutor.cpp new file mode 100644 index 00000000000..c34195a0793 --- /dev/null +++ b/src/Processors/Executors/PullingPipelineExecutor.cpp @@ -0,0 +1,198 @@ +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +struct PullingPipelineExecutor::Data +{ + PipelineExecutorPtr executor; + std::exception_ptr exception; + std::atomic_bool is_executed = false; + std::atomic_bool has_exception = false; + ThreadFromGlobalPool thread; + + ~Data() + { + if (thread.joinable()) + thread.join(); + } + + void rethrowExceptionIfHas() + { + if (has_exception) + { + has_exception = false; + std::rethrow_exception(std::move(exception)); + } + } +}; + +PullingPipelineExecutor::PullingPipelineExecutor(QueryPipeline & pipeline_) : pipeline(pipeline_) +{ + lazy_format = std::make_shared(pipeline.getHeader()); + pipeline.setOutput(lazy_format); +} + +PullingPipelineExecutor::~PullingPipelineExecutor() +{ + try + { + cancel(); + } + catch (...) + { + tryLogCurrentException("PullingPipelineExecutor"); + } +} + +static void threadFunction(PullingPipelineExecutor::Data & data, ThreadGroupStatusPtr thread_group, size_t num_threads) +{ + if (thread_group) + CurrentThread::attachTo(thread_group); + + SCOPE_EXIT( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); + + setThreadName("QueryPipelineEx"); + + try + { + data.executor->execute(num_threads); + } + catch (...) + { + data.exception = std::current_exception(); + data.has_exception = true; + } +} + + +bool PullingPipelineExecutor::pull(Chunk & chunk, uint64_t milliseconds) +{ + if (!data) + { + data = std::make_unique(); + data->executor = pipeline.execute(); + + auto func = [&, thread_group = CurrentThread::getGroup()]() + { + threadFunction(*data, thread_group, pipeline.getNumThreads()); + }; + + data->thread = ThreadFromGlobalPool(std::move(func)); + } + + if (data->has_exception) + { + /// Finish lazy format in case of exception. Otherwise thread.join() may hung. + lazy_format->finish(); + data->has_exception = false; + std::rethrow_exception(std::move(data->exception)); + } + + if (lazy_format->isFinished()) + { + data->is_executed = true; + /// Wait thread ant rethrow exception if any. + cancel(); + return false; + } + + chunk = lazy_format->getChunk(milliseconds); + return true; +} + +bool PullingPipelineExecutor::pull(Block & block, uint64_t milliseconds) +{ + Chunk chunk; + + if (!pull(chunk, milliseconds)) + return false; + + if (!chunk) + { + /// In case if timeout exceeded. + block.clear(); + return true; + } + + block = lazy_format->getPort(IOutputFormat::PortKind::Main).getHeader().cloneWithColumns(chunk.detachColumns()); + + if (auto chunk_info = chunk.getChunkInfo()) + { + if (const auto * agg_info = typeid_cast(chunk_info.get())) + { + block.info.bucket_num = agg_info->bucket_num; + block.info.is_overflows = agg_info->is_overflows; + } + } + + return true; +} + +void PullingPipelineExecutor::cancel() +{ + /// Cancel execution if it wasn't finished. + if (data && !data->is_executed && data->executor) + data->executor->cancel(); + + /// Finish lazy format. Otherwise thread.join() may hung. + if (!lazy_format->isFinished()) + lazy_format->finish(); + + /// Join thread here to wait for possible exception. + if (data && data->thread.joinable()) + data->thread.join(); + + /// Rethrow exception to not swallow it in destructor. + if (data) + data->rethrowExceptionIfHas(); +} + +Chunk PullingPipelineExecutor::getTotals() +{ + return lazy_format->getTotals(); +} + +Chunk PullingPipelineExecutor::getExtremes() +{ + return lazy_format->getExtremes(); +} + +Block PullingPipelineExecutor::getTotalsBlock() +{ + auto totals = getTotals(); + + if (totals.empty()) + return {}; + + const auto & header = lazy_format->getPort(IOutputFormat::PortKind::Totals).getHeader(); + return header.cloneWithColumns(totals.detachColumns()); +} + +Block PullingPipelineExecutor::getExtremesBlock() +{ + auto extremes = getExtremes(); + + if (extremes.empty()) + return {}; + + const auto & header = lazy_format->getPort(IOutputFormat::PortKind::Extremes).getHeader(); + return header.cloneWithColumns(extremes.detachColumns()); +} + +BlockStreamProfileInfo & PullingPipelineExecutor::getProfileInfo() +{ + return lazy_format->getProfileInfo(); +} + +} diff --git a/src/Processors/Executors/PullingPipelineExecutor.h b/src/Processors/Executors/PullingPipelineExecutor.h new file mode 100644 index 00000000000..f3b06fc618a --- /dev/null +++ b/src/Processors/Executors/PullingPipelineExecutor.h @@ -0,0 +1,54 @@ +#pragma once +#include + +namespace DB +{ + +class QueryPipeline; +class Block; +class Chunk; +class LazyOutputFormat; +struct BlockStreamProfileInfo; + +/// Pulling executor for QueryPipeline. +/// Typical usage is: +/// +/// PullingPipelineExecutor executor(query_pipeline); +/// while (executor.pull(chunk)) +/// ... process chunk ... +class PullingPipelineExecutor +{ +public: + explicit PullingPipelineExecutor(QueryPipeline & pipeline_); + ~PullingPipelineExecutor(); + + /// Methods return false if query is finished. + /// If milliseconds > 0, returns empty object and `true` after timeout exceeded. + /// You can use any pull method. + bool pull(Chunk & chunk, uint64_t milliseconds = 0); + bool pull(Block & block, uint64_t milliseconds = 0); + + /// Stop execution. It is not necessary, but helps to stop execution before executor is destroyed. + void cancel(); + + /// Get totals and extremes. Returns empty chunk if doesn't have any. + Chunk getTotals(); + Chunk getExtremes(); + + /// Get totals and extremes. Returns empty chunk if doesn't have any. + Block getTotalsBlock(); + Block getExtremesBlock(); + + /// Get query profile info. + BlockStreamProfileInfo & getProfileInfo(); + + /// Internal executor data. + struct Data; + +private: + QueryPipeline & pipeline; + std::shared_ptr lazy_format; + std::unique_ptr data; +}; + +} diff --git a/src/Processors/Executors/SequentialPipelineExecutor.cpp b/src/Processors/Executors/SequentialPipelineExecutor.cpp deleted file mode 100644 index 9e3b398f372..00000000000 --- a/src/Processors/Executors/SequentialPipelineExecutor.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - -//SequentialPipelineExecutor::SequentialPipelineExecutor(const Processors & processors) -// : processors(processors) -//{ -//} -// -// -//SequentialPipelineExecutor::Status SequentialPipelineExecutor::prepare() -//{ -// current_processor = nullptr; -// -// bool has_someone_to_wait = false; -// Status found_status = Status::Finished; -// -// for (auto & element : processors) -// { -// traverse(*element, -// [&] (IProcessor & processor) -// { -// Status status = processor.prepare(); -// -// if (status == Status::Wait) -// has_someone_to_wait = true; -// -// if (status == Status::Ready || status == Status::Async) -// { -// current_processor = &processor; -// found_status = status; -// } -// -// return status; -// }); -// -// if (current_processor) -// break; -// } -// -// if (current_processor) -// return found_status; -// if (has_someone_to_wait) -// return Status::Wait; -// -// for (auto & element : processors) -// { -// if (element->prepare() == Status::NeedData) -// throw Exception("Pipeline stuck: " + element->getName() + " processor needs input data but no one is going to generate it", ErrorCodes::LOGICAL_ERROR); -// if (element->prepare() == Status::PortFull) -// throw Exception("Pipeline stuck: " + element->getName() + " processor has data in output port but no one is going to consume it", ErrorCodes::LOGICAL_ERROR); -// } -// -// return Status::Finished; -//} -// -// -//void SequentialPipelineExecutor::work() -//{ -// if (!current_processor) -// throw Exception("Bad pipeline", ErrorCodes::LOGICAL_ERROR); -// -// current_processor->work(); -//} -// -// -//void SequentialPipelineExecutor::schedule(EventCounter & watch) -//{ -// if (!current_processor) -// throw Exception("Bad pipeline", ErrorCodes::LOGICAL_ERROR); -// -// current_processor->schedule(watch); -//} - -} - diff --git a/src/Processors/Executors/SequentialPipelineExecutor.h b/src/Processors/Executors/SequentialPipelineExecutor.h deleted file mode 100644 index 5dbd8b73fee..00000000000 --- a/src/Processors/Executors/SequentialPipelineExecutor.h +++ /dev/null @@ -1,31 +0,0 @@ -#pragma once - -#include - -#include - - -namespace DB -{ - -/** Wraps pipeline in a single processor. - * This processor has no inputs and outputs and just executes the pipeline, - * performing all synchronous work from the current thread. - */ -//class SequentialPipelineExecutor : public IProcessor -//{ -//private: -// Processors processors; -// IProcessor * current_processor = nullptr; -// -//public: -// SequentialPipelineExecutor(const Processors & processors); -// -// String getName() const override { return "SequentialPipelineExecutor"; } -// -// Status prepare() override; -// void work() override; -// void schedule(EventCounter & watch) override; -//}; - -} diff --git a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp index 25e93215ddf..2a4d451d45a 100644 --- a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp @@ -70,7 +70,7 @@ Chunk ValuesBlockInputFormat::generate() if (!templates[i] || !templates[i]->rowsCount()) continue; if (columns[i]->empty()) - columns[i] = std::move(*templates[i]->evaluateAll(block_missing_values, i)).mutate(); + columns[i] = IColumn::mutate(templates[i]->evaluateAll(block_missing_values, i)); else { ColumnPtr evaluated = templates[i]->evaluateAll(block_missing_values, i, columns[i]->size()); @@ -134,7 +134,7 @@ bool ValuesBlockInputFormat::tryParseExpressionUsingTemplate(MutableColumnPtr & /// Expression in the current row is not match template deduced on the first row. /// Evaluate expressions, which were parsed using this template. if (column->empty()) - column = std::move(*templates[column_idx]->evaluateAll(block_missing_values, column_idx)).mutate(); + column = IColumn::mutate(templates[column_idx]->evaluateAll(block_missing_values, column_idx)); else { ColumnPtr evaluated = templates[column_idx]->evaluateAll(block_missing_values, column_idx, column->size()); diff --git a/src/Processors/Formats/LazyOutputFormat.cpp b/src/Processors/Formats/LazyOutputFormat.cpp index 920b34b3813..46287d1cce9 100644 --- a/src/Processors/Formats/LazyOutputFormat.cpp +++ b/src/Processors/Formats/LazyOutputFormat.cpp @@ -7,7 +7,7 @@ namespace DB WriteBuffer LazyOutputFormat::out(nullptr, 0); -Block LazyOutputFormat::getBlock(UInt64 milliseconds) +Chunk LazyOutputFormat::getChunk(UInt64 milliseconds) { if (finished_processing) { @@ -19,38 +19,20 @@ Block LazyOutputFormat::getBlock(UInt64 milliseconds) if (!queue.tryPop(chunk, milliseconds)) return {}; - if (!chunk) - return {}; + if (chunk) + info.update(chunk.getNumRows(), chunk.allocatedBytes()); - auto block = getPort(PortKind::Main).getHeader().cloneWithColumns(chunk.detachColumns()); - info.update(block); - - if (auto chunk_info = chunk.getChunkInfo()) - { - if (const auto * agg_info = typeid_cast(chunk_info.get())) - { - block.info.bucket_num = agg_info->bucket_num; - block.info.is_overflows = agg_info->is_overflows; - } - } - - return block; + return chunk; } -Block LazyOutputFormat::getTotals() +Chunk LazyOutputFormat::getTotals() { - if (!totals) - return {}; - - return getPort(PortKind::Totals).getHeader().cloneWithColumns(totals.detachColumns()); + return std::move(totals); } -Block LazyOutputFormat::getExtremes() +Chunk LazyOutputFormat::getExtremes() { - if (!extremes) - return {}; - - return getPort(PortKind::Extremes).getHeader().cloneWithColumns(extremes.detachColumns()); + return std::move(extremes); } void LazyOutputFormat::setRowsBeforeLimit(size_t rows_before_limit) diff --git a/src/Processors/Formats/LazyOutputFormat.h b/src/Processors/Formats/LazyOutputFormat.h index 59cdbc4e81e..14d7a7f47d7 100644 --- a/src/Processors/Formats/LazyOutputFormat.h +++ b/src/Processors/Formats/LazyOutputFormat.h @@ -8,8 +8,8 @@ namespace DB { /// LazyOutputFormat is used to retrieve ready data from executing pipeline. -/// You can periodically call `getBlock` from separate thread. -/// Used in TCPHandler. +/// You can periodically call `getChunk` from separate thread. +/// Used in PullingPipelineExecutor. class LazyOutputFormat : public IOutputFormat { @@ -19,9 +19,9 @@ public: String getName() const override { return "LazyOutputFormat"; } - Block getBlock(UInt64 milliseconds = 0); - Block getTotals(); - Block getExtremes(); + Chunk getChunk(UInt64 milliseconds = 0); + Chunk getTotals(); + Chunk getExtremes(); bool isFinished() { return finished_processing && queue.size() == 0; } diff --git a/src/Processors/Merges/Algorithms/MergedData.h b/src/Processors/Merges/Algorithms/MergedData.h index c96cc9ad6f4..bb530908c1e 100644 --- a/src/Processors/Merges/Algorithms/MergedData.h +++ b/src/Processors/Merges/Algorithms/MergedData.h @@ -44,7 +44,7 @@ public: { num_rows = limit_rows; for (auto & column : columns) - column = (*column->cut(0, num_rows)).mutate(); + column = IColumn::mutate(column->cut(0, num_rows)); } need_flush = true; diff --git a/src/Processors/Transforms/TotalsHavingTransform.cpp b/src/Processors/Transforms/TotalsHavingTransform.cpp index 451415f7133..083066a72d9 100644 --- a/src/Processors/Transforms/TotalsHavingTransform.cpp +++ b/src/Processors/Transforms/TotalsHavingTransform.cpp @@ -24,7 +24,7 @@ void finalizeChunk(Chunk & chunk) { if (typeid_cast(column.get())) { - auto mut_column = (*std::move(column)).mutate(); + auto mut_column = IColumn::mutate(std::move(column)); column = ColumnAggregateFunction::convertToValues(std::move(mut_column)); } } diff --git a/src/Processors/tests/processors_test.cpp b/src/Processors/tests/processors_test.cpp index b0270932234..3c73223e59c 100644 --- a/src/Processors/tests/processors_test.cpp +++ b/src/Processors/tests/processors_test.cpp @@ -9,8 +9,6 @@ #include #include #include -#include -#include #include #include diff --git a/src/Processors/tests/processors_test_aggregation.cpp b/src/Processors/tests/processors_test_aggregation.cpp index af809fab9f2..e3316432ba8 100644 --- a/src/Processors/tests/processors_test_aggregation.cpp +++ b/src/Processors/tests/processors_test_aggregation.cpp @@ -9,8 +9,6 @@ #include #include #include -#include -#include #include #include diff --git a/src/Processors/ya.make b/src/Processors/ya.make index 7bcc25edcb8..02402df56e3 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -10,9 +10,8 @@ SRCS( Chunk.cpp ConcatProcessor.cpp DelayedPortsProcessor.cpp - Executors/ParallelPipelineExecutor.cpp Executors/PipelineExecutor.cpp - Executors/SequentialPipelineExecutor.cpp + Executors/PullingPipelineExecutor.cpp Executors/TreeExecutorBlockInputStream.cpp ForkProcessor.cpp Formats/IInputFormat.cpp diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 67bd88d10a8..91a48f13a39 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -733,7 +733,6 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con auto all_columns = metadata.columns; /// Default expression for all added/modified columns ASTPtr default_expr_list = std::make_shared(); - NameToNameMap renames_map; for (size_t i = 0; i < size(); ++i) { const auto & command = (*this)[i]; @@ -809,31 +808,40 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con } else if (command.type == AlterCommand::RENAME_COLUMN) { + for (size_t j = i + 1; j < size(); ++j) + { + auto next_command = (*this)[j]; + if (next_command.type == AlterCommand::RENAME_COLUMN) + { + if (next_command.column_name == command.rename_to) + throw Exception{"Transitive renames in a single ALTER query are not allowed (don't make sense)", + ErrorCodes::NOT_IMPLEMENTED}; + else if (next_command.column_name == command.column_name) + throw Exception{"Cannot rename column '" + backQuote(command.column_name) + + "' to two different names in a single ALTER query", + ErrorCodes::BAD_ARGUMENTS}; + } + } + /// TODO Implement nested rename - if (metadata.columns.hasNested(command.column_name)) + if (all_columns.hasNested(command.column_name)) { throw Exception{"Cannot rename whole Nested struct", ErrorCodes::NOT_IMPLEMENTED}; } - if (!metadata.columns.has(command.column_name)) + if (!all_columns.has(command.column_name)) { if (!command.if_exists) throw Exception{"Wrong column name. Cannot find column " + backQuote(command.column_name) + " to rename", ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK}; + else + continue; } - if (metadata.columns.has(command.rename_to)) + if (all_columns.has(command.rename_to)) throw Exception{"Cannot rename to " + backQuote(command.rename_to) + ": column with this name already exists", ErrorCodes::DUPLICATE_COLUMN}; - - if (renames_map.count(command.column_name)) - throw Exception{"Cannot rename column '" + backQuote(command.column_name) + "' to two different names in a single ALTER query", ErrorCodes::BAD_ARGUMENTS}; - - if (renames_map.count(command.rename_to)) - throw Exception{"Rename loop detected in ALTER query", - ErrorCodes::BAD_ARGUMENTS}; - String from_nested_table_name = Nested::extractTableName(command.column_name); String to_nested_table_name = Nested::extractTableName(command.rename_to); bool from_nested = from_nested_table_name != command.column_name; @@ -846,7 +854,7 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con } else if (!from_nested && !to_nested) { - renames_map[command.column_name] = command.rename_to; + all_columns.rename(command.column_name, command.rename_to); } else { diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index d83fbd84b4b..6b6a8ed5ba8 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -215,8 +215,10 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri auto pools = createPoolsForAddresses(name, pool_factory); const auto settings = storage.global_context.getSettings(); - return pools.size() == 1 ? pools.front() : std::make_shared(pools, LoadBalancing::RANDOM, - settings.distributed_replica_error_half_life.totalSeconds(), settings.distributed_replica_error_cap); + return pools.size() == 1 ? pools.front() : std::make_shared(pools, + settings.load_balancing, + settings.distributed_replica_error_half_life.totalSeconds(), + settings.distributed_replica_error_cap); } @@ -666,7 +668,10 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map batch.send(); } - Poco::File{current_batch_file_path}.remove(); + /// current_batch.txt will not exist if there was no send + /// (this is the case when all batches that was pending has been marked as pending) + if (Poco::File{current_batch_file_path}.exists()) + Poco::File{current_batch_file_path}.remove(); } bool StorageDistributedDirectoryMonitor::isFileBrokenErrorCode(int code) diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index ae0af5f9cf4..4e4bbf188b2 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -167,6 +167,7 @@ std::string DistributedBlockOutputStream::getCurrentStateDescription() void DistributedBlockOutputStream::initWritingJobs(const Block & first_block) { + const Settings & settings = context.getSettingsRef(); const auto & addresses_with_failovers = cluster->getShardsAddresses(); const auto & shards_info = cluster->getShardsInfo(); size_t num_shards = shards_info.size(); @@ -180,14 +181,14 @@ void DistributedBlockOutputStream::initWritingJobs(const Block & first_block) const auto & shard_info = shards_info[shard_index]; auto & shard_jobs = per_shard_jobs[shard_index]; - /// If hasInternalReplication, than prefer local replica - if (!shard_info.hasInternalReplication() || !shard_info.isLocal()) + /// If hasInternalReplication, than prefer local replica (if !prefer_localhost_replica) + if (!shard_info.hasInternalReplication() || !shard_info.isLocal() || !settings.prefer_localhost_replica) { const auto & replicas = addresses_with_failovers[shard_index]; for (size_t replica_index : ext::range(0, replicas.size())) { - if (!replicas[replica_index].is_local) + if (!replicas[replica_index].is_local || !settings.prefer_localhost_replica) { shard_jobs.replicas_jobs.emplace_back(shard_index, replica_index, false, first_block); ++remote_jobs_count; @@ -198,7 +199,7 @@ void DistributedBlockOutputStream::initWritingJobs(const Block & first_block) } } - if (shard_info.isLocal()) + if (shard_info.isLocal() && settings.prefer_localhost_replica) { shard_jobs.replicas_jobs.emplace_back(shard_index, 0, true, first_block); ++local_jobs_count; @@ -275,12 +276,12 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp } const Block & shard_block = (num_shards > 1) ? job.current_shard_block : current_block; + const Settings & settings = context.getSettingsRef(); - if (!job.is_local_job) + if (!job.is_local_job || !settings.prefer_localhost_replica) { if (!job.stream) { - const Settings & settings = context.getSettingsRef(); auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(settings); if (shard_info.hasInternalReplication()) { @@ -318,7 +319,7 @@ ThreadPool::Job DistributedBlockOutputStream::runWritingJob(DistributedBlockOutp CurrentMetrics::Increment metric_increment{CurrentMetrics::DistributedSend}; job.stream->write(shard_block); } - else + else // local { if (!job.stream) { @@ -507,31 +508,25 @@ void DistributedBlockOutputStream::writeSplitAsync(const Block & block) void DistributedBlockOutputStream::writeAsyncImpl(const Block & block, const size_t shard_id) { const auto & shard_info = cluster->getShardsInfo()[shard_id]; + const auto & settings = context.getSettingsRef(); if (shard_info.hasInternalReplication()) { - if (shard_info.getLocalNodeCount() > 0) - { + if (shard_info.isLocal() && settings.prefer_localhost_replica) /// Prefer insert into current instance directly writeToLocal(block, shard_info.getLocalNodeCount()); - } else - { - if (shard_info.dir_name_for_internal_replication.empty()) - throw Exception("Directory name for async inserts is empty, table " + storage.getStorageID().getNameForLogs(), ErrorCodes::LOGICAL_ERROR); - - writeToShard(block, {shard_info.dir_name_for_internal_replication}); - } + writeToShard(block, {shard_info.pathForInsert(settings.prefer_localhost_replica)}); } else { - if (shard_info.getLocalNodeCount() > 0) + if (shard_info.isLocal()) writeToLocal(block, shard_info.getLocalNodeCount()); std::vector dir_names; for (const auto & address : cluster->getShardsAddresses()[shard_id]) - if (!address.is_local) - dir_names.push_back(address.toFullString(context.getSettingsRef().use_compact_format_in_distributed_parts_names)); + if (!address.is_local || !settings.prefer_localhost_replica) + dir_names.push_back(address.toFullString(settings.use_compact_format_in_distributed_parts_names)); if (!dir_names.empty()) writeToShard(block, dir_names); @@ -574,31 +569,34 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: first_file_tmp_path = tmp_path + file_name; - WriteBufferFromFile out{first_file_tmp_path}; - CompressedWriteBuffer compress{out}; - NativeBlockOutputStream stream{compress, ClickHouseRevision::get(), block.cloneEmpty()}; + /// Write batch to temporary location + { + WriteBufferFromFile out{first_file_tmp_path}; + CompressedWriteBuffer compress{out}; + NativeBlockOutputStream stream{compress, ClickHouseRevision::get(), block.cloneEmpty()}; - /// Prepare the header. - /// We wrap the header into a string for compatibility with older versions: - /// a shard will able to read the header partly and ignore other parts based on its version. - WriteBufferFromOwnString header_buf; - writeVarUInt(ClickHouseRevision::get(), header_buf); - writeStringBinary(query_string, header_buf); - context.getSettingsRef().serialize(header_buf); - context.getClientInfo().write(header_buf, ClickHouseRevision::get()); + /// Prepare the header. + /// We wrap the header into a string for compatibility with older versions: + /// a shard will able to read the header partly and ignore other parts based on its version. + WriteBufferFromOwnString header_buf; + writeVarUInt(ClickHouseRevision::get(), header_buf); + writeStringBinary(query_string, header_buf); + context.getSettingsRef().serialize(header_buf); + context.getClientInfo().write(header_buf, ClickHouseRevision::get()); - /// Add new fields here, for example: - /// writeVarUInt(my_new_data, header_buf); + /// Add new fields here, for example: + /// writeVarUInt(my_new_data, header_buf); - /// Write the header. - const StringRef header = header_buf.stringRef(); - writeVarUInt(DBMS_DISTRIBUTED_SIGNATURE_HEADER, out); - writeStringBinary(header, out); - writePODBinary(CityHash_v1_0_2::CityHash128(header.data, header.size), out); + /// Write the header. + const StringRef header = header_buf.stringRef(); + writeVarUInt(DBMS_DISTRIBUTED_SIGNATURE_HEADER, out); + writeStringBinary(header, out); + writePODBinary(CityHash_v1_0_2::CityHash128(header.data, header.size), out); - stream.writePrefix(); - stream.write(block); - stream.writeSuffix(); + stream.writePrefix(); + stream.write(block); + stream.writeSuffix(); + } // Create hardlink here to reuse increment number const std::string block_file_path(path + '/' + file_name); diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 62feea8fe34..eff4161ffb6 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -80,22 +80,9 @@ ReadBufferFromKafkaConsumer::ReadBufferFromKafkaConsumer( }); } -ReadBufferFromKafkaConsumer::~ReadBufferFromKafkaConsumer() -{ - /// NOTE: see https://github.com/edenhill/librdkafka/issues/2077 - try - { - if (!consumer->get_subscription().empty()) - consumer->unsubscribe(); - if (!assignment.empty()) - consumer->unassign(); - while (consumer->get_consumer_queue().next_event(100ms)); - } - catch (const cppkafka::HandleException & e) - { - LOG_ERROR(log, "Exception from ReadBufferFromKafkaConsumer destructor: " << e.what()); - } -} +// NOTE on removed desctuctor: There is no need to unsubscribe prior to calling rd_kafka_consumer_close(). +// check: https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#termination +// manual destruction was source of weird errors (hangs during droping kafka table, etc.) void ReadBufferFromKafkaConsumer::commit() { @@ -226,8 +213,13 @@ void ReadBufferFromKafkaConsumer::unsubscribe() // it should not raise exception as used in destructor try { - if (!consumer->get_subscription().empty()) - consumer->unsubscribe(); + // From docs: Any previous subscription will be unassigned and unsubscribed first. + consumer->subscribe(topics); + + // I wanted to avoid explicit unsubscribe as it requires draining the messages + // to close the consumer safely after unsubscribe + // see https://github.com/edenhill/librdkafka/issues/2077 + // https://github.com/confluentinc/confluent-kafka-go/issues/189 etc. } catch (const cppkafka::HandleException & e) { diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h index 700a69cf49b..c5b72ed6d7c 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h @@ -28,7 +28,6 @@ public: const std::atomic & stopped_, const Names & _topics ); - ~ReadBufferFromKafkaConsumer() override; void allowNext() { allowed = true; } // Allow to read next message. void commit(); // Commit all processed messages. @@ -64,10 +63,13 @@ private: const std::atomic & stopped; + // order is important, need to be destructed before consumer Messages messages; Messages::const_iterator current; bool rebalance_happened = false; + + // order is important, need to be destructed before consumer cppkafka::TopicPartitionList assignment; const Names topics; diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index e3e28563906..2e2797983b6 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -235,14 +235,19 @@ ProducerBufferPtr StorageKafka::createWriteBuffer(const Block & header) ConsumerBufferPtr StorageKafka::createReadBuffer() { cppkafka::Configuration conf; + conf.set("metadata.broker.list", brokers); conf.set("group.id", group); conf.set("client.id", VERSION_FULL); + conf.set("auto.offset.reset", "smallest"); // If no offset stored for this group, read all messages from the start + + updateConfiguration(conf); + + // those settings should not be changed by users. conf.set("enable.auto.commit", "false"); // We manually commit offsets after a stream successfully finished conf.set("enable.auto.offset.store", "false"); // Update offset automatically - to commit them all at once. conf.set("enable.partition.eof", "false"); // Ignore EOF messages - updateConfiguration(conf); // Create a consumer and subscribe to topics auto consumer = std::make_shared(conf); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index a5a7f1f4494..59bd2830789 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -356,19 +356,26 @@ size_t IMergeTreeDataPart::getFileSizeOrZero(const String & file_name) const String IMergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const { const auto & storage_columns = storage.getColumns().getAllPhysical(); - const std::string * minimum_size_column = nullptr; + auto alter_conversions = storage.getAlterConversionsForPart(shared_from_this()); + + std::optional minimum_size_column; UInt64 minimum_size = std::numeric_limits::max(); for (const auto & column : storage_columns) { - if (!hasColumnFiles(column.name, *column.type)) + auto column_name = column.name; + auto column_type = column.type; + if (alter_conversions.isColumnRenamed(column.name)) + column_name = alter_conversions.getColumnOldName(column.name); + + if (!hasColumnFiles(column_name, *column_type)) continue; - const auto size = getColumnSize(column.name, *column.type).data_compressed; + const auto size = getColumnSize(column_name, *column_type).data_compressed; if (size < minimum_size) { minimum_size = size; - minimum_size_column = &column.name; + minimum_size_column = column_name; } } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 784a3ff047b..338c1db1f06 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -128,6 +128,10 @@ public: String getNewName(const MergeTreePartInfo & new_part_info) const; /// Returns column position in part structure or std::nullopt if it's missing in part. + /// + /// NOTE: Doesn't take column renames into account, if some column renames + /// take place, you must take original name of column for this part from + /// storage and pass it to this method. std::optional getColumnPosition(const String & column_name) const; /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). @@ -291,7 +295,11 @@ public: /// Makes full clone of part in detached/ on another disk void makeCloneOnDiskDetached(const ReservationPtr & reservation) const; - /// Checks that .bin and .mrk files exist + /// Checks that .bin and .mrk files exist. + /// + /// NOTE: Doesn't take column renames into account, if some column renames + /// take place, you must take original name of column for this part from + /// storage and pass it to this method. virtual bool hasColumnFiles(const String & /* column */, const IDataType & /* type */) const{ return false; } static UInt64 calculateTotalSizeOnDisk(const DiskPtr & disk_, const String & from); diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 8243983d837..5d2a5ac3616 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -187,17 +187,17 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns NameAndTypePair IMergeTreeReader::getColumnFromPart(const NameAndTypePair & required_column) const { - auto it = columns_from_part.find(required_column.name); - if (it != columns_from_part.end()) - return {it->first, it->second}; - if (alter_conversions.isColumnRenamed(required_column.name)) { String old_name = alter_conversions.getColumnOldName(required_column.name); - it = columns_from_part.find(old_name); + auto it = columns_from_part.find(old_name); if (it != columns_from_part.end()) return {it->first, it->second}; } + else if (auto it = columns_from_part.find(required_column.name); it != columns_from_part.end()) + { + return {it->first, it->second}; + } return required_column; } diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index cd0827c8a0e..310c566fb19 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -21,18 +21,23 @@ NameSet injectRequiredColumns(const MergeTreeData & storage, const MergeTreeData auto all_column_files_missing = true; const auto & storage_columns = storage.getColumns(); + auto alter_conversions = storage.getAlterConversionsForPart(part); for (size_t i = 0; i < columns.size(); ++i) { - const auto & column_name = columns[i]; + /// possibly renamed + auto column_name_in_part = columns[i]; + + if (alter_conversions.isColumnRenamed(column_name_in_part)) + column_name_in_part = alter_conversions.getColumnOldName(column_name_in_part); /// column has files and hence does not require evaluation - if (part->hasColumnFiles(column_name, *storage_columns.getPhysical(column_name).type)) + if (part->hasColumnFiles(column_name_in_part, *storage_columns.getPhysical(columns[i]).type)) { all_column_files_missing = false; continue; } - const auto column_default = storage_columns.getDefault(column_name); + const auto column_default = storage_columns.getDefault(columns[i]); if (!column_default) continue; diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index acdb7616175..61eb038f6b0 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1085,7 +1085,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor auto indices_to_recalc = getIndicesToRecalculate(in, storage_from_source_part, updated_header.getNamesAndTypesList(), context); NameSet files_to_skip = collectFilesToSkip(updated_header, indices_to_recalc, mrk_extension); - NameToNameMap files_to_rename = collectFilesForRenames(source_part, for_file_renames, mrk_extension); + NameToNameVector files_to_rename = collectFilesForRenames(source_part, for_file_renames, mrk_extension); if (need_remove_expired_values) files_to_skip.insert("ttl.txt"); @@ -1097,7 +1097,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor continue; String destination = new_part_tmp_path + "/"; - auto rename_it = files_to_rename.find(it->name()); + String file_name = it->name(); + auto rename_it = std::find_if(files_to_rename.begin(), files_to_rename.end(), [&file_name](const auto & rename_pair) { return rename_pair.first == file_name; }); if (rename_it != files_to_rename.end()) { if (rename_it->second.empty()) @@ -1328,7 +1329,7 @@ void MergeTreeDataMergerMutator::splitMutationCommands( } -NameToNameMap MergeTreeDataMergerMutator::collectFilesForRenames( +NameToNameVector MergeTreeDataMergerMutator::collectFilesForRenames( MergeTreeData::DataPartPtr source_part, const MutationCommands & commands_for_removes, const String & mrk_extension) { /// Collect counts for shared streams of different columns. As an example, Nested columns have shared stream with array sizes. @@ -1343,14 +1344,14 @@ NameToNameMap MergeTreeDataMergerMutator::collectFilesForRenames( {}); } - NameToNameMap rename_map; + NameToNameVector rename_vector; /// Remove old indices for (const auto & command : commands_for_removes) { if (command.type == MutationCommand::Type::DROP_INDEX) { - rename_map.emplace("skp_idx_" + command.column_name + ".idx", ""); - rename_map.emplace("skp_idx_" + command.column_name + mrk_extension, ""); + rename_vector.emplace_back("skp_idx_" + command.column_name + ".idx", ""); + rename_vector.emplace_back("skp_idx_" + command.column_name + mrk_extension, ""); } else if (command.type == MutationCommand::Type::DROP_COLUMN) { @@ -1360,8 +1361,8 @@ NameToNameMap MergeTreeDataMergerMutator::collectFilesForRenames( /// Delete files if they are no longer shared with another column. if (--stream_counts[stream_name] == 0) { - rename_map.emplace(stream_name + ".bin", ""); - rename_map.emplace(stream_name + mrk_extension, ""); + rename_vector.emplace_back(stream_name + ".bin", ""); + rename_vector.emplace_back(stream_name + mrk_extension, ""); } }; @@ -1383,8 +1384,8 @@ NameToNameMap MergeTreeDataMergerMutator::collectFilesForRenames( if (stream_from != stream_to) { - rename_map.emplace(stream_from + ".bin", stream_to + ".bin"); - rename_map.emplace(stream_from + mrk_extension, stream_to + mrk_extension); + rename_vector.emplace_back(stream_from + ".bin", stream_to + ".bin"); + rename_vector.emplace_back(stream_from + mrk_extension, stream_to + mrk_extension); } }; IDataType::SubstreamPath stream_path; @@ -1394,7 +1395,7 @@ NameToNameMap MergeTreeDataMergerMutator::collectFilesForRenames( } } - return rename_map; + return rename_vector; } NameSet MergeTreeDataMergerMutator::collectFilesToSkip( diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index f2f4ea95fd0..78de7375d70 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -143,10 +143,11 @@ private: MutationCommands & for_interpreter, MutationCommands & for_file_renames); - - /// Apply commands to source_part i.e. remove some columns in source_part - /// and return set of files, that have to be removed from filesystem and checksums - static NameToNameMap collectFilesForRenames(MergeTreeData::DataPartPtr source_part, const MutationCommands & commands_for_removes, const String & mrk_extension); + /// Apply commands to source_part i.e. remove and rename some columns in + /// source_part and return set of files, that have to be removed or renamed + /// from filesystem and in-memory checksums. Ordered result is important, + /// because we can apply renames that affects each other: x -> z, y -> x. + static NameToNameVector collectFilesForRenames(MergeTreeData::DataPartPtr source_part, const MutationCommands & commands_for_removes, const String & mrk_extension); /// Files, that we don't need to remove and don't need to hardlink, for example columns.txt and checksums.txt. /// Because we will generate new versions of them after we perform mutation. diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index f7563df318a..e71d75816e8 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -286,7 +286,7 @@ static void appendBlock(const Block & from, Block & to) for (size_t column_no = 0, columns = to.columns(); column_no < columns; ++column_no) { const IColumn & col_from = *from.getByPosition(column_no).column.get(); - MutableColumnPtr col_to = (*std::move(to.getByPosition(column_no).column)).mutate(); + MutableColumnPtr col_to = IColumn::mutate(std::move(to.getByPosition(column_no).column)); col_to->insertRangeFrom(col_from, 0, rows); @@ -302,7 +302,7 @@ static void appendBlock(const Block & from, Block & to) { ColumnPtr & col_to = to.getByPosition(column_no).column; if (col_to->size() != old_rows) - col_to = (*std::move(col_to)).mutate()->cut(0, old_rows); + col_to = col_to->cut(0, old_rows); } } catch (...) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 00fc22c34aa..f8202cbcac3 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -550,7 +550,8 @@ void StorageDistributed::checkAlterIsPossible(const AlterCommands & commands, co if (command.type != AlterCommand::Type::ADD_COLUMN && command.type != AlterCommand::Type::MODIFY_COLUMN && command.type != AlterCommand::Type::DROP_COLUMN - && command.type != AlterCommand::Type::COMMENT_COLUMN) + && command.type != AlterCommand::Type::COMMENT_COLUMN + && command.type != AlterCommand::Type::RENAME_COLUMN) throw Exception("Alter of type '" + alterTypeToString(command.type) + "' is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 68d468233a8..91add9a8104 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -93,9 +93,18 @@ void StorageMergeTree::startup() /// NOTE background task will also do the above cleanups periodically. time_after_previous_cleanup.restart(); - merging_mutating_task_handle = global_context.getBackgroundPool().addTask([this] { return mergeMutateTask(); }); + + auto & pool = global_context.getBackgroundPool(); + + merging_mutating_task_handle = pool.createTask([this] { return mergeMutateTask(); }); + /// Ensure that thread started only after assignment to 'merging_mutating_task_handle' is done. + pool.startTask(merging_mutating_task_handle); + if (areBackgroundMovesNeeded()) - moving_task_handle = global_context.getBackgroundMovePool().addTask([this] { return movePartsTask(); }); + { + moving_task_handle = pool.createTask([this] { return movePartsTask(); }); + pool.startTask(moving_task_handle); + } } diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 6c6f79b50e7..a69e140fe5a 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -112,53 +112,21 @@ namespace BlockInputStreamPtr reader; bool initialized = false; }; - - class StorageURLBlockOutputStream : public IBlockOutputStream - { - public: - StorageURLBlockOutputStream(const Poco::URI & uri, - const String & format, - const Block & sample_block_, - const Context & context, - const ConnectionTimeouts & timeouts, - const CompressionMethod compression_method) - : sample_block(sample_block_) - { - write_buf = wrapWriteBufferWithCompressionMethod( - std::make_unique(uri, Poco::Net::HTTPRequest::HTTP_POST, timeouts), - compression_method, 3); - writer = FormatFactory::instance().getOutput(format, *write_buf, sample_block, context); - } - - Block getHeader() const override - { - return sample_block; - } - - void write(const Block & block) override - { - writer->write(block); - } - - void writePrefix() override - { - writer->writePrefix(); - } - - void writeSuffix() override - { - writer->writeSuffix(); - writer->flush(); - write_buf->finalize(); - } - - private: - Block sample_block; - std::unique_ptr write_buf; - BlockOutputStreamPtr writer; - }; } +StorageURLBlockOutputStream::StorageURLBlockOutputStream(const Poco::URI & uri, + const String & format, + const Block & sample_block_, + const Context & context, + const ConnectionTimeouts & timeouts, + const CompressionMethod compression_method) + : sample_block(sample_block_) +{ + write_buf = wrapWriteBufferWithCompressionMethod( + std::make_unique(uri, Poco::Net::HTTPRequest::HTTP_POST, timeouts), + compression_method, 3); + writer = FormatFactory::instance().getOutput(format, *write_buf, sample_block, context); +} std::string IStorageURLBase::getReadMethod() const { diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 0ff84bc4d37..5a6584f0301 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -3,6 +3,9 @@ #include #include #include +#include +#include +#include namespace DB @@ -39,10 +42,9 @@ protected: Poco::URI uri; const Context & context_global; String compression_method; - -private: String format_name; +private: virtual std::string getReadMethod() const; virtual std::vector> getReadURIParams( @@ -62,6 +64,43 @@ private: virtual Block getHeaderBlock(const Names & column_names) const = 0; }; +class StorageURLBlockOutputStream : public IBlockOutputStream +{ +public: + StorageURLBlockOutputStream(const Poco::URI & uri, + const String & format, + const Block & sample_block_, + const Context & context, + const ConnectionTimeouts & timeouts, + const CompressionMethod compression_method); + + Block getHeader() const override + { + return sample_block; + } + + void write(const Block & block) override + { + writer->write(block); + } + + void writePrefix() override + { + writer->writePrefix(); + } + + void writeSuffix() override + { + writer->writeSuffix(); + writer->flush(); + write_buf->finalize(); + } + +private: + Block sample_block; + std::unique_ptr write_buf; + BlockOutputStreamPtr writer; +}; class StorageURL final : public ext::shared_ptr_helper, public IStorageURLBase { diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index b17cec3afe8..08538798389 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -3,17 +3,15 @@ #include #include #include +#include #include -#include #include -#include #include #include -#include #include #include #include -#include +#include #include @@ -97,6 +95,30 @@ Pipes StorageXDBC::read(const Names & column_names, return IStorageURLBase::read(column_names, query_info, context, processed_stage, max_block_size, num_streams); } +BlockOutputStreamPtr StorageXDBC::write(const ASTPtr & /*query*/, const Context & context) +{ + bridge_helper->startBridgeSync(); + + NamesAndTypesList cols; + Poco::URI request_uri = uri; + request_uri.setPath("/write"); + for (const String & name : getSampleBlock().getNames()) + { + auto column_data = getColumns().getPhysical(name); + cols.emplace_back(column_data.name, column_data.type); + } + auto url_params = bridge_helper->getURLParams(cols.toString(), 65536); + for (const auto & [param, value] : url_params) + request_uri.addQueryParameter(param, value); + request_uri.addQueryParameter("db_name", remote_database_name); + request_uri.addQueryParameter("table_name", remote_table_name); + request_uri.addQueryParameter("format_name", format_name); + + return std::make_shared( + request_uri, format_name, getSampleBlock(), context, + ConnectionTimeouts::getHTTPTimeouts(context), + chooseCompressionMethod(uri.toString(), compression_method)); +} Block StorageXDBC::getHeaderBlock(const Names & column_names) const { diff --git a/src/Storages/StorageXDBC.h b/src/Storages/StorageXDBC.h index c602415d8a5..afc61dac5cd 100644 --- a/src/Storages/StorageXDBC.h +++ b/src/Storages/StorageXDBC.h @@ -29,6 +29,8 @@ public: const ColumnsDescription & columns_, const Context & context_, BridgeHelperPtr bridge_helper_); + BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; + private: BridgeHelperPtr bridge_helper; diff --git a/src/Storages/System/StorageSystemCurrentRoles.cpp b/src/Storages/System/StorageSystemCurrentRoles.cpp new file mode 100644 index 00000000000..b0667f2f3ca --- /dev/null +++ b/src/Storages/System/StorageSystemCurrentRoles.cpp @@ -0,0 +1,53 @@ +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +NamesAndTypesList StorageSystemCurrentRoles::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"role_name", std::make_shared()}, + {"with_admin_option", std::make_shared()}, + {"is_default", std::make_shared()}, + }; + return names_and_types; +} + + +void StorageSystemCurrentRoles::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + auto roles_info = context.getRolesInfo(); + auto user = context.getUser(); + if (!roles_info || !user) + return; + + size_t column_index = 0; + auto & column_role_name = assert_cast(*res_columns[column_index++]); + auto & column_admin_option = assert_cast(*res_columns[column_index++]).getData(); + auto & column_is_default = assert_cast(*res_columns[column_index++]).getData(); + + auto add_row = [&](const String & role_name, bool admin_option, bool is_default) + { + column_role_name.insertData(role_name.data(), role_name.length()); + column_admin_option.push_back(admin_option); + column_is_default.push_back(is_default); + }; + + for (const auto & role_id : roles_info->current_roles) + { + const String & role_name = roles_info->names_of_roles.at(role_id); + bool admin_option = roles_info->enabled_roles_with_admin_option.count(role_id); + bool is_default = user->default_roles.match(role_id); + add_row(role_name, admin_option, is_default); + } +} + +} diff --git a/src/Storages/System/StorageSystemCurrentRoles.h b/src/Storages/System/StorageSystemCurrentRoles.h new file mode 100644 index 00000000000..807db661371 --- /dev/null +++ b/src/Storages/System/StorageSystemCurrentRoles.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `current_roles` system table, which allows you to get information about current roles. +class StorageSystemCurrentRoles final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemCurrentRoles"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemEnabledRoles.cpp b/src/Storages/System/StorageSystemEnabledRoles.cpp new file mode 100644 index 00000000000..27a42ca6f8b --- /dev/null +++ b/src/Storages/System/StorageSystemEnabledRoles.cpp @@ -0,0 +1,57 @@ +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +NamesAndTypesList StorageSystemEnabledRoles::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"role_name", std::make_shared()}, + {"with_admin_option", std::make_shared()}, + {"is_current", std::make_shared()}, + {"is_default", std::make_shared()}, + }; + return names_and_types; +} + + +void StorageSystemEnabledRoles::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + auto roles_info = context.getRolesInfo(); + auto user = context.getUser(); + if (!roles_info || !user) + return; + + size_t column_index = 0; + auto & column_role_name = assert_cast(*res_columns[column_index++]); + auto & column_admin_option = assert_cast(*res_columns[column_index++]).getData(); + auto & column_is_current = assert_cast(*res_columns[column_index++]).getData(); + auto & column_is_default = assert_cast(*res_columns[column_index++]).getData(); + + auto add_row = [&](const String & role_name, bool admin_option, bool is_current, bool is_default) + { + column_role_name.insertData(role_name.data(), role_name.length()); + column_admin_option.push_back(admin_option); + column_is_current.push_back(is_current); + column_is_default.push_back(is_default); + }; + + for (const auto & role_id : roles_info->enabled_roles) + { + const String & role_name = roles_info->names_of_roles.at(role_id); + bool admin_option = roles_info->enabled_roles_with_admin_option.count(role_id); + bool is_current = roles_info->current_roles.count(role_id); + bool is_default = user->default_roles.match(role_id); + add_row(role_name, admin_option, is_current, is_default); + } +} + +} diff --git a/src/Storages/System/StorageSystemEnabledRoles.h b/src/Storages/System/StorageSystemEnabledRoles.h new file mode 100644 index 00000000000..18df31c646a --- /dev/null +++ b/src/Storages/System/StorageSystemEnabledRoles.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `enabled_roles` system table, which allows you to get information about enabled roles. +class StorageSystemEnabledRoles final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemEnabledRoles"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemGrants.cpp b/src/Storages/System/StorageSystemGrants.cpp new file mode 100644 index 00000000000..a663e3307fe --- /dev/null +++ b/src/Storages/System/StorageSystemGrants.cpp @@ -0,0 +1,180 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +using EntityType = IAccessEntity::Type; + + +NamesAndTypesList StorageSystemGrants::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"user_name", std::make_shared(std::make_shared())}, + {"role_name", std::make_shared(std::make_shared())}, + {"access_type", std::make_shared(StorageSystemPrivileges::getAccessTypeEnumValues())}, + {"database", std::make_shared(std::make_shared())}, + {"table", std::make_shared(std::make_shared())}, + {"column", std::make_shared(std::make_shared())}, + {"is_partial_revoke", std::make_shared()}, + {"grant_option", std::make_shared()}, + }; + return names_and_types; +} + + +void StorageSystemGrants::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_USERS | AccessType::SHOW_ROLES); + const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + boost::range::push_back(ids, access_control.findAll()); + + size_t column_index = 0; + auto & column_user_name = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_user_name_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_role_name = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_role_name_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_access_type = assert_cast(*res_columns[column_index++]).getData(); + auto & column_database = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_database_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_table = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_table_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_column = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_column_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_is_partial_revoke = assert_cast(*res_columns[column_index++]).getData(); + auto & column_grant_option = assert_cast(*res_columns[column_index++]).getData(); + + auto add_row = [&](const String & grantee_name, + EntityType grantee_type, + AccessType access_type, + const String * database, + const String * table, + const String * column, + bool is_partial_revoke, + bool grant_option) + { + if (grantee_type == EntityType::USER) + { + column_user_name.insertData(grantee_name.data(), grantee_name.length()); + column_user_name_null_map.push_back(false); + column_role_name.insertDefault(); + column_role_name_null_map.push_back(true); + } + else if (grantee_type == EntityType::ROLE) + { + column_user_name.insertDefault(); + column_user_name_null_map.push_back(true); + column_role_name.insertData(grantee_name.data(), grantee_name.length()); + column_role_name_null_map.push_back(false); + } + else + assert(false); + + column_access_type.push_back(static_cast(access_type)); + + if (database) + { + column_database.insertData(database->data(), database->length()); + column_database_null_map.push_back(false); + } + else + { + column_database.insertDefault(); + column_database_null_map.push_back(true); + } + + if (table) + { + column_table.insertData(table->data(), table->length()); + column_table_null_map.push_back(false); + } + else + { + column_table.insertDefault(); + column_table_null_map.push_back(true); + } + + if (column) + { + column_column.insertData(column->data(), column->length()); + column_column_null_map.push_back(false); + } + else + { + column_column.insertDefault(); + column_column_null_map.push_back(true); + } + + column_is_partial_revoke.push_back(is_partial_revoke); + column_grant_option.push_back(grant_option); + }; + + auto add_rows = [&](const String & grantee_name, + IAccessEntity::Type grantee_type, + const AccessRightsElements & elements, + bool is_partial_revoke, + bool grant_option) + { + for (const auto & element : elements) + { + auto access_types = element.access_flags.toAccessTypes(); + if (access_types.empty() || (!element.any_column && element.columns.empty())) + continue; + + const auto * database = element.any_database ? nullptr : &element.database; + const auto * table = element.any_table ? nullptr : &element.table; + + if (element.any_column) + { + for (const auto & access_type : access_types) + add_row(grantee_name, grantee_type, access_type, database, table, nullptr, is_partial_revoke, grant_option); + } + else + { + for (const auto & access_type : access_types) + for (const auto & column : element.columns) + add_row(grantee_name, grantee_type, access_type, database, table, &column, is_partial_revoke, grant_option); + } + } + }; + + for (const auto & id : ids) + { + auto entity = access_control.tryRead(id); + if (!entity) + continue; + + const GrantedAccess * access = nullptr; + if (auto role = typeid_cast(entity)) + access = &role->access; + else if (auto user = typeid_cast(entity)) + access = &user->access; + else + continue; + + const String & grantee_name = entity->getName(); + const auto grantee_type = entity->getType(); + auto grants_and_revokes = access->access.getGrantsAndPartialRevokes(); + auto grants_and_revokes_with_grant_option = access->access_with_grant_option.getGrantsAndPartialRevokes(); + + add_rows(grantee_name, grantee_type, grants_and_revokes.grants, /* is_partial_revoke = */ false, /* grant_option = */ false); + add_rows(grantee_name, grantee_type, grants_and_revokes.revokes, /* is_partial_revoke = */ true, /* grant_option = */ false); + add_rows(grantee_name, grantee_type, grants_and_revokes_with_grant_option.grants, /* is_partial_revoke = */ false, /* grant_option = */ true); + add_rows(grantee_name, grantee_type, grants_and_revokes_with_grant_option.revokes, /* is_partial_revoke = */ true, /* grant_option = */ true); + } +} + +} diff --git a/src/Storages/System/StorageSystemGrants.h b/src/Storages/System/StorageSystemGrants.h new file mode 100644 index 00000000000..39c38deed85 --- /dev/null +++ b/src/Storages/System/StorageSystemGrants.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `grants` system table, which allows you to get information about grants. +class StorageSystemGrants final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemGrants"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemPrivileges.cpp b/src/Storages/System/StorageSystemPrivileges.cpp new file mode 100644 index 00000000000..f0a1b21368e --- /dev/null +++ b/src/Storages/System/StorageSystemPrivileges.cpp @@ -0,0 +1,137 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace +{ + enum Level + { + GROUP = -1, + GLOBAL, + DATABASE, + TABLE, + DICTIONARY, + VIEW, + COLUMN, + }; + + DataTypeEnum8::Values getLevelEnumValues() + { + DataTypeEnum8::Values enum_values; + enum_values.emplace_back("GLOBAL", static_cast(GLOBAL)); + enum_values.emplace_back("DATABASE", static_cast(DATABASE)); + enum_values.emplace_back("TABLE", static_cast(TABLE)); + enum_values.emplace_back("DICTIONARY", static_cast(DICTIONARY)); + enum_values.emplace_back("VIEW", static_cast(VIEW)); + enum_values.emplace_back("COLUMN", static_cast(COLUMN)); + return enum_values; + } +} + + +const std::vector> & StorageSystemPrivileges::getAccessTypeEnumValues() +{ + static const std::vector> values = [] + { + std::vector> res; + +#define ADD_ACCESS_TYPE_ENUM_VALUE(name, aliases, node_type, parent_group_name) \ + res.emplace_back(toString(AccessType::name), static_cast(AccessType::name)); + + APPLY_FOR_ACCESS_TYPES(ADD_ACCESS_TYPE_ENUM_VALUE) +#undef ADD_ACCESS_TYPE_ENUM_VALUE + + return res; + }(); + return values; +} + + +NamesAndTypesList StorageSystemPrivileges::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"privilege", std::make_shared(getAccessTypeEnumValues())}, + {"aliases", std::make_shared(std::make_shared())}, + {"level", std::make_shared(std::make_shared(getLevelEnumValues()))}, + {"parent_group", std::make_shared(std::make_shared(getAccessTypeEnumValues()))}, + }; + return names_and_types; +} + + +void StorageSystemPrivileges::fillData(MutableColumns & res_columns, const Context &, const SelectQueryInfo &) const +{ + size_t column_index = 0; + auto & column_access_type = assert_cast(*res_columns[column_index++]).getData(); + auto & column_aliases = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_aliases_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_level = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()).getData(); + auto & column_level_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_parent_group = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()).getData(); + auto & column_parent_group_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + + auto add_row = [&](AccessType access_type, const std::string_view & aliases, Level max_level, AccessType parent_group) + { + column_access_type.push_back(static_cast(access_type)); + + for (size_t pos = 0; pos < aliases.length();) + { + size_t next_pos = aliases.find_first_of(',', pos); + std::string_view alias = aliases.substr(pos, next_pos - pos); + pos = ((next_pos == std::string_view::npos) ? next_pos : next_pos + 1); + + while (alias.starts_with(' ')) + alias.remove_prefix(1); + while (alias.ends_with(' ')) + alias.remove_suffix(1); + column_aliases.insertData(alias.data(), alias.length()); + } + column_aliases_offsets.push_back(column_aliases.size()); + + if (max_level == GROUP) + { + column_level.push_back(0); + column_level_null_map.push_back(true); + } + else + { + column_level.push_back(static_cast(max_level)); + column_level_null_map.push_back(false); + } + + if (parent_group == AccessType::NONE) + { + column_parent_group.push_back(0); + column_parent_group_null_map.push_back(true); + } + else + { + column_parent_group.push_back(static_cast(parent_group)); + column_parent_group_null_map.push_back(false); + } + }; + +#define STORAGE_SYSTEM_PRIVILEGES_ADD_ROW(name, aliases, node_type, parent_group_name) \ + add_row(AccessType::name, aliases, node_type, AccessType::parent_group_name); + + APPLY_FOR_ACCESS_TYPES(STORAGE_SYSTEM_PRIVILEGES_ADD_ROW) + +#undef STORAGE_SYSTEM_PRIVILEGES_ADD_ROW +} + +} diff --git a/src/Storages/System/StorageSystemPrivileges.h b/src/Storages/System/StorageSystemPrivileges.h new file mode 100644 index 00000000000..8540e3d7ec3 --- /dev/null +++ b/src/Storages/System/StorageSystemPrivileges.h @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `privileges` system table, which allows you to get information about access types. +class StorageSystemPrivileges final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemPrivileges"; } + static NamesAndTypesList getNamesAndTypes(); + static const std::vector> & getAccessTypeEnumValues(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemQuotaLimits.cpp b/src/Storages/System/StorageSystemQuotaLimits.cpp new file mode 100644 index 00000000000..fcfe11a0d23 --- /dev/null +++ b/src/Storages/System/StorageSystemQuotaLimits.cpp @@ -0,0 +1,122 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +using ResourceAmount = Quota::ResourceAmount; +using ResourceType = Quota::ResourceType; +using ResourceTypeInfo = Quota::ResourceTypeInfo; +constexpr auto MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; + + +namespace +{ + void addValue(IColumn & out_column, NullMap & out_column_null_map, ResourceAmount amount, const ResourceTypeInfo & type_info) + { + out_column_null_map.push_back(false); + if (type_info.output_as_float) + static_cast(out_column).getData().push_back(double(amount) / type_info.output_denominator); + else + static_cast(out_column).getData().push_back(amount / type_info.output_denominator); + } + + void addValue(IColumn & out_column, NullMap & out_column_null_map, std::optional amount, const ResourceTypeInfo & type_info) + { + if (amount) + addValue(out_column, out_column_null_map, *amount, type_info); + else + { + out_column_null_map.push_back(true); + out_column.insertDefault(); + } + } +} + + +NamesAndTypesList StorageSystemQuotaLimits::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"quota_name", std::make_shared()}, + {"duration", std::make_shared()}, + {"is_randomized_interval", std::make_shared()}, + }; + + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) + { + const auto & type_info = ResourceTypeInfo::get(resource_type); + String column_name = "max_" + type_info.name; + DataTypePtr data_type; + if (type_info.output_as_float) + data_type = std::make_shared(); + else + data_type = std::make_shared(); + names_and_types.push_back({column_name, std::make_shared(data_type)}); + } + + return names_and_types; +} + + +void StorageSystemQuotaLimits::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_QUOTAS); + const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + + size_t column_index = 0; + auto & column_quota_name = assert_cast(*res_columns[column_index++]); + auto & column_duration = assert_cast(*res_columns[column_index++]).getData(); + auto & column_is_randomized_interval = assert_cast(*res_columns[column_index++]).getData(); + + IColumn * column_max[MAX_RESOURCE_TYPE]; + NullMap * column_max_null_map[MAX_RESOURCE_TYPE]; + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) + { + column_max[resource_type] = &assert_cast(*res_columns[column_index]).getNestedColumn(); + column_max_null_map[resource_type] = &assert_cast(*res_columns[column_index++]).getNullMapData(); + } + + auto add_row = [&](const String & quota_name, const Quota::Limits & limits) + { + column_quota_name.insertData(quota_name.data(), quota_name.length()); + column_duration.push_back(limits.duration.count()); + column_is_randomized_interval.push_back(limits.randomize_interval); + + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) + { + const auto & type_info = ResourceTypeInfo::get(resource_type); + addValue(*column_max[resource_type], *column_max_null_map[resource_type], limits.max[resource_type], type_info); + } + }; + + auto add_rows = [&](const String & quota_name, const std::vector & all_limits) + { + for (const auto & limits : all_limits) + add_row(quota_name, limits); + }; + + for (const auto & id : ids) + { + auto quota = access_control.tryRead(id); + if (!quota) + continue; + const auto * storage = access_control.findStorage(id); + if (!storage) + continue; + + add_rows(quota->getName(), quota->all_limits); + } +} +} diff --git a/src/Storages/System/StorageSystemQuotaLimits.h b/src/Storages/System/StorageSystemQuotaLimits.h new file mode 100644 index 00000000000..e9ae7fc09d0 --- /dev/null +++ b/src/Storages/System/StorageSystemQuotaLimits.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `quota_limits` system table, which allows you to get information about the limits set for quotas. +class StorageSystemQuotaLimits final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemQuotaLimits"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemQuotaUsage.cpp b/src/Storages/System/StorageSystemQuotaUsage.cpp index 1f943d02446..002ab081bcf 100644 --- a/src/Storages/System/StorageSystemQuotaUsage.cpp +++ b/src/Storages/System/StorageSystemQuotaUsage.cpp @@ -1,40 +1,82 @@ #include #include #include -#include #include #include +#include +#include +#include #include #include -#include -#include +#include #include #include namespace DB { +using ResourceAmount = Quota::ResourceAmount; +using ResourceType = Quota::ResourceType; +using ResourceTypeInfo = Quota::ResourceTypeInfo; +constexpr auto MAX_RESOURCE_TYPE = Quota::MAX_RESOURCE_TYPE; + + +namespace +{ + void addValue(IColumn & out_column, NullMap & out_column_null_map, ResourceAmount amount, const ResourceTypeInfo & type_info) + { + out_column_null_map.push_back(false); + if (type_info.output_as_float) + static_cast(out_column).getData().push_back(double(amount) / type_info.output_denominator); + else + static_cast(out_column).getData().push_back(amount / type_info.output_denominator); + } + + void addValue(IColumn & out_column, NullMap & out_column_null_map, std::optional amount, const ResourceTypeInfo & type_info) + { + if (amount) + addValue(out_column, out_column_null_map, *amount, type_info); + else + { + out_column_null_map.push_back(true); + out_column.insertDefault(); + } + } +} + + NamesAndTypesList StorageSystemQuotaUsage::getNamesAndTypes() { - NamesAndTypesList names_and_types{ - {"name", std::make_shared()}, - {"id", std::make_shared()}, - {"key", std::make_shared()}, - {"duration", std::make_shared(std::make_shared())}, - {"end_of_interval", std::make_shared(std::make_shared())}}; + return getNamesAndTypesImpl(/* add_column_is_current = */ false); +} - for (auto resource_type : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) +NamesAndTypesList StorageSystemQuotaUsage::getNamesAndTypesImpl(bool add_column_is_current) +{ + NamesAndTypesList names_and_types{ + {"quota_name", std::make_shared()}, + {"quota_key", std::make_shared()} + }; + + if (add_column_is_current) + names_and_types.push_back({"is_current", std::make_shared()}); + + names_and_types.push_back({"start_time", std::make_shared(std::make_shared())}); + names_and_types.push_back({"end_time", std::make_shared(std::make_shared())}); + names_and_types.push_back({"duration", std::make_shared(std::make_shared())}); + + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) { + const auto & type_info = ResourceTypeInfo::get(resource_type); + String column_name = type_info.name; DataTypePtr data_type; - if (resource_type == Quota::EXECUTION_TIME) + if (type_info.output_as_float) data_type = std::make_shared(); else data_type = std::make_shared(); - - String column_name = Quota::resourceTypeToColumnName(resource_type); names_and_types.push_back({column_name, std::make_shared(data_type)}); names_and_types.push_back({String("max_") + column_name, std::make_shared(data_type)}); } + return names_and_types; } @@ -42,42 +84,111 @@ NamesAndTypesList StorageSystemQuotaUsage::getNamesAndTypes() void StorageSystemQuotaUsage::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { context.checkAccess(AccessType::SHOW_QUOTAS); - const auto & access_control = context.getAccessControlManager(); + auto usage = context.getQuotaUsage(); + if (!usage) + return; - for (const auto & info : access_control.getQuotaUsageInfo()) + fillDataImpl(res_columns, context, /* add_column_is_current = */ false, {std::move(usage).value()}); +} + + +void StorageSystemQuotaUsage::fillDataImpl( + MutableColumns & res_columns, + const Context & context, + bool add_column_is_current, + const std::vector & quotas_usage) +{ + size_t column_index = 0; + auto & column_quota_name = assert_cast(*res_columns[column_index++]); + auto & column_quota_key = assert_cast(*res_columns[column_index++]); + + ColumnUInt8::Container * column_is_current = nullptr; + if (add_column_is_current) + column_is_current = &assert_cast(*res_columns[column_index++]).getData(); + + auto & column_start_time = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_start_time_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_end_time = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_end_time_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_duration = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_duration_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + + IColumn * column_usage[MAX_RESOURCE_TYPE]; + NullMap * column_usage_null_map[MAX_RESOURCE_TYPE]; + IColumn * column_max[MAX_RESOURCE_TYPE]; + NullMap * column_max_null_map[MAX_RESOURCE_TYPE]; + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) { - for (const auto & interval : info.intervals) + column_usage[resource_type] = &assert_cast(*res_columns[column_index]).getNestedColumn(); + column_usage_null_map[resource_type] = &assert_cast(*res_columns[column_index++]).getNullMapData(); + column_max[resource_type] = &assert_cast(*res_columns[column_index]).getNestedColumn(); + column_max_null_map[resource_type] = &assert_cast(*res_columns[column_index++]).getNullMapData(); + } + + std::optional current_quota_id; + if (add_column_is_current) + { + if (auto current_usage = context.getQuotaUsage()) + current_quota_id = current_usage->quota_id; + } + + auto add_row = [&](const String & quota_name, const UUID & quota_id, const String & quota_key, const QuotaUsage::Interval * interval) + { + column_quota_name.insertData(quota_name.data(), quota_name.length()); + column_quota_key.insertData(quota_key.data(), quota_key.length()); + + if (!interval) { - size_t i = 0; - res_columns[i++]->insert(info.quota_name); - res_columns[i++]->insert(info.quota_id); - res_columns[i++]->insert(info.quota_key); - res_columns[i++]->insert(std::chrono::seconds{interval.duration}.count()); - res_columns[i++]->insert(std::chrono::system_clock::to_time_t(interval.end_of_interval)); - for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) + column_start_time.insertDefault(); + column_start_time_null_map.push_back(true); + column_end_time.insertDefault(); + column_end_time_null_map.push_back(true); + column_duration.insertDefault(); + column_duration_null_map.push_back(true); + for (auto resource_type : ext::range(MAX_RESOURCE_TYPE)) { - if (resource_type == Quota::EXECUTION_TIME) - { - res_columns[i++]->insert(Quota::executionTimeToSeconds(interval.used[resource_type])); - res_columns[i++]->insert(Quota::executionTimeToSeconds(interval.max[resource_type])); - } - else - { - res_columns[i++]->insert(interval.used[resource_type]); - res_columns[i++]->insert(interval.max[resource_type]); - } + column_usage[resource_type]->insertDefault(); + column_usage_null_map[resource_type]->push_back(true); + column_max[resource_type]->insertDefault(); + column_max_null_map[resource_type]->push_back(true); } + return; } - if (info.intervals.empty()) + time_t end_time = std::chrono::system_clock::to_time_t(interval->end_of_interval); + UInt32 duration = static_cast(std::chrono::duration_cast(interval->duration).count()); + time_t start_time = end_time - duration; + column_start_time.getData().push_back(start_time); + column_end_time.getData().push_back(end_time); + column_duration.getData().push_back(duration); + column_start_time_null_map.push_back(false); + column_end_time_null_map.push_back(false); + column_duration_null_map.push_back(false); + + for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) { - size_t i = 0; - res_columns[i++]->insert(info.quota_name); - res_columns[i++]->insert(info.quota_id); - res_columns[i++]->insert(info.quota_key); - for (size_t j = 0; j != Quota::MAX_RESOURCE_TYPE * 2 + 2; ++j) - res_columns[i++]->insertDefault(); + const auto & type_info = ResourceTypeInfo::get(resource_type); + addValue(*column_max[resource_type], *column_max_null_map[resource_type], interval->max[resource_type], type_info); + addValue(*column_usage[resource_type], *column_usage_null_map[resource_type], interval->used[resource_type], type_info); } - } + + if (add_column_is_current) + column_is_current->push_back(quota_id == current_quota_id); + }; + + auto add_rows = [&](const String & quota_name, const UUID & quota_id, const String & quota_key, const std::vector & intervals) + { + if (intervals.empty()) + { + add_row(quota_name, quota_id, quota_key, nullptr); + return; + } + + for (const auto & interval : intervals) + add_row(quota_name, quota_id, quota_key, &interval); + }; + + for (const auto & usage : quotas_usage) + add_rows(usage.quota_name, usage.quota_id, usage.quota_key, usage.intervals); } } diff --git a/src/Storages/System/StorageSystemQuotaUsage.h b/src/Storages/System/StorageSystemQuotaUsage.h index d410decf394..abb9505eb5a 100644 --- a/src/Storages/System/StorageSystemQuotaUsage.h +++ b/src/Storages/System/StorageSystemQuotaUsage.h @@ -6,12 +6,12 @@ namespace DB { - class Context; +struct QuotaUsage; -/** Implements the `quota_usage` system tables, which allows you to get information about - * how the quotas are used by all users. +/** Implements the `quota_usage` system table, which allows you to get information about + * how the current user uses the quota. */ class StorageSystemQuotaUsage final : public ext::shared_ptr_helper, public IStorageSystemOneBlock { @@ -19,6 +19,9 @@ public: std::string getName() const override { return "SystemQuotaUsage"; } static NamesAndTypesList getNamesAndTypes(); + static NamesAndTypesList getNamesAndTypesImpl(bool add_column_is_current); + static void fillDataImpl(MutableColumns & res_columns, const Context & context, bool add_column_is_current, const std::vector & quotas_usage); + protected: friend struct ext::shared_ptr_helper; using IStorageSystemOneBlock::IStorageSystemOneBlock; diff --git a/src/Storages/System/StorageSystemQuotas.cpp b/src/Storages/System/StorageSystemQuotas.cpp index a22bb11bbc3..a3b687dc011 100644 --- a/src/Storages/System/StorageSystemQuotas.cpp +++ b/src/Storages/System/StorageSystemQuotas.cpp @@ -3,10 +3,12 @@ #include #include #include -#include #include #include +#include +#include #include +#include #include #include #include @@ -17,11 +19,14 @@ namespace DB { namespace { + using KeyType = Quota::KeyType; + using KeyTypeInfo = Quota::KeyTypeInfo; + DataTypeEnum8::Values getKeyTypeEnumValues() { DataTypeEnum8::Values enum_values; - for (auto key_type : ext::range_with_static_cast(Quota::MAX_KEY_TYPE)) - enum_values.push_back({Quota::getNameOfKeyType(key_type), static_cast(key_type)}); + for (auto key_type : ext::range(KeyType::MAX)) + enum_values.push_back({KeyTypeInfo::get(key_type).name, static_cast(key_type)}); return enum_values; } } @@ -34,21 +39,11 @@ NamesAndTypesList StorageSystemQuotas::getNamesAndTypes() {"id", std::make_shared()}, {"source", std::make_shared()}, {"key_type", std::make_shared(getKeyTypeEnumValues())}, - {"roles", std::make_shared(std::make_shared())}, - {"intervals.duration", std::make_shared(std::make_shared())}, - {"intervals.randomize_interval", std::make_shared(std::make_shared())}}; - - for (auto resource_type : ext::range_with_static_cast(Quota::MAX_RESOURCE_TYPE)) - { - DataTypePtr data_type; - if (resource_type == Quota::EXECUTION_TIME) - data_type = std::make_shared(); - else - data_type = std::make_shared(); - - String column_name = String("intervals.max_") + Quota::resourceTypeToColumnName(resource_type); - names_and_types.push_back({column_name, std::make_shared(data_type)}); - } + {"durations", std::make_shared(std::make_shared())}, + {"apply_to_all", std::make_shared()}, + {"apply_to_list", std::make_shared(std::make_shared())}, + {"apply_to_except", std::make_shared(std::make_shared())} + }; return names_and_types; } @@ -56,61 +51,60 @@ NamesAndTypesList StorageSystemQuotas::getNamesAndTypes() void StorageSystemQuotas::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { context.checkAccess(AccessType::SHOW_QUOTAS); - - size_t i = 0; - auto & name_column = *res_columns[i++]; - auto & id_column = *res_columns[i++]; - auto & storage_name_column = *res_columns[i++]; - auto & key_type_column = *res_columns[i++]; - auto & roles_data = assert_cast(*res_columns[i]).getData(); - auto & roles_offsets = assert_cast(*res_columns[i++]).getOffsets(); - auto & durations_data = assert_cast(*res_columns[i]).getData(); - auto & durations_offsets = assert_cast(*res_columns[i++]).getOffsets(); - auto & randomize_intervals_data = assert_cast(*res_columns[i]).getData(); - auto & randomize_intervals_offsets = assert_cast(*res_columns[i++]).getOffsets(); - IColumn * limits_data[Quota::MAX_RESOURCE_TYPE]; - ColumnArray::Offsets * limits_offsets[Quota::MAX_RESOURCE_TYPE]; - for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) - { - limits_data[resource_type] = &assert_cast(*res_columns[i]).getData(); - limits_offsets[resource_type] = &assert_cast(*res_columns[i++]).getOffsets(); - } - const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + + size_t column_index = 0; + auto & column_name = assert_cast(*res_columns[column_index++]); + auto & column_id = assert_cast(*res_columns[column_index++]).getData(); + auto & column_storage = assert_cast(*res_columns[column_index++]); + auto & column_key_type = assert_cast(*res_columns[column_index++]).getData(); + auto & column_durations = assert_cast(assert_cast(*res_columns[column_index]).getData()).getData(); + auto & column_durations_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_apply_to_all = assert_cast(*res_columns[column_index++]).getData(); + auto & column_apply_to_list = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_apply_to_list_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_apply_to_except = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_apply_to_except_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + + auto add_row = [&](const String & name, + const UUID & id, + const String & storage_name, + const std::vector & all_limits, + KeyType key_type, + const ExtendedRoleSet & apply_to) + { + column_name.insertData(name.data(), name.length()); + column_id.push_back(id); + column_storage.insertData(storage_name.data(), storage_name.length()); + column_key_type.push_back(static_cast(key_type)); + + for (const auto & limits : all_limits) + column_durations.push_back(std::chrono::duration_cast(limits.duration).count()); + column_durations_offsets.push_back(column_durations.size()); + + auto apply_to_ast = apply_to.toASTWithNames(access_control); + column_apply_to_all.push_back(apply_to_ast->all); + + for (const auto & role_name : apply_to_ast->names) + column_apply_to_list.insertData(role_name.data(), role_name.length()); + column_apply_to_list_offsets.push_back(column_apply_to_list.size()); + + for (const auto & role_name : apply_to_ast->except_names) + column_apply_to_except.insertData(role_name.data(), role_name.length()); + column_apply_to_except_offsets.push_back(column_apply_to_except.size()); + }; + for (const auto & id : access_control.findAll()) { auto quota = access_control.tryRead(id); if (!quota) continue; const auto * storage = access_control.findStorage(id); - String storage_name = storage ? storage->getStorageName() : ""; + if (!storage) + continue; - name_column.insert(quota->getName()); - id_column.insert(id); - storage_name_column.insert(storage_name); - key_type_column.insert(static_cast(quota->key_type)); - - for (const String & role : quota->to_roles.toStringsWithNames(access_control)) - roles_data.insert(role); - roles_offsets.push_back(roles_data.size()); - - for (const auto & limits : quota->all_limits) - { - durations_data.insert(std::chrono::seconds{limits.duration}.count()); - randomize_intervals_data.insert(static_cast(limits.randomize_interval)); - for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) - { - if (resource_type == Quota::EXECUTION_TIME) - limits_data[resource_type]->insert(Quota::executionTimeToSeconds(limits.max[resource_type])); - else - limits_data[resource_type]->insert(limits.max[resource_type]); - } - } - - durations_offsets.push_back(durations_data.size()); - randomize_intervals_offsets.push_back(randomize_intervals_data.size()); - for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) - limits_offsets[resource_type]->push_back(limits_data[resource_type]->size()); + add_row(quota->getName(), id, storage->getStorageName(), quota->all_limits, quota->key_type, quota->to_roles); } } } diff --git a/src/Storages/System/StorageSystemQuotas.h b/src/Storages/System/StorageSystemQuotas.h index dda188c7ab7..8d1da53d641 100644 --- a/src/Storages/System/StorageSystemQuotas.h +++ b/src/Storages/System/StorageSystemQuotas.h @@ -6,10 +6,8 @@ namespace DB { - class Context; - /** Implements the `quotas` system tables, which allows you to get information about quotas. */ class StorageSystemQuotas final : public ext::shared_ptr_helper, public IStorageSystemOneBlock diff --git a/src/Storages/System/StorageSystemQuotasUsage.cpp b/src/Storages/System/StorageSystemQuotasUsage.cpp new file mode 100644 index 00000000000..5c6879cd143 --- /dev/null +++ b/src/Storages/System/StorageSystemQuotasUsage.cpp @@ -0,0 +1,22 @@ +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +NamesAndTypesList StorageSystemQuotasUsage::getNamesAndTypes() +{ + return StorageSystemQuotaUsage::getNamesAndTypesImpl(/* add_column_is_current = */ true); +} + +void StorageSystemQuotasUsage::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_QUOTAS); + auto all_quotas_usage = context.getAccessControlManager().getAllQuotasUsage(); + StorageSystemQuotaUsage::fillDataImpl(res_columns, context, /* add_column_is_current = */ true, all_quotas_usage); +} +} diff --git a/src/Storages/System/StorageSystemQuotasUsage.h b/src/Storages/System/StorageSystemQuotasUsage.h new file mode 100644 index 00000000000..d4fd93b577d --- /dev/null +++ b/src/Storages/System/StorageSystemQuotasUsage.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/** Implements the `quotas_usage` system table, which allows you to get information about + * how all users use the quotas. + */ +class StorageSystemQuotasUsage final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemQuotasUsage"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemRoleGrants.cpp b/src/Storages/System/StorageSystemRoleGrants.cpp new file mode 100644 index 00000000000..00147a0dae6 --- /dev/null +++ b/src/Storages/System/StorageSystemRoleGrants.cpp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +using EntityType = IAccessEntity::Type; + + +NamesAndTypesList StorageSystemRoleGrants::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"user_name", std::make_shared(std::make_shared())}, + {"role_name", std::make_shared(std::make_shared())}, + {"granted_role_name", std::make_shared()}, + {"granted_role_is_default", std::make_shared()}, + {"with_admin_option", std::make_shared()}, + }; + return names_and_types; +} + + +void StorageSystemRoleGrants::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_USERS | AccessType::SHOW_ROLES); + const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + boost::range::push_back(ids, access_control.findAll()); + + size_t column_index = 0; + auto & column_user_name = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_user_name_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_role_name = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + auto & column_role_name_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); + auto & column_granted_role_name = assert_cast(*res_columns[column_index++]); + auto & column_is_default = assert_cast(*res_columns[column_index++]).getData(); + auto & column_admin_option = assert_cast(*res_columns[column_index++]).getData(); + + auto add_row = [&](const String & grantee_name, + IAccessEntity::Type grantee_type, + const String & granted_role_name, + bool is_default, + bool with_admin_option) + { + if (grantee_type == EntityType::USER) + { + column_user_name.insertData(grantee_name.data(), grantee_name.length()); + column_user_name_null_map.push_back(false); + column_role_name.insertDefault(); + column_role_name_null_map.push_back(true); + } + else if (grantee_type == EntityType::ROLE) + { + column_user_name.insertDefault(); + column_user_name_null_map.push_back(true); + column_role_name.insertData(grantee_name.data(), grantee_name.length()); + column_role_name_null_map.push_back(false); + } + else + assert(false); + + column_granted_role_name.insertData(granted_role_name.data(), granted_role_name.length()); + column_is_default.push_back(is_default); + column_admin_option.push_back(with_admin_option); + }; + + auto add_rows = [&](const String & grantee_name, + IAccessEntity::Type grantee_type, + const GrantedRoles & granted_roles, + const ExtendedRoleSet * default_roles) + { + for (const auto & role_id : granted_roles.roles) + { + auto role_name = access_control.tryReadName(role_id); + if (!role_name) + continue; + + bool is_default = !default_roles || default_roles->match(role_id); + bool with_admin_option = granted_roles.roles_with_admin_option.count(role_id); + add_row(grantee_name, grantee_type, *role_name, is_default, with_admin_option); + } + }; + + for (const auto & id : ids) + { + auto entity = access_control.tryRead(id); + if (!entity) + continue; + + const GrantedRoles * granted_roles = nullptr; + const ExtendedRoleSet * default_roles = nullptr; + if (auto role = typeid_cast(entity)) + granted_roles = &role->granted_roles; + else if (auto user = typeid_cast(entity)) + { + granted_roles = &user->granted_roles; + default_roles = &user->default_roles; + } + else + continue; + + add_rows(entity->getName(), entity->getType(), *granted_roles, default_roles); + } +} + +} diff --git a/src/Storages/System/StorageSystemRoleGrants.h b/src/Storages/System/StorageSystemRoleGrants.h new file mode 100644 index 00000000000..0a02303abc3 --- /dev/null +++ b/src/Storages/System/StorageSystemRoleGrants.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `role_grants` system table, which allows you to get information about granted roles. +class StorageSystemRoleGrants final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemRoleGrants"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemRoles.cpp b/src/Storages/System/StorageSystemRoles.cpp new file mode 100644 index 00000000000..c04cb6641a6 --- /dev/null +++ b/src/Storages/System/StorageSystemRoles.cpp @@ -0,0 +1,60 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +NamesAndTypesList StorageSystemRoles::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"name", std::make_shared()}, + {"id", std::make_shared()}, + {"storage", std::make_shared()}, + }; + return names_and_types; +} + + +void StorageSystemRoles::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_ROLES); + const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + + size_t column_index = 0; + auto & column_name = assert_cast(*res_columns[column_index++]); + auto & column_id = assert_cast(*res_columns[column_index++]).getData(); + auto & column_storage = assert_cast(*res_columns[column_index++]); + + auto add_row = [&](const String & name, + const UUID & id, + const String & storage_name) + { + column_name.insertData(name.data(), name.length()); + column_id.push_back(id); + column_storage.insertData(storage_name.data(), storage_name.length()); + }; + + for (const auto & id : ids) + { + auto role = access_control.tryRead(id); + if (!role) + continue; + + const auto * storage = access_control.findStorage(id); + if (!storage) + continue; + + add_row(role->getName(), id, storage->getStorageName()); + } +} + +} diff --git a/src/Storages/System/StorageSystemRoles.h b/src/Storages/System/StorageSystemRoles.h new file mode 100644 index 00000000000..fb44194baff --- /dev/null +++ b/src/Storages/System/StorageSystemRoles.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `roles` system table, which allows you to get information about roles. +class StorageSystemRoles final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemRoles"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemRowPolicies.cpp b/src/Storages/System/StorageSystemRowPolicies.cpp index 12221cc52de..ca77a5182a5 100644 --- a/src/Storages/System/StorageSystemRowPolicies.cpp +++ b/src/Storages/System/StorageSystemRowPolicies.cpp @@ -2,32 +2,52 @@ #include #include #include -#include #include +#include +#include +#include +#include +#include #include +#include #include #include #include #include +#include namespace DB { +using ConditionTypeInfo = RowPolicy::ConditionTypeInfo; +constexpr auto MAX_CONDITION_TYPE = RowPolicy::MAX_CONDITION_TYPE; + + NamesAndTypesList StorageSystemRowPolicies::getNamesAndTypes() { NamesAndTypesList names_and_types{ + {"name", std::make_shared()}, + {"short_name", std::make_shared()}, {"database", std::make_shared()}, {"table", std::make_shared()}, - {"name", std::make_shared()}, - {"full_name", std::make_shared()}, {"id", std::make_shared()}, {"source", std::make_shared()}, - {"restrictive", std::make_shared()}, }; - for (auto index : ext::range_with_static_cast(RowPolicy::MAX_CONDITION_TYPE)) - names_and_types.push_back({RowPolicy::conditionTypeToColumnName(index), std::make_shared()}); + for (auto type : ext::range(MAX_CONDITION_TYPE)) + { + const String & column_name = ConditionTypeInfo::get(type).name; + names_and_types.push_back({column_name, std::make_shared(std::make_shared())}); + } + NamesAndTypesList extra_names_and_types{ + {"is_restrictive", std::make_shared()}, + {"apply_to_all", std::make_shared()}, + {"apply_to_list", std::make_shared(std::make_shared())}, + {"apply_to_except", std::make_shared(std::make_shared())} + }; + + boost::range::push_back(names_and_types, std::move(extra_names_and_types)); return names_and_types; } @@ -38,24 +58,83 @@ void StorageSystemRowPolicies::fillData(MutableColumns & res_columns, const Cont const auto & access_control = context.getAccessControlManager(); std::vector ids = access_control.findAll(); + size_t column_index = 0; + auto & column_name = assert_cast(*res_columns[column_index++]); + auto & column_short_name = assert_cast(*res_columns[column_index++]); + auto & column_database = assert_cast(*res_columns[column_index++]); + auto & column_table = assert_cast(*res_columns[column_index++]); + auto & column_id = assert_cast(*res_columns[column_index++]).getData(); + auto & column_storage = assert_cast(*res_columns[column_index++]); + + ColumnString * column_condition[MAX_CONDITION_TYPE]; + NullMap * column_condition_null_map[MAX_CONDITION_TYPE]; + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + { + column_condition[condition_type] = &assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); + column_condition_null_map[condition_type] = &assert_cast(*res_columns[column_index++]).getNullMapData(); + } + + auto & column_is_restrictive = assert_cast(*res_columns[column_index++]).getData(); + auto & column_apply_to_all = assert_cast(*res_columns[column_index++]).getData(); + auto & column_apply_to_list = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_apply_to_list_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_apply_to_except = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_apply_to_except_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + + auto add_row = [&](const String & name, + const RowPolicy::NameParts & name_parts, + const UUID & id, + const String & storage_name, + const std::array & conditions, + bool is_restrictive, + const ExtendedRoleSet & apply_to) + { + column_name.insertData(name.data(), name.length()); + column_short_name.insertData(name_parts.short_name.data(), name_parts.short_name.length()); + column_database.insertData(name_parts.database.data(), name_parts.database.length()); + column_table.insertData(name_parts.table_name.data(), name_parts.table_name.length()); + column_id.push_back(id); + column_storage.insertData(storage_name.data(), storage_name.length()); + + for (auto condition_type : ext::range(MAX_CONDITION_TYPE)) + { + const String & condition = conditions[condition_type]; + if (condition.empty()) + { + column_condition[condition_type]->insertDefault(); + column_condition_null_map[condition_type]->push_back(true); + } + else + { + column_condition[condition_type]->insertData(condition.data(), condition.length()); + column_condition_null_map[condition_type]->push_back(false); + } + } + + column_is_restrictive.push_back(is_restrictive); + + auto apply_to_ast = apply_to.toASTWithNames(access_control); + column_apply_to_all.push_back(apply_to_ast->all); + + for (const auto & role_name : apply_to_ast->names) + column_apply_to_list.insertData(role_name.data(), role_name.length()); + column_apply_to_list_offsets.push_back(column_apply_to_list.size()); + + for (const auto & role_name : apply_to_ast->except_names) + column_apply_to_except.insertData(role_name.data(), role_name.length()); + column_apply_to_except_offsets.push_back(column_apply_to_except.size()); + }; + for (const auto & id : ids) { auto policy = access_control.tryRead(id); if (!policy) continue; const auto * storage = access_control.findStorage(id); + if (!storage) + continue; - size_t i = 0; - res_columns[i++]->insert(policy->getDatabase()); - res_columns[i++]->insert(policy->getTableName()); - res_columns[i++]->insert(policy->getName()); - res_columns[i++]->insert(policy->getFullName()); - res_columns[i++]->insert(id); - res_columns[i++]->insert(storage ? storage->getStorageName() : ""); - res_columns[i++]->insert(policy->isRestrictive()); - - for (auto index : ext::range(RowPolicy::MAX_CONDITION_TYPE)) - res_columns[i++]->insert(policy->conditions[index]); + add_row(policy->getName(), policy->getNameParts(), id, storage->getStorageName(), policy->conditions, policy->isRestrictive(), policy->to_roles); } } } diff --git a/src/Storages/System/StorageSystemSettingsProfileElements.cpp b/src/Storages/System/StorageSystemSettingsProfileElements.cpp new file mode 100644 index 00000000000..2e4d1ad1e05 --- /dev/null +++ b/src/Storages/System/StorageSystemSettingsProfileElements.cpp @@ -0,0 +1,216 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +using EntityType = IAccessEntity::Type; + + +NamesAndTypesList StorageSystemSettingsProfileElements::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"profile_name", std::make_shared(std::make_shared())}, + {"user_name", std::make_shared(std::make_shared())}, + {"role_name", std::make_shared(std::make_shared())}, + {"index", std::make_shared()}, + {"setting_name", std::make_shared(std::make_shared())}, + {"value", std::make_shared(std::make_shared())}, + {"min", std::make_shared(std::make_shared())}, + {"max", std::make_shared(std::make_shared())}, + {"readonly", std::make_shared(std::make_shared())}, + {"inherit_profile", std::make_shared(std::make_shared())}, + }; + return names_and_types; +} + + +void StorageSystemSettingsProfileElements::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_SETTINGS_PROFILES); + const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + boost::range::push_back(ids, access_control.findAll()); + boost::range::push_back(ids, access_control.findAll()); + + size_t i = 0; + auto & column_profile_name = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_profile_name_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_user_name = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_user_name_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_role_name = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_role_name_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_index = assert_cast(*res_columns[i++]).getData(); + auto & column_setting_name = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_setting_name_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_value = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_value_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_min = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_min_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_max = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_max_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_readonly = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()).getData(); + auto & column_readonly_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + auto & column_inherit_profile = assert_cast(assert_cast(*res_columns[i]).getNestedColumn()); + auto & column_inherit_profile_null_map = assert_cast(*res_columns[i++]).getNullMapData(); + + auto add_rows_for_single_element = [&](const String & owner_name, EntityType owner_type, const SettingsProfileElement & element, size_t & index) + { + switch (owner_type) + { + case EntityType::SETTINGS_PROFILE: + { + column_user_name.insertDefault(); + column_user_name_null_map.push_back(true); + column_role_name.insertDefault(); + column_role_name_null_map.push_back(true); + column_profile_name.insertData(owner_name.data(), owner_name.length()); + column_profile_name_null_map.push_back(false); + break; + } + case EntityType::USER: + { + column_user_name.insertData(owner_name.data(), owner_name.length()); + column_user_name_null_map.push_back(false); + column_profile_name.insertDefault(); + column_profile_name_null_map.push_back(true); + column_role_name.insertDefault(); + column_role_name_null_map.push_back(true); + break; + } + case EntityType::ROLE: + { + column_user_name.insertDefault(); + column_user_name_null_map.push_back(true); + column_role_name.insertData(owner_name.data(), owner_name.length()); + column_role_name_null_map.push_back(false); + column_profile_name.insertDefault(); + column_profile_name_null_map.push_back(true); + break; + } + default: + assert(false); + } + + if (element.parent_profile) + { + auto parent_profile = access_control.tryReadName(*element.parent_profile); + if (parent_profile) + { + column_index.push_back(index++); + column_setting_name.insertDefault(); + column_setting_name_null_map.push_back(true); + column_value.insertDefault(); + column_value_null_map.push_back(true); + column_min.insertDefault(); + column_min_null_map.push_back(true); + column_max.insertDefault(); + column_max_null_map.push_back(true); + column_readonly.push_back(0); + column_readonly_null_map.push_back(true); + const String & parent_profile_str = *parent_profile; + column_inherit_profile.insertData(parent_profile_str.data(), parent_profile_str.length()); + column_inherit_profile_null_map.push_back(false); + } + } + + if ((element.setting_index != static_cast(-1)) + && (!element.value.isNull() || !element.min_value.isNull() || !element.max_value.isNull() || element.readonly)) + { + auto setting_name = Settings::getName(element.setting_index); + column_index.push_back(index++); + column_setting_name.insertData(setting_name.data, setting_name.size); + column_setting_name_null_map.push_back(false); + + if (element.value.isNull()) + { + column_value.insertDefault(); + column_value_null_map.push_back(true); + } + else + { + String str = Settings::valueToString(element.setting_index, element.value); + column_value.insertData(str.data(), str.length()); + column_value_null_map.push_back(false); + } + + if (element.min_value.isNull()) + { + column_min.insertDefault(); + column_min_null_map.push_back(true); + } + else + { + String str = Settings::valueToString(element.setting_index, element.min_value); + column_min.insertData(str.data(), str.length()); + column_min_null_map.push_back(false); + } + + if (element.max_value.isNull()) + { + column_max.insertDefault(); + column_max_null_map.push_back(true); + } + else + { + String str = Settings::valueToString(element.setting_index, element.max_value); + column_max.insertData(str.data(), str.length()); + column_max_null_map.push_back(false); + } + + if (element.readonly) + { + column_readonly.push_back(*element.readonly); + column_readonly_null_map.push_back(false); + } + else + { + column_readonly.push_back(0); + column_readonly_null_map.push_back(true); + } + + column_inherit_profile.insertDefault(); + column_inherit_profile_null_map.push_back(true); + } + }; + + auto add_rows = [&](const String & owner_name, IAccessEntity::Type owner_type, const SettingsProfileElements & elements) + { + size_t index = 0; + for (const auto & element : elements) + add_rows_for_single_element(owner_name, owner_type, element, index); + }; + + for (const auto & id : ids) + { + auto entity = access_control.tryRead(id); + if (!entity) + continue; + + const SettingsProfileElements * settings = nullptr; + if (auto role = typeid_cast(entity)) + settings = &role->settings; + else if (auto user = typeid_cast(entity)) + settings = &user->settings; + else if (auto profile = typeid_cast(entity)) + settings = &profile->elements; + else + continue; + + add_rows(entity->getName(), entity->getType(), *settings); + } +} + +} diff --git a/src/Storages/System/StorageSystemSettingsProfileElements.h b/src/Storages/System/StorageSystemSettingsProfileElements.h new file mode 100644 index 00000000000..2dc79fed0e7 --- /dev/null +++ b/src/Storages/System/StorageSystemSettingsProfileElements.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `settings_profile_elements` system table, which allows you to get information about elements of settings profiles. +class StorageSystemSettingsProfileElements final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemSettingsProfileElements"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemSettingsProfiles.cpp b/src/Storages/System/StorageSystemSettingsProfiles.cpp new file mode 100644 index 00000000000..d02c5910608 --- /dev/null +++ b/src/Storages/System/StorageSystemSettingsProfiles.cpp @@ -0,0 +1,87 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +NamesAndTypesList StorageSystemSettingsProfiles::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"name", std::make_shared()}, + {"id", std::make_shared()}, + {"storage", std::make_shared()}, + {"num_elements", std::make_shared()}, + {"apply_to_all", std::make_shared()}, + {"apply_to_list", std::make_shared(std::make_shared())}, + {"apply_to_except", std::make_shared(std::make_shared())}, + }; + return names_and_types; +} + + +void StorageSystemSettingsProfiles::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_SETTINGS_PROFILES); + const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + + size_t column_index = 0; + auto & column_name = assert_cast(*res_columns[column_index++]); + auto & column_id = assert_cast(*res_columns[column_index++]).getData(); + auto & column_storage = assert_cast(*res_columns[column_index++]); + auto & column_num_elements = assert_cast(*res_columns[column_index++]).getData(); + auto & column_apply_to_all = assert_cast(*res_columns[column_index++]).getData(); + auto & column_apply_to_list = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_apply_to_list_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_apply_to_except = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_apply_to_except_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + + auto add_row = [&](const String & name, + const UUID & id, + const String & storage_name, + const SettingsProfileElements & elements, + const ExtendedRoleSet & apply_to) + { + column_name.insertData(name.data(), name.length()); + column_id.push_back(id); + column_storage.insertData(storage_name.data(), storage_name.length()); + column_num_elements.push_back(elements.size()); + + auto apply_to_ast = apply_to.toASTWithNames(access_control); + column_apply_to_all.push_back(apply_to_ast->all); + + for (const auto & role_name : apply_to_ast->names) + column_apply_to_list.insertData(role_name.data(), role_name.length()); + column_apply_to_list_offsets.push_back(column_apply_to_list.size()); + + for (const auto & role_name : apply_to_ast->except_names) + column_apply_to_except.insertData(role_name.data(), role_name.length()); + column_apply_to_except_offsets.push_back(column_apply_to_except.size()); + }; + + for (const auto & id : ids) + { + auto profile = access_control.tryRead(id); + if (!profile) + continue; + + const auto * storage = access_control.findStorage(id); + if (!storage) + continue; + + add_row(profile->getName(), id, storage->getStorageName(), profile->elements, profile->to_roles); + } +} + +} diff --git a/src/Storages/System/StorageSystemSettingsProfiles.h b/src/Storages/System/StorageSystemSettingsProfiles.h new file mode 100644 index 00000000000..c6b887c99df --- /dev/null +++ b/src/Storages/System/StorageSystemSettingsProfiles.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `settings_profiles` system table, which allows you to get information about profiles. +class StorageSystemSettingsProfiles final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemSettingsProfiles"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp new file mode 100644 index 00000000000..d0e042d054f --- /dev/null +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -0,0 +1,135 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +NamesAndTypesList StorageSystemUsers::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"name", std::make_shared()}, + {"id", std::make_shared()}, + {"storage", std::make_shared()}, + {"host_ip", std::make_shared(std::make_shared())}, + {"host_names", std::make_shared(std::make_shared())}, + {"host_names_regexp", std::make_shared(std::make_shared())}, + {"host_names_like", std::make_shared(std::make_shared())}, + {"default_roles_all", std::make_shared()}, + {"default_roles_list", std::make_shared(std::make_shared())}, + {"default_roles_except", std::make_shared(std::make_shared())}, + }; + return names_and_types; +} + + +void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + context.checkAccess(AccessType::SHOW_USERS); + const auto & access_control = context.getAccessControlManager(); + std::vector ids = access_control.findAll(); + + size_t column_index = 0; + auto & column_name = assert_cast(*res_columns[column_index++]); + auto & column_id = assert_cast(*res_columns[column_index++]).getData(); + auto & column_storage = assert_cast(*res_columns[column_index++]); + auto & column_host_ip = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_host_ip_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_host_names = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_host_names_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_host_names_regexp = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_host_names_regexp_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_host_names_like = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_host_names_like_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_default_roles_all = assert_cast(*res_columns[column_index++]).getData(); + auto & column_default_roles_list = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_default_roles_list_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_default_roles_except = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_default_roles_except_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + + auto add_row = [&](const String & name, + const UUID & id, + const String & storage_name, + const AllowedClientHosts & allowed_hosts, + const ExtendedRoleSet & default_roles) + { + column_name.insertData(name.data(), name.length()); + column_id.push_back(id); + column_storage.insertData(storage_name.data(), storage_name.length()); + + if (allowed_hosts.containsAnyHost()) + { + static constexpr std::string_view str{"::/0"}; + column_host_ip.insertData(str.data(), str.length()); + } + else + { + if (allowed_hosts.containsLocalHost()) + { + static constexpr std::string_view str{"localhost"}; + column_host_names.insertData(str.data(), str.length()); + } + + for (const auto & ip : allowed_hosts.getAddresses()) + { + String str = ip.toString(); + column_host_ip.insertData(str.data(), str.length()); + } + for (const auto & subnet : allowed_hosts.getSubnets()) + { + String str = subnet.toString(); + column_host_ip.insertData(str.data(), str.length()); + } + + for (const auto & host_name : allowed_hosts.getNames()) + column_host_names.insertData(host_name.data(), host_name.length()); + + for (const auto & name_regexp : allowed_hosts.getNameRegexps()) + column_host_names_regexp.insertData(name_regexp.data(), name_regexp.length()); + + for (const auto & like_pattern : allowed_hosts.getLikePatterns()) + column_host_names_like.insertData(like_pattern.data(), like_pattern.length()); + } + + column_host_ip_offsets.push_back(column_host_ip.size()); + column_host_names_offsets.push_back(column_host_names.size()); + column_host_names_regexp_offsets.push_back(column_host_names_regexp.size()); + column_host_names_like_offsets.push_back(column_host_names_like.size()); + + auto default_roles_ast = default_roles.toASTWithNames(access_control); + column_default_roles_all.push_back(default_roles_ast->all); + + for (const auto & role_name : default_roles_ast->names) + column_default_roles_list.insertData(role_name.data(), role_name.length()); + column_default_roles_list_offsets.push_back(column_default_roles_list.size()); + + for (const auto & role_name : default_roles_ast->except_names) + column_default_roles_except.insertData(role_name.data(), role_name.length()); + column_default_roles_except_offsets.push_back(column_default_roles_except.size()); + }; + + for (const auto & id : ids) + { + auto user = access_control.tryRead(id); + if (!user) + continue; + + const auto * storage = access_control.findStorage(id); + if (!storage) + continue; + + add_row(user->getName(), id, storage->getStorageName(), user->allowed_client_hosts, user->default_roles); + } +} + +} diff --git a/src/Storages/System/StorageSystemUsers.h b/src/Storages/System/StorageSystemUsers.h new file mode 100644 index 00000000000..707ea94591d --- /dev/null +++ b/src/Storages/System/StorageSystemUsers.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class Context; + +/// Implements `users` system table, which allows you to get information about users. +class StorageSystemUsers final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemUsers"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct ext::shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 4c9f34b82e1..585eab2b4d8 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -15,6 +15,7 @@ #include #include #include + #include #include #include @@ -25,11 +26,8 @@ #include #include #include -#include -#include #include #include -#include #include #include #include @@ -44,6 +42,21 @@ #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #ifdef OS_LINUX #include #endif @@ -66,9 +79,6 @@ void attachSystemTablesLocal(IDatabase & system_database) system_database.attachTable("functions", StorageSystemFunctions::create("functions")); system_database.attachTable("events", StorageSystemEvents::create("events")); system_database.attachTable("settings", StorageSystemSettings::create("settings")); - system_database.attachTable("quotas", StorageSystemQuotas::create("quotas")); - system_database.attachTable("quota_usage", StorageSystemQuotaUsage::create("quota_usage")); - system_database.attachTable("row_policies", StorageSystemRowPolicies::create("row_policies")); system_database.attachTable("merge_tree_settings", SystemMergeTreeSettings::create("merge_tree_settings")); system_database.attachTable("build_options", StorageSystemBuildOptions::create("build_options")); system_database.attachTable("formats", StorageSystemFormats::create("formats")); @@ -78,6 +88,20 @@ void attachSystemTablesLocal(IDatabase & system_database) system_database.attachTable("collations", StorageSystemCollations::create("collations")); system_database.attachTable("table_engines", StorageSystemTableEngines::create("table_engines")); system_database.attachTable("contributors", StorageSystemContributors::create("contributors")); + system_database.attachTable("users", StorageSystemUsers::create("users")); + system_database.attachTable("roles", StorageSystemRoles::create("roles")); + system_database.attachTable("grants", StorageSystemGrants::create("grants")); + system_database.attachTable("role_grants", StorageSystemRoleGrants::create("role_grants")); + system_database.attachTable("current_roles", StorageSystemCurrentRoles::create("current_roles")); + system_database.attachTable("enabled_roles", StorageSystemEnabledRoles::create("enabled_roles")); + system_database.attachTable("settings_profiles", StorageSystemSettingsProfiles::create("settings_profiles")); + system_database.attachTable("settings_profile_elements", StorageSystemSettingsProfileElements::create("settings_profile_elements")); + system_database.attachTable("row_policies", StorageSystemRowPolicies::create("row_policies")); + system_database.attachTable("quotas", StorageSystemQuotas::create("quotas")); + system_database.attachTable("quota_limits", StorageSystemQuotaLimits::create("quota_limits")); + system_database.attachTable("quota_usage", StorageSystemQuotaUsage::create("quota_usage")); + system_database.attachTable("quotas_usage", StorageSystemQuotasUsage::create("all_quotas_usage")); + system_database.attachTable("privileges", StorageSystemPrivileges::create("privileges")); #if !defined(ARCADIA_BUILD) system_database.attachTable("licenses", StorageSystemLicenses::create("licenses")); #endif diff --git a/src/Storages/ya.make b/src/Storages/ya.make index 5ed6fcbf637..ffa3924d11a 100644 --- a/src/Storages/ya.make +++ b/src/Storages/ya.make @@ -86,6 +86,7 @@ SRCS( System/StorageSystemAsynchronousMetrics.cpp System/StorageSystemBuildOptions.cpp System/StorageSystemClusters.cpp + System/StorageSystemCurrentRoles.cpp System/StorageSystemCollations.cpp System/StorageSystemColumns.cpp System/StorageSystemContributors.cpp @@ -95,9 +96,11 @@ SRCS( System/StorageSystemDetachedParts.cpp System/StorageSystemDictionaries.cpp System/StorageSystemDisks.cpp + System/StorageSystemEnabledRoles.cpp System/StorageSystemEvents.cpp System/StorageSystemFormats.cpp System/StorageSystemFunctions.cpp + System/StorageSystemGrants.cpp System/StorageSystemGraphite.cpp System/StorageSystemMacros.cpp System/StorageSystemMerges.cpp @@ -110,18 +113,26 @@ SRCS( System/StorageSystemParts.cpp System/StorageSystemPartsBase.cpp System/StorageSystemPartsColumns.cpp + System/StorageSystemPrivileges.cpp System/StorageSystemProcesses.cpp - System/StorageSystemQuotas.cpp + System/StorageSystemQuotaLimits.cpp System/StorageSystemQuotaUsage.cpp + System/StorageSystemQuotas.cpp + System/StorageSystemQuotasUsage.cpp System/StorageSystemReplicas.cpp System/StorageSystemReplicationQueue.cpp + System/StorageSystemRoleGrants.cpp + System/StorageSystemRoles.cpp System/StorageSystemRowPolicies.cpp System/StorageSystemSettings.cpp + System/StorageSystemSettingsProfileElements.cpp + System/StorageSystemSettingsProfiles.cpp System/StorageSystemStackTrace.cpp System/StorageSystemStoragePolicies.cpp System/StorageSystemTableEngines.cpp System/StorageSystemTableFunctions.cpp System/StorageSystemTables.cpp + System/StorageSystemUsers.cpp System/StorageSystemZeros.cpp System/StorageSystemZooKeeper.cpp AlterCommands.cpp diff --git a/tests/integration/helpers/test_tools.py b/tests/integration/helpers/test_tools.py index e90580de44c..93265d280df 100644 --- a/tests/integration/helpers/test_tools.py +++ b/tests/integration/helpers/test_tools.py @@ -5,16 +5,29 @@ class TSV: """Helper to get pretty diffs between expected and actual tab-separated value files""" def __init__(self, contents): - raw_lines = contents.readlines() if isinstance(contents, file) else contents.splitlines(True) + if isinstance(contents, file): + raw_lines = contents.readlines() + elif isinstance(contents, str) or isinstance(contents, unicode): + raw_lines = contents.splitlines(True) + elif isinstance(contents, list): + raw_lines = ['\t'.join(map(str, l)) if isinstance(l, list) else str(l) for l in contents] + else: + raise TypeError("contents must be either file or string or list, actual type: " + type(contents).__name__) self.lines = [l.strip() for l in raw_lines if l.strip()] def __eq__(self, other): + if not isinstance(other, TSV): + return self == TSV(other) return self.lines == other.lines def __ne__(self, other): + if not isinstance(other, TSV): + return self != TSV(other) return self.lines != other.lines def diff(self, other, n1=None, n2=None): + if not isinstance(other, TSV): + return self.diff(TSV(other), n1=n1, n2=n2) return list(line.rstrip() for line in difflib.unified_diff(self.lines, other.lines, fromfile=n1, tofile=n2))[2:] def __str__(self): diff --git a/tests/integration/test_access_control_on_cluster/test.py b/tests/integration/test_access_control_on_cluster/test.py index 6ca4ac15398..4dc9baca0a0 100644 --- a/tests/integration/test_access_control_on_cluster/test.py +++ b/tests/integration/test_access_control_on_cluster/test.py @@ -35,7 +35,7 @@ def test_access_control_on_cluster(): assert ch3.query("SHOW GRANTS FOR Alex") == "" ch2.query("DROP USER Alex ON CLUSTER 'cluster'") - assert "User `Alex` not found" in ch1.query_and_get_error("SHOW CREATE USER Alex") - assert "User `Alex` not found" in ch2.query_and_get_error("SHOW CREATE USER Alex") - assert "User `Alex` not found" in ch3.query_and_get_error("SHOW CREATE USER Alex") + assert "There is no user `Alex`" in ch1.query_and_get_error("SHOW CREATE USER Alex") + assert "There is no user `Alex`" in ch2.query_and_get_error("SHOW CREATE USER Alex") + assert "There is no user `Alex`" in ch3.query_and_get_error("SHOW CREATE USER Alex") diff --git a/tests/integration/test_aggregation_memory_efficient/test.py b/tests/integration/test_aggregation_memory_efficient/test.py index a4e8e2b6295..3a7ada5f02e 100644 --- a/tests/integration/test_aggregation_memory_efficient/test.py +++ b/tests/integration/test_aggregation_memory_efficient/test.py @@ -29,21 +29,18 @@ def start_cluster(): def test_remote(start_cluster): - for flag in (0, 1): - node1.query("set experimental_use_processors = {}".format(flag)) + node1.query("set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") + res = node1.query("select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") + assert res == '200000\n' - node1.query("set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") - res = node1.query("select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") - assert res == '200000\n' + node1.query("set distributed_aggregation_memory_efficient = 0") + res = node1.query("select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") + assert res == '200000\n' - node1.query("set distributed_aggregation_memory_efficient = 0") - res = node1.query("select sum(a) from (SELECT B, uniqExact(A) a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY B)") - assert res == '200000\n' + node1.query("set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") + res = node1.query("SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") + assert res == 'node1\t100000\nnode2\t100000\n' - node1.query("set distributed_aggregation_memory_efficient = 1, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes=1") - res = node1.query("SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") - assert res == 'node1\t100000\nnode2\t100000\n' - - node1.query("set distributed_aggregation_memory_efficient = 0") - res = node1.query("SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") - assert res == 'node1\t100000\nnode2\t100000\n' + node1.query("set distributed_aggregation_memory_efficient = 0") + res = node1.query("SELECT fullHostName() AS h, uniqExact(A) AS a FROM remote('node{1,2}', default.da_memory_efficient_shard) GROUP BY h ORDER BY h;") + assert res == 'node1\t100000\nnode2\t100000\n' diff --git a/tests/integration/test_backward_compatability/test_short_strings_aggregation.py b/tests/integration/test_backward_compatability/test_short_strings_aggregation.py new file mode 100644 index 00000000000..1c264d1e636 --- /dev/null +++ b/tests/integration/test_backward_compatability/test_short_strings_aggregation.py @@ -0,0 +1,28 @@ +import pytest + +import helpers.client as client +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', with_zookeeper=False, image='yandex/clickhouse-server:19.16.9.37', stay_alive=True, with_installed_binary=True) +node2 = cluster.add_instance('node2', with_zookeeper=False, image='yandex/clickhouse-server:19.16.9.37', stay_alive=True, with_installed_binary=True) +node3 = cluster.add_instance('node3', with_zookeeper=False) + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_backward_compatability(start_cluster): + node1.query("create table tab (s String) engine = MergeTree order by s") + node2.query("create table tab (s String) engine = MergeTree order by s") + node1.query("insert into tab select number from numbers(50)") + node2.query("insert into tab select number from numbers(1000000)") + res = node3.query("select s, count() from remote('node{1,2}', default, tab) group by s order by toUInt64(s) limit 50") + print(res) + assert res == ''.join('{}\t2\n'.format(i) for i in range(50)) diff --git a/tests/integration/test_backward_compatability/test_string_aggregation.py b/tests/integration/test_backward_compatability/test_string_aggregation.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_dictionaries_access/__init__.py b/tests/integration/test_dictionaries_access/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_dictionaries_access/test.py b/tests/integration/test_dictionaries_access/test.py new file mode 100644 index 00000000000..cbba651a3b7 --- /dev/null +++ b/tests/integration/test_dictionaries_access/test.py @@ -0,0 +1,92 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance('instance') + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + + instance.query("CREATE USER mira") + instance.query("CREATE TABLE test_table(x Int32, y Int32) ENGINE=Log") + instance.query("INSERT INTO test_table VALUES (5,6)") + + yield cluster + + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def clear_after_test(): + try: + yield + finally: + instance.query("CREATE USER OR REPLACE mira") + instance.query("DROP DICTIONARY IF EXISTS test_dict") + + +create_query = """ + CREATE DICTIONARY test_dict(x Int32, y Int32) PRIMARY KEY x + LAYOUT(FLAT()) + SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'test_table' DB 'default')) + LIFETIME(0) + """ + +drop_query = "DROP DICTIONARY test_dict" + + +def test_create(): + assert instance.query("SHOW GRANTS FOR mira") == "" + assert "Not enough privileges" in instance.query_and_get_error(create_query, user="mira") + + instance.query("GRANT CREATE DICTIONARY ON *.* TO mira") + instance.query(create_query, user="mira") + instance.query(drop_query) + + instance.query("REVOKE CREATE DICTIONARY ON *.* FROM mira") + assert instance.query("SHOW GRANTS FOR mira") == "" + assert "Not enough privileges" in instance.query_and_get_error(create_query, user="mira") + + instance.query("GRANT CREATE DICTIONARY ON default.* TO mira") + instance.query(create_query, user="mira") + instance.query(drop_query) + + instance.query("REVOKE CREATE DICTIONARY ON default.* FROM mira") + assert instance.query("SHOW GRANTS FOR mira") == "" + assert "Not enough privileges" in instance.query_and_get_error(create_query, user="mira") + + instance.query("GRANT CREATE DICTIONARY ON default.test_dict TO mira") + instance.query(create_query, user="mira") + + +def test_drop(): + instance.query(create_query) + + assert instance.query("SHOW GRANTS FOR mira") == "" + assert "Not enough privileges" in instance.query_and_get_error(drop_query, user="mira") + + instance.query("GRANT DROP DICTIONARY ON *.* TO mira") + instance.query(drop_query, user="mira") + instance.query(create_query) + + +def test_dictget(): + instance.query(create_query) + + dictget_query = "SELECT dictGet('default.test_dict', 'y', toUInt64(5))" + instance.query(dictget_query) == "6\n" + assert "Not enough privileges" in instance.query_and_get_error(dictget_query, user='mira') + + instance.query("GRANT dictGet ON default.test_dict TO mira") + instance.query(dictget_query, user='mira') == "6\n" + + dictget_query = "SELECT dictGet('default.test_dict', 'y', toUInt64(1))" + instance.query(dictget_query) == "0\n" + instance.query(dictget_query, user='mira') == "0\n" + + instance.query("REVOKE dictGet ON *.* FROM mira") + assert "Not enough privileges" in instance.query_and_get_error(dictget_query, user='mira') diff --git a/tests/integration/test_disk_access_storage/test.py b/tests/integration/test_disk_access_storage/test.py index babceee7c76..315440b4358 100644 --- a/tests/integration/test_disk_access_storage/test.py +++ b/tests/integration/test_disk_access_storage/test.py @@ -97,9 +97,9 @@ def test_drop(): def check(): assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1\n" assert instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE s2\n" - assert "User `u2` not found" in instance.query_and_get_error("SHOW CREATE USER u2") - assert "Row policy `p ON mydb.mytable` not found" in instance.query_and_get_error("SHOW CREATE ROW POLICY p ON mydb.mytable") - assert "Quota `q` not found" in instance.query_and_get_error("SHOW CREATE QUOTA q") + assert "There is no user `u2`" in instance.query_and_get_error("SHOW CREATE USER u2") + assert "There is no row policy `p ON mydb.mytable`" in instance.query_and_get_error("SHOW CREATE ROW POLICY p ON mydb.mytable") + assert "There is no quota `q`" in instance.query_and_get_error("SHOW CREATE QUOTA q") check() instance.restart_clickhouse() # Check persistency diff --git a/tests/integration/test_distributed_backward_compatability/__init__.py b/tests/integration/test_distributed_backward_compatability/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_distributed_backward_compatability/configs/remote_servers.xml b/tests/integration/test_distributed_backward_compatability/configs/remote_servers.xml new file mode 100644 index 00000000000..ebce4697529 --- /dev/null +++ b/tests/integration/test_distributed_backward_compatability/configs/remote_servers.xml @@ -0,0 +1,18 @@ + + + + + + node1 + 9000 + + + + + node2 + 9000 + + + + + diff --git a/tests/integration/test_distributed_backward_compatability/test.py b/tests/integration/test_distributed_backward_compatability/test.py new file mode 100644 index 00000000000..a0362cea49e --- /dev/null +++ b/tests/integration/test_distributed_backward_compatability/test.py @@ -0,0 +1,47 @@ +import pytest +import time + +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager +from helpers.test_tools import TSV + + +cluster = ClickHouseCluster(__file__) + +node_old = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml'], image='yandex/clickhouse-server:19.17.8.54', stay_alive=True, with_installed_binary=True) +node_new = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml']) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + for node in (node_old, node_new): + node.query("CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id") + + node_old.query("INSERT INTO local_table VALUES (1, 'node1')") + node_new.query("INSERT INTO local_table VALUES (2, 'node2')") + + node_old.query("CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)") + node_new.query("CREATE TABLE distributed(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table)") + + yield cluster + + finally: + cluster.shutdown() + +def test_distributed_in_tuple(started_cluster): + query1 = "SELECT count() FROM distributed WHERE (id, val) IN ((1, 'node1'), (2, 'a'), (3, 'b'))" + query2 = "SELECT sum((id, val) IN ((1, 'node1'), (2, 'a'), (3, 'b'))) FROM distributed" + assert node_old.query(query1) == "1\n" + assert node_old.query(query2) == "1\n" + assert node_new.query(query1) == "1\n" + assert node_new.query(query2) == "1\n" + + large_set = '(' + ','.join([str(i) for i in range(1000)]) + ')' + query3 = "SELECT count() FROM distributed WHERE id IN " + large_set + query4 = "SELECT sum(id IN {}) FROM distributed".format(large_set) + assert node_old.query(query3) == "2\n" + assert node_old.query(query4) == "2\n" + assert node_new.query(query3) == "2\n" + assert node_new.query(query4) == "2\n" diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index 6f4b0be5325..8df68547f38 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -1,5 +1,6 @@ import pytest from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV import re cluster = ClickHouseCluster(__file__) @@ -127,3 +128,53 @@ def test_admin_option(): instance.query('GRANT R1 TO A WITH ADMIN OPTION') instance.query("GRANT R1 TO B", user='A') assert instance.query("SELECT * FROM test_table", user='B') == "1\t5\n2\t10\n" + + +def test_introspection(): + instance.query("CREATE USER A") + instance.query("CREATE USER B") + instance.query('CREATE ROLE R1') + instance.query('CREATE ROLE R2') + instance.query('GRANT R1 TO A') + instance.query('GRANT R2 TO B WITH ADMIN OPTION') + instance.query('GRANT SELECT ON test.table TO A, R2') + instance.query('GRANT CREATE ON *.* TO B WITH GRANT OPTION') + instance.query('REVOKE SELECT(x) ON test.table FROM R2') + + assert instance.query("SHOW USERS") == TSV([ "A", "B", "default" ]) + assert instance.query("SHOW ROLES") == TSV([ "R1", "R2" ]) + assert instance.query("SHOW GRANTS FOR A") == TSV([ "GRANT SELECT ON test.table TO A", "GRANT R1 TO A" ]) + assert instance.query("SHOW GRANTS FOR B") == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT R2 TO B WITH ADMIN OPTION" ]) + assert instance.query("SHOW GRANTS FOR R1") == "" + assert instance.query("SHOW GRANTS FOR R2") == TSV([ "GRANT SELECT ON test.table TO R2", "REVOKE SELECT(x) ON test.table FROM R2" ]) + + assert instance.query("SHOW GRANTS", user='A') == TSV([ "GRANT SELECT ON test.table TO A", "GRANT R1 TO A" ]) + assert instance.query("SHOW GRANTS", user='B') == TSV([ "GRANT CREATE ON *.* TO B WITH GRANT OPTION", "GRANT R2 TO B WITH ADMIN OPTION" ]) + assert instance.query("SHOW CURRENT ROLES", user='A') == TSV([[ "R1", 0, 1 ]]) + assert instance.query("SHOW CURRENT ROLES", user='B') == TSV([[ "R2", 1, 1 ]]) + assert instance.query("SHOW ENABLED ROLES", user='A') == TSV([[ "R1", 0, 1, 1 ]]) + assert instance.query("SHOW ENABLED ROLES", user='B') == TSV([[ "R2", 1, 1, 1 ]]) + + assert instance.query("SELECT name, storage, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\ + TSV([[ "A", "disk", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ], + [ "B", "disk", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]]) + + assert instance.query("SELECT name, storage from system.roles WHERE name IN ('R1', 'R2') ORDER BY name") ==\ + TSV([[ "R1", "disk" ], + [ "R2", "disk" ]]) + + assert instance.query("SELECT * from system.grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, access_type, grant_option") ==\ + TSV([[ "A", "\N", "SELECT", "test", "table", "\N", 0, 0 ], + [ "B", "\N", "CREATE", "\N", "\N", "\N", 0, 0 ], + [ "B", "\N", "CREATE", "\N", "\N", "\N", 0, 1 ], + [ "\N", "R2", "SELECT", "test", "table", "\N", 0, 0 ], + [ "\N", "R2", "SELECT", "test", "table", "x", 1, 0 ]]) + + assert instance.query("SELECT * from system.role_grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, granted_role_name") ==\ + TSV([[ "A", "\N", "R1", 1, 0 ], + [ "B", "\N", "R2", 1, 1 ]]) + + assert instance.query("SELECT * from system.current_roles ORDER BY role_name", user='A') == TSV([[ "R1", 0, 1 ]]) + assert instance.query("SELECT * from system.current_roles ORDER BY role_name", user='B') == TSV([[ "R2", 1, 1 ]]) + assert instance.query("SELECT * from system.enabled_roles ORDER BY role_name", user='A') == TSV([[ "R1", 0, 1, 1 ]]) + assert instance.query("SELECT * from system.enabled_roles ORDER BY role_name", user='B') == TSV([[ "R2", 1, 1, 1 ]]) diff --git a/tests/integration/test_insert_distributed_load_balancing/__init__.py b/tests/integration/test_insert_distributed_load_balancing/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_insert_distributed_load_balancing/configs/remote_servers.xml b/tests/integration/test_insert_distributed_load_balancing/configs/remote_servers.xml new file mode 100644 index 00000000000..61bc5af1f7d --- /dev/null +++ b/tests/integration/test_insert_distributed_load_balancing/configs/remote_servers.xml @@ -0,0 +1,18 @@ + + + + + true + + n2 + 9000 + + + n1 + 9000 + + + + + + diff --git a/tests/integration/test_insert_distributed_load_balancing/test.py b/tests/integration/test_insert_distributed_load_balancing/test.py new file mode 100644 index 00000000000..99d74ddc8df --- /dev/null +++ b/tests/integration/test_insert_distributed_load_balancing/test.py @@ -0,0 +1,67 @@ +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name +# pylint: disable=line-too-long + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +n1 = cluster.add_instance('n1', main_configs=['configs/remote_servers.xml']) +n2 = cluster.add_instance('n2', main_configs=['configs/remote_servers.xml']) + +@pytest.fixture(scope='module', autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +def create_tables(**dist_settings): + n1.query('DROP TABLE IF EXISTS data') + n2.query('DROP TABLE IF EXISTS data') + n1.query('DROP TABLE IF EXISTS dist') + + n1.query('CREATE TABLE data (key Int) Engine=Memory()') + n2.query('CREATE TABLE data (key Int) Engine=Memory()') + n1.query(""" + CREATE TABLE dist AS data + Engine=Distributed( + integration_test_cluster, + currentDatabase(), + data, + rand() + ) + """, settings=dist_settings) + +def insert_data(**settings): + create_tables(**settings) + n1.query('INSERT INTO dist SELECT * FROM numbers(10)', settings=settings) + n1.query('SYSTEM FLUSH DISTRIBUTED dist') + +def test_prefer_localhost_replica_1(): + insert_data() + assert int(n1.query('SELECT count() FROM data')) == 10 + assert int(n2.query('SELECT count() FROM data')) == 0 + +def test_prefer_localhost_replica_1_load_balancing_in_order(): + insert_data(load_balancing='in_order') + assert int(n1.query('SELECT count() FROM data')) == 10 + assert int(n2.query('SELECT count() FROM data')) == 0 + +def test_prefer_localhost_replica_0_load_balancing_nearest_hostname(): + insert_data(load_balancing='nearest_hostname', prefer_localhost_replica=0) + assert int(n1.query('SELECT count() FROM data')) == 10 + assert int(n2.query('SELECT count() FROM data')) == 0 + +def test_prefer_localhost_replica_0_load_balancing_in_order(): + insert_data(load_balancing='in_order', prefer_localhost_replica=0) + assert int(n1.query('SELECT count() FROM data')) == 0 + assert int(n2.query('SELECT count() FROM data')) == 10 + +def test_prefer_localhost_replica_0_load_balancing_in_order_sync(): + insert_data(load_balancing='in_order', prefer_localhost_replica=0, insert_distributed_sync=1) + assert int(n1.query('SELECT count() FROM data')) == 0 + assert int(n2.query('SELECT count() FROM data')) == 10 diff --git a/tests/integration/test_odbc_interaction/test.py b/tests/integration/test_odbc_interaction/test.py index 41f54ddd0e6..c6b4e0eb5b0 100644 --- a/tests/integration/test_odbc_interaction/test.py +++ b/tests/integration/test_odbc_interaction/test.py @@ -111,6 +111,21 @@ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32, column_x Nulla conn.close() +def test_mysql_insert(started_cluster): + mysql_setup = node1.odbc_drivers["MySQL"] + table_name = 'test_insert' + conn = get_mysql_conn() + create_mysql_table(conn, table_name) + odbc_args = "'DSN={}', '{}', '{}'".format(mysql_setup["DSN"], mysql_setup["Database"], table_name) + + node1.query("create table mysql_insert (id Int64, name String, age UInt8, money Float, column_x Nullable(Int16)) Engine=ODBC({})".format(odbc_args)) + node1.query("insert into mysql_insert values (1, 'test', 11, 111, 1111), (2, 'odbc', 22, 222, NULL)") + assert node1.query("select * from mysql_insert") == "1\ttest\t11\t111\t1111\n2\todbc\t22\t222\t\\N\n" + + node1.query("insert into table function odbc({}) values (3, 'insert', 33, 333, 3333)".format(odbc_args)) + node1.query("insert into table function odbc({}) (id, name, age, money) select id*4, upper(name), age*4, money*4 from odbc({}) where id=1".format(odbc_args, odbc_args)) + assert node1.query("select * from mysql_insert where id in (3, 4)") == "3\tinsert\t33\t333\t3333\n4\tTEST\t44\t444\t\\N\n" + def test_sqlite_simple_select_function_works(started_cluster): sqlite_setup = node1.odbc_drivers["SQLite3"] @@ -170,7 +185,11 @@ def test_sqlite_odbc_cached_dictionary(started_cluster): assert node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(1))") == "3\n" - node1.exec_in_container(["bash", "-c", "echo 'INSERT INTO t3 values(200, 2, 7);' | sqlite3 {}".format(sqlite_db)], privileged=True, user='root') + # Allow insert + node1.exec_in_container(["bash", "-c", "chmod a+rw /tmp"], privileged=True, user='root') + node1.exec_in_container(["bash", "-c", "chmod a+rw {}".format(sqlite_db)], privileged=True, user='root') + + node1.query("insert into table function odbc('DSN={};', '', 't3') values (200, 2, 7)".format(node1.odbc_drivers["SQLite3"]["DSN"])) assert node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(200))") == "7\n" # new value @@ -200,6 +219,17 @@ def test_postgres_odbc_hached_dictionary_no_tty_pipe_overflow(started_cluster): assert node1.query("select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(3))") == "xxx\n" +def test_postgres_insert(started_cluster): + conn = get_postgres_conn() + conn.cursor().execute("truncate table clickhouse.test_table") + node1.query("create table pg_insert (column1 UInt8, column2 String) engine=ODBC('DSN=postgresql_odbc;', 'clickhouse', 'test_table')") + node1.query("insert into pg_insert values (1, 'hello'), (2, 'world')") + assert node1.query("select * from pg_insert") == '1\thello\n2\tworld\n' + node1.query("insert into table function odbc('DSN=postgresql_odbc;', 'clickhouse', 'test_table') format CSV 3,test") + node1.query("insert into table function odbc('DSN=postgresql_odbc;', 'clickhouse', 'test_table') select number, 's' || toString(number) from numbers (4, 7)") + assert node1.query("select sum(column1), count(column1) from pg_insert") == "55\t10\n" + assert node1.query("select sum(n), count(n) from (select (*,).1 as n from (select * from odbc('DSN=postgresql_odbc;', 'clickhouse', 'test_table')))") == "55\t10\n" + def test_bridge_dies_with_parent(started_cluster): node1.query("select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(1))") diff --git a/tests/integration/test_quota/test.py b/tests/integration/test_quota/test.py index ae68a34a03e..6e00bf7241b 100644 --- a/tests/integration/test_quota/test.py +++ b/tests/integration/test_quota/test.py @@ -1,6 +1,6 @@ import pytest from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry +from helpers.test_tools import assert_eq_with_retry, TSV import os import re import time @@ -9,17 +9,24 @@ cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance', config_dir="configs") -query_from_system_quotas = "SELECT * FROM system.quotas ORDER BY name"; - -query_from_system_quota_usage = "SELECT id, key, duration, "\ - "queries, errors, result_rows, result_bytes, read_rows, read_bytes "\ - "FROM system.quota_usage ORDER BY id, key, duration"; def system_quotas(): - return instance.query(query_from_system_quotas).rstrip('\n') + return TSV(instance.query("SELECT * FROM system.quotas ORDER BY name")) + +def system_quota_limits(): + return TSV(instance.query("SELECT * FROM system.quota_limits ORDER BY quota_name, duration")) def system_quota_usage(): - return instance.query(query_from_system_quota_usage).rstrip('\n') + query = "SELECT quota_name, quota_key, duration, queries, max_queries, errors, max_errors, result_rows, max_result_rows,"\ + "result_bytes, max_result_bytes, read_rows, max_read_rows, read_bytes, max_read_bytes, max_execution_time "\ + "FROM system.quota_usage ORDER BY duration" + return TSV(instance.query(query)) + +def system_quotas_usage(): + query = "SELECT quota_name, quota_key, is_current, duration, queries, max_queries, errors, max_errors, result_rows, max_result_rows, "\ + "result_bytes, max_result_bytes, read_rows, max_read_rows, read_bytes, max_read_bytes, max_execution_time "\ + "FROM system.quotas_usage ORDER BY quota_name, quota_key, duration" + return TSV(instance.query(query)) def copy_quota_xml(local_file_name, reload_immediately = True): @@ -54,197 +61,225 @@ def reset_quotas_and_usage_info(): def test_quota_from_users_xml(): - assert instance.query("SELECT currentQuota()") == "myQuota\n" - assert instance.query("SELECT currentQuotaID()") == "e651da9c-a748-8703-061a-7e5e5096dae7\n" - assert instance.query("SELECT currentQuotaKey()") == "default\n" - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", [31556952], 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] + assert system_quotas_usage() == [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] instance.query("SELECT * from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200" + assert system_quota_usage() == [["myQuota", "default", 31556952, 1, 1000, 0, "\N", 50, "\N", 200, "\N", 50, 1000, 200, "\N", "\N"]] instance.query("SELECT COUNT() from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t0\t51\t208\t50\t200" + assert system_quota_usage() == [["myQuota", "default", 31556952, 2, 1000, 0, "\N", 51, "\N", 208, "\N", 50, 1000, 200, "\N", "\N"]] def test_simpliest_quota(): # Simpliest quota doesn't even track usage. copy_quota_xml('simpliest.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[]", 0, "['default']", "[]"]] + assert system_quota_limits() == "" + assert system_quota_usage() == [["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]] instance.query("SELECT * from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + assert system_quota_usage() == [["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]] def test_tracking_quota(): # Now we're tracking usage. copy_quota_xml('tracking.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[0]\t[0]\t[0]\t[0]\t[0]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, "\N", "\N", "\N", "\N", "\N", "\N", "\N"]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 0, "\N", 0, "\N", 0, "\N", 0, "\N", 0, "\N", 0, "\N", "\N"]] instance.query("SELECT * from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200" + assert system_quota_usage() == [["myQuota", "default", 31556952, 1, "\N", 0, "\N", 50, "\N", 200, "\N", 50, "\N", 200, "\N", "\N"]] instance.query("SELECT COUNT() from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t0\t51\t208\t50\t200" + assert system_quota_usage() == [["myQuota", "default", 31556952, 2, "\N", 0, "\N", 51, "\N", 208, "\N", 50, "\N", 200, "\N", "\N"]] def test_exceed_quota(): # Change quota, now the limits are tiny so we will exceed the quota. copy_quota_xml('tiny_limits.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1]\t[1]\t[1]\t[0]\t[1]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1, 1, 1, "\N", 1, "\N", "\N"]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 0, 1, 0, 1, 0, 1, 0, "\N", 0, 1, 0, "\N", "\N"]] assert re.search("Quota.*has\ been\ exceeded", instance.query_and_get_error("SELECT * from test_table")) - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t1\t0\t0\t50\t0" + assert system_quota_usage() == [["myQuota", "default", 31556952, 1, 1, 1, 1, 0, 1, 0, "\N", 50, 1, 0, "\N", "\N"]] # Change quota, now the limits are enough to execute queries. copy_quota_xml('normal_limits.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t1\t0\t0\t50\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 1, 1000, 1, "\N", 0, "\N", 0, "\N", 50, 1000, 0, "\N", "\N"]] instance.query("SELECT * from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t1\t50\t200\t100\t200" + assert system_quota_usage() == [["myQuota", "default", 31556952, 2, 1000, 1, "\N", 50, "\N", 200, "\N", 100, 1000, 200, "\N", "\N"]] def test_add_remove_interval(): - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", [31556952], 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] # Add interval. copy_quota_xml('two_intervals.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952,63113904]\t[0,1]\t[1000,0]\t[0,0]\t[0,0]\t[0,30000]\t[1000,0]\t[0,20000]\t[0,120]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0\n"\ - "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t63113904\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952,63113904]", 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"], + ["myQuota", 63113904, 1, "\N", "\N", "\N", 30000, "\N", 20000, 120]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"], + ["myQuota", "default", 63113904, 0, "\N", 0, "\N", 0, "\N", 0, 30000, 0, "\N", 0, 20000, 120]] instance.query("SELECT * from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200\n"\ - "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t63113904\t1\t0\t50\t200\t50\t200" + assert system_quota_usage() == [["myQuota", "default", 31556952, 1, 1000, 0, "\N", 50, "\N", 200, "\N", 50, 1000, 200, "\N", "\N"], + ["myQuota", "default", 63113904, 1, "\N", 0, "\N", 50, "\N", 200, 30000, 50, "\N", 200, 20000, 120]] # Remove interval. copy_quota_xml('normal_limits.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t1\t0\t50\t200\t50\t200" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", [31556952], 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 1, 1000, 0, "\N", 50, "\N", 200, "\N", 50, 1000, 200, "\N", "\N"]] instance.query("SELECT * from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t2\t0\t100\t400\t100\t400" + assert system_quota_usage() == [["myQuota", "default", 31556952, 2, 1000, 0, "\N", 100, "\N", 400, "\N", 100, 1000, 400, "\N", "\N"]] # Remove all intervals. copy_quota_xml('simpliest.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]\t[]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[]", 0, "['default']", "[]"]] + assert system_quota_limits() == "" + assert system_quota_usage() == [["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]] instance.query("SELECT * from test_table") - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N" + assert system_quota_usage() == [["myQuota", "default", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N", "\N"]] # Add one interval back. copy_quota_xml('normal_limits.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", [31556952], 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quota_usage() == [["myQuota", "default", 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] def test_add_remove_quota(): - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", [31556952], 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quotas_usage() == [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] # Add quota. copy_quota_xml('two_quotas.xml') - assert system_quotas() ==\ - "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]\n"\ - "myQuota2\t4590510c-4d13-bf21-ec8a-c2187b092e73\tusers.xml\tclient key or user name\t[]\t[3600,2629746]\t[1,0]\t[0,0]\t[0,0]\t[4000,0]\t[400000,0]\t[4000,0]\t[400000,0]\t[60,1800]" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"], + ["myQuota2", "4590510c-4d13-bf21-ec8a-c2187b092e73", "users.xml", "client key or user name", "[3600,2629746]", 0, "[]", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"], + ["myQuota2", 3600, 1, "\N", "\N", 4000, 400000, 4000, 400000, 60], + ["myQuota2", 2629746, 0, "\N", "\N", "\N", "\N", "\N", "\N", 1800]] + assert system_quotas_usage() == [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] # Drop quota. copy_quota_xml('normal_limits.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quotas_usage() == [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] # Drop all quotas. copy_quota_xml('no_quotas.xml') assert system_quotas() == "" - assert system_quota_usage() == "" + assert system_quota_limits() == "" + assert system_quotas_usage() == "" # Add one quota back. copy_quota_xml('normal_limits.xml') - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" - assert system_quota_usage() == "e651da9c-a748-8703-061a-7e5e5096dae7\tdefault\t31556952\t0\t0\t0\t0\t0\t0" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] + assert system_quotas_usage() == [["myQuota", "default", 1, 31556952, 0, 1000, 0, "\N", 0, "\N", 0, "\N", 0, 1000, 0, "\N", "\N"]] def test_reload_users_xml_by_timer(): - assert system_quotas() == "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1000]\t[0]\t[0]\t[0]\t[1000]\t[0]\t[0]" + assert system_quotas() == [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]] + assert system_quota_limits() == [["myQuota", 31556952, 0, 1000, "\N", "\N", "\N", 1000, "\N", "\N"]] time.sleep(1) # The modification time of the 'quota.xml' file should be different, # because config files are reload by timer only when the modification time is changed. copy_quota_xml('tiny_limits.xml', reload_immediately=False) - assert_eq_with_retry(instance, query_from_system_quotas, "myQuota\te651da9c-a748-8703-061a-7e5e5096dae7\tusers.xml\tuser name\t['default']\t[31556952]\t[0]\t[1]\t[1]\t[1]\t[0]\t[1]\t[0]\t[0]") + assert_eq_with_retry(instance, "SELECT * FROM system.quotas", [["myQuota", "e651da9c-a748-8703-061a-7e5e5096dae7", "users.xml", "user name", "[31556952]", 0, "['default']", "[]"]]) + assert_eq_with_retry(instance, "SELECT * FROM system.quota_limits", [["myQuota", 31556952, 0, 1, 1, 1, "\N", 1, "\N", "\N"]]) def test_dcl_introspection(): assert instance.query("SHOW QUOTAS") == "myQuota\n" - assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000 TO default\n" - expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=0/1000 errors=0 result_rows=0 result_bytes=0 read_rows=0/1000 read_bytes=0 execution_time=0" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE CURRENT")) - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE ALL")) + assert instance.query("SHOW CREATE QUOTA") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000 TO default\n" + assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t0\\t1000\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t1000\\t0\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("SELECT * from test_table") - expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=1/1000 errors=0 result_rows=50 result_bytes=200 read_rows=50/1000 read_bytes=200 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) # Add interval. copy_quota_xml('two_intervals.xml') assert instance.query("SHOW QUOTAS") == "myQuota\n" - assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000, FOR RANDOMIZED INTERVAL 2 YEAR MAX RESULT BYTES 30000, READ BYTES 20000, EXECUTION TIME 120 TO default\n" - expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=1/1000 errors=0 result_rows=50 result_bytes=200 read_rows=50/1000 read_bytes=200 execution_time=.*\n"\ - "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=0 errors=0 result_rows=0 result_bytes=0/30000 read_rows=0 read_bytes=0/20000 execution_time=0/120" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert instance.query("SHOW CREATE QUOTA") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000, FOR RANDOMIZED INTERVAL 2 YEAR MAX RESULT BYTES 30000, READ BYTES 20000, EXECUTION TIME 120 TO default\n" + assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n" + "myQuota\\tdefault\\t.*\\t63113904\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t30000\\t0\\t\\\\N\\t0\\t20000\\t.*\\t120", + instance.query("SHOW QUOTA")) # Drop interval, add quota. copy_quota_xml('two_quotas.xml') assert instance.query("SHOW QUOTAS") == "myQuota\nmyQuota2\n" assert instance.query("SHOW CREATE QUOTA myQuota") == "CREATE QUOTA myQuota KEYED BY \\'user name\\' FOR INTERVAL 1 YEAR MAX QUERIES 1000, READ ROWS 1000 TO default\n" assert instance.query("SHOW CREATE QUOTA myQuota2") == "CREATE QUOTA myQuota2 KEYED BY \\'client key or user name\\' FOR RANDOMIZED INTERVAL 1 HOUR MAX RESULT ROWS 4000, RESULT BYTES 400000, READ ROWS 4000, READ BYTES 400000, EXECUTION TIME 60, FOR INTERVAL 1 MONTH MAX EXECUTION TIME 1800\n" - expected_usage = "myQuota key=\\\\'default\\\\' interval=\[.*\] queries=1/1000 errors=0 result_rows=50 result_bytes=200 read_rows=50/1000 read_bytes=200 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert re.match("myQuota\\tdefault\\t.*\\t31556952\\t1\\t1000\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t1000\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) + + # Drop all quotas. + copy_quota_xml('no_quotas.xml') + assert instance.query("SHOW QUOTAS") == "" + assert instance.query("SHOW CREATE QUOTA") == "" + assert instance.query("SHOW QUOTA") == "" def test_dcl_management(): copy_quota_xml('no_quotas.xml') - assert instance.query("SHOW QUOTAS") == "" - assert instance.query("SHOW QUOTA USAGE") == "" + assert instance.query("SHOW QUOTA") == "" instance.query("CREATE QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 123 TO CURRENT_USER") - assert instance.query("SHOW QUOTAS") == "qA\n" assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR INTERVAL 5 QUARTER MAX QUERIES 123 TO default\n" - expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=0/123 errors=0 result_rows=0 result_bytes=0 read_rows=0 read_bytes=0 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) - + assert re.match("qA\\t\\t.*\\t39446190\\t0\\t123\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) + instance.query("SELECT * from test_table") - expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=1/123 errors=0 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert re.match("qA\\t\\t.*\\t39446190\\t1\\t123\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH MAX QUERIES 321, MAX ERRORS 10, FOR INTERVAL 0.5 HOUR MAX EXECUTION TIME 0.5") assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR INTERVAL 30 MINUTE MAX EXECUTION TIME 0.5, FOR INTERVAL 5 QUARTER MAX QUERIES 321, ERRORS 10 TO default\n" - expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=0 errors=0 result_rows=0 result_bytes=0 read_rows=0 read_bytes=0 execution_time=.*/0.5\n"\ - "qA key=\\\\'\\\\' interval=\[.*\] queries=1/321 errors=0/10 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) - - instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH NO LIMITS, FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY, FOR INTERVAL 1800 SECOND NO LIMITS") - assert instance.query("SHOW CREATE QUOTA qA") == "CREATE QUOTA qA KEYED BY \\'none\\' FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY TO default\n" - expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=0 errors=0 result_rows=0 result_bytes=0 read_rows=0 read_bytes=0 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert re.match("qA\\t\\t.*\\t1800\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t0.5\n" + "qA\\t\\t.*\\t39446190\\t1\\t321\\t0\\t10\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("SELECT * from test_table") - expected_usage = "qA key=\\\\'\\\\' interval=\[.*\] queries=1 errors=0 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert re.match("qA\\t\\t.*\\t1800\\t1\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t0.5\n" + "qA\\t\\t.*\\t39446190\\t2\\t321\\t0\\t10\\t100\\t\\\\N\\t400\\t\\\\N\\t100\\t\\\\N\\t400\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) + + instance.query("ALTER QUOTA qA FOR INTERVAL 15 MONTH NO LIMITS, FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY, FOR INTERVAL 1800 SECOND NO LIMITS") + assert re.match("qA\\t\\t.*\\t42075936\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t0\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) + + instance.query("SELECT * from test_table") + assert re.match("qA\\t\\t.*\\t42075936\\t1\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("ALTER QUOTA qA RENAME TO qB") assert instance.query("SHOW CREATE QUOTA qB") == "CREATE QUOTA qB KEYED BY \\'none\\' FOR RANDOMIZED INTERVAL 16 MONTH TRACKING ONLY TO default\n" - expected_usage = "qB key=\\\\'\\\\' interval=\[.*\] queries=1 errors=0 result_rows=50 result_bytes=200 read_rows=50 read_bytes=200 execution_time=.*" - assert re.match(expected_usage, instance.query("SHOW QUOTA USAGE")) + assert re.match("qB\\t\\t.*\\t42075936\\t1\\t\\\\N\\t0\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t50\\t\\\\N\\t200\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) + + instance.query("SELECT * from test_table") + assert re.match("qB\\t\\t.*\\t42075936\\t2\\t\\\\N\\t0\\t\\\\N\\t100\\t\\\\N\\t400\\t\\\\N\\t100\\t\\\\N\\t400\\t\\\\N\\t.*\\t\\\\N\n", + instance.query("SHOW QUOTA")) instance.query("DROP QUOTA qB") - assert instance.query("SHOW QUOTAS") == "" - assert instance.query("SHOW QUOTA USAGE") == "" + assert instance.query("SHOW QUOTA") == "" def test_users_xml_is_readonly(): diff --git a/tests/integration/test_rename_column/__init__.py b/tests/integration/test_rename_column/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_rename_column/configs/config.d/instant_moves.xml b/tests/integration/test_rename_column/configs/config.d/instant_moves.xml new file mode 100644 index 00000000000..7b68c6946ca --- /dev/null +++ b/tests/integration/test_rename_column/configs/config.d/instant_moves.xml @@ -0,0 +1,4 @@ + + 0.5 + 0.5 + diff --git a/tests/integration/test_rename_column/configs/config.d/part_log.xml b/tests/integration/test_rename_column/configs/config.d/part_log.xml new file mode 100644 index 00000000000..53ea0a8fc13 --- /dev/null +++ b/tests/integration/test_rename_column/configs/config.d/part_log.xml @@ -0,0 +1,7 @@ + + + system +
part_log
+ 500 + + diff --git a/tests/integration/test_rename_column/configs/config.d/storage_configuration.xml b/tests/integration/test_rename_column/configs/config.d/storage_configuration.xml new file mode 100644 index 00000000000..131219abf3d --- /dev/null +++ b/tests/integration/test_rename_column/configs/config.d/storage_configuration.xml @@ -0,0 +1,28 @@ + + + + + + + + /internal/ + + + /external/ + + + + + + + internal + + + external + + + + + + + diff --git a/tests/integration/test_rename_column/configs/config.d/zookeeper_session_timeout.xml b/tests/integration/test_rename_column/configs/config.d/zookeeper_session_timeout.xml new file mode 100644 index 00000000000..caa0ff11137 --- /dev/null +++ b/tests/integration/test_rename_column/configs/config.d/zookeeper_session_timeout.xml @@ -0,0 +1,6 @@ + + + + 15000 + + diff --git a/tests/integration/test_rename_column/configs/remote_servers.xml b/tests/integration/test_rename_column/configs/remote_servers.xml new file mode 100644 index 00000000000..4c3de4b3905 --- /dev/null +++ b/tests/integration/test_rename_column/configs/remote_servers.xml @@ -0,0 +1,28 @@ + + + + + true + + node1 + 9000 + + + node2 + 9000 + + + + true + + node3 + 9000 + + + node4 + 9000 + + + + + diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py new file mode 100644 index 00000000000..9fac783e712 --- /dev/null +++ b/tests/integration/test_rename_column/test.py @@ -0,0 +1,555 @@ +from __future__ import print_function + +import time +import random +import pytest + +from multiprocessing.dummy import Pool +from helpers.cluster import ClickHouseCluster +from helpers.client import QueryRuntimeException +from helpers.network import PartitionManager +from helpers.test_tools import TSV + +node_options = dict( + with_zookeeper=True, + main_configs=['configs/remote_servers.xml'], + config_dir='configs', + tmpfs=['/external:size=200M', '/internal:size=1M']) + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', macros={"shard": 0, "replica": 1}, **node_options) +node2 = cluster.add_instance('node2', macros={"shard": 0, "replica": 2}, **node_options) +node3 = cluster.add_instance('node3', macros={"shard": 1, "replica": 1}, **node_options) +node4 = cluster.add_instance('node4', macros={"shard": 1, "replica": 2}, **node_options) +nodes = [node1, node2, node3, node4] + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + except Exception as ex: + print(ex) + finally: + cluster.shutdown() + + +def drop_table(nodes, table_name): + for node in nodes: + node.query("DROP TABLE IF EXISTS {} NO DELAY".format(table_name)) + time.sleep(1) + + +def create_table(nodes, table_name, with_storage_policy=False, with_time_column=False, + with_ttl_move=False, with_ttl_delete=False): + extra_columns = "" + settings = [] + + for node in nodes: + sql = """ + CREATE TABLE {table_name} + ( + num UInt32, + num2 UInt32 DEFAULT num + 1{extra_columns} + ) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{table_name}', '{replica}') + ORDER BY num PARTITION BY num % 100 + """ + if with_ttl_move: + sql += """ + TTL time + INTERVAL (num2 % 1) SECOND TO DISK 'external' + """ + if with_ttl_delete: + sql += """ + TTL time + INTERVAL (num2 % 1) SECOND DELETE + """ + settings.append("merge_with_ttl_timeout = 1") + + if with_storage_policy: + settings.append("storage_policy='default_with_external'") + + if settings: + sql += """ + SETTINGS {} + """.format(", ".join(settings)) + + if with_time_column: + extra_columns = """, + time DateTime + """ + node.query(sql.format(table_name=table_name, replica=node.name, extra_columns=extra_columns)) + + +def create_distributed_table(node, table_name): + sql = """ + CREATE TABLE %(table_name)s_replicated ON CLUSTER test_cluster + ( + num UInt32, + num2 UInt32 DEFAULT num + 1 + ) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/{shard}/%(table_name)s_replicated', '{replica}') + ORDER BY num PARTITION BY num %% 100; + """ % dict(table_name=table_name) + node.query(sql) + sql = """ + CREATE TABLE %(table_name)s ON CLUSTER test_cluster AS %(table_name)s_replicated + ENGINE = Distributed(test_cluster, default, %(table_name)s_replicated, rand()) + """ % dict(table_name=table_name) + node.query(sql) + + +def drop_distributed_table(node, table_name): + node.query("DROP TABLE IF EXISTS {} ON CLUSTER test_cluster".format(table_name)) + node.query("DROP TABLE IF EXISTS {}_replicated ON CLUSTER test_cluster".format(table_name)) + time.sleep(1) + + +def insert(node, table_name, chunk=1000, col_names=None, iterations=1, ignore_exception=False, + slow=False, with_many_parts=False, offset=0, with_time_column=False): + if col_names is None: + col_names = ['num', 'num2'] + for i in range(iterations): + try: + query = ["SET max_partitions_per_insert_block = 10000000"] + if with_many_parts: + query.append("SET max_insert_block_size = 64") + if with_time_column: + query.append( + "INSERT INTO {table_name} ({col0}, {col1}, time) SELECT number AS {col0}, number + 1 AS {col1}, now() + 10 AS time FROM numbers_mt({chunk})" + .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1])) + elif slow: + query.append( + "INSERT INTO {table_name} ({col0}, {col1}) SELECT number + sleepEachRow(0.001) AS {col0}, number + 1 AS {col1} FROM numbers_mt({chunk})" + .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1])) + else: + query.append( + "INSERT INTO {table_name} ({col0},{col1}) SELECT number + {offset} AS {col0}, number + 1 + {offset} AS {col1} FROM numbers_mt({chunk})" + .format(table_name=table_name, chunk=chunk, col0=col_names[0], col1=col_names[1], offset=str(offset))) + node.query(";\n".join(query)) + except QueryRuntimeException as ex: + if not ignore_exception: + raise + + +def select(node, table_name, col_name="num", expected_result=None, iterations=1, ignore_exception=False, slow=False, poll=None): + for i in range(iterations): + start_time = time.time() + while True: + try: + if slow: + r = node.query("SELECT count() FROM (SELECT num2, sleepEachRow(0.5) FROM {} WHERE {} % 1000 > 0)".format(table_name, col_name)) + else: + r = node.query("SELECT count() FROM {} WHERE {} % 1000 > 0".format(table_name, col_name)) + if expected_result: + if r != expected_result and poll and time.time() - start_time < poll: + continue + assert r == expected_result + except QueryRuntimeException as ex: + if not ignore_exception: + raise + break + + +def rename_column(node, table_name, name, new_name, iterations=1, ignore_exception=False): + for i in range(iterations): + try: + node.query("ALTER TABLE {table_name} RENAME COLUMN {name} to {new_name}".format( + table_name=table_name, name=name, new_name=new_name + )) + except QueryRuntimeException as ex: + if not ignore_exception: + raise + + +def rename_column_on_cluster(node, table_name, name, new_name, iterations=1, ignore_exception=False): + for i in range(iterations): + try: + node.query("ALTER TABLE {table_name} ON CLUSTER test_cluster RENAME COLUMN {name} to {new_name}".format( + table_name=table_name, name=name, new_name=new_name + )) + except QueryRuntimeException as ex: + if not ignore_exception: + raise + + +def alter_move(node, table_name, iterations=1, ignore_exception=False): + for i in range(iterations): + move_part = random.randint(0, 99) + move_volume = 'external' + try: + node.query("ALTER TABLE {table_name} MOVE PARTITION '{move_part}' TO VOLUME '{move_volume}'" + .format(table_name=table_name, move_part=move_part, move_volume=move_volume)) + except QueryRuntimeException as ex: + if not ignore_exception: + raise + + +def test_rename_parallel_same_node(started_cluster): + table_name = "test_rename_parallel_same_node" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name) + insert(node1, table_name, 1000) + + p = Pool(15) + tasks = [] + for i in range(1): + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) + tasks.append(p.apply_async(rename_column, (node1, table_name, "foo2", "foo3", 5, True))) + tasks.append(p.apply_async(rename_column, (node1, table_name, "foo3", "num2", 5, True))) + for task in tasks: + task.get(timeout=240) + + # rename column back to original + rename_column(node1, table_name, "foo3", "num2", 1, True) + rename_column(node1, table_name, "foo2", "num2", 1, True) + + # check that select still works + select(node1, table_name, "num2", "999\n") + finally: + drop_table(nodes, table_name) + + +def test_rename_parallel(started_cluster): + table_name = "test_rename_parallel" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name) + insert(node1, table_name, 1000) + + p = Pool(15) + tasks = [] + for i in range(1): + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 5, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "foo3", "num2", 5, True))) + for task in tasks: + task.get(timeout=240) + + # rename column back to original + rename_column(node1, table_name, "foo3", "num2", 1, True) + rename_column(node1, table_name, "foo2", "num2", 1, True) + + # check that select still works + select(node1, table_name, "num2", "999\n") + finally: + drop_table(nodes, table_name) + + +def test_rename_with_parallel_select(started_cluster): + table_name = "test_rename_with_parallel_select" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name) + insert(node1, table_name, 1000) + + select(node1, table_name, "num2", "999\n", poll=30) + select(node2, table_name, "num2", "999\n", poll=30) + select(node3, table_name, "num2", "999\n", poll=30) + + p = Pool(15) + tasks = [] + for i in range(1): + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 5, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "foo3", "num2", 5, True))) + tasks.append(p.apply_async(select, (node1, table_name, "foo3", "999\n", 5, True))) + tasks.append(p.apply_async(select, (node2, table_name, "num2", "999\n", 5, True))) + tasks.append(p.apply_async(select, (node3, table_name, "foo2", "999\n", 5, True))) + for task in tasks: + task.get(timeout=240) + + # rename column back to original name + rename_column(node1, table_name, "foo3", "num2", 1, True) + rename_column(node1, table_name, "foo2", "num2", 1, True) + + # check that select still works + select(node1, table_name, "num2", "999\n") + finally: + drop_table(nodes, table_name) + + +def test_rename_with_parallel_insert(started_cluster): + table_name = "test_rename_with_parallel_insert" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name) + insert(node1, table_name, 1000) + + p = Pool(15) + tasks = [] + for i in range(1): + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 5, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "foo3", "num2", 5, True))) + tasks.append(p.apply_async(insert, (node1, table_name, 100, ["num", "foo3"], 5, True))) + tasks.append(p.apply_async(insert, (node2, table_name, 100, ["num", "num2"], 5, True))) + tasks.append(p.apply_async(insert, (node3, table_name, 100, ["num", "foo2"], 5, True))) + for task in tasks: + task.get(timeout=240) + + # rename column back to original + rename_column(node1, table_name, "foo3", "num2", 1, True) + rename_column(node1, table_name, "foo2", "num2", 1, True) + + # check that select still works + select(node1, table_name, "num2") + finally: + drop_table(nodes, table_name) + + +@pytest.mark.skip(reason="For unknown reason one of these tests kill Zookeeper") +def test_rename_with_parallel_merges(started_cluster): + table_name = "test_rename_with_parallel_merges" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name) + for i in range(20): + insert(node1, table_name, 100, ["num","num2"], 1, False, False, True, offset=i*100) + + def merge_parts(node, table_name, iterations=1): + for i in range(iterations): + node.query("OPTIMIZE TABLE %s FINAL" % table_name) + + p = Pool(15) + tasks = [] + for i in range(1): + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 5, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 5, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "foo3", "num2", 5, True))) + tasks.append(p.apply_async(merge_parts, (node1, table_name, 5))) + tasks.append(p.apply_async(merge_parts, (node2, table_name, 5))) + tasks.append(p.apply_async(merge_parts, (node3, table_name, 5))) + + for task in tasks: + task.get(timeout=240) + + # rename column back to the original name + rename_column(node1, table_name, "foo3", "num2", 1, True) + rename_column(node1, table_name, "foo2", "num2", 1, True) + + # check that select still works + select(node1, table_name, "num2", "1998\n") + select(node2, table_name, "num2", "1998\n") + select(node3, table_name, "num2", "1998\n") + finally: + drop_table(nodes, table_name) + + +@pytest.mark.skip(reason="For unknown reason one of these tests kill Zookeeper") +def test_rename_with_parallel_slow_insert(started_cluster): + table_name = "test_rename_with_parallel_slow_insert" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name) + insert(node1, table_name, 1000) + + p = Pool(15) + tasks = [] + tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, False, True))) + tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, True, True))) # deduplicated + time.sleep(0.5) + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2"))) + + for task in tasks: + task.get(timeout=240) + + insert(node1, table_name, 100, ["num", "foo2"]) + + # rename column back to original + rename_column(node1, table_name, "foo2", "num2") + + # check that select still works + select(node1, table_name, "num2", "11089\n") + select(node2, table_name, "num2", "11089\n", poll=30) + select(node3, table_name, "num2", "11089\n", poll=30) + finally: + drop_table(nodes, table_name) + + +def test_rename_with_parallel_slow_select(started_cluster): + table_name = "test_rename_with_parallel_slow_select" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name) + insert(node1, table_name, 1000) + + p = Pool(15) + tasks = [] + + tasks.append(p.apply_async(select, (node1, table_name, "num2", "999\n", 1, True, True))) + time.sleep(0.5) + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2"))) + + for task in tasks: + task.get(timeout=240) + + insert(node1, table_name, 100, ["num", "foo2"]) + + # rename column back to original + rename_column(node1, table_name, "foo2", "num2") + + # check that select still works + select(node1, table_name, "num2", "1099\n") + select(node2, table_name, "num2", "1099\n", poll=30) + select(node3, table_name, "num2", "1099\n", poll=30) + finally: + drop_table(nodes, table_name) + + +def test_rename_with_parallel_moves(started_cluster): + table_name = "test_rename_with_parallel_moves" + drop_table(nodes, table_name) + try: + create_table(nodes, table_name, with_storage_policy=True) + insert(node1, table_name, 1000) + + p = Pool(15) + tasks = [] + + tasks.append(p.apply_async(alter_move, (node1, table_name, 20, True))) + tasks.append(p.apply_async(alter_move, (node2, table_name, 20, True))) + tasks.append(p.apply_async(alter_move, (node3, table_name, 20, True))) + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 20, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 20, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "num3", "num2", 20, True))) + + for task in tasks: + task.get(timeout=240) + + # rename column back to original + rename_column(node1, table_name, "foo2", "num2", 1, True) + rename_column(node1, table_name, "foo3", "num2", 1, True) + + # check that select still works + select(node1, table_name, "num2", "999\n") + select(node2, table_name, "num2", "999\n", poll=30) + select(node3, table_name, "num2", "999\n", poll=30) + finally: + drop_table(nodes, table_name) + + +def test_rename_with_parallel_ttl_move(started_cluster): + table_name = 'test_rename_with_parallel_ttl_move' + try: + create_table(nodes, table_name, with_storage_policy=True, with_time_column=True, with_ttl_move=True) + rename_column(node1, table_name, "time", "time2", 1, False) + rename_column(node1, table_name, "time2", "time", 1, False) + + p = Pool(15) + tasks = [] + + tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, False, False, True, 0, True))) + time.sleep(5) + rename_column(node1, table_name, "time", "time2", 1, False) + time.sleep(4) + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 20, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 20, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "num3", "num2", 20, True))) + + for task in tasks: + task.get(timeout=240) + + # check some parts got moved + assert "external" in set(node1.query("SELECT disk_name FROM system.parts WHERE table == '{}' AND active=1 ORDER BY modification_time".format(table_name)).strip().splitlines()) + + # rename column back to original + rename_column(node1, table_name, "foo2", "num2", 1, True) + rename_column(node1, table_name, "foo3", "num2", 1, True) + + # check that select still works + select(node1, table_name, "num2", "9990\n") + finally: + drop_table(nodes, table_name) + + +def test_rename_with_parallel_ttl_delete(started_cluster): + table_name = 'test_rename_with_parallel_ttl_delete' + try: + create_table(nodes, table_name, with_time_column=True, with_ttl_delete=True) + rename_column(node1, table_name, "time", "time2", 1, False) + rename_column(node1, table_name, "time2", "time", 1, False) + + def merge_parts(node, table_name, iterations=1): + for i in range(iterations): + node.query("OPTIMIZE TABLE {}".format(table_name)) + + p = Pool(15) + tasks = [] + + tasks.append(p.apply_async(insert, (node1, table_name, 10000, ["num", "num2"], 1, False, False, True, 0, True))) + time.sleep(15) + tasks.append(p.apply_async(rename_column, (node1, table_name, "num2", "foo2", 20, True))) + tasks.append(p.apply_async(rename_column, (node2, table_name, "foo2", "foo3", 20, True))) + tasks.append(p.apply_async(rename_column, (node3, table_name, "num3", "num2", 20, True))) + tasks.append(p.apply_async(merge_parts, (node1, table_name, 20))) + tasks.append(p.apply_async(merge_parts, (node2, table_name, 20))) + tasks.append(p.apply_async(merge_parts, (node3, table_name, 20))) + + for task in tasks: + task.get(timeout=240) + + # rename column back to original + rename_column(node1, table_name, "foo2", "num2", 1, True) + rename_column(node1, table_name, "foo3", "num2", 1, True) + + assert int(node1.query("SELECT count() FROM {}".format(table_name)).strip()) < 10000 + finally: + drop_table(nodes, table_name) + + +@pytest.mark.skip(reason="For unknown reason one of these tests kill Zookeeper") +def test_rename_distributed(started_cluster): + table_name = 'test_rename_distributed' + try: + create_distributed_table(node1, table_name) + insert(node1, table_name, 1000) + + rename_column_on_cluster(node1, table_name, 'num2', 'foo2') + rename_column_on_cluster(node1, '%s_replicated' % table_name, 'num2', 'foo2') + + insert(node1, table_name, 1000, col_names=['num','foo2']) + + select(node1, table_name, "foo2", '1998\n', poll=30) + finally: + drop_distributed_table(node1, table_name) + + +@pytest.mark.skip(reason="For unknown reason one of these tests kill Zookeeper") +def test_rename_distributed_parallel_insert_and_select(started_cluster): + table_name = 'test_rename_distributed_parallel_insert_and_select' + try: + create_distributed_table(node1, table_name) + insert(node1, table_name, 1000) + + p = Pool(15) + tasks = [] + for i in range(1): + tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'num2', 'foo2', 3, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'num2', 'foo2', 3, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'foo2', 'foo3', 3, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'foo2', 'foo3', 3, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, table_name, 'foo3', 'num2', 3, True))) + tasks.append(p.apply_async(rename_column_on_cluster, (node1, '%s_replicated' % table_name, 'foo3', 'num2', 3, True))) + tasks.append(p.apply_async(insert, (node1, table_name, 10, ["num", "foo3"], 5, True))) + tasks.append(p.apply_async(insert, (node2, table_name, 10, ["num", "num2"], 5, True))) + tasks.append(p.apply_async(insert, (node3, table_name, 10, ["num", "foo2"], 5, True))) + tasks.append(p.apply_async(select, (node1, table_name, "foo2", None, 5, True))) + tasks.append(p.apply_async(select, (node2, table_name, "foo3", None, 5, True))) + tasks.append(p.apply_async(select, (node3, table_name, "num2", None, 5, True))) + for task in tasks: + task.get(timeout=240) + + rename_column_on_cluster(node1, table_name, 'foo2', 'num2', 1, True) + rename_column_on_cluster(node1, '%s_replicated' % table_name, 'foo2', 'num2', 1, True) + rename_column_on_cluster(node1, table_name, 'foo3', 'num2', 1, True) + rename_column_on_cluster(node1, '%s_replicated' % table_name, 'foo3', 'num2', 1, True) + + insert(node1, table_name, 1000, col_names=['num','num2']) + select(node1, table_name, "num2") + select(node2, table_name, "num2") + select(node3, table_name, "num2") + select(node4, table_name, "num2") + finally: + drop_distributed_table(node1, table_name) diff --git a/tests/integration/test_row_policy/configs/config.d/remote_servers.xml b/tests/integration/test_row_policy/configs/config.d/remote_servers.xml index 8eea90f523b..395e97b97ac 100644 --- a/tests/integration/test_row_policy/configs/config.d/remote_servers.xml +++ b/tests/integration/test_row_policy/configs/config.d/remote_servers.xml @@ -3,13 +3,13 @@ - instance1 + node 9000 - instance2 + node2 9000 diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py index ec7304f47d3..a1884d059c7 100644 --- a/tests/integration/test_row_policy/test.py +++ b/tests/integration/test_row_policy/test.py @@ -1,21 +1,22 @@ import pytest from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry +from helpers.test_tools import assert_eq_with_retry, TSV import os import re import time cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance('instance1', config_dir="configs", with_zookeeper=True) -instance2 = cluster.add_instance('instance2', config_dir="configs", with_zookeeper=True) +node = cluster.add_instance('node', config_dir="configs", with_zookeeper=True) +node2 = cluster.add_instance('node2', config_dir="configs", with_zookeeper=True) +nodes = [node, node2] def copy_policy_xml(local_file_name, reload_immediately = True): script_dir = os.path.dirname(os.path.realpath(__file__)) - instance.copy_file_to_container(os.path.join(script_dir, local_file_name), '/etc/clickhouse-server/users.d/row_policy.xml') - instance2.copy_file_to_container(os.path.join(script_dir, local_file_name), '/etc/clickhouse-server/users.d/row_policy.xml') - if reload_immediately: - instance.query("SYSTEM RELOAD CONFIG") + for current_node in nodes: + current_node.copy_file_to_container(os.path.join(script_dir, local_file_name), '/etc/clickhouse-server/users.d/row_policy.xml') + if reload_immediately: + current_node.query("SYSTEM RELOAD CONFIG") @pytest.fixture(scope="module", autouse=True) @@ -23,42 +24,25 @@ def started_cluster(): try: cluster.start() - instance.query(''' - CREATE DATABASE mydb ENGINE=Ordinary; + for current_node in nodes: + current_node.query(''' + CREATE DATABASE mydb ENGINE=Ordinary; - CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table1 values (0, 0), (0, 1), (1, 0), (1, 1); + CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; + INSERT INTO mydb.filtered_table1 values (0, 0), (0, 1), (1, 0), (1, 1); - CREATE TABLE mydb.table (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.table values (0, 0), (0, 1), (1, 0), (1, 1); + CREATE TABLE mydb.table (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; + INSERT INTO mydb.table values (0, 0), (0, 1), (1, 0), (1, 1); - CREATE TABLE mydb.filtered_table2 (a UInt8, b UInt8, c UInt8, d UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table2 values (0, 0, 0, 0), (1, 2, 3, 4), (4, 3, 2, 1), (0, 0, 6, 0); + CREATE TABLE mydb.filtered_table2 (a UInt8, b UInt8, c UInt8, d UInt8) ENGINE MergeTree ORDER BY a; + INSERT INTO mydb.filtered_table2 values (0, 0, 0, 0), (1, 2, 3, 4), (4, 3, 2, 1), (0, 0, 6, 0); - CREATE TABLE mydb.filtered_table3 (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table3 values (0, 0), (0, 1), (1, 0), (1, 1); + CREATE TABLE mydb.filtered_table3 (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; + INSERT INTO mydb.filtered_table3 values (0, 0), (0, 1), (1, 0), (1, 1); - CREATE TABLE mydb.`.filtered_table4` (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.`.filtered_table4` values (0, 0), (0, 1), (1, 0), (1, 1); - ''') - instance2.query(''' - CREATE DATABASE mydb ENGINE=Ordinary; - - CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table1 values (0, 0), (0, 1), (1, 0), (1, 1); - - CREATE TABLE mydb.table (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.table values (0, 0), (0, 1), (1, 0), (1, 1); - - CREATE TABLE mydb.filtered_table2 (a UInt8, b UInt8, c UInt8, d UInt8) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table2 values (0, 0, 0, 0), (1, 2, 3, 4), (4, 3, 2, 1), (0, 0, 6, 0); - - CREATE TABLE mydb.filtered_table3 (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.filtered_table3 values (0, 0), (0, 1), (1, 0), (1, 1); - - CREATE TABLE mydb.`.filtered_table4` (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; - INSERT INTO mydb.`.filtered_table4` values (0, 0), (0, 1), (1, 0), (1, 1); - ''') + CREATE TABLE mydb.`.filtered_table4` (a UInt8, b UInt8, c UInt16 ALIAS a + b) ENGINE MergeTree ORDER BY a; + INSERT INTO mydb.`.filtered_table4` values (0, 0), (0, 1), (1, 0), (1, 1); + ''') yield cluster @@ -72,243 +56,239 @@ def reset_policies(): yield finally: copy_policy_xml('normal_filters.xml') - instance.query("DROP POLICY IF EXISTS pA, pB ON mydb.filtered_table1") + for current_node in nodes: + current_node.query("DROP POLICY IF EXISTS pA, pB ON mydb.filtered_table1") def test_smoke(): - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t1\n1\t0\n" + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1,0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) - assert instance.query("SELECT a FROM mydb.filtered_table1") == "1\n1\n" - assert instance.query("SELECT b FROM mydb.filtered_table1") == "0\n1\n" - assert instance.query("SELECT a FROM mydb.filtered_table1 WHERE a = 1") == "1\n1\n" - assert instance.query("SELECT a FROM mydb.filtered_table1 WHERE a IN (1)") == "1\n1\n" - assert instance.query("SELECT a = 1 FROM mydb.filtered_table1") == "1\n1\n" + assert node.query("SELECT a FROM mydb.filtered_table1") == TSV([[1], [1]]) + assert node.query("SELECT b FROM mydb.filtered_table1") == TSV([[0], [1]]) + assert node.query("SELECT a FROM mydb.filtered_table1 WHERE a = 1") == TSV([[1], [1]]) + assert node.query("SELECT a FROM mydb.filtered_table1 WHERE a IN (1)") == TSV([[1], [1]]) + assert node.query("SELECT a = 1 FROM mydb.filtered_table1") == TSV([[1], [1]]) - assert instance.query("SELECT a FROM mydb.filtered_table3") == "0\n1\n" - assert instance.query("SELECT b FROM mydb.filtered_table3") == "1\n0\n" - assert instance.query("SELECT c FROM mydb.filtered_table3") == "1\n1\n" - assert instance.query("SELECT a + b FROM mydb.filtered_table3") == "1\n1\n" - assert instance.query("SELECT a FROM mydb.filtered_table3 WHERE c = 1") == "0\n1\n" - assert instance.query("SELECT c = 1 FROM mydb.filtered_table3") == "1\n1\n" - assert instance.query("SELECT a + b = 1 FROM mydb.filtered_table3") == "1\n1\n" + assert node.query("SELECT a FROM mydb.filtered_table3") == TSV([[0], [1]]) + assert node.query("SELECT b FROM mydb.filtered_table3") == TSV([[1], [0]]) + assert node.query("SELECT c FROM mydb.filtered_table3") == TSV([[1], [1]]) + assert node.query("SELECT a + b FROM mydb.filtered_table3") == TSV([[1], [1]]) + assert node.query("SELECT a FROM mydb.filtered_table3 WHERE c = 1") == TSV([[0], [1]]) + assert node.query("SELECT c = 1 FROM mydb.filtered_table3") == TSV([[1], [1]]) + assert node.query("SELECT a + b = 1 FROM mydb.filtered_table3") == TSV([[1], [1]]) def test_join(): - assert instance.query("SELECT * FROM mydb.filtered_table1 as t1 ANY LEFT JOIN mydb.filtered_table1 as t2 ON t1.a = t2.b") == "1\t0\t1\t1\n1\t1\t1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table1 as t2 ANY RIGHT JOIN mydb.filtered_table1 as t1 ON t2.b = t1.a") == "1\t1\t1\t0\n" + assert node.query("SELECT * FROM mydb.filtered_table1 as t1 ANY LEFT JOIN mydb.filtered_table1 as t2 ON t1.a = t2.b") == TSV([[1, 0, 1, 1], [1, 1, 1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table1 as t2 ANY RIGHT JOIN mydb.filtered_table1 as t1 ON t2.b = t1.a") == TSV([[1, 1, 1, 0]]) def test_cannot_trick_row_policy_with_keyword_with(): - assert instance.query("WITH 0 AS a SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" - assert instance.query("WITH 0 AS a SELECT a, b FROM mydb.filtered_table1") == "0\t0\n0\t1\n" - assert instance.query("WITH 0 AS a SELECT a FROM mydb.filtered_table1") == "0\n0\n" - assert instance.query("WITH 0 AS a SELECT b FROM mydb.filtered_table1") == "0\n1\n" + assert node.query("WITH 0 AS a SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) + assert node.query("WITH 0 AS a SELECT a, b FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1]]) + assert node.query("WITH 0 AS a SELECT a FROM mydb.filtered_table1") == TSV([[0], [0]]) + assert node.query("WITH 0 AS a SELECT b FROM mydb.filtered_table1") == TSV([[0], [1]]) def test_prewhere_not_supported(): expected_error = "PREWHERE is not supported if the table is filtered by row-level security" - assert expected_error in instance.query_and_get_error("SELECT * FROM mydb.filtered_table1 PREWHERE 1") - assert expected_error in instance.query_and_get_error("SELECT * FROM mydb.filtered_table2 PREWHERE 1") - assert expected_error in instance.query_and_get_error("SELECT * FROM mydb.filtered_table3 PREWHERE 1") + assert expected_error in node.query_and_get_error("SELECT * FROM mydb.filtered_table1 PREWHERE 1") + assert expected_error in node.query_and_get_error("SELECT * FROM mydb.filtered_table2 PREWHERE 1") + assert expected_error in node.query_and_get_error("SELECT * FROM mydb.filtered_table3 PREWHERE 1") # However PREWHERE should still work for user without filtering. - assert instance.query("SELECT * FROM mydb.filtered_table1 PREWHERE 1", user="another") == "0\t0\n0\t1\n1\t0\n1\t1\n" + assert node.query("SELECT * FROM mydb.filtered_table1 PREWHERE 1", user="another") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) def test_single_table_name(): copy_policy_xml('tag_with_table_name.xml') - assert instance.query("SELECT * FROM mydb.table") == "1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t1\n1\t0\n" + assert node.query("SELECT * FROM mydb.table") == TSV([[1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) - assert instance.query("SELECT a FROM mydb.table") == "1\n1\n" - assert instance.query("SELECT b FROM mydb.table") == "0\n1\n" - assert instance.query("SELECT a FROM mydb.table WHERE a = 1") == "1\n1\n" - assert instance.query("SELECT a = 1 FROM mydb.table") == "1\n1\n" + assert node.query("SELECT a FROM mydb.table") == TSV([[1], [1]]) + assert node.query("SELECT b FROM mydb.table") == TSV([[0], [1]]) + assert node.query("SELECT a FROM mydb.table WHERE a = 1") == TSV([[1], [1]]) + assert node.query("SELECT a = 1 FROM mydb.table") == TSV([[1], [1]]) - assert instance.query("SELECT a FROM mydb.filtered_table3") == "0\n1\n" - assert instance.query("SELECT b FROM mydb.filtered_table3") == "1\n0\n" - assert instance.query("SELECT c FROM mydb.filtered_table3") == "1\n1\n" - assert instance.query("SELECT a + b FROM mydb.filtered_table3") == "1\n1\n" - assert instance.query("SELECT a FROM mydb.filtered_table3 WHERE c = 1") == "0\n1\n" - assert instance.query("SELECT c = 1 FROM mydb.filtered_table3") == "1\n1\n" - assert instance.query("SELECT a + b = 1 FROM mydb.filtered_table3") == "1\n1\n" + assert node.query("SELECT a FROM mydb.filtered_table3") == TSV([[0], [1]]) + assert node.query("SELECT b FROM mydb.filtered_table3") == TSV([[1], [0]]) + assert node.query("SELECT c FROM mydb.filtered_table3") == TSV([[1], [1]]) + assert node.query("SELECT a + b FROM mydb.filtered_table3") == TSV([[1], [1]]) + assert node.query("SELECT a FROM mydb.filtered_table3 WHERE c = 1") == TSV([[0], [1]]) + assert node.query("SELECT c = 1 FROM mydb.filtered_table3") == TSV([[1], [1]]) + assert node.query("SELECT a + b = 1 FROM mydb.filtered_table3") == TSV([[1], [1]]) def test_custom_table_name(): copy_policy_xml('multiple_tags_with_table_names.xml') - assert instance.query("SELECT * FROM mydb.table") == "1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n" - assert instance.query("SELECT * FROM mydb.`.filtered_table4`") == "0\t1\n1\t0\n" + assert node.query("SELECT * FROM mydb.table") == TSV([[1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.`.filtered_table4`") == TSV([[0, 1], [1, 0]]) - assert instance.query("SELECT a FROM mydb.table") == "1\n1\n" - assert instance.query("SELECT b FROM mydb.table") == "0\n1\n" - assert instance.query("SELECT a FROM mydb.table WHERE a = 1") == "1\n1\n" - assert instance.query("SELECT a = 1 FROM mydb.table") == "1\n1\n" + assert node.query("SELECT a FROM mydb.table") == TSV([[1], [1]]) + assert node.query("SELECT b FROM mydb.table") == TSV([[0], [1]]) + assert node.query("SELECT a FROM mydb.table WHERE a = 1") == TSV([[1], [1]]) + assert node.query("SELECT a = 1 FROM mydb.table") == TSV([[1], [1]]) - assert instance.query("SELECT a FROM mydb.`.filtered_table4`") == "0\n1\n" - assert instance.query("SELECT b FROM mydb.`.filtered_table4`") == "1\n0\n" - assert instance.query("SELECT c FROM mydb.`.filtered_table4`") == "1\n1\n" - assert instance.query("SELECT a + b FROM mydb.`.filtered_table4`") == "1\n1\n" - assert instance.query("SELECT a FROM mydb.`.filtered_table4` WHERE c = 1") == "0\n1\n" - assert instance.query("SELECT c = 1 FROM mydb.`.filtered_table4`") == "1\n1\n" - assert instance.query("SELECT a + b = 1 FROM mydb.`.filtered_table4`") == "1\n1\n" + assert node.query("SELECT a FROM mydb.`.filtered_table4`") == TSV([[0], [1]]) + assert node.query("SELECT b FROM mydb.`.filtered_table4`") == TSV([[1], [0]]) + assert node.query("SELECT c FROM mydb.`.filtered_table4`") == TSV([[1], [1]]) + assert node.query("SELECT a + b FROM mydb.`.filtered_table4`") == TSV([[1], [1]]) + assert node.query("SELECT a FROM mydb.`.filtered_table4` WHERE c = 1") == TSV([[0], [1]]) + assert node.query("SELECT c = 1 FROM mydb.`.filtered_table4`") == TSV([[1], [1]]) + assert node.query("SELECT a + b = 1 FROM mydb.`.filtered_table4`") == TSV([[1], [1]]) def test_change_of_users_xml_changes_row_policies(): copy_policy_xml('normal_filters.xml') - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t1\n1\t0\n" + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) copy_policy_xml('all_rows.xml') - assert instance.query("SELECT * FROM mydb.filtered_table1") == "0\t0\n0\t1\n1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n1\t2\t3\t4\n4\t3\t2\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t0\n0\t1\n1\t0\n1\t1\n" + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) copy_policy_xml('no_rows.xml') - assert instance.query("SELECT * FROM mydb.filtered_table1") == "" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "" + assert node.query("SELECT * FROM mydb.filtered_table1") == "" + assert node.query("SELECT * FROM mydb.filtered_table2") == "" + assert node.query("SELECT * FROM mydb.filtered_table3") == "" copy_policy_xml('normal_filters.xml') - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t1\n1\t0\n" + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) copy_policy_xml('no_filters.xml') - assert instance.query("SELECT * FROM mydb.filtered_table1") == "0\t0\n0\t1\n1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n1\t2\t3\t4\n4\t3\t2\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t0\n0\t1\n1\t0\n1\t1\n" + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) copy_policy_xml('normal_filters.xml') - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t1\n1\t0\n" + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) def test_reload_users_xml_by_timer(): copy_policy_xml('normal_filters.xml') - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" - assert instance.query("SELECT * FROM mydb.filtered_table2") == "0\t0\t0\t0\n0\t0\t6\t0\n" - assert instance.query("SELECT * FROM mydb.filtered_table3") == "0\t1\n1\t0\n" + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) + assert node.query("SELECT * FROM mydb.filtered_table2") == TSV([[0, 0, 0, 0], [0, 0, 6, 0]]) + assert node.query("SELECT * FROM mydb.filtered_table3") == TSV([[0, 1], [1, 0]]) time.sleep(1) # The modification time of the 'row_policy.xml' file should be different. copy_policy_xml('all_rows.xml', False) - assert_eq_with_retry(instance, "SELECT * FROM mydb.filtered_table1", "0\t0\n0\t1\n1\t0\n1\t1") - assert_eq_with_retry(instance, "SELECT * FROM mydb.filtered_table2", "0\t0\t0\t0\n0\t0\t6\t0\n1\t2\t3\t4\n4\t3\t2\t1") - assert_eq_with_retry(instance, "SELECT * FROM mydb.filtered_table3", "0\t0\n0\t1\n1\t0\n1\t1") + assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table1", [[0, 0], [0, 1], [1, 0], [1, 1]]) + assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table2", [[0, 0, 0, 0], [0, 0, 6, 0], [1, 2, 3, 4], [4, 3, 2, 1]]) + assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table3", [[0, 0], [0, 1], [1, 0], [1, 1]]) time.sleep(1) # The modification time of the 'row_policy.xml' file should be different. copy_policy_xml('normal_filters.xml', False) - assert_eq_with_retry(instance, "SELECT * FROM mydb.filtered_table1", "1\t0\n1\t1") - assert_eq_with_retry(instance, "SELECT * FROM mydb.filtered_table2", "0\t0\t0\t0\n0\t0\t6\t0") - assert_eq_with_retry(instance, "SELECT * FROM mydb.filtered_table3", "0\t1\n1\t0") + assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table1", [[1, 0], [1, 1]]) + assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table2", [[0, 0, 0, 0], [0, 0, 6, 0]]) + assert_eq_with_retry(node, "SELECT * FROM mydb.filtered_table3", [[0, 1], [1, 0]]) def test_introspection(): - assert instance.query("SELECT currentRowPolicies('mydb', 'filtered_table1')") == "['default']\n" - assert instance.query("SELECT currentRowPolicies('mydb', 'filtered_table2')") == "['default']\n" - assert instance.query("SELECT currentRowPolicies('mydb', 'filtered_table3')") == "['default']\n" - assert instance.query("SELECT arraySort(currentRowPolicies())") == "[('mydb','filtered_table1','default'),('mydb','filtered_table2','default'),('mydb','filtered_table3','default'),('mydb','local','default')]\n" - - policy1 = "mydb\tfiltered_table1\tdefault\tdefault ON mydb.filtered_table1\t9e8a8f62-4965-2b5e-8599-57c7b99b3549\tusers.xml\t0\ta = 1\t\t\t\t\n" - policy2 = "mydb\tfiltered_table2\tdefault\tdefault ON mydb.filtered_table2\tcffae79d-b9bf-a2ef-b798-019c18470b25\tusers.xml\t0\ta + b < 1 or c - d > 5\t\t\t\t\n" - policy3 = "mydb\tfiltered_table3\tdefault\tdefault ON mydb.filtered_table3\t12fc5cef-e3da-3940-ec79-d8be3911f42b\tusers.xml\t0\tc = 1\t\t\t\t\n" - policy4 = "mydb\tlocal\tdefault\tdefault ON mydb.local\tcdacaeb5-1d97-f99d-2bb0-4574f290629c\tusers.xml\t0\t1\t\t\t\t\n" - assert instance.query("SELECT * from system.row_policies WHERE has(currentRowPolicyIDs('mydb', 'filtered_table1'), id) ORDER BY table, name") == policy1 - assert instance.query("SELECT * from system.row_policies WHERE has(currentRowPolicyIDs('mydb', 'filtered_table2'), id) ORDER BY table, name") == policy2 - assert instance.query("SELECT * from system.row_policies WHERE has(currentRowPolicyIDs('mydb', 'filtered_table3'), id) ORDER BY table, name") == policy3 - assert instance.query("SELECT * from system.row_policies WHERE has(currentRowPolicyIDs('mydb', 'local'), id) ORDER BY table, name") == policy4 - assert instance.query("SELECT * from system.row_policies WHERE has(currentRowPolicyIDs(), id) ORDER BY table, name") == policy1 + policy2 + policy3 + policy4 + policies = [ + ["another ON mydb.filtered_table1", "another", "mydb", "filtered_table1", "6068883a-0e9d-f802-7e22-0144f8e66d3c", "users.xml", "1", 0, 0, "['another']", "[]"], + ["another ON mydb.filtered_table2", "another", "mydb", "filtered_table2", "c019e957-c60b-d54e-cc52-7c90dac5fb01", "users.xml", "1", 0, 0, "['another']", "[]"], + ["another ON mydb.filtered_table3", "another", "mydb", "filtered_table3", "4cb080d0-44e8-dbef-6026-346655143628", "users.xml", "1", 0, 0, "['another']", "[]"], + ["another ON mydb.local", "another", "mydb", "local", "5b23c389-7e18-06bf-a6bc-dd1afbbc0a97", "users.xml", "a = 1", 0, 0, "['another']", "[]"], + ["default ON mydb.filtered_table1", "default", "mydb", "filtered_table1", "9e8a8f62-4965-2b5e-8599-57c7b99b3549", "users.xml", "a = 1", 0, 0, "['default']", "[]"], + ["default ON mydb.filtered_table2", "default", "mydb", "filtered_table2", "cffae79d-b9bf-a2ef-b798-019c18470b25", "users.xml", "a + b < 1 or c - d > 5", 0, 0, "['default']", "[]"], + ["default ON mydb.filtered_table3", "default", "mydb", "filtered_table3", "12fc5cef-e3da-3940-ec79-d8be3911f42b", "users.xml", "c = 1", 0, 0, "['default']", "[]"], + ["default ON mydb.local", "default", "mydb", "local", "cdacaeb5-1d97-f99d-2bb0-4574f290629c", "users.xml", "1", 0, 0, "['default']", "[]"] + ] + assert node.query("SELECT * from system.row_policies ORDER BY short_name, database, table") == TSV(policies) def test_dcl_introspection(): - assert instance.query("SHOW POLICIES ON mydb.filtered_table1") == "another\ndefault\n" - assert instance.query("SHOW POLICIES CURRENT ON mydb.filtered_table2") == "default\n" - assert instance.query("SHOW POLICIES") == "another ON mydb.filtered_table1\nanother ON mydb.filtered_table2\nanother ON mydb.filtered_table3\nanother ON mydb.local\ndefault ON mydb.filtered_table1\ndefault ON mydb.filtered_table2\ndefault ON mydb.filtered_table3\ndefault ON mydb.local\n" - assert instance.query("SHOW POLICIES CURRENT") == "default ON mydb.filtered_table1\ndefault ON mydb.filtered_table2\ndefault ON mydb.filtered_table3\ndefault ON mydb.local\n" + assert node.query("SHOW POLICIES") == TSV(["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", "another ON mydb.local", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3", "default ON mydb.local"]) + assert node.query("SHOW POLICIES ON mydb.filtered_table1") == TSV(["another", "default"]) + assert node.query("SHOW POLICIES ON mydb.local") == TSV(["another", "default"]) - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.local") == "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING a = 1 TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING ((a + b) < 1) OR ((c - d) > 5) TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING c = 1 TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.local") == "CREATE ROW POLICY default ON mydb.local FOR SELECT USING 1 TO default\n" copy_policy_xml('all_rows.xml') - assert instance.query("SHOW POLICIES CURRENT") == "default ON mydb.filtered_table1\ndefault ON mydb.filtered_table2\ndefault ON mydb.filtered_table3\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING 1 TO default\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING 1 TO default\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING 1 TO default\n" + assert node.query("SHOW POLICIES") == TSV(["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3"]) + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING 1 TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING 1 TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING 1 TO default\n" copy_policy_xml('no_rows.xml') - assert instance.query("SHOW POLICIES CURRENT") == "default ON mydb.filtered_table1\ndefault ON mydb.filtered_table2\ndefault ON mydb.filtered_table3\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING NULL TO default\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING NULL TO default\n" - assert instance.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING NULL TO default\n" + assert node.query("SHOW POLICIES") == TSV(["another ON mydb.filtered_table1", "another ON mydb.filtered_table2", "another ON mydb.filtered_table3", "default ON mydb.filtered_table1", "default ON mydb.filtered_table2", "default ON mydb.filtered_table3"]) + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table1") == "CREATE ROW POLICY default ON mydb.filtered_table1 FOR SELECT USING NULL TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table2") == "CREATE ROW POLICY default ON mydb.filtered_table2 FOR SELECT USING NULL TO default\n" + assert node.query("SHOW CREATE POLICY default ON mydb.filtered_table3") == "CREATE ROW POLICY default ON mydb.filtered_table3 FOR SELECT USING NULL TO default\n" copy_policy_xml('no_filters.xml') - assert instance.query("SHOW POLICIES") == "" + assert node.query("SHOW POLICIES") == "" def test_dcl_management(): copy_policy_xml('no_filters.xml') - assert instance.query("SHOW POLICIES") == "" + assert node.query("SHOW POLICIES") == "" - instance.query("CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING ab") - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n" + node.query("ALTER POLICY pA ON mydb.filtered_table1 FOR SELECT USING a>b") + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0]]) - instance.query("ALTER POLICY pA ON mydb.filtered_table1 RENAME TO pB") - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n" - assert instance.query("SHOW POLICIES CURRENT ON mydb.filtered_table1") == "pB\n" - assert instance.query("SHOW CREATE POLICY pB ON mydb.filtered_table1") == "CREATE ROW POLICY pB ON mydb.filtered_table1 FOR SELECT USING a > b TO default\n" + node.query("ALTER POLICY pA ON mydb.filtered_table1 RENAME TO pB") + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0]]) + assert node.query("SHOW POLICIES ON mydb.filtered_table1") == "pB\n" + assert node.query("SHOW CREATE POLICY pB ON mydb.filtered_table1") == "CREATE ROW POLICY pB ON mydb.filtered_table1 FOR SELECT USING a > b TO default\n" - instance.query("DROP POLICY pB ON mydb.filtered_table1") - assert instance.query("SELECT * FROM mydb.filtered_table1") == "0\t0\n0\t1\n1\t0\n1\t1\n" - assert instance.query("SHOW POLICIES") == "" + node.query("DROP POLICY pB ON mydb.filtered_table1") + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[0, 0], [0, 1], [1, 0], [1, 1]]) + assert node.query("SHOW POLICIES") == "" def test_users_xml_is_readonly(): - assert re.search("storage is readonly", instance.query_and_get_error("DROP POLICY default ON mydb.filtered_table1")) + assert re.search("storage is readonly", node.query_and_get_error("DROP POLICY default ON mydb.filtered_table1")) def test_miscellaneous_engines(): copy_policy_xml('normal_filters.xml') # ReplicatedMergeTree - instance.query("DROP TABLE mydb.filtered_table1") - instance.query("CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE ReplicatedMergeTree('/clickhouse/tables/00-00/filtered_table1', 'replica1') ORDER BY a") - instance.query("INSERT INTO mydb.filtered_table1 values (0, 0), (0, 1), (1, 0), (1, 1)") - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t0\n1\t1\n" + node.query("DROP TABLE mydb.filtered_table1") + node.query("CREATE TABLE mydb.filtered_table1 (a UInt8, b UInt8) ENGINE ReplicatedMergeTree('/clickhouse/tables/00-00/filtered_table1', 'replica1') ORDER BY a") + node.query("INSERT INTO mydb.filtered_table1 values (0, 0), (0, 1), (1, 0), (1, 1)") + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 0], [1, 1]]) # CollapsingMergeTree - instance.query("DROP TABLE mydb.filtered_table1") - instance.query("CREATE TABLE mydb.filtered_table1 (a UInt8, b Int8) ENGINE CollapsingMergeTree(b) ORDER BY a") - instance.query("INSERT INTO mydb.filtered_table1 values (0, 1), (0, 1), (1, 1), (1, 1)") - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t1\n1\t1\n" + node.query("DROP TABLE mydb.filtered_table1") + node.query("CREATE TABLE mydb.filtered_table1 (a UInt8, b Int8) ENGINE CollapsingMergeTree(b) ORDER BY a") + node.query("INSERT INTO mydb.filtered_table1 values (0, 1), (0, 1), (1, 1), (1, 1)") + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 1], [1, 1]]) # ReplicatedCollapsingMergeTree - instance.query("DROP TABLE mydb.filtered_table1") - instance.query("CREATE TABLE mydb.filtered_table1 (a UInt8, b Int8) ENGINE ReplicatedCollapsingMergeTree('/clickhouse/tables/00-00/filtered_table1', 'replica1', b) ORDER BY a") - instance.query("INSERT INTO mydb.filtered_table1 values (0, 1), (0, 1), (1, 1), (1, 1)") - assert instance.query("SELECT * FROM mydb.filtered_table1") == "1\t1\n1\t1\n" + node.query("DROP TABLE mydb.filtered_table1") + node.query("CREATE TABLE mydb.filtered_table1 (a UInt8, b Int8) ENGINE ReplicatedCollapsingMergeTree('/clickhouse/tables/00-00/filtered_table1', 'replica1', b) ORDER BY a") + node.query("INSERT INTO mydb.filtered_table1 values (0, 1), (0, 1), (1, 1), (1, 1)") + assert node.query("SELECT * FROM mydb.filtered_table1") == TSV([[1, 1], [1, 1]]) # DistributedMergeTree - instance.query("DROP TABLE IF EXISTS mydb.not_filtered_table") - instance.query("CREATE TABLE mydb.not_filtered_table (a UInt8, b UInt8) ENGINE Distributed('test_local_cluster', mydb, local)") - instance.query("CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a") - instance2.query("CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a") - instance.query("INSERT INTO mydb.local values (2, 0), (2, 1), (1, 0), (1, 1)") - instance2.query("INSERT INTO mydb.local values (3, 0), (3, 1), (1, 0), (1, 1)") - assert instance.query("SELECT * FROM mydb.not_filtered_table", user="another") == "1\t0\n1\t1\n1\t0\n1\t1\n" - assert instance.query("SELECT sum(a), b FROM mydb.not_filtered_table GROUP BY b ORDER BY b", user="another") == "2\t0\n2\t1\n" + node.query("DROP TABLE IF EXISTS mydb.not_filtered_table") + node.query("CREATE TABLE mydb.not_filtered_table (a UInt8, b UInt8) ENGINE Distributed('test_local_cluster', mydb, local)") + node.query("CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a") + node2.query("CREATE TABLE mydb.local (a UInt8, b UInt8) ENGINE MergeTree ORDER BY a") + node.query("INSERT INTO mydb.local values (2, 0), (2, 1), (1, 0), (1, 1)") + node2.query("INSERT INTO mydb.local values (3, 0), (3, 1), (1, 0), (1, 1)") + assert node.query("SELECT * FROM mydb.not_filtered_table", user="another") == TSV([[1, 0], [1, 1], [1, 0], [1, 1]]) + assert node.query("SELECT sum(a), b FROM mydb.not_filtered_table GROUP BY b ORDER BY b", user="another") == TSV([[2, 0], [2, 1]]) diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index 8b9d023d56f..9c8b116e6d2 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -1,10 +1,25 @@ import pytest from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) instance = cluster.add_instance('instance') +def system_settings_profile(profile_name): + return TSV(instance.query("SELECT name, storage, num_elements, apply_to_all, apply_to_list, apply_to_except FROM system.settings_profiles WHERE name='" + profile_name + "'")) + +def system_settings_profile_elements(profile_name=None, user_name=None, role_name=None): + where = "" + if profile_name: + where = " WHERE profile_name='" + profile_name + "'" + elif user_name: + where = " WHERE user_name='" + user_name + "'" + elif role_name: + where = " WHERE role_name='" + role_name + "'" + return TSV(instance.query("SELECT * FROM system.settings_profile_elements" + where)) + + @pytest.fixture(scope="module", autouse=True) def setup_nodes(): try: @@ -28,19 +43,23 @@ def reset_after_test(): instance.query("DROP SETTINGS PROFILE IF EXISTS xyz, alpha") -def test_settings_profile(): +def test_smoke(): # Set settings and constraints via CREATE SETTINGS PROFILE ... TO user instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin") assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile("xyz") == [[ "xyz", "disk", 1, 0, "['robin']", "[]" ]] + assert system_settings_profile_elements(profile_name="xyz") == [[ "xyz", "\N", "\N", 0, "max_memory_usage", 100000001, 90000000, 110000000, "\N", "\N" ]] instance.query("ALTER SETTINGS PROFILE xyz TO NONE") assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile("xyz") == [[ "xyz", "disk", 1, 0, "[]", "[]" ]] + assert system_settings_profile_elements(user_name="robin") == [] # Set settings and constraints via CREATE USER ... SETTINGS PROFILE instance.query("ALTER USER robin SETTINGS PROFILE xyz") @@ -48,52 +67,57 @@ def test_settings_profile(): assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile_elements(user_name="robin") == [[ "\N", "robin", "\N", 0, "\N", "\N", "\N", "\N", "\N", "xyz" ]] instance.query("ALTER USER robin SETTINGS NONE") assert instance.query("SHOW CREATE USER robin") == "CREATE USER robin\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile_elements(user_name="robin") == [] -def test_settings_profile_from_granted_role(): +def test_settings_from_granted_role(): # Set settings and constraints via granted role - instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000") + instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000") instance.query("CREATE ROLE worker SETTINGS PROFILE xyz") instance.query("GRANT worker TO robin") - assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000\n" + assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000\n" assert instance.query("SHOW CREATE ROLE worker") == "CREATE ROLE worker SETTINGS PROFILE xyz\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" - assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") + assert instance.query("SELECT value FROM system.settings WHERE name = 'max_ast_depth'", user="robin") == "2000\n" assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile("xyz") == [[ "xyz", "disk", 2, 0, "[]", "[]" ]] + assert system_settings_profile_elements(profile_name="xyz") == [[ "xyz", "\N", "\N", 0, "max_memory_usage", 100000001, "\N", 110000000, "\N", "\N" ], + [ "xyz", "\N", "\N", 1, "max_ast_depth", 2000, "\N", "\N", "\N", "\N" ]] + assert system_settings_profile_elements(role_name="worker") == [[ "\N", "\N", "worker", 0, "\N", "\N", "\N", "\N", "\N", "xyz" ]] instance.query("REVOKE worker FROM robin") assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" - instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") instance.query("ALTER ROLE worker SETTINGS NONE") instance.query("GRANT worker TO robin") assert instance.query("SHOW CREATE ROLE worker") == "CREATE ROLE worker\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" - instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile_elements(role_name="worker") == [] # Set settings and constraints via CREATE SETTINGS PROFILE ... TO granted role instance.query("ALTER SETTINGS PROFILE xyz TO worker") - assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO worker\n" + assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000 TO worker\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000001\n" - assert "Setting max_memory_usage shouldn't be less than 90000000" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") assert "Setting max_memory_usage shouldn't be greater than 110000000" in instance.query_and_get_error("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile("xyz") == [[ "xyz", "disk", 2, 0, "['worker']", "[]" ]] instance.query("ALTER SETTINGS PROFILE xyz TO NONE") - assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000\n" + assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000001 MAX 110000000, max_ast_depth = 2000\n" assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "10000000000\n" - instance.query("SET max_memory_usage = 80000000", user="robin") instance.query("SET max_memory_usage = 120000000", user="robin") + assert system_settings_profile("xyz") == [[ "xyz", "disk", 2, 0, "[]", "[]" ]] -def test_inheritance_of_settings_profile(): +def test_inheritance(): instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY") instance.query("CREATE SETTINGS PROFILE alpha SETTINGS PROFILE xyz TO robin") assert instance.query("SHOW CREATE SETTINGS PROFILE xyz") == "CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000002 READONLY\n" @@ -101,6 +125,12 @@ def test_inheritance_of_settings_profile(): assert instance.query("SELECT value FROM system.settings WHERE name = 'max_memory_usage'", user="robin") == "100000002\n" assert "Setting max_memory_usage should not be changed" in instance.query_and_get_error("SET max_memory_usage = 80000000", user="robin") + assert system_settings_profile("xyz") == [[ "xyz", "disk", 1, 0, "[]", "[]" ]] + assert system_settings_profile_elements(profile_name="xyz") == [[ "xyz", "\N", "\N", 0, "max_memory_usage", 100000002, "\N", "\N", 1, "\N" ]] + assert system_settings_profile("alpha") == [[ "alpha", "disk", 1, 0, "['robin']", "[]" ]] + assert system_settings_profile_elements(profile_name="alpha") == [[ "alpha", "\N", "\N", 0, "\N", "\N", "\N", "\N", "\N", "xyz" ]] + assert system_settings_profile_elements(user_name="robin") == [] + def test_alter_and_drop(): instance.query("CREATE SETTINGS PROFILE xyz SETTINGS max_memory_usage = 100000003 MIN 90000000 MAX 110000000 TO robin") @@ -117,6 +147,12 @@ def test_alter_and_drop(): instance.query("SET max_memory_usage = 120000000", user="robin") +def test_show_profiles(): + instance.query("CREATE SETTINGS PROFILE xyz") + assert instance.query("SHOW SETTINGS PROFILES") == "default\nreadonly\nxyz\n" + assert instance.query("SHOW PROFILES") == "default\nreadonly\nxyz\n" + + def test_allow_introspection(): assert "Not enough privileges" in instance.query_and_get_error("SELECT demangle('a')", user="robin") diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 92fa06c262e..cbe96df3c29 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -198,7 +198,6 @@ def test_kafka_settings_new_syntax(kafka_cluster): kafka_check_result(result, True) -@pytest.mark.skip(reason="https://github.com/edenhill/librdkafka/issues/2077") @pytest.mark.timeout(180) def test_kafka_consumer_hang(kafka_cluster): @@ -219,7 +218,7 @@ def test_kafka_consumer_hang(kafka_cluster): CREATE MATERIALIZED VIEW test.consumer TO test.view AS SELECT * FROM test.kafka; ''') - time.sleep(12) + time.sleep(10) instance.query('SELECT * FROM test.view') # This should trigger heartbeat fail, @@ -229,19 +228,23 @@ def test_kafka_consumer_hang(kafka_cluster): time.sleep(0.5) kafka_cluster.unpause_container('kafka1') + # print("Attempt to drop") instance.query('DROP TABLE test.kafka') + #kafka_cluster.open_bash_shell('instance') + instance.query(''' DROP TABLE test.consumer; DROP TABLE test.view; ''') - log = '/var/log/clickhouse-server/stderr.log' - instance.exec_in_container(['grep', '-q', 'BROKERFAIL', log]) - instance.exec_in_container(['grep', '-q', '|ASSIGN|', log]) - instance.exec_in_container(['grep', '-q', 'Heartbeat failed: REBALANCE_IN_PROGRESS: group is rebalancing', log]) - instance.exec_in_container(['grep', '-q', 'Group "consumer_hang": waiting for rebalance_cb', log]) + # original problem appearance was a sequence of the following messages in librdkafka logs: + # BROKERFAIL -> |ASSIGN| -> REBALANCE_IN_PROGRESS -> "waiting for rebalance_cb" (repeated forever) + # so it was waiting forever while the application will execute queued rebalance callback + # from a user perspective: we expect no hanging 'drop' queries + # 'dr'||'op' to avoid self matching + assert int(instance.query("select count() from system.processes where position(lower(query),'dr'||'op')>0")) == 0 @pytest.mark.timeout(180) def test_kafka_csv_with_delimiter(kafka_cluster): @@ -1234,7 +1237,7 @@ def test_exception_from_destructor(kafka_cluster): DROP TABLE test.kafka; ''') - kafka_cluster.open_bash_shell('instance') + #kafka_cluster.open_bash_shell('instance') assert TSV(instance.query('SELECT 1')) == TSV('1') diff --git a/tests/performance/decimal_parse.xml b/tests/performance/decimal_parse.xml new file mode 100644 index 00000000000..19e940b13df --- /dev/null +++ b/tests/performance/decimal_parse.xml @@ -0,0 +1,3 @@ + + SELECT count() FROM zeros(10000000) WHERE NOT ignore(toDecimal32OrZero(toString(rand() % 10000), 5)) + diff --git a/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh b/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh index 21014b56b67..5173b5f5772 100755 --- a/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh +++ b/tests/queries/0_stateless/00634_performance_introspection_and_logging.sh @@ -13,7 +13,7 @@ server_logs_file=${CLICKHOUSE_TMP}/$cur_name"_server.logs" server_logs="--server_logs_file=$server_logs_file" rm -f "$server_logs_file" -settings="$server_logs --log_queries=1 --log_query_threads=1 --log_profile_events=1 --log_query_settings=1 --experimental_use_processors=0" +settings="$server_logs --log_queries=1 --log_query_threads=1 --log_profile_events=1 --log_query_settings=1" # Test insert logging on each block and checkPacket() method diff --git a/tests/queries/0_stateless/00720_with_cube.sql b/tests/queries/0_stateless/00720_with_cube.sql index 42b65c8222c..d236e9da34b 100644 --- a/tests/queries/0_stateless/00720_with_cube.sql +++ b/tests/queries/0_stateless/00720_with_cube.sql @@ -1,8 +1,6 @@ DROP TABLE IF EXISTS cube; CREATE TABLE cube(a String, b Int32, s Int32) ENGINE = Memory; --- SET experimental_use_processors=1; - INSERT INTO cube VALUES ('a', 1, 10), ('a', 1, 15), ('a', 2, 20); INSERT INTO cube VALUES ('a', 2, 25), ('b', 1, 10), ('b', 1, 5); INSERT INTO cube VALUES ('b', 2, 20), ('b', 2, 15); diff --git a/tests/queries/0_stateless/00963_achimbab.reference b/tests/queries/0_stateless/00963_achimbab.reference index 5248223efdd..a5f501d1a6a 100644 --- a/tests/queries/0_stateless/00963_achimbab.reference +++ b/tests/queries/0_stateless/00963_achimbab.reference @@ -20,7 +20,7 @@ { "total": 1, "arrayElement(k, 1)": null, - "arrayElement(k, 2)": 4 + "arrayElement(k, 2)": 1 }, { "total": 1, @@ -30,12 +30,12 @@ { "total": 1, "arrayElement(k, 1)": null, - "arrayElement(k, 2)": 1 + "arrayElement(k, 2)": 3 }, { "total": 1, "arrayElement(k, 1)": null, - "arrayElement(k, 2)": 3 + "arrayElement(k, 2)": 4 }, { "total": 1, @@ -53,5 +53,5 @@ "rows": 5, - "rows_before_limit_at_least": 5 + "rows_before_limit_at_least": 1000000 } diff --git a/tests/queries/0_stateless/00963_achimbab.sql b/tests/queries/0_stateless/00963_achimbab.sql index 3f66cfbbf31..758ecf5acf3 100644 --- a/tests/queries/0_stateless/00963_achimbab.sql +++ b/tests/queries/0_stateless/00963_achimbab.sql @@ -1,5 +1,4 @@ SET output_format_write_statistics = 0; -SET experimental_use_processors = 0; select sum(cnt) > 0 as total, @@ -14,6 +13,6 @@ select limit 1000000 ) group by k with totals -order by total desc +order by k[2] SETTINGS max_threads = 100, max_execution_time = 120 format JSON; diff --git a/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.reference b/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.reference index d80fc78e03d..b261da18d51 100644 --- a/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.reference +++ b/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.reference @@ -1,4 +1,2 @@ 1 0 -1 -0 diff --git a/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.sql b/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.sql index 6d2bf5f4863..dcec82461a5 100644 --- a/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.sql +++ b/tests/queries/0_stateless/01018_optimize_read_in_order_with_in_subquery.sql @@ -8,8 +8,4 @@ INSERT INTO TESTTABLE4 VALUES (0,'1','1'), (1,'0','1'); SELECT _id FROM TESTTABLE4 PREWHERE l IN (select '1') ORDER BY _id DESC LIMIT 10; -SET experimental_use_processors=1; - -SELECT _id FROM TESTTABLE4 PREWHERE l IN (select '1') ORDER BY _id DESC LIMIT 10; - -DROP TABLE TESTTABLE4; \ No newline at end of file +DROP TABLE TESTTABLE4; diff --git a/tests/queries/0_stateless/01073_grant_and_revoke.reference b/tests/queries/0_stateless/01073_grant_and_revoke.reference index d7d97fa28fe..134256c8113 100644 --- a/tests/queries/0_stateless/01073_grant_and_revoke.reference +++ b/tests/queries/0_stateless/01073_grant_and_revoke.reference @@ -1,11 +1,11 @@ CREATE USER test_user_01073 A B -GRANT ALTER DELETE, INSERT ON *.* TO test_user_01073 GRANT SELECT ON db1.* TO test_user_01073 GRANT SELECT ON db2.table TO test_user_01073 GRANT SELECT(col1) ON db3.table TO test_user_01073 GRANT SELECT(col1, col2) ON db4.table TO test_user_01073 +GRANT INSERT, ALTER DELETE ON *.* TO test_user_01073 C -GRANT ALTER DELETE ON *.* TO test_user_01073 GRANT SELECT(col1) ON db4.table TO test_user_01073 +GRANT ALTER DELETE ON *.* TO test_user_01073 diff --git a/tests/queries/0_stateless/01104_distributed_numbers_test.sql b/tests/queries/0_stateless/01104_distributed_numbers_test.sql index 7f56a4e08fd..101342b4c34 100644 --- a/tests/queries/0_stateless/01104_distributed_numbers_test.sql +++ b/tests/queries/0_stateless/01104_distributed_numbers_test.sql @@ -1,8 +1,6 @@ DROP TABLE IF EXISTS d_numbers; CREATE TABLE d_numbers (number UInt32) ENGINE = Distributed(test_cluster_two_shards, system, numbers, rand()); -SET experimental_use_processors = 1; - SELECT '100' AS number FROM d_numbers AS n WHERE n.number = 100 LIMIT 2; SET distributed_product_mode = 'local'; diff --git a/tests/queries/0_stateless/01107_atomic_db_detach_attach.reference b/tests/queries/0_stateless/01107_atomic_db_detach_attach.reference index bf34eef313a..b78efbbb592 100644 --- a/tests/queries/0_stateless/01107_atomic_db_detach_attach.reference +++ b/tests/queries/0_stateless/01107_atomic_db_detach_attach.reference @@ -2,3 +2,5 @@ OK OK 5 10 5 10 +dropped +end diff --git a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh index 6ff36318d30..5fa6e194670 100755 --- a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh +++ b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh @@ -21,6 +21,7 @@ $CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" $CLICKHOUSE_CLIENT --allow_experimental_database_atomic=1 -q "ATTACH DATABASE test_01107 ENGINE=Atomic" $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01107.mt" -$CLICKHOUSE_CLIENT -q "INSERT INTO test_01107.mt SELECT number + sleepEachRow(3) FROM numbers(100)" & +$CLICKHOUSE_CLIENT -q "INSERT INTO test_01107.mt SELECT number + sleepEachRow(1) FROM numbers(5)" && echo "end" & sleep 1 -$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01107" +$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01107" && sleep 1 && echo "dropped" +wait diff --git a/tests/queries/0_stateless/01114_database_atomic.reference b/tests/queries/0_stateless/01114_database_atomic.reference index a6c9fc3c4f4..dece5bd9b8b 100644 --- a/tests/queries/0_stateless/01114_database_atomic.reference +++ b/tests/queries/0_stateless/01114_database_atomic.reference @@ -12,5 +12,6 @@ mt 00001114-0000-4000-8000-000000000002 CREATE TABLE test_01114_2.mt UUID \'0000 CREATE TABLE test_01114_1.mt UUID \'00001114-0000-4000-8000-000000000001\'\n(\n `n` UInt64\n)\nENGINE = MergeTree()\nPARTITION BY n % 5\nORDER BY tuple()\nSETTINGS index_granularity = 8192 CREATE TABLE test_01114_2.mt UUID \'00001114-0000-4000-8000-000000000002\'\n(\n `n` UInt64\n)\nENGINE = MergeTree()\nPARTITION BY n % 5\nORDER BY tuple()\nSETTINGS index_granularity = 8192 5 +dropped 20 190 30 435 diff --git a/tests/queries/0_stateless/01114_database_atomic.sh b/tests/queries/0_stateless/01114_database_atomic.sh index 0ef7844b2d1..58c5b2d287d 100755 --- a/tests/queries/0_stateless/01114_database_atomic.sh +++ b/tests/queries/0_stateless/01114_database_atomic.sh @@ -57,7 +57,7 @@ $CLICKHOUSE_CLIENT -q "SELECT count() FROM test_01114_1.mt" # result: 5 $CLICKHOUSE_CLIENT -q "SELECT tuple(s, sleepEachRow(3)) FROM test_01114_1.mt" > /dev/null & # 15s sleep 1 -$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_1" +$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_1" && echo "dropped" wait # for INSERT diff --git a/tests/queries/0_stateless/01213_alter_rename_column.sql b/tests/queries/0_stateless/01213_alter_rename_column.sql index 59e4191d7d5..1732ea88274 100644 --- a/tests/queries/0_stateless/01213_alter_rename_column.sql +++ b/tests/queries/0_stateless/01213_alter_rename_column.sql @@ -24,7 +24,7 @@ SELECT * FROM table_for_rename WHERE key = 1 FORMAT TSVWithNames; ALTER TABLE table_for_rename RENAME COLUMN value3 to value2; --{serverError 15} ALTER TABLE table_for_rename RENAME COLUMN value3 TO r1, RENAME COLUMN value3 TO r2; --{serverError 36} -ALTER TABLE table_for_rename RENAME COLUMN value3 TO r1, RENAME COLUMN r1 TO value1; --{serverError 10} +ALTER TABLE table_for_rename RENAME COLUMN value3 TO r1, RENAME COLUMN r1 TO value1; --{serverError 48} ALTER TABLE table_for_rename RENAME COLUMN value2 TO renamed_value2, RENAME COLUMN value3 TO renamed_value3; diff --git a/tests/queries/0_stateless/01232_extremes.sql b/tests/queries/0_stateless/01232_extremes.sql index 9379dc1cd38..e46a6602766 100644 --- a/tests/queries/0_stateless/01232_extremes.sql +++ b/tests/queries/0_stateless/01232_extremes.sql @@ -1,6 +1,5 @@ set send_logs_level = 'error'; set extremes = 1; --- set experimental_use_processors=0; select * from remote('127.0.0.1', numbers(2)); select '-'; diff --git a/tests/queries/0_stateless/01260_ubsan_decimal_parse.reference b/tests/queries/0_stateless/01260_ubsan_decimal_parse.reference new file mode 100644 index 00000000000..945da8ffd36 --- /dev/null +++ b/tests/queries/0_stateless/01260_ubsan_decimal_parse.reference @@ -0,0 +1 @@ +0.000000 diff --git a/tests/queries/0_stateless/01260_ubsan_decimal_parse.sql b/tests/queries/0_stateless/01260_ubsan_decimal_parse.sql new file mode 100644 index 00000000000..2c7cda512e8 --- /dev/null +++ b/tests/queries/0_stateless/01260_ubsan_decimal_parse.sql @@ -0,0 +1 @@ +SELECT toDecimal32OrZero(CAST(-7174046, 'String'), 6); diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference new file mode 100644 index 00000000000..e85dbd89801 --- /dev/null +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -0,0 +1,113 @@ +SHOW DATABASES [] DATABASE SHOW +SHOW TABLES [] TABLE SHOW +SHOW COLUMNS [] COLUMN SHOW +SHOW DICTIONARIES [] DICTIONARY SHOW +SHOW [] \N ALL +SELECT [] COLUMN ALL +INSERT [] COLUMN ALL +ALTER UPDATE ['UPDATE'] COLUMN ALTER TABLE +ALTER DELETE ['DELETE'] COLUMN ALTER TABLE +ALTER ADD COLUMN ['ADD COLUMN'] COLUMN ALTER COLUMN +ALTER MODIFY COLUMN ['MODIFY COLUMN'] COLUMN ALTER COLUMN +ALTER DROP COLUMN ['DROP COLUMN'] COLUMN ALTER COLUMN +ALTER COMMENT COLUMN ['COMMENT COLUMN'] COLUMN ALTER COLUMN +ALTER CLEAR COLUMN ['CLEAR COLUMN'] COLUMN ALTER COLUMN +ALTER RENAME COLUMN ['RENAME COLUMN'] COLUMN ALTER COLUMN +ALTER COLUMN [] \N ALTER TABLE +ALTER ORDER BY ['ALTER MODIFY ORDER BY','MODIFY ORDER BY'] TABLE ALTER INDEX +ALTER ADD INDEX ['ADD INDEX'] TABLE ALTER INDEX +ALTER DROP INDEX ['DROP INDEX'] TABLE ALTER INDEX +ALTER MATERIALIZE INDEX ['MATERIALIZE INDEX'] TABLE ALTER INDEX +ALTER CLEAR INDEX ['CLEAR INDEX'] TABLE ALTER INDEX +ALTER INDEX ['INDEX'] \N ALTER TABLE +ALTER ADD CONSTRAINT ['ADD CONSTRAINT'] TABLE ALTER CONSTRAINT +ALTER DROP CONSTRAINT ['DROP CONSTRAINT'] TABLE ALTER CONSTRAINT +ALTER CONSTRAINT ['CONSTRAINT'] \N ALTER TABLE +ALTER TTL ['ALTER MODIFY TTL','MODIFY TTL'] TABLE ALTER TABLE +ALTER MATERIALIZE TTL ['MATERIALIZE TTL'] TABLE ALTER TABLE +ALTER SETTINGS ['ALTER SETTING','ALTER MODIFY SETTING','MODIFY SETTING'] TABLE ALTER TABLE +ALTER MOVE PARTITION ['ALTER MOVE PART','MOVE PARTITION','MOVE PART'] TABLE ALTER TABLE +ALTER FETCH PARTITION ['FETCH PARTITION'] TABLE ALTER TABLE +ALTER FREEZE PARTITION ['FREEZE PARTITION'] TABLE ALTER TABLE +ALTER TABLE [] \N ALTER +ALTER VIEW REFRESH ['ALTER LIVE VIEW REFRESH','REFRESH VIEW'] VIEW ALTER VIEW +ALTER VIEW MODIFY QUERY ['ALTER TABLE MODIFY QUERY'] VIEW ALTER VIEW +ALTER VIEW [] \N ALTER +ALTER [] \N ALL +CREATE DATABASE [] DATABASE CREATE +CREATE TABLE [] TABLE CREATE +CREATE VIEW [] VIEW CREATE +CREATE DICTIONARY [] DICTIONARY CREATE +CREATE TEMPORARY TABLE [] GLOBAL CREATE +CREATE [] \N ALL +DROP DATABASE [] DATABASE DROP +DROP TABLE [] TABLE DROP +DROP VIEW [] VIEW DROP +DROP DICTIONARY [] DICTIONARY DROP +DROP [] \N ALL +TRUNCATE ['TRUNCATE TABLE'] TABLE ALL +OPTIMIZE ['OPTIMIZE TABLE'] TABLE ALL +KILL QUERY [] GLOBAL ALL +CREATE USER [] GLOBAL ACCESS MANAGEMENT +ALTER USER [] GLOBAL ACCESS MANAGEMENT +DROP USER [] GLOBAL ACCESS MANAGEMENT +CREATE ROLE [] GLOBAL ACCESS MANAGEMENT +ALTER ROLE [] GLOBAL ACCESS MANAGEMENT +DROP ROLE [] GLOBAL ACCESS MANAGEMENT +ROLE ADMIN [] GLOBAL ACCESS MANAGEMENT +CREATE ROW POLICY ['CREATE POLICY'] GLOBAL ACCESS MANAGEMENT +ALTER ROW POLICY ['ALTER POLICY'] GLOBAL ACCESS MANAGEMENT +DROP ROW POLICY ['DROP POLICY'] GLOBAL ACCESS MANAGEMENT +CREATE QUOTA [] GLOBAL ACCESS MANAGEMENT +ALTER QUOTA [] GLOBAL ACCESS MANAGEMENT +DROP QUOTA [] GLOBAL ACCESS MANAGEMENT +CREATE SETTINGS PROFILE ['CREATE PROFILE'] GLOBAL ACCESS MANAGEMENT +ALTER SETTINGS PROFILE ['ALTER PROFILE'] GLOBAL ACCESS MANAGEMENT +DROP SETTINGS PROFILE ['DROP PROFILE'] GLOBAL ACCESS MANAGEMENT +SHOW USERS ['SHOW CREATE USER'] GLOBAL SHOW ACCESS +SHOW ROLES ['SHOW CREATE ROLE'] GLOBAL SHOW ACCESS +SHOW ROW POLICIES ['SHOW POLICIES','SHOW CREATE ROW POLICY','SHOW CREATE POLICY'] GLOBAL SHOW ACCESS +SHOW QUOTAS ['SHOW CREATE QUOTA'] GLOBAL SHOW ACCESS +SHOW SETTINGS PROFILES ['SHOW PROFILES','SHOW CREATE SETTINGS PROFILE','SHOW CREATE PROFILE'] GLOBAL SHOW ACCESS +SHOW ACCESS [] \N ACCESS MANAGEMENT +ACCESS MANAGEMENT [] \N ALL +SYSTEM SHUTDOWN ['SYSTEM KILL','SHUTDOWN'] GLOBAL SYSTEM +SYSTEM DROP DNS CACHE ['SYSTEM DROP DNS','DROP DNS CACHE','DROP DNS'] GLOBAL SYSTEM DROP CACHE +SYSTEM DROP MARK CACHE ['SYSTEM DROP MARK','DROP MARK CACHE','DROP MARKS'] GLOBAL SYSTEM DROP CACHE +SYSTEM DROP UNCOMPRESSED CACHE ['SYSTEM DROP UNCOMPRESSED','DROP UNCOMPRESSED CACHE','DROP UNCOMPRESSED'] GLOBAL SYSTEM DROP CACHE +SYSTEM DROP COMPILED EXPRESSION CACHE ['SYSTEM DROP COMPILED EXPRESSION','DROP COMPILED EXPRESSION CACHE','DROP COMPILED EXPRESSIONS'] GLOBAL SYSTEM DROP CACHE +SYSTEM DROP CACHE ['DROP CACHE'] \N SYSTEM +SYSTEM RELOAD CONFIG ['RELOAD CONFIG'] GLOBAL SYSTEM RELOAD +SYSTEM RELOAD DICTIONARY ['SYSTEM RELOAD DICTIONARIES','RELOAD DICTIONARY','RELOAD DICTIONARIES'] GLOBAL SYSTEM RELOAD +SYSTEM RELOAD EMBEDDED DICTIONARIES ['RELOAD EMBEDDED DICTIONARIES'] GLOBAL SYSTEM RELOAD +SYSTEM RELOAD [] \N SYSTEM +SYSTEM MERGES ['SYSTEM STOP MERGES','SYSTEM START MERGES','STOP_MERGES','START MERGES'] TABLE SYSTEM +SYSTEM TTL MERGES ['SYSTEM STOP TTL MERGES','SYSTEM START TTL MERGES','STOP TTL MERGES','START TTL MERGES'] TABLE SYSTEM +SYSTEM FETCHES ['SYSTEM STOP FETCHES','SYSTEM START FETCHES','STOP FETCHES','START FETCHES'] TABLE SYSTEM +SYSTEM MOVES ['SYSTEM STOP MOVES','SYSTEM START MOVES','STOP MOVES','START MOVES'] TABLE SYSTEM +SYSTEM DISTRIBUTED SENDS ['SYSTEM STOP DISTRIBUTED SENDS','SYSTEM START DISTRIBUTED SENDS','STOP DISTRIBUTED SENDS','START DISTRIBUTED SENDS'] TABLE SYSTEM SENDS +SYSTEM REPLICATED SENDS ['SYSTEM STOP REPLICATED SENDS','SYSTEM START REPLICATED SENDS','STOP_REPLICATED_SENDS','START REPLICATED SENDS'] TABLE SYSTEM SENDS +SYSTEM SENDS ['SYSTEM STOP SENDS','SYSTEM START SENDS','STOP SENDS','START SENDS'] \N SYSTEM +SYSTEM REPLICATION QUEUES ['SYSTEM STOP REPLICATION QUEUES','SYSTEM START REPLICATION QUEUES','STOP_REPLICATION_QUEUES','START REPLICATION QUEUES'] TABLE SYSTEM +SYSTEM SYNC REPLICA ['SYNC REPLICA'] TABLE SYSTEM +SYSTEM RESTART REPLICA ['RESTART REPLICA'] TABLE SYSTEM +SYSTEM FLUSH DISTRIBUTED ['FLUSH DISTRIBUTED'] TABLE SYSTEM FLUSH +SYSTEM FLUSH LOGS ['FLUSH LOGS'] GLOBAL SYSTEM FLUSH +SYSTEM FLUSH [] \N SYSTEM +SYSTEM [] \N ALL +dictGet ['dictHas','dictGetHierarchy','dictIsIn'] DICTIONARY ALL +addressToLine [] GLOBAL INTROSPECTION +addressToSymbol [] GLOBAL INTROSPECTION +demangle [] GLOBAL INTROSPECTION +INTROSPECTION ['INTROSPECTION FUNCTIONS'] \N ALL +FILE [] GLOBAL SOURCES +URL [] GLOBAL SOURCES +REMOTE [] GLOBAL SOURCES +MYSQL [] GLOBAL SOURCES +ODBC [] GLOBAL SOURCES +JDBC [] GLOBAL SOURCES +HDFS [] GLOBAL SOURCES +S3 [] GLOBAL SOURCES +SOURCES [] \N ALL +ALL ['ALL PRIVILEGES'] \N \N +NONE ['USAGE','NO PRIVILEGES'] \N \N diff --git a/tests/queries/0_stateless/01271_show_privileges.sql b/tests/queries/0_stateless/01271_show_privileges.sql new file mode 100644 index 00000000000..efd6ddb200c --- /dev/null +++ b/tests/queries/0_stateless/01271_show_privileges.sql @@ -0,0 +1 @@ +SHOW PRIVILEGES; \ No newline at end of file diff --git a/tests/queries/0_stateless/01274_alter_rename_column_distributed.reference b/tests/queries/0_stateless/01274_alter_rename_column_distributed.reference new file mode 100644 index 00000000000..7ed221867c1 --- /dev/null +++ b/tests/queries/0_stateless/01274_alter_rename_column_distributed.reference @@ -0,0 +1,4 @@ +2020-01-01 hello +2020-01-01 hello +2020-01-02 hello2 +2020-01-02 hello2 diff --git a/tests/queries/0_stateless/01274_alter_rename_column_distributed.sql b/tests/queries/0_stateless/01274_alter_rename_column_distributed.sql new file mode 100644 index 00000000000..a35dc7cca56 --- /dev/null +++ b/tests/queries/0_stateless/01274_alter_rename_column_distributed.sql @@ -0,0 +1,17 @@ +DROP TABLE IF EXISTS visits; +DROP TABLE IF EXISTS visits_dist; + +CREATE TABLE visits(StartDate Date, Name String) ENGINE MergeTree ORDER BY(StartDate); +CREATE TABLE visits_dist AS visits ENGINE Distributed(test_cluster_two_shards_localhost, currentDatabase(), 'visits', rand()); + +INSERT INTO visits_dist (StartDate, Name) VALUES ('2020-01-01', 'hello'); +INSERT INTO visits_dist (StartDate, Name) VALUES ('2020-01-02', 'hello2'); + +ALTER TABLE visits RENAME COLUMN Name TO Name2; +ALTER TABLE visits_dist RENAME COLUMN Name TO Name2; + +SELECT * FROM visits_dist ORDER BY StartDate, Name2; + +DROP TABLE visits; +DROP TABLE visits_dist; + diff --git a/tests/queries/0_stateless/01275_alter_rename_column_default_expr.reference b/tests/queries/0_stateless/01275_alter_rename_column_default_expr.reference new file mode 100644 index 00000000000..d81601b92c5 --- /dev/null +++ b/tests/queries/0_stateless/01275_alter_rename_column_default_expr.reference @@ -0,0 +1,87 @@ +2019-10-01 0 0 1 0 + 1 +2019-10-02 1 1 2 1 + 2 +2019-10-03 2 2 3 2 + 3 +2019-10-01 3 3 4 3 + 4 +2019-10-02 4 4 5 4 + 5 +2019-10-03 5 5 6 5 + 6 +2019-10-01 6 6 7 6 + 7 +2019-10-02 7 7 8 7 + 8 +2019-10-03 8 8 9 8 + 9 +CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value4` String, \n `value5` String, \n `value3` String DEFAULT concat(value4, \' + \', value5)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +2019-10-01 0 0 1 0 + 1 +2019-10-02 1 1 2 1 + 2 +2019-10-03 2 2 3 2 + 3 +2019-10-01 3 3 4 3 + 4 +2019-10-02 4 4 5 4 + 5 +2019-10-03 5 5 6 5 + 6 +2019-10-01 6 6 7 6 + 7 +2019-10-02 7 7 8 7 + 8 +2019-10-03 8 8 9 8 + 9 +2019-10-01 0 0 1 0 + 1 +2019-10-02 1 1 2 1 + 2 +2019-10-03 2 2 3 2 + 3 +2019-10-01 3 3 4 3 + 4 +2019-10-02 4 4 5 4 + 5 +2019-10-03 5 5 6 5 + 6 +2019-10-01 6 6 7 6 + 7 +2019-10-02 7 7 8 7 + 8 +2019-10-03 8 8 9 8 + 9 +2019-10-02 10 10 11 10 + 11 +2019-10-03 11 11 12 11 + 12 +2019-10-01 12 12 13 12 + 13 +2019-10-02 13 13 14 13 + 14 +2019-10-03 14 14 15 14 + 15 +2019-10-01 15 15 16 15 + 16 +2019-10-02 16 16 17 16 + 17 +2019-10-03 17 17 18 17 + 18 +2019-10-01 18 18 19 18 + 19 +2019-10-02 19 19 20 19 + 20 +CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value1` String, \n `value2` String, \n `value3` String DEFAULT concat(value1, \' + \', value2)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +2019-10-01 0 0 1 0 + 1 +2019-10-02 1 1 2 1 + 2 +2019-10-03 2 2 3 2 + 3 +2019-10-01 3 3 4 3 + 4 +2019-10-02 4 4 5 4 + 5 +2019-10-03 5 5 6 5 + 6 +2019-10-01 6 6 7 6 + 7 +2019-10-02 7 7 8 7 + 8 +2019-10-03 8 8 9 8 + 9 +2019-10-02 10 10 11 10 + 11 +2019-10-03 11 11 12 11 + 12 +2019-10-01 12 12 13 12 + 13 +2019-10-02 13 13 14 13 + 14 +2019-10-03 14 14 15 14 + 15 +2019-10-01 15 15 16 15 + 16 +2019-10-02 16 16 17 16 + 17 +2019-10-03 17 17 18 17 + 18 +2019-10-01 18 18 19 18 + 19 +2019-10-02 19 19 20 19 + 20 +2019-10-01 0 0 1 0 + 1 +2019-10-02 1 1 2 1 + 2 +2019-10-03 2 2 3 2 + 3 +2019-10-01 3 3 4 3 + 4 +2019-10-02 4 4 5 4 + 5 +2019-10-03 5 5 6 5 + 6 +2019-10-01 6 6 7 6 + 7 +2019-10-02 7 7 8 7 + 8 +2019-10-03 8 8 9 8 + 9 +2019-10-02 10 10 11 10 + 11 +2019-10-03 11 11 12 11 + 12 +2019-10-01 12 12 13 12 + 13 +2019-10-02 13 13 14 13 + 14 +2019-10-03 14 14 15 14 + 15 +2019-10-01 15 15 16 15 + 16 +2019-10-02 16 16 17 16 + 17 +2019-10-03 17 17 18 17 + 18 +2019-10-01 18 18 19 18 + 19 +2019-10-02 19 19 20 19 + 20 +2019-10-03 20 20 21 20 + 21 +2019-10-01 21 21 22 21 + 22 +2019-10-02 22 22 23 22 + 23 +2019-10-03 23 23 24 23 + 24 +2019-10-01 24 24 25 24 + 25 +2019-10-02 25 25 26 25 + 26 +2019-10-03 26 26 27 26 + 27 +2019-10-01 27 27 28 27 + 28 +2019-10-02 28 28 29 28 + 29 +2019-10-03 29 29 30 29 + 30 diff --git a/tests/queries/0_stateless/01275_alter_rename_column_default_expr.sql b/tests/queries/0_stateless/01275_alter_rename_column_default_expr.sql new file mode 100644 index 00000000000..21106d200af --- /dev/null +++ b/tests/queries/0_stateless/01275_alter_rename_column_default_expr.sql @@ -0,0 +1,34 @@ +DROP TABLE IF EXISTS table_for_rename; + +CREATE TABLE table_for_rename +( + date Date, + key UInt64, + value1 String, + value2 String, + value3 String DEFAULT concat(value1, ' + ', value2) +) +ENGINE = MergeTree() +PARTITION BY date +ORDER BY key; + +INSERT INTO table_for_rename (date, key, value1, value2) SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1) from numbers(9); +SELECT * FROM table_for_rename ORDER BY key; + +ALTER TABLE table_for_rename RENAME COLUMN value1 TO value4; +ALTER TABLE table_for_rename RENAME COLUMN value2 TO value5; +SHOW CREATE TABLE table_for_rename; +SELECT * FROM table_for_rename ORDER BY key; + +INSERT INTO table_for_rename (date, key, value4, value5) SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1) from numbers(10, 10); +SELECT * FROM table_for_rename ORDER BY key; + +ALTER TABLE table_for_rename RENAME COLUMN value4 TO value1; +ALTER TABLE table_for_rename RENAME COLUMN value5 TO value2; +SHOW CREATE TABLE table_for_rename; +SELECT * FROM table_for_rename ORDER BY key; + +INSERT INTO table_for_rename (date, key, value1, value2) SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1) from numbers(20,10); +SELECT * FROM table_for_rename ORDER BY key; + +DROP TABLE IF EXISTS table_for_rename; diff --git a/tests/queries/0_stateless/01275_parallel_mv.reference b/tests/queries/0_stateless/01275_parallel_mv.reference new file mode 100644 index 00000000000..898d3f7266e --- /dev/null +++ b/tests/queries/0_stateless/01275_parallel_mv.reference @@ -0,0 +1,4 @@ +10 +10 +0 +10 diff --git a/tests/queries/0_stateless/01275_parallel_mv.sql b/tests/queries/0_stateless/01275_parallel_mv.sql new file mode 100644 index 00000000000..b67fbf02f8d --- /dev/null +++ b/tests/queries/0_stateless/01275_parallel_mv.sql @@ -0,0 +1,18 @@ +drop table if exists testX; +drop table if exists testXA; +drop table if exists testXB; +drop table if exists testXC; + +create table testX (A Int64) engine=MergeTree order by tuple(); + +create materialized view testXA engine=MergeTree order by tuple() as select sleep(1) from testX; +create materialized view testXB engine=MergeTree order by tuple() as select sleep(2), throwIf(A=1) from testX; +create materialized view testXC engine=MergeTree order by tuple() as select sleep(1) from testX; + +set parallel_view_processing=1; +insert into testX select number from numbers(10); -- {serverError 395} + +select count() from testX; +select count() from testXA; +select count() from testXB; +select count() from testXC; diff --git a/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.reference b/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.reference new file mode 100644 index 00000000000..5d721230db3 --- /dev/null +++ b/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.reference @@ -0,0 +1,90 @@ +2019-10-01 0 0 1 +2019-10-02 1 1 2 +2019-10-03 2 2 3 +2019-10-01 3 3 4 +2019-10-02 4 4 5 +2019-10-03 5 5 6 +2019-10-01 6 6 7 +2019-10-02 7 7 8 +2019-10-03 8 8 9 +CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value4` String, \n `value5` String, \n `value3` String MATERIALIZED concat(value4, \' + \', value5)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +2019-10-01 0 0 1 +2019-10-02 1 1 2 +2019-10-03 2 2 3 +2019-10-01 3 3 4 +2019-10-02 4 4 5 +2019-10-03 5 5 6 +2019-10-01 6 6 7 +2019-10-02 7 7 8 +2019-10-03 8 8 9 +-- insert after rename -- +2019-10-01 0 0 1 +2019-10-02 1 1 2 +2019-10-03 2 2 3 +2019-10-01 3 3 4 +2019-10-02 4 4 5 +2019-10-03 5 5 6 +2019-10-01 6 6 7 +2019-10-02 7 7 8 +2019-10-03 8 8 9 +2019-10-02 10 10 11 +2019-10-03 11 11 12 +2019-10-01 12 12 13 +2019-10-02 13 13 14 +2019-10-03 14 14 15 +2019-10-01 15 15 16 +2019-10-02 16 16 17 +2019-10-03 17 17 18 +2019-10-01 18 18 19 +2019-10-02 19 19 20 +-- rename columns back -- +CREATE TABLE default.table_for_rename\n(\n `date` Date, \n `key` UInt64, \n `value1` String, \n `value2` String, \n `value3` String MATERIALIZED concat(value1, \' + \', value2)\n)\nENGINE = MergeTree()\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +2019-10-01 0 0 1 +2019-10-02 1 1 2 +2019-10-03 2 2 3 +2019-10-01 3 3 4 +2019-10-02 4 4 5 +2019-10-03 5 5 6 +2019-10-01 6 6 7 +2019-10-02 7 7 8 +2019-10-03 8 8 9 +2019-10-02 10 10 11 +2019-10-03 11 11 12 +2019-10-01 12 12 13 +2019-10-02 13 13 14 +2019-10-03 14 14 15 +2019-10-01 15 15 16 +2019-10-02 16 16 17 +2019-10-03 17 17 18 +2019-10-01 18 18 19 +2019-10-02 19 19 20 +-- insert after rename column -- +2019-10-01 0 0 1 +2019-10-02 1 1 2 +2019-10-03 2 2 3 +2019-10-01 3 3 4 +2019-10-02 4 4 5 +2019-10-03 5 5 6 +2019-10-01 6 6 7 +2019-10-02 7 7 8 +2019-10-03 8 8 9 +2019-10-02 10 10 11 +2019-10-03 11 11 12 +2019-10-01 12 12 13 +2019-10-02 13 13 14 +2019-10-03 14 14 15 +2019-10-01 15 15 16 +2019-10-02 16 16 17 +2019-10-03 17 17 18 +2019-10-01 18 18 19 +2019-10-02 19 19 20 +2019-10-03 20 20 21 +2019-10-01 21 21 22 +2019-10-02 22 22 23 +2019-10-03 23 23 24 +2019-10-01 24 24 25 +2019-10-02 25 25 26 +2019-10-03 26 26 27 +2019-10-01 27 27 28 +2019-10-02 28 28 29 +2019-10-03 29 29 30 diff --git a/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.sql b/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.sql new file mode 100644 index 00000000000..9089c52edf6 --- /dev/null +++ b/tests/queries/0_stateless/01276_alter_rename_column_materialized_expr.sql @@ -0,0 +1,37 @@ +DROP TABLE IF EXISTS table_for_rename; + +CREATE TABLE table_for_rename +( + date Date, + key UInt64, + value1 String, + value2 String, + value3 String MATERIALIZED concat(value1, ' + ', value2) +) +ENGINE = MergeTree() +PARTITION BY date +ORDER BY key; + +INSERT INTO table_for_rename (date, key, value1, value2) SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1) from numbers(9); +SELECT * FROM table_for_rename ORDER BY key; + +ALTER TABLE table_for_rename RENAME COLUMN value1 TO value4; +ALTER TABLE table_for_rename RENAME COLUMN value2 TO value5; +SHOW CREATE TABLE table_for_rename; +SELECT * FROM table_for_rename ORDER BY key; + +SELECT '-- insert after rename --'; +INSERT INTO table_for_rename (date, key, value4, value5) SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1) from numbers(10, 10); +SELECT * FROM table_for_rename ORDER BY key; + +SELECT '-- rename columns back --'; +ALTER TABLE table_for_rename RENAME COLUMN value4 TO value1; +ALTER TABLE table_for_rename RENAME COLUMN value5 TO value2; +SHOW CREATE TABLE table_for_rename; +SELECT * FROM table_for_rename ORDER BY key; + +SELECT '-- insert after rename column --'; +INSERT INTO table_for_rename (date, key, value1, value2) SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number + 1) from numbers(20,10); +SELECT * FROM table_for_rename ORDER BY key; + +DROP TABLE IF EXISTS table_for_rename; diff --git a/tests/queries/0_stateless/01278_alter_rename_combination.reference b/tests/queries/0_stateless/01278_alter_rename_combination.reference new file mode 100644 index 00000000000..3f00378b4b7 --- /dev/null +++ b/tests/queries/0_stateless/01278_alter_rename_combination.reference @@ -0,0 +1,15 @@ +CREATE TABLE default.rename_table\n(\n `key` Int32, \n `old_value1` Int32, \n `value1` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +key old_value1 value1 +1 2 3 +CREATE TABLE default.rename_table\n(\n `k` Int32, \n `v1` Int32, \n `v2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +k v1 v2 +1 2 3 +4 5 6 +---polymorphic--- +CREATE TABLE default.rename_table_polymorphic\n(\n `key` Int32, \n `old_value1` Int32, \n `value1` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 10000, index_granularity = 8192 +key old_value1 value1 +1 2 3 +CREATE TABLE default.rename_table_polymorphic\n(\n `k` Int32, \n `v1` Int32, \n `v2` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS min_rows_for_wide_part = 10000, index_granularity = 8192 +k v1 v2 +1 2 3 +4 5 6 diff --git a/tests/queries/0_stateless/01278_alter_rename_combination.sql b/tests/queries/0_stateless/01278_alter_rename_combination.sql new file mode 100644 index 00000000000..fa73362622c --- /dev/null +++ b/tests/queries/0_stateless/01278_alter_rename_combination.sql @@ -0,0 +1,55 @@ +DROP TABLE IF EXISTS rename_table; + +CREATE TABLE rename_table (key Int32, value1 Int32, value2 Int32) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO rename_table VALUES (1, 2, 3); + +-- replace one with other +ALTER TABLE rename_table RENAME COLUMN value1 TO old_value1, RENAME COLUMN value2 TO value1; + +SHOW CREATE TABLE rename_table; + +SELECT * FROM rename_table FORMAT TSVWithNames; + +INSERT INTO rename_table VALUES (4, 5, 6); + +-- rename all columns simultaneously +ALTER TABLE rename_table RENAME COLUMN old_value1 TO v1, RENAME COLUMN value1 TO v2, RENAME COLUMN key to k; + +SHOW CREATE TABLE rename_table; + +SELECT * FROM rename_table ORDER BY k FORMAT TSVWithNames; + +DROP TABLE IF EXISTS rename_table; + +SELECT '---polymorphic---'; + +DROP TABLE IF EXISTS rename_table_polymorphic; + +CREATE TABLE rename_table_polymorphic ( + key Int32, + value1 Int32, + value2 Int32 +) +ENGINE = MergeTree +ORDER BY tuple() +SETTINGS min_rows_for_wide_part = 10000; + +INSERT INTO rename_table_polymorphic VALUES (1, 2, 3); + +ALTER TABLE rename_table_polymorphic RENAME COLUMN value1 TO old_value1, RENAME COLUMN value2 TO value1; + +SHOW CREATE TABLE rename_table_polymorphic; + +SELECT * FROM rename_table_polymorphic FORMAT TSVWithNames; + +INSERT INTO rename_table_polymorphic VALUES (4, 5, 6); + +-- rename all columns simultaneously +ALTER TABLE rename_table_polymorphic RENAME COLUMN old_value1 TO v1, RENAME COLUMN value1 TO v2, RENAME COLUMN key to k; + +SHOW CREATE TABLE rename_table_polymorphic; + +SELECT * FROM rename_table_polymorphic ORDER BY k FORMAT TSVWithNames; + +DROP TABLE IF EXISTS rename_table_polymorphic; diff --git a/tests/queries/0_stateless/01278_variance_nonnegative.reference b/tests/queries/0_stateless/01278_variance_nonnegative.reference new file mode 100644 index 00000000000..405d3348775 --- /dev/null +++ b/tests/queries/0_stateless/01278_variance_nonnegative.reference @@ -0,0 +1,8 @@ +0 +0 +0 +0 +0 +0 +0 +0 diff --git a/tests/queries/0_stateless/01278_variance_nonnegative.sql b/tests/queries/0_stateless/01278_variance_nonnegative.sql new file mode 100644 index 00000000000..aa676d8b269 --- /dev/null +++ b/tests/queries/0_stateless/01278_variance_nonnegative.sql @@ -0,0 +1,9 @@ +SELECT varSamp(0.1) FROM numbers(1000000); +SELECT varPop(0.1) FROM numbers(1000000); +SELECT stddevSamp(0.1) FROM numbers(1000000); +SELECT stddevPop(0.1) FROM numbers(1000000); + +SELECT varSampStable(0.1) FROM numbers(1000000); +SELECT varPopStable(0.1) FROM numbers(1000000); +SELECT stddevSampStable(0.1) FROM numbers(1000000); +SELECT stddevPopStable(0.1) FROM numbers(1000000); diff --git a/tests/queries/0_stateless/01280_null_in.reference b/tests/queries/0_stateless/01280_null_in.reference new file mode 100644 index 00000000000..04402256766 --- /dev/null +++ b/tests/queries/0_stateless/01280_null_in.reference @@ -0,0 +1,8 @@ +0 +0 +1 +1 +0 +0 +1 +1 diff --git a/tests/queries/0_stateless/01280_null_in.sql b/tests/queries/0_stateless/01280_null_in.sql new file mode 100644 index 00000000000..76fe4db6786 --- /dev/null +++ b/tests/queries/0_stateless/01280_null_in.sql @@ -0,0 +1,9 @@ +SELECT count(in(NULL, [])); +SELECT count(notIn(NULL, [])); +SELECT count(nullIn(NULL, [])); +SELECT count(notNullIn(NULL, [])); + +SELECT count(in(NULL, tuple(NULL))); +SELECT count(notIn(NULL, tuple(NULL))); +SELECT count(nullIn(NULL, tuple(NULL))); +SELECT count(notNullIn(NULL, tuple(NULL)));