mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-22 07:31:57 +00:00
Merge branch 'master' of github.com:yandex/ClickHouse
This commit is contained in:
Alexey Milovidov
commit
8766dab54e
@ -13,6 +13,9 @@ list(APPEND dirs ${dirs1})
|
||||
get_property (dirs1 TARGET cityhash PROPERTY INCLUDE_DIRECTORIES)
|
||||
list(APPEND dirs ${dirs1})
|
||||
|
||||
get_property (dirs1 TARGET roaring PROPERTY INCLUDE_DIRECTORIES)
|
||||
list(APPEND dirs ${dirs1})
|
||||
|
||||
if (USE_INTERNAL_BOOST_LIBRARY)
|
||||
get_property (dirs1 TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY} PROPERTY INCLUDE_DIRECTORIES)
|
||||
list(APPEND dirs ${dirs1})
|
||||
|
||||
1
contrib/CMakeLists.txt
vendored
1
contrib/CMakeLists.txt
vendored
@ -46,6 +46,7 @@ if (USE_INTERNAL_METROHASH_LIBRARY)
|
||||
endif ()
|
||||
|
||||
add_subdirectory (murmurhash)
|
||||
add_subdirectory (croaring)
|
||||
|
||||
if (USE_INTERNAL_BTRIE_LIBRARY)
|
||||
add_subdirectory (libbtrie)
|
||||
|
||||
@ -198,7 +198,7 @@ list(APPEND PARQUET_SRCS
|
||||
add_library(${PARQUET_LIBRARY} ${LINK_MODE} ${PARQUET_SRCS})
|
||||
target_include_directories(${PARQUET_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src)
|
||||
include(${ClickHouse_SOURCE_DIR}/contrib/thrift/build/cmake/ConfigureChecks.cmake) # makes config.h
|
||||
target_link_libraries(${PARQUET_LIBRARY} PRIVATE ${ARROW_LIBRARY} ${THRIFT_LIBRARY} ${Boost_REGEX_LIBRARY})
|
||||
target_link_libraries(${PARQUET_LIBRARY} PUBLIC ${ARROW_LIBRARY} PRIVATE ${THRIFT_LIBRARY} ${Boost_REGEX_LIBRARY})
|
||||
target_include_directories(${PARQUET_LIBRARY} PRIVATE ${Boost_INCLUDE_DIRS})
|
||||
|
||||
if(SANITIZE STREQUAL "undefined")
|
||||
|
||||
6
contrib/croaring/CMakeLists.txt
Normal file
6
contrib/croaring/CMakeLists.txt
Normal file
@ -0,0 +1,6 @@
|
||||
add_library(roaring
|
||||
roaring.c
|
||||
roaring.h
|
||||
roaring.hh)
|
||||
|
||||
target_include_directories (roaring PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
202
contrib/croaring/LICENSE
Normal file
202
contrib/croaring/LICENSE
Normal file
@ -0,0 +1,202 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "{}"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2016 The CRoaring authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
2
contrib/croaring/README.txt
Normal file
2
contrib/croaring/README.txt
Normal file
@ -0,0 +1,2 @@
|
||||
download from https://github.com/RoaringBitmap/CRoaring/archive/v0.2.57.tar.gz
|
||||
and use ./amalgamation.sh generate
|
||||
11093
contrib/croaring/roaring.c
Normal file
11093
contrib/croaring/roaring.c
Normal file
File diff suppressed because it is too large
Load Diff
7187
contrib/croaring/roaring.h
Normal file
7187
contrib/croaring/roaring.h
Normal file
File diff suppressed because it is too large
Load Diff
1732
contrib/croaring/roaring.hh
Normal file
1732
contrib/croaring/roaring.hh
Normal file
File diff suppressed because it is too large
Load Diff
@ -137,12 +137,10 @@ endif ()
|
||||
add_subdirectory(src/Common/ZooKeeper)
|
||||
add_subdirectory(src/Common/Config)
|
||||
|
||||
if (MAKE_STATIC_LIBRARIES)
|
||||
if (MAKE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES)
|
||||
add_library(dbms ${dbms_headers} ${dbms_sources})
|
||||
else ()
|
||||
add_library(dbms SHARED ${dbms_headers} ${dbms_sources})
|
||||
set_target_properties (dbms PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse)
|
||||
install (TARGETS dbms LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse)
|
||||
endif ()
|
||||
|
||||
if (USE_EMBEDDED_COMPILER)
|
||||
@ -201,6 +199,8 @@ target_link_libraries (clickhouse_common_io
|
||||
Threads::Threads
|
||||
PRIVATE
|
||||
${CMAKE_DL_LIBS}
|
||||
PUBLIC
|
||||
roaring
|
||||
)
|
||||
|
||||
target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR})
|
||||
|
||||
@ -7,7 +7,7 @@ option (ENABLE_CLICKHOUSE_SERVER "Enable clickhouse-server" ${ENABLE_CLICKHOUSE_
|
||||
option (ENABLE_CLICKHOUSE_CLIENT "Enable clickhouse-client" ${ENABLE_CLICKHOUSE_ALL})
|
||||
option (ENABLE_CLICKHOUSE_LOCAL "Enable clickhouse-local" ${ENABLE_CLICKHOUSE_ALL})
|
||||
option (ENABLE_CLICKHOUSE_BENCHMARK "Enable clickhouse-benchmark" ${ENABLE_CLICKHOUSE_ALL})
|
||||
option (ENABLE_CLICKHOUSE_PERFORMANCE "Enable clickhouse-performance-test" ${ENABLE_CLICKHOUSE_ALL})
|
||||
option (ENABLE_CLICKHOUSE_PERFORMANCE_TEST "Enable clickhouse-performance-test" ${ENABLE_CLICKHOUSE_ALL})
|
||||
option (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG "Enable clickhouse-extract-from-config" ${ENABLE_CLICKHOUSE_ALL})
|
||||
option (ENABLE_CLICKHOUSE_COMPRESSOR "Enable clickhouse-compressor" ${ENABLE_CLICKHOUSE_ALL})
|
||||
option (ENABLE_CLICKHOUSE_COPIER "Enable clickhouse-copier" ${ENABLE_CLICKHOUSE_ALL})
|
||||
@ -15,8 +15,64 @@ option (ENABLE_CLICKHOUSE_FORMAT "Enable clickhouse-format" ${ENABLE_CLICKHOUSE_
|
||||
option (ENABLE_CLICKHOUSE_OBFUSCATOR "Enable clickhouse-obfuscator" ${ENABLE_CLICKHOUSE_ALL})
|
||||
option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "Enable clickhouse-odbc-bridge" ${ENABLE_CLICKHOUSE_ALL})
|
||||
|
||||
if(NOT (MAKE_STATIC_LIBRARIES OR SPLIT_SHARED_LIBRARIES))
|
||||
set(CLICKHOUSE_ONE_SHARED 1)
|
||||
endif()
|
||||
|
||||
configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h)
|
||||
|
||||
|
||||
|
||||
macro(clickhouse_target_link_split_lib target name)
|
||||
if(NOT CLICKHOUSE_ONE_SHARED)
|
||||
target_link_libraries(${target} PRIVATE clickhouse-${name}-lib)
|
||||
else()
|
||||
target_link_libraries(${target} PRIVATE clickhouse-lib)
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
macro(clickhouse_program_link_split_binary name)
|
||||
clickhouse_target_link_split_lib(clickhouse-${name} ${name})
|
||||
endmacro()
|
||||
|
||||
macro(clickhouse_program_add_library name)
|
||||
string(TOUPPER ${name} name_uc)
|
||||
string(REPLACE "-" "_" name_uc ${name_uc})
|
||||
|
||||
# Some dark magic
|
||||
set(CLICKHOUSE_${name_uc}_SOURCES ${CLICKHOUSE_${name_uc}_SOURCES} PARENT_SCOPE)
|
||||
set(CLICKHOUSE_${name_uc}_LINK ${CLICKHOUSE_${name_uc}_LINK} PARENT_SCOPE)
|
||||
set(CLICKHOUSE_${name_uc}_INCLUDE ${CLICKHOUSE_${name_uc}_INCLUDE} PARENT_SCOPE)
|
||||
|
||||
if(NOT CLICKHOUSE_ONE_SHARED)
|
||||
add_library(clickhouse-${name}-lib ${LINK_MODE} ${CLICKHOUSE_${name_uc}_SOURCES})
|
||||
|
||||
set(_link ${CLICKHOUSE_${name_uc}_LINK}) # can't use ${} in if()
|
||||
if(_link)
|
||||
target_link_libraries(clickhouse-${name}-lib ${CLICKHOUSE_${name_uc}_LINK})
|
||||
endif()
|
||||
|
||||
set(_include ${CLICKHOUSE_${name_uc}_INCLUDE}) # can't use ${} in if()
|
||||
if (_include)
|
||||
target_include_directories(clickhouse-${name}-lib ${CLICKHOUSE_${name_uc}_INCLUDE})
|
||||
endif()
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
macro(clickhouse_program_add_executable name)
|
||||
if(CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable(clickhouse-${name} clickhouse-${name}.cpp)
|
||||
clickhouse_program_link_split_binary(${name})
|
||||
install(TARGETS clickhouse-${name} ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
macro(clickhouse_program_add name)
|
||||
clickhouse_program_add_library(${name})
|
||||
clickhouse_program_add_executable(${name})
|
||||
endmacro()
|
||||
|
||||
|
||||
add_subdirectory (server)
|
||||
add_subdirectory (client)
|
||||
add_subdirectory (local)
|
||||
@ -33,6 +89,13 @@ if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
|
||||
add_subdirectory (odbc-bridge)
|
||||
endif ()
|
||||
|
||||
if (CLICKHOUSE_ONE_SHARED)
|
||||
add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_PERFORMANCE_TEST_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_COMPILER_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES})
|
||||
target_link_libraries(clickhouse-lib PUBLIC ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_PERFORMANCE_TEST_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_COMPILER_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK})
|
||||
set_target_properties(clickhouse-lib PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse)
|
||||
target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_PERFORMANCE_TEST_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_COMPILER_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE})
|
||||
endif()
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-performance-test
|
||||
clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-copier)
|
||||
@ -71,7 +134,7 @@ else ()
|
||||
if (ENABLE_CLICKHOUSE_BENCHMARK)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse-benchmark-lib)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_PERFORMANCE)
|
||||
if (ENABLE_CLICKHOUSE_PERFORMANCE_TEST)
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse-performance-test-lib)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_COPIER)
|
||||
@ -114,7 +177,7 @@ else ()
|
||||
install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-benchmark DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-benchmark)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_PERFORMANCE)
|
||||
if (ENABLE_CLICKHOUSE_PERFORMANCE_TEST)
|
||||
add_custom_target (clickhouse-performance-test ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-performance-test DEPENDS clickhouse)
|
||||
install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-performance-test DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-performance-test)
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
add_library (clickhouse-benchmark-lib ${LINK_MODE} Benchmark.cpp)
|
||||
target_link_libraries (clickhouse-benchmark-lib PRIVATE clickhouse_aggregate_functions clickhouse-client-lib clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
target_include_directories (clickhouse-benchmark-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
|
||||
set(CLICKHOUSE_BENCHMARK_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Benchmark.cpp)
|
||||
set(CLICKHOUSE_BENCHMARK_LINK PRIVATE clickhouse_aggregate_functions clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
set(CLICKHOUSE_BENCHMARK_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-benchmark clickhouse-benchmark.cpp)
|
||||
target_link_libraries (clickhouse-benchmark PRIVATE clickhouse-benchmark-lib clickhouse_aggregate_functions)
|
||||
install (TARGETS clickhouse-benchmark ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif ()
|
||||
clickhouse_program_add(benchmark)
|
||||
|
||||
if(NOT CLICKHOUSE_ONE_SHARED)
|
||||
target_link_libraries (clickhouse-benchmark-lib PRIVATE clickhouse-client-lib)
|
||||
endif()
|
||||
|
||||
@ -1,13 +1,7 @@
|
||||
add_library (clickhouse-client-lib ${LINK_MODE} Client.cpp)
|
||||
target_link_libraries (clickhouse-client-lib PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
if (READLINE_INCLUDE_DIR)
|
||||
target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
|
||||
endif ()
|
||||
set(CLICKHOUSE_CLIENT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Client.cpp)
|
||||
set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
set(CLICKHOUSE_CLIENT_INCLUDE SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-client clickhouse-client.cpp)
|
||||
target_link_libraries (clickhouse-client PRIVATE clickhouse-client-lib)
|
||||
install (TARGETS clickhouse-client ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif ()
|
||||
clickhouse_program_add(client)
|
||||
|
||||
install (FILES clickhouse-client.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-client COMPONENT clickhouse-client RENAME config.xml)
|
||||
|
||||
@ -1,9 +1,7 @@
|
||||
add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp)
|
||||
target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_compression clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
# Also in utils
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
# Also in utils
|
||||
add_executable (clickhouse-compressor clickhouse-compressor.cpp)
|
||||
target_link_libraries (clickhouse-compressor PRIVATE clickhouse-compressor-lib)
|
||||
install (TARGETS clickhouse-compressor ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif ()
|
||||
set(CLICKHOUSE_COMPRESSOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Compressor.cpp)
|
||||
set(CLICKHOUSE_COMPRESSOR_LINK PRIVATE clickhouse_compression clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
#set(CLICKHOUSE_COMPRESSOR_INCLUDE SYSTEM PRIVATE ...)
|
||||
|
||||
clickhouse_program_add(compressor)
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
#cmakedefine01 ENABLE_CLICKHOUSE_CLIENT
|
||||
#cmakedefine01 ENABLE_CLICKHOUSE_LOCAL
|
||||
#cmakedefine01 ENABLE_CLICKHOUSE_BENCHMARK
|
||||
#cmakedefine01 ENABLE_CLICKHOUSE_PERFORMANCE
|
||||
#cmakedefine01 ENABLE_CLICKHOUSE_PERFORMANCE_TEST
|
||||
#cmakedefine01 ENABLE_CLICKHOUSE_COPIER
|
||||
#cmakedefine01 ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG
|
||||
#cmakedefine01 ENABLE_CLICKHOUSE_COMPRESSOR
|
||||
|
||||
@ -1,8 +1,5 @@
|
||||
add_library (clickhouse-copier-lib ${LINK_MODE} ClusterCopier.cpp)
|
||||
target_link_libraries (clickhouse-copier-lib PRIVATE clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions daemon)
|
||||
set(CLICKHOUSE_COPIER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ClusterCopier.cpp)
|
||||
set(CLICKHOUSE_COPIER_LINK PRIVATE clickhouse_functions clickhouse_aggregate_functions daemon)
|
||||
#set(CLICKHOUSE_COPIER_INCLUDE SYSTEM PRIVATE ...)
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-copier clickhouse-copier.cpp)
|
||||
target_link_libraries (clickhouse-copier clickhouse-copier-lib)
|
||||
install (TARGETS clickhouse-copier ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif ()
|
||||
clickhouse_program_add(copier)
|
||||
|
||||
@ -1,8 +1,5 @@
|
||||
add_library (clickhouse-extract-from-config-lib ${LINK_MODE} ExtractFromConfig.cpp)
|
||||
target_link_libraries (clickhouse-extract-from-config-lib PRIVATE clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
set(CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ExtractFromConfig.cpp)
|
||||
set(CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK PRIVATE clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
#set(CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE SYSTEM PRIVATE ...)
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-extract-from-config clickhouse-extract-from-config.cpp)
|
||||
target_link_libraries (clickhouse-extract-from-config PRIVATE clickhouse-extract-from-config-lib)
|
||||
install (TARGETS clickhouse-extract-from-config ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif ()
|
||||
clickhouse_program_add(extract-from-config)
|
||||
|
||||
@ -1,7 +1,5 @@
|
||||
add_library (clickhouse-format-lib ${LINK_MODE} Format.cpp)
|
||||
target_link_libraries (clickhouse-format-lib PRIVATE dbms clickhouse_common_io clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-format clickhouse-format.cpp)
|
||||
target_link_libraries (clickhouse-format PRIVATE clickhouse-format-lib)
|
||||
install (TARGETS clickhouse-format ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif ()
|
||||
set(CLICKHOUSE_FORMAT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Format.cpp)
|
||||
set(CLICKHOUSE_FORMAT_LINK PRIVATE dbms clickhouse_common_io clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
#set(CLICKHOUSE_FORMAT_INCLUDE SYSTEM PRIVATE ...)
|
||||
|
||||
clickhouse_program_add(format)
|
||||
|
||||
@ -1,8 +1,9 @@
|
||||
add_library (clickhouse-local-lib ${LINK_MODE} LocalServer.cpp)
|
||||
target_link_libraries (clickhouse-local-lib PRIVATE clickhouse_dictionaries clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
set(CLICKHOUSE_LOCAL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/LocalServer.cpp)
|
||||
set(CLICKHOUSE_LOCAL_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
#set(CLICKHOUSE_LOCAL_INCLUDE SYSTEM PRIVATE ...)
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-local clickhouse-local.cpp)
|
||||
target_link_libraries (clickhouse-local PRIVATE clickhouse-local-lib)
|
||||
install (TARGETS clickhouse-local ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
clickhouse_program_add(local)
|
||||
|
||||
if(NOT CLICKHOUSE_ONE_SHARED)
|
||||
target_link_libraries(clickhouse-local-lib PRIVATE clickhouse-server-lib)
|
||||
endif ()
|
||||
|
||||
@ -38,7 +38,7 @@ int mainEntryClickHouseLocal(int argc, char ** argv);
|
||||
#if ENABLE_CLICKHOUSE_BENCHMARK || !defined(ENABLE_CLICKHOUSE_BENCHMARK)
|
||||
int mainEntryClickHouseBenchmark(int argc, char ** argv);
|
||||
#endif
|
||||
#if ENABLE_CLICKHOUSE_PERFORMANCE || !defined(ENABLE_CLICKHOUSE_PERFORMANCE)
|
||||
#if ENABLE_CLICKHOUSE_PERFORMANCE_TEST || !defined(ENABLE_CLICKHOUSE_PERFORMANCE_TEST)
|
||||
int mainEntryClickHousePerformanceTest(int argc, char ** argv);
|
||||
#endif
|
||||
#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG || !defined(ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
|
||||
@ -84,7 +84,7 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
|
||||
#if ENABLE_CLICKHOUSE_SERVER || !defined(ENABLE_CLICKHOUSE_SERVER)
|
||||
{"server", mainEntryClickHouseServer},
|
||||
#endif
|
||||
#if ENABLE_CLICKHOUSE_PERFORMANCE || !defined(ENABLE_CLICKHOUSE_PERFORMANCE)
|
||||
#if ENABLE_CLICKHOUSE_PERFORMANCE_TEST || !defined(ENABLE_CLICKHOUSE_PERFORMANCE_TEST)
|
||||
{"performance-test", mainEntryClickHousePerformanceTest},
|
||||
#endif
|
||||
#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG || !defined(ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
|
||||
|
||||
@ -1,9 +1,5 @@
|
||||
add_library (clickhouse-obfuscator-lib ${LINK_MODE} Obfuscator.cpp)
|
||||
target_link_libraries (clickhouse-obfuscator-lib PRIVATE dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
set(CLICKHOUSE_OBFUSCATOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Obfuscator.cpp)
|
||||
set(CLICKHOUSE_OBFUSCATOR_LINK PRIVATE dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
#set(CLICKHOUSE_OBFUSCATOR_INCLUDE SYSTEM PRIVATE ...)
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-obfuscator clickhouse-obfuscator.cpp)
|
||||
set_target_properties(clickhouse-obfuscator PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
|
||||
target_link_libraries (clickhouse-obfuscator PRIVATE clickhouse-obfuscator-lib)
|
||||
install (TARGETS clickhouse-obfuscator ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif ()
|
||||
clickhouse_program_add(obfuscator)
|
||||
|
||||
@ -1,30 +1,36 @@
|
||||
add_headers_and_sources(clickhouse_odbc_bridge .)
|
||||
set(CLICKHOUSE_ODBC_BRIDGE_SOURCES
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/ColumnInfoHandler.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/getIdentifierQuote.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/HandlerFactory.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/IdentifierQuoteHandler.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/MainHandler.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/ODBCBlockInputStream.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/odbc-bridge.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/ODBCBridge.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/PingHandler.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/validateODBCConnectionString.cpp
|
||||
)
|
||||
|
||||
add_library (clickhouse-odbc-bridge-lib ${LINK_MODE} ${clickhouse_odbc_bridge_sources})
|
||||
|
||||
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE daemon dbms clickhouse_common_io)
|
||||
target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
|
||||
set(CLICKHOUSE_ODBC_BRIDGE_LINK PRIVATE daemon dbms clickhouse_common_io)
|
||||
set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
|
||||
|
||||
if (USE_POCO_SQLODBC)
|
||||
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQLODBC_LIBRARY})
|
||||
target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR})
|
||||
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQLODBC_LIBRARY})
|
||||
set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR})
|
||||
endif ()
|
||||
if (Poco_SQL_FOUND)
|
||||
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQL_LIBRARY})
|
||||
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQL_LIBRARY})
|
||||
endif ()
|
||||
|
||||
if (USE_POCO_DATAODBC)
|
||||
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_DataODBC_LIBRARY})
|
||||
target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
|
||||
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_DataODBC_LIBRARY})
|
||||
set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
|
||||
endif()
|
||||
if (Poco_Data_FOUND)
|
||||
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_Data_LIBRARY})
|
||||
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_Data_LIBRARY})
|
||||
endif ()
|
||||
|
||||
|
||||
if (ENABLE_TESTS)
|
||||
add_subdirectory (tests)
|
||||
endif ()
|
||||
clickhouse_program_add_library(odbc-bridge)
|
||||
|
||||
# clickhouse-odbc-bridge is always a separate binary.
|
||||
# Reason: it must not export symbols from SSL, mariadb-client, etc. to not break ABI compatibility with ODBC drivers.
|
||||
@ -32,5 +38,11 @@ endif ()
|
||||
SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "")
|
||||
|
||||
add_executable (clickhouse-odbc-bridge odbc-bridge.cpp)
|
||||
target_link_libraries (clickhouse-odbc-bridge PRIVATE clickhouse-odbc-bridge-lib)
|
||||
|
||||
clickhouse_program_link_split_binary(odbc-bridge)
|
||||
|
||||
install (TARGETS clickhouse-odbc-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
|
||||
if (ENABLE_TESTS)
|
||||
add_subdirectory (tests)
|
||||
endif ()
|
||||
|
||||
@ -1,2 +1,3 @@
|
||||
add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp)
|
||||
target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse-odbc-bridge-lib clickhouse_common_io)
|
||||
clickhouse_target_link_split_lib(validate-odbc-connection-string odbc-bridge)
|
||||
target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse_common_io)
|
||||
|
||||
@ -1,21 +1,18 @@
|
||||
add_library (clickhouse-performance-test-lib ${LINK_MODE}
|
||||
JSONString.cpp
|
||||
StopConditionsSet.cpp
|
||||
TestStopConditions.cpp
|
||||
TestStats.cpp
|
||||
ConfigPreprocessor.cpp
|
||||
PerformanceTest.cpp
|
||||
PerformanceTestInfo.cpp
|
||||
executeQuery.cpp
|
||||
applySubstitutions.cpp
|
||||
ReportBuilder.cpp
|
||||
PerformanceTestSuite.cpp
|
||||
)
|
||||
target_link_libraries (clickhouse-performance-test-lib PRIVATE dbms clickhouse_common_io clickhouse_common_config ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
target_include_directories (clickhouse-performance-test-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
|
||||
set(CLICKHOUSE_PERFORMANCE_TEST_SOURCES
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/JSONString.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/StopConditionsSet.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/TestStopConditions.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/TestStats.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/ConfigPreprocessor.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/PerformanceTest.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/PerformanceTestInfo.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/executeQuery.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/applySubstitutions.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/ReportBuilder.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/PerformanceTestSuite.cpp
|
||||
)
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-performance-test clickhouse-performance-test.cpp)
|
||||
target_link_libraries (clickhouse-performance-test PRIVATE clickhouse-performance-test-lib)
|
||||
install (TARGETS clickhouse-performance-test ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif ()
|
||||
set(CLICKHOUSE_PERFORMANCE_TEST_LINK PRIVATE dbms clickhouse_common_io clickhouse_common_config ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||
set(CLICKHOUSE_PERFORMANCE_TEST_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
|
||||
|
||||
clickhouse_program_add(performance-test)
|
||||
|
||||
@ -1,27 +1,22 @@
|
||||
add_library (clickhouse-server-lib ${LINK_MODE}
|
||||
HTTPHandler.cpp
|
||||
InterserverIOHTTPHandler.cpp
|
||||
MetricsTransmitter.cpp
|
||||
NotFoundHandler.cpp
|
||||
PingRequestHandler.cpp
|
||||
ReplicasStatusHandler.cpp
|
||||
RootRequestHandler.cpp
|
||||
Server.cpp
|
||||
TCPHandler.cpp
|
||||
)
|
||||
set(CLICKHOUSE_SERVER_SOURCES
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/HTTPHandler.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/InterserverIOHTTPHandler.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/MetricsTransmitter.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/NotFoundHandler.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/PingRequestHandler.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/ReplicasStatusHandler.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/RootRequestHandler.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/Server.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/TCPHandler.cpp
|
||||
)
|
||||
|
||||
target_link_libraries (clickhouse-server-lib PRIVATE clickhouse_dictionaries clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY})
|
||||
set(CLICKHOUSE_SERVER_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY})
|
||||
if (USE_POCO_NETSSL)
|
||||
target_link_libraries (clickhouse-server-lib PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
|
||||
set(CLICKHOUSE_SERVER_LINK ${CLICKHOUSE_SERVER_LINK} PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
|
||||
endif ()
|
||||
set(CLICKHOUSE_SERVER_INCLUDE PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
|
||||
|
||||
target_include_directories (clickhouse-server-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
add_executable (clickhouse-server clickhouse-server.cpp)
|
||||
target_link_libraries (clickhouse-server PRIVATE clickhouse-server-lib)
|
||||
install (TARGETS clickhouse-server ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif ()
|
||||
clickhouse_program_add(server)
|
||||
|
||||
if (GLIBC_COMPATIBILITY)
|
||||
set (GLIBC_MAX_REQUIRED 2.4 CACHE INTERNAL "")
|
||||
@ -31,7 +26,4 @@ if (GLIBC_COMPATIBILITY)
|
||||
#add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-server | grep '@GLIBC' | grep -oP 'GLIBC_[\\d\\.]+' | sort | uniq | sort --version-sort --reverse | perl -lnE 'warn($_), exit 1 if $_ gt q{GLIBC_${GLIBC_MAX_REQUIRED}}'") # old
|
||||
endif ()
|
||||
|
||||
install (
|
||||
FILES config.xml users.xml
|
||||
DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server
|
||||
COMPONENT clickhouse)
|
||||
install(FILES config.xml users.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server COMPONENT clickhouse)
|
||||
|
||||
40
dbms/src/AggregateFunctions/AggregateFunctionGroupBitmap.cpp
Normal file
40
dbms/src/AggregateFunctions/AggregateFunctionGroupBitmap.cpp
Normal file
@ -0,0 +1,40 @@
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <AggregateFunctions/AggregateFunctionGroupBitmap.h>
|
||||
#include <AggregateFunctions/Helpers.h>
|
||||
#include <AggregateFunctions/FactoryHelpers.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
template <template <typename> class Data>
|
||||
AggregateFunctionPtr createAggregateFunctionBitmap(const std::string & name, const DataTypes & argument_types, const Array & parameters)
|
||||
{
|
||||
assertNoParameters(name, parameters);
|
||||
assertUnary(name, argument_types);
|
||||
|
||||
if (!argument_types[0]->canBeUsedInBitOperations())
|
||||
throw Exception("The type " + argument_types[0]->getName() + " of argument for aggregate function " + name
|
||||
+ " is illegal, because it cannot be used in Bitmap operations",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
AggregateFunctionPtr res(createWithUnsignedIntegerType<AggregateFunctionBitmap, Data>(*argument_types[0], argument_types[0]));
|
||||
|
||||
if (!res)
|
||||
throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void registerAggregateFunctionsBitmap(AggregateFunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction("groupBitmap", createAggregateFunctionBitmap<AggregateFunctionGroupBitmapData>);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
56
dbms/src/AggregateFunctions/AggregateFunctionGroupBitmap.h
Normal file
56
dbms/src/AggregateFunctions/AggregateFunctionGroupBitmap.h
Normal file
@ -0,0 +1,56 @@
|
||||
#pragma once
|
||||
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <boost/noncopyable.hpp>
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
#include <AggregateFunctions/AggregateFunctionGroupBitmapData.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Counts bitmap operation on numbers.
|
||||
template <typename T, typename Data>
|
||||
class AggregateFunctionBitmap final : public IAggregateFunctionDataHelper<Data, AggregateFunctionBitmap<T, Data>>
|
||||
{
|
||||
public:
|
||||
AggregateFunctionBitmap(const DataTypePtr & type)
|
||||
: IAggregateFunctionDataHelper<Data, AggregateFunctionBitmap<T, Data>>({type}, {}) {}
|
||||
|
||||
String getName() const override { return Data::name(); }
|
||||
|
||||
DataTypePtr getReturnType() const override
|
||||
{
|
||||
return std::make_shared<DataTypeNumber<T>>();
|
||||
}
|
||||
|
||||
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
|
||||
{
|
||||
this->data(place).rbs.add(static_cast<const ColumnVector<T> &>(*columns[0]).getData()[row_num]);
|
||||
}
|
||||
|
||||
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
|
||||
{
|
||||
this->data(place).rbs.merge(this->data(rhs).rbs);
|
||||
}
|
||||
|
||||
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
|
||||
{
|
||||
this->data(place).rbs.write(buf);
|
||||
}
|
||||
|
||||
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
|
||||
{
|
||||
this->data(place).rbs.read(buf);
|
||||
}
|
||||
|
||||
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
|
||||
{
|
||||
static_cast<ColumnVector<T> &>(to).getData().push_back(this->data(place).rbs.size());
|
||||
}
|
||||
|
||||
const char * getHeaderFilePath() const override { return __FILE__; }
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
516
dbms/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h
Normal file
516
dbms/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h
Normal file
@ -0,0 +1,516 @@
|
||||
#pragma once
|
||||
|
||||
#include <roaring.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <boost/noncopyable.hpp>
|
||||
#include <roaring.hh>
|
||||
#include <Common/HashTable/SmallTable.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
/**
|
||||
* For a small number of values - an array of fixed size "on the stack".
|
||||
* For large, roaring_bitmap_t is allocated.
|
||||
* For a description of the roaring_bitmap_t, see: https://github.com/RoaringBitmap/CRoaring
|
||||
*/
|
||||
template <typename T, UInt8 small_set_size>
|
||||
class RoaringBitmapWithSmallSet : private boost::noncopyable
|
||||
{
|
||||
private:
|
||||
using Small = SmallSet<T, small_set_size>;
|
||||
using ValueBuffer = std::vector<T>;
|
||||
Small small;
|
||||
roaring_bitmap_t * rb = nullptr;
|
||||
|
||||
void toLarge()
|
||||
{
|
||||
rb = roaring_bitmap_create();
|
||||
|
||||
for (const auto & x : small)
|
||||
roaring_bitmap_add(rb, x.getValue());
|
||||
}
|
||||
|
||||
public:
|
||||
bool isLarge() const { return rb != nullptr; }
|
||||
|
||||
bool isSmall() const { return rb == nullptr; }
|
||||
|
||||
~RoaringBitmapWithSmallSet()
|
||||
{
|
||||
if (isLarge())
|
||||
roaring_bitmap_free(rb);
|
||||
}
|
||||
|
||||
void add(T value)
|
||||
{
|
||||
if (isSmall())
|
||||
{
|
||||
if (small.find(value) == small.end())
|
||||
{
|
||||
if (!small.full())
|
||||
small.insert(value);
|
||||
else
|
||||
{
|
||||
toLarge();
|
||||
roaring_bitmap_add(rb, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
roaring_bitmap_add(rb, value);
|
||||
}
|
||||
|
||||
UInt64 size() const { return isSmall() ? small.size() : roaring_bitmap_get_cardinality(rb); }
|
||||
|
||||
void merge(const RoaringBitmapWithSmallSet & r1)
|
||||
{
|
||||
if (r1.isLarge())
|
||||
{
|
||||
if (isSmall())
|
||||
toLarge();
|
||||
|
||||
roaring_bitmap_or_inplace(rb, r1.rb);
|
||||
}
|
||||
else
|
||||
{
|
||||
for (const auto & x : r1.small)
|
||||
add(x.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
void read(DB::ReadBuffer & in)
|
||||
{
|
||||
bool is_large;
|
||||
readBinary(is_large, in);
|
||||
|
||||
if (is_large)
|
||||
{
|
||||
toLarge();
|
||||
UInt32 cardinality;
|
||||
readBinary(cardinality, in);
|
||||
db_roaring_bitmap_add_many(in, rb, cardinality);
|
||||
}
|
||||
else
|
||||
small.read(in);
|
||||
}
|
||||
|
||||
void write(DB::WriteBuffer & out) const
|
||||
{
|
||||
writeBinary(isLarge(), out);
|
||||
|
||||
if (isLarge())
|
||||
{
|
||||
UInt32 cardinality = roaring_bitmap_get_cardinality(rb);
|
||||
writePODBinary(cardinality, out);
|
||||
db_ra_to_uint32_array(out, &rb->high_low_container);
|
||||
}
|
||||
else
|
||||
small.write(out);
|
||||
}
|
||||
|
||||
|
||||
roaring_bitmap_t * getRb() const { return rb; }
|
||||
|
||||
Small & getSmall() const { return small; }
|
||||
|
||||
/**
|
||||
* Get a new roaring_bitmap_t from elements of small
|
||||
*/
|
||||
roaring_bitmap_t * getNewRbFromSmall() const
|
||||
{
|
||||
roaring_bitmap_t * smallRb = roaring_bitmap_create();
|
||||
for (const auto & x : small)
|
||||
roaring_bitmap_add(smallRb, x.getValue());
|
||||
return smallRb;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the intersection between two bitmaps
|
||||
*/
|
||||
void rb_and(const RoaringBitmapWithSmallSet & r1)
|
||||
{
|
||||
ValueBuffer buffer;
|
||||
if (isSmall() && r1.isSmall())
|
||||
{
|
||||
// intersect
|
||||
for (const auto & x : small)
|
||||
if (r1.small.find(x.getValue()) != r1.small.end())
|
||||
buffer.push_back(x.getValue());
|
||||
|
||||
// Clear out the original values
|
||||
small.clear();
|
||||
|
||||
for (const auto & value : buffer)
|
||||
small.insert(value);
|
||||
|
||||
buffer.clear();
|
||||
}
|
||||
else if (isSmall() && r1.isLarge())
|
||||
{
|
||||
for (const auto & x : small)
|
||||
if (roaring_bitmap_contains(r1.rb, x.getValue()))
|
||||
buffer.push_back(x.getValue());
|
||||
|
||||
// Clear out the original values
|
||||
small.clear();
|
||||
|
||||
for (const auto & value : buffer)
|
||||
small.insert(value);
|
||||
|
||||
buffer.clear();
|
||||
}
|
||||
else
|
||||
{
|
||||
roaring_bitmap_t * rb1 = r1.isSmall() ? r1.getNewRbFromSmall() : r1.getRb();
|
||||
roaring_bitmap_and_inplace(rb, rb1);
|
||||
if (r1.isSmall())
|
||||
roaring_bitmap_free(rb1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the union between two bitmaps.
|
||||
*/
|
||||
void rb_or(const RoaringBitmapWithSmallSet & r1) { merge(r1); }
|
||||
|
||||
/**
|
||||
* Computes the symmetric difference (xor) between two bitmaps.
|
||||
*/
|
||||
void rb_xor(const RoaringBitmapWithSmallSet & r1)
|
||||
{
|
||||
if (isSmall())
|
||||
toLarge();
|
||||
roaring_bitmap_t * rb1 = r1.isSmall() ? r1.getNewRbFromSmall() : r1.getRb();
|
||||
roaring_bitmap_xor_inplace(rb, rb1);
|
||||
if (r1.isSmall())
|
||||
roaring_bitmap_free(rb1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the difference (andnot) between two bitmaps
|
||||
*/
|
||||
void rb_andnot(const RoaringBitmapWithSmallSet & r1)
|
||||
{
|
||||
ValueBuffer buffer;
|
||||
if (isSmall() && r1.isSmall())
|
||||
{
|
||||
// subtract
|
||||
for (const auto & x : small)
|
||||
if (r1.small.find(x.getValue()) == r1.small.end())
|
||||
buffer.push_back(x.getValue());
|
||||
|
||||
// Clear out the original values
|
||||
small.clear();
|
||||
|
||||
for (const auto & value : buffer)
|
||||
small.insert(value);
|
||||
|
||||
buffer.clear();
|
||||
}
|
||||
else if (isSmall() && r1.isLarge())
|
||||
{
|
||||
for (const auto & x : small)
|
||||
if (!roaring_bitmap_contains(r1.rb, x.getValue()))
|
||||
buffer.push_back(x.getValue());
|
||||
|
||||
// Clear out the original values
|
||||
small.clear();
|
||||
|
||||
for (const auto & value : buffer)
|
||||
small.insert(value);
|
||||
|
||||
buffer.clear();
|
||||
}
|
||||
else
|
||||
{
|
||||
roaring_bitmap_t * rb1 = r1.isSmall() ? r1.getNewRbFromSmall() : r1.getRb();
|
||||
roaring_bitmap_andnot_inplace(rb, rb1);
|
||||
if (r1.isSmall())
|
||||
roaring_bitmap_free(rb1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the cardinality of the intersection between two bitmaps.
|
||||
*/
|
||||
UInt64 rb_and_cardinality(const RoaringBitmapWithSmallSet & r1) const
|
||||
{
|
||||
UInt64 retSize = 0;
|
||||
if (isSmall() && r1.isSmall())
|
||||
{
|
||||
for (const auto & x : small)
|
||||
if (r1.small.find(x.getValue()) != r1.small.end())
|
||||
retSize++;
|
||||
}
|
||||
else if (isSmall() && r1.isLarge())
|
||||
{
|
||||
for (const auto & x : small)
|
||||
if (roaring_bitmap_contains(r1.rb, x.getValue()))
|
||||
retSize++;
|
||||
}
|
||||
else
|
||||
{
|
||||
roaring_bitmap_t * rb1 = r1.isSmall() ? r1.getNewRbFromSmall() : r1.getRb();
|
||||
retSize = roaring_bitmap_and_cardinality(rb, rb1);
|
||||
if (r1.isSmall())
|
||||
roaring_bitmap_free(rb1);
|
||||
}
|
||||
return retSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the cardinality of the union between two bitmaps.
|
||||
*/
|
||||
UInt64 rb_or_cardinality(const RoaringBitmapWithSmallSet & r1) const
|
||||
{
|
||||
UInt64 c1 = size();
|
||||
UInt64 c2 = r1.size();
|
||||
UInt64 inter = rb_and_cardinality(r1);
|
||||
return c1 + c2 - inter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the cardinality of the symmetric difference (andnot) between two bitmaps.
|
||||
*/
|
||||
UInt64 rb_xor_cardinality(const RoaringBitmapWithSmallSet & r1) const
|
||||
{
|
||||
UInt64 c1 = size();
|
||||
UInt64 c2 = r1.size();
|
||||
UInt64 inter = rb_and_cardinality(r1);
|
||||
return c1 + c2 - 2 * inter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the cardinality of the difference (andnot) between two bitmaps.
|
||||
*/
|
||||
UInt64 rb_andnot_cardinality(const RoaringBitmapWithSmallSet & r1) const
|
||||
{
|
||||
UInt64 c1 = size();
|
||||
UInt64 inter = rb_and_cardinality(r1);
|
||||
return c1 - inter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return 1 if the two bitmaps contain the same elements.
|
||||
*/
|
||||
UInt8 rb_equals(const RoaringBitmapWithSmallSet & r1)
|
||||
{
|
||||
if (isSmall())
|
||||
toLarge();
|
||||
roaring_bitmap_t * rb1 = r1.isSmall() ? r1.getNewRbFromSmall() : r1.getRb();
|
||||
UInt8 is_true = roaring_bitmap_equals(rb, rb1);
|
||||
if (r1.isSmall())
|
||||
roaring_bitmap_free(rb1);
|
||||
return is_true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether two bitmaps intersect.
|
||||
*/
|
||||
UInt8 rb_intersect(const RoaringBitmapWithSmallSet & r1)
|
||||
{
|
||||
if (isSmall())
|
||||
toLarge();
|
||||
roaring_bitmap_t * rb1 = r1.isSmall() ? r1.getNewRbFromSmall() : r1.getRb();
|
||||
UInt8 is_true = roaring_bitmap_intersect(rb, rb1);
|
||||
if (r1.isSmall())
|
||||
roaring_bitmap_free(rb1);
|
||||
return is_true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove value
|
||||
*/
|
||||
void rb_remove(UInt64 offsetid)
|
||||
{
|
||||
if (isSmall())
|
||||
toLarge();
|
||||
roaring_bitmap_remove(rb, offsetid);
|
||||
}
|
||||
|
||||
/**
|
||||
* compute (in place) the negation of the roaring bitmap within a specified
|
||||
* interval: [range_start, range_end). The number of negated values is
|
||||
* range_end - range_start.
|
||||
* Areas outside the range are passed through unchanged.
|
||||
*/
|
||||
void rb_flip(UInt64 offsetstart, UInt64 offsetend)
|
||||
{
|
||||
if (isSmall())
|
||||
toLarge();
|
||||
roaring_bitmap_flip_inplace(rb, offsetstart, offsetend);
|
||||
}
|
||||
|
||||
/**
|
||||
* returns the number of integers that are smaller or equal to offsetid.
|
||||
*/
|
||||
UInt64 rb_rank(UInt64 offsetid)
|
||||
{
|
||||
if (isSmall())
|
||||
toLarge();
|
||||
return roaring_bitmap_rank(rb, offsetid);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert elements to integer array, return number of elements
|
||||
*/
|
||||
template <typename Element>
|
||||
UInt64 rb_to_array(PaddedPODArray<Element> & res_data) const
|
||||
{
|
||||
UInt64 count = 0;
|
||||
if (isSmall())
|
||||
{
|
||||
for (const auto & x : small)
|
||||
{
|
||||
res_data.emplace_back(x.getValue());
|
||||
count++;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
roaring_uint32_iterator_t iterator;
|
||||
roaring_init_iterator(rb, &iterator);
|
||||
while (iterator.has_value)
|
||||
{
|
||||
res_data.emplace_back(iterator.current_value);
|
||||
roaring_advance_uint32_iterator(&iterator);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private:
|
||||
/// To read and write the DB Buffer directly, migrate code from CRoaring
|
||||
void db_roaring_bitmap_add_many(DB::ReadBuffer & dbBuf, roaring_bitmap_t * r, size_t n_args)
|
||||
{
|
||||
void * container = NULL; // hold value of last container touched
|
||||
uint8_t typecode = 0; // typecode of last container touched
|
||||
uint32_t prev = 0; // previous valued inserted
|
||||
size_t i = 0; // index of value
|
||||
int containerindex = 0;
|
||||
if (n_args == 0)
|
||||
return;
|
||||
uint32_t val;
|
||||
readBinary(val, dbBuf);
|
||||
container = containerptr_roaring_bitmap_add(r, val, &typecode, &containerindex);
|
||||
prev = val;
|
||||
i++;
|
||||
for (; i < n_args; i++)
|
||||
{
|
||||
readBinary(val, dbBuf);
|
||||
if (((prev ^ val) >> 16) == 0)
|
||||
{ // no need to seek the container, it is at hand
|
||||
// because we already have the container at hand, we can do the
|
||||
// insertion
|
||||
// automatically, bypassing the roaring_bitmap_add call
|
||||
uint8_t newtypecode = typecode;
|
||||
void * container2 = container_add(container, val & 0xFFFF, typecode, &newtypecode);
|
||||
// rare instance when we need to
|
||||
if (container2 != container)
|
||||
{
|
||||
// change the container type
|
||||
container_free(container, typecode);
|
||||
ra_set_container_at_index(&r->high_low_container, containerindex, container2, newtypecode);
|
||||
typecode = newtypecode;
|
||||
container = container2;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
container = containerptr_roaring_bitmap_add(r, val, &typecode, &containerindex);
|
||||
}
|
||||
prev = val;
|
||||
}
|
||||
}
|
||||
|
||||
void db_ra_to_uint32_array(DB::WriteBuffer & dbBuf, roaring_array_t * ra) const
|
||||
{
|
||||
size_t ctr = 0;
|
||||
for (Int32 i = 0; i < ra->size; ++i)
|
||||
{
|
||||
Int32 num_added = db_container_to_uint32_array(dbBuf, ra->containers[i], ra->typecodes[i], ((UInt32)ra->keys[i]) << 16);
|
||||
ctr += num_added;
|
||||
}
|
||||
}
|
||||
|
||||
UInt32 db_container_to_uint32_array(DB::WriteBuffer & dbBuf, const void * container, UInt8 typecode, UInt32 base) const
|
||||
{
|
||||
container = container_unwrap_shared(container, &typecode);
|
||||
switch (typecode)
|
||||
{
|
||||
case BITSET_CONTAINER_TYPE_CODE:
|
||||
return db_bitset_container_to_uint32_array(dbBuf, (const bitset_container_t *)container, base);
|
||||
case ARRAY_CONTAINER_TYPE_CODE:
|
||||
return db_array_container_to_uint32_array(dbBuf, (const array_container_t *)container, base);
|
||||
case RUN_CONTAINER_TYPE_CODE:
|
||||
return db_run_container_to_uint32_array(dbBuf, (const run_container_t *)container, base);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
UInt32 db_bitset_container_to_uint32_array(DB::WriteBuffer & dbBuf, const bitset_container_t * cont, UInt32 base) const
|
||||
{
|
||||
return (UInt32)db_bitset_extract_setbits(dbBuf, cont->array, BITSET_CONTAINER_SIZE_IN_WORDS, base);
|
||||
}
|
||||
|
||||
size_t db_bitset_extract_setbits(DB::WriteBuffer & dbBuf, UInt64 * bitset, size_t length, UInt32 base) const
|
||||
{
|
||||
UInt32 outpos = 0;
|
||||
for (size_t i = 0; i < length; ++i)
|
||||
{
|
||||
UInt64 w = bitset[i];
|
||||
while (w != 0)
|
||||
{
|
||||
UInt64 t = w & (~w + 1); // on x64, should compile to BLSI (careful: the Intel compiler seems to fail)
|
||||
UInt32 r = __builtin_ctzll(w); // on x64, should compile to TZCNT
|
||||
UInt32 val = r + base;
|
||||
writePODBinary(val, dbBuf);
|
||||
outpos++;
|
||||
w ^= t;
|
||||
}
|
||||
base += 64;
|
||||
}
|
||||
return outpos;
|
||||
}
|
||||
|
||||
int db_array_container_to_uint32_array(DB::WriteBuffer & dbBuf, const array_container_t * cont, UInt32 base) const
|
||||
{
|
||||
UInt32 outpos = 0;
|
||||
for (Int32 i = 0; i < cont->cardinality; ++i)
|
||||
{
|
||||
const UInt32 val = base + cont->array[i];
|
||||
writePODBinary(val, dbBuf);
|
||||
outpos++;
|
||||
}
|
||||
return outpos;
|
||||
}
|
||||
|
||||
int db_run_container_to_uint32_array(DB::WriteBuffer & dbBuf, const run_container_t * cont, UInt32 base) const
|
||||
{
|
||||
UInt32 outpos = 0;
|
||||
for (Int32 i = 0; i < cont->n_runs; ++i)
|
||||
{
|
||||
UInt32 run_start = base + cont->runs[i].value;
|
||||
UInt16 le = cont->runs[i].length;
|
||||
for (Int32 j = 0; j <= le; ++j)
|
||||
{
|
||||
UInt32 val = run_start + j;
|
||||
writePODBinary(val, dbBuf);
|
||||
outpos++;
|
||||
}
|
||||
}
|
||||
return outpos;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct AggregateFunctionGroupBitmapData
|
||||
{
|
||||
RoaringBitmapWithSmallSet<T, 32> rbs;
|
||||
static const char * name() { return "groupBitmap"; }
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
@ -26,6 +26,7 @@ void registerAggregateFunctionUniqCombined(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionTopK(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsBitwise(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsBitmap(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsMaxIntersections(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionEntropy(AggregateFunctionFactory &);
|
||||
|
||||
@ -63,6 +64,7 @@ void registerAggregateFunctions()
|
||||
registerAggregateFunctionUniqUpTo(factory);
|
||||
registerAggregateFunctionTopK(factory);
|
||||
registerAggregateFunctionsBitwise(factory);
|
||||
registerAggregateFunctionsBitmap(factory);
|
||||
registerAggregateFunctionsMaxIntersections(factory);
|
||||
registerAggregateFunctionHistogram(factory);
|
||||
registerAggregateFunctionRetention(factory);
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
#include <Common/Arena.h>
|
||||
#include <Common/SipHash.h>
|
||||
#include <Common/memcpySmall.h>
|
||||
#include <Common/memcmpSmall.h>
|
||||
|
||||
#include <DataStreams/ColumnGathererStream.h>
|
||||
|
||||
@ -106,8 +107,7 @@ struct ColumnFixedString::less
|
||||
explicit less(const ColumnFixedString & parent_) : parent(parent_) {}
|
||||
bool operator()(size_t lhs, size_t rhs) const
|
||||
{
|
||||
/// TODO: memcmp slows down.
|
||||
int res = memcmp(&parent.chars[lhs * parent.n], &parent.chars[rhs * parent.n], parent.n);
|
||||
int res = memcmpSmallAllowOverflow15(parent.chars.data() + lhs * parent.n, parent.chars.data() + rhs * parent.n, parent.n);
|
||||
return positive ? (res < 0) : (res > 0);
|
||||
}
|
||||
};
|
||||
|
||||
@ -1,8 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <string.h> // memcmp
|
||||
|
||||
#include <Common/PODArray.h>
|
||||
#include <Common/memcmpSmall.h>
|
||||
#include <Columns/IColumn.h>
|
||||
#include <Columns/ColumnVectorHelper.h>
|
||||
|
||||
@ -98,7 +97,7 @@ public:
|
||||
int compareAt(size_t p1, size_t p2, const IColumn & rhs_, int /*nan_direction_hint*/) const override
|
||||
{
|
||||
const ColumnFixedString & rhs = static_cast<const ColumnFixedString &>(rhs_);
|
||||
return memcmp(&chars[p1 * n], &rhs.chars[p2 * n], n);
|
||||
return memcmpSmallAllowOverflow15(chars.data() + p1 * n, rhs.chars.data() + p2 * n, n);
|
||||
}
|
||||
|
||||
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#include <Core/Defines.h>
|
||||
#include <Common/Arena.h>
|
||||
#include <Common/memcmpSmall.h>
|
||||
#include <Columns/Collator.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnsCommon.h>
|
||||
@ -239,15 +240,11 @@ struct ColumnString::less
|
||||
explicit less(const ColumnString & parent_) : parent(parent_) {}
|
||||
bool operator()(size_t lhs, size_t rhs) const
|
||||
{
|
||||
size_t left_len = parent.sizeAt(lhs);
|
||||
size_t right_len = parent.sizeAt(rhs);
|
||||
int res = memcmpSmallAllowOverflow15(
|
||||
parent.chars.data() + parent.offsetAt(lhs), parent.sizeAt(lhs) - 1,
|
||||
parent.chars.data() + parent.offsetAt(rhs), parent.sizeAt(rhs) - 1);
|
||||
|
||||
int res = memcmp(&parent.chars[parent.offsetAt(lhs)], &parent.chars[parent.offsetAt(rhs)], std::min(left_len, right_len));
|
||||
|
||||
if (res != 0)
|
||||
return positive ? (res < 0) : (res > 0);
|
||||
else
|
||||
return positive ? (left_len < right_len) : (left_len > right_len);
|
||||
return positive ? (res < 0) : (res > 0);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#include <Common/PODArray.h>
|
||||
#include <Common/SipHash.h>
|
||||
#include <Common/memcpySmall.h>
|
||||
#include <Common/memcmpSmall.h>
|
||||
|
||||
|
||||
class Collator;
|
||||
@ -210,16 +211,7 @@ public:
|
||||
int compareAt(size_t n, size_t m, const IColumn & rhs_, int /*nan_direction_hint*/) const override
|
||||
{
|
||||
const ColumnString & rhs = static_cast<const ColumnString &>(rhs_);
|
||||
|
||||
const size_t size = sizeAt(n);
|
||||
const size_t rhs_size = rhs.sizeAt(m);
|
||||
|
||||
int cmp = memcmp(&chars[offsetAt(n)], &rhs.chars[rhs.offsetAt(m)], std::min(size, rhs_size));
|
||||
|
||||
if (cmp != 0)
|
||||
return cmp;
|
||||
else
|
||||
return size > rhs_size ? 1 : (size < rhs_size ? -1 : 0);
|
||||
return memcmpSmallAllowOverflow15(chars.data() + offsetAt(n), sizeAt(n) - 1, rhs.chars.data() + rhs.offsetAt(m), rhs.sizeAt(m) - 1);
|
||||
}
|
||||
|
||||
/// Variant of compareAt for string comparison with respect of collation.
|
||||
|
||||
239
dbms/src/Common/memcmpSmall.h
Normal file
239
dbms/src/Common/memcmpSmall.h
Normal file
@ -0,0 +1,239 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <algorithm>
|
||||
|
||||
#ifdef __SSE2__
|
||||
#include <emmintrin.h>
|
||||
|
||||
|
||||
namespace detail
|
||||
{
|
||||
|
||||
template <typename T>
|
||||
inline int cmp(T a, T b)
|
||||
{
|
||||
if (a < b)
|
||||
return -1;
|
||||
if (a > b)
|
||||
return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/** All functions works under the following assumptions:
|
||||
* - it's possible to read up to 15 excessive bytes after end of 'a' and 'b' region;
|
||||
* - memory regions are relatively small and extra loop unrolling is not worth to do.
|
||||
*/
|
||||
|
||||
/** Variant when memory regions may have different sizes.
|
||||
*/
|
||||
template <typename Char>
|
||||
inline int memcmpSmallAllowOverflow15(const Char * a, size_t a_size, const Char * b, size_t b_size)
|
||||
{
|
||||
size_t min_size = std::min(a_size, b_size);
|
||||
|
||||
for (size_t offset = 0; offset < min_size; offset += 16)
|
||||
{
|
||||
uint16_t mask = _mm_movemask_epi8(_mm_cmpeq_epi8(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(a + offset)),
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(b + offset))));
|
||||
mask = ~mask;
|
||||
|
||||
if (mask)
|
||||
{
|
||||
offset += __builtin_ctz(mask);
|
||||
|
||||
if (offset >= min_size)
|
||||
break;
|
||||
|
||||
return detail::cmp(a[offset], b[offset]);
|
||||
}
|
||||
}
|
||||
|
||||
return detail::cmp(a_size, b_size);
|
||||
}
|
||||
|
||||
|
||||
/** Variant when memory regions have same size.
|
||||
* TODO Check if the compiler can optimize previous function when the caller pass identical sizes.
|
||||
*/
|
||||
template <typename Char>
|
||||
inline int memcmpSmallAllowOverflow15(const Char * a, const Char * b, size_t size)
|
||||
{
|
||||
for (size_t offset = 0; offset < size; offset += 16)
|
||||
{
|
||||
uint16_t mask = _mm_movemask_epi8(_mm_cmpeq_epi8(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(a + offset)),
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(b + offset))));
|
||||
mask = ~mask;
|
||||
|
||||
if (mask)
|
||||
{
|
||||
offset += __builtin_ctz(mask);
|
||||
|
||||
if (offset >= size)
|
||||
return 0;
|
||||
|
||||
return detail::cmp(a[offset], b[offset]);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/** Compare memory regions for equality.
|
||||
*/
|
||||
template <typename Char>
|
||||
inline bool memequalSmallAllowOverflow15(const Char * a, size_t a_size, const Char * b, size_t b_size)
|
||||
{
|
||||
if (a_size != b_size)
|
||||
return false;
|
||||
|
||||
for (size_t offset = 0; offset < a_size; offset += 16)
|
||||
{
|
||||
uint16_t mask = _mm_movemask_epi8(_mm_cmpeq_epi8(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(a + offset)),
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(b + offset))));
|
||||
mask = ~mask;
|
||||
|
||||
if (mask)
|
||||
{
|
||||
offset += __builtin_ctz(mask);
|
||||
return offset >= a_size;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
/** Variant when the caller know in advance that the size is a multiple of 16.
|
||||
*/
|
||||
template <typename Char>
|
||||
inline int memcmpSmallMultipleOf16(const Char * a, const Char * b, size_t size)
|
||||
{
|
||||
for (size_t offset = 0; offset < size; offset += 16)
|
||||
{
|
||||
uint16_t mask = _mm_movemask_epi8(_mm_cmpeq_epi8(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(a + offset)),
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(b + offset))));
|
||||
mask = ~mask;
|
||||
|
||||
if (mask)
|
||||
{
|
||||
offset += __builtin_ctz(mask);
|
||||
return detail::cmp(a[offset], b[offset]);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/** Variant when the size is 16 exactly.
|
||||
*/
|
||||
template <typename Char>
|
||||
inline int memcmp16(const Char * a, const Char * b)
|
||||
{
|
||||
uint16_t mask = _mm_movemask_epi8(_mm_cmpeq_epi8(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(a)),
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(b))));
|
||||
mask = ~mask;
|
||||
|
||||
if (mask)
|
||||
{
|
||||
auto offset = __builtin_ctz(mask);
|
||||
return detail::cmp(a[offset], b[offset]);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/** Variant when the size is 16 exactly.
|
||||
*/
|
||||
inline bool memequal16(const void * a, const void * b)
|
||||
{
|
||||
return 0xFFFF == _mm_movemask_epi8(_mm_cmpeq_epi8(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(a)),
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(b))));
|
||||
}
|
||||
|
||||
|
||||
/** Compare memory region to zero */
|
||||
inline bool memoryIsZeroSmallAllowOverflow15(const void * data, size_t size)
|
||||
{
|
||||
const __m128i zero16 = _mm_setzero_si128();
|
||||
|
||||
for (size_t offset = 0; offset < size; offset += 16)
|
||||
{
|
||||
uint16_t mask = _mm_movemask_epi8(_mm_cmpeq_epi8(zero16,
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(reinterpret_cast<const char *>(data) + offset))));
|
||||
mask = ~mask;
|
||||
|
||||
if (mask)
|
||||
{
|
||||
offset += __builtin_ctz(mask);
|
||||
return offset >= size;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
#else
|
||||
|
||||
#include <cstring>
|
||||
|
||||
template <typename Char>
|
||||
inline int memcmpSmallAllowOverflow15(const Char * a, size_t a_size, const Char * b, size_t b_size)
|
||||
{
|
||||
return memcmp(a, b, std::min(a_size, b_size));
|
||||
}
|
||||
|
||||
template <typename Char>
|
||||
inline int memcmpSmallAllowOverflow15(const Char * a, const Char * b, size_t size)
|
||||
{
|
||||
return memcmp(a, b, size);
|
||||
}
|
||||
|
||||
template <typename Char>
|
||||
inline bool memequalSmallAllowOverflow15(const Char * a, size_t a_size, const Char * b, size_t b_size)
|
||||
{
|
||||
return a_size == b_size && 0 == memcmp(a, b, a_size);
|
||||
}
|
||||
|
||||
template <typename Char>
|
||||
inline int memcmpSmallMultipleOf16(const Char * a, const Char * b, size_t size)
|
||||
{
|
||||
return memcmp(a, b, size);
|
||||
}
|
||||
|
||||
template <typename Char>
|
||||
inline int memcmp16(const Char * a, const Char * b)
|
||||
{
|
||||
return memcmp(a, b, 16);
|
||||
}
|
||||
|
||||
inline bool memequal16(const void * a, const void * b)
|
||||
{
|
||||
return 0 == memcmp(a, b, 16);
|
||||
}
|
||||
|
||||
inline bool memoryIsZeroSmallAllowOverflow15(const void * data, size_t size)
|
||||
{
|
||||
const char * pos = reinterpret_cast<const char *>(data);
|
||||
const char * end = pos + size;
|
||||
|
||||
for (; pos < end; ++pos)
|
||||
if (*pos)
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
#endif
|
||||
@ -1,7 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <string.h>
|
||||
#include <Core/Defines.h>
|
||||
|
||||
#ifdef __SSE2__
|
||||
#include <emmintrin.h>
|
||||
|
||||
@ -3,6 +3,7 @@ add_headers_and_sources(clickhouse_compression .)
|
||||
add_library(clickhouse_compression ${LINK_MODE} ${clickhouse_compression_headers} ${clickhouse_compression_sources})
|
||||
target_link_libraries(clickhouse_compression PRIVATE clickhouse_parsers clickhouse_common_io ${ZSTD_LIBRARY} ${LZ4_LIBRARY})
|
||||
target_include_directories(clickhouse_compression PUBLIC ${DBMS_INCLUDE_DIR})
|
||||
target_include_directories(clickhouse_compression SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR})
|
||||
|
||||
if (NOT USE_INTERNAL_LZ4_LIBRARY)
|
||||
target_include_directories(clickhouse_compression SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR})
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
#include <cstring>
|
||||
|
||||
#include <Common/memcmpSmall.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
|
||||
@ -38,11 +39,9 @@ struct EmptyImpl
|
||||
|
||||
static void vector_fixed_to_vector(const ColumnString::Chars & data, size_t n, PaddedPODArray<UInt8> & res)
|
||||
{
|
||||
std::vector<char> empty_chars(n);
|
||||
size_t size = data.size() / n;
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
res[i] = negative ^ (0 == memcmp(&data[i * n], empty_chars.data(), n));
|
||||
res[i] = negative ^ memoryIsZeroSmallAllowOverflow15(data.data() + i * n, n);
|
||||
}
|
||||
|
||||
static void array(const ColumnString::Offsets & offsets, PaddedPODArray<UInt8> & res)
|
||||
|
||||
25
dbms/src/Functions/FunctionsBitmap.cpp
Normal file
25
dbms/src/Functions/FunctionsBitmap.cpp
Normal file
@ -0,0 +1,25 @@
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/FunctionsBitmap.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
void registerFunctionsBitmap(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionBitmapBuild>();
|
||||
factory.registerFunction<FunctionBitmapToArray>();
|
||||
|
||||
factory.registerFunction<FunctionBitmapSelfCardinality>();
|
||||
factory.registerFunction<FunctionBitmapAndCardinality>();
|
||||
factory.registerFunction<FunctionBitmapOrCardinality>();
|
||||
factory.registerFunction<FunctionBitmapXorCardinality>();
|
||||
factory.registerFunction<FunctionBitmapAndnotCardinality>();
|
||||
|
||||
factory.registerFunction<FunctionBitmapAnd>();
|
||||
factory.registerFunction<FunctionBitmapOr>();
|
||||
factory.registerFunction<FunctionBitmapXor>();
|
||||
factory.registerFunction<FunctionBitmapAndnot>();
|
||||
|
||||
}
|
||||
}
|
||||
593
dbms/src/Functions/FunctionsBitmap.h
Normal file
593
dbms/src/Functions/FunctionsBitmap.h
Normal file
@ -0,0 +1,593 @@
|
||||
#pragma once
|
||||
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <AggregateFunctions/AggregateFunctionGroupBitmapData.h>
|
||||
#include <Columns/ColumnAggregateFunction.h>
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Columns/ColumnFunction.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <DataTypes/DataTypeAggregateFunction.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
}
|
||||
|
||||
/** Bitmap functions.
|
||||
* Build a bitmap from integer array:
|
||||
* bitmapBuild: integer[] -> bitmap
|
||||
*
|
||||
* Convert bitmap to integer array:
|
||||
* bitmapToArray: bitmap -> integer[]
|
||||
*
|
||||
* Two bitmap and calculation:
|
||||
* bitmapAnd: bitmap,bitmap -> bitmap
|
||||
*
|
||||
* Two bitmap or calculation:
|
||||
* bitmapOr: bitmap,bitmap -> bitmap
|
||||
*
|
||||
* Two bitmap xor calculation:
|
||||
* bitmapXor: bitmap,bitmap -> bitmap
|
||||
*
|
||||
* Two bitmap andnot calculation:
|
||||
* bitmapAndnot: bitmap,bitmap -> bitmap
|
||||
*
|
||||
* Retrun bitmap cardinality:
|
||||
* bitmapCardinality: bitmap -> integer
|
||||
*
|
||||
* Two bitmap and calculation, return cardinality:
|
||||
* bitmapAndCardinality: bitmap,bitmap -> integer
|
||||
*
|
||||
* Two bitmap or calculation, return cardinality:
|
||||
* bitmapOrCardinality: bitmap,bitmap -> integer
|
||||
*
|
||||
* Two bitmap xor calculation, return cardinality:
|
||||
* bitmapXorCardinality: bitmap,bitmap -> integer
|
||||
*
|
||||
* Two bitmap andnot calculation, return cardinality:
|
||||
* bitmapAndnotCardinality: bitmap,bitmap -> integer
|
||||
*/
|
||||
|
||||
template <typename Name>
|
||||
class FunctionBitmapBuildImpl : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = Name::name;
|
||||
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionBitmapBuildImpl>(); }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
bool isVariadic() const override { return false; }
|
||||
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
if (arguments[0]->onlyNull())
|
||||
return arguments[0];
|
||||
|
||||
auto array_type = typeid_cast<const DataTypeArray *>(arguments[0].get());
|
||||
if (!array_type)
|
||||
throw Exception(
|
||||
"First argument for function " + getName() + " must be an array but it has type " + arguments[0]->getName() + ".",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
auto nested_type = array_type->getNestedType();
|
||||
DataTypes argument_types = {nested_type};
|
||||
Array params_row;
|
||||
AggregateFunctionPtr bitmap_function
|
||||
= AggregateFunctionFactory::instance().get(AggregateFunctionGroupBitmapData<UInt32>::name(), argument_types, params_row);
|
||||
|
||||
return std::make_shared<DataTypeAggregateFunction>(bitmap_function, argument_types, params_row);
|
||||
}
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /* input_rows_count */) override
|
||||
{
|
||||
const IDataType * from_type = block.getByPosition(arguments[0]).type.get();
|
||||
auto array_type = typeid_cast<const DataTypeArray *>(from_type);
|
||||
auto nested_type = array_type->getNestedType();
|
||||
|
||||
DataTypes argument_types = {nested_type};
|
||||
|
||||
WhichDataType which(nested_type);
|
||||
if (which.isUInt8())
|
||||
executeBitmapData<UInt8>(block, argument_types, arguments, result);
|
||||
else if (which.isUInt16())
|
||||
executeBitmapData<UInt16>(block, argument_types, arguments, result);
|
||||
else if (which.isUInt32())
|
||||
executeBitmapData<UInt32>(block, argument_types, arguments, result);
|
||||
else if (which.isUInt64())
|
||||
executeBitmapData<UInt64>(block, argument_types, arguments, result);
|
||||
else
|
||||
throw Exception(
|
||||
"Unexpected type " + from_type->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
|
||||
private:
|
||||
template <typename T>
|
||||
void executeBitmapData(Block & block, DataTypes & argument_types, const ColumnNumbers & arguments, size_t result)
|
||||
{
|
||||
// input data
|
||||
const ColumnArray * array = typeid_cast<const ColumnArray *>(block.getByPosition(arguments[0]).column.get());
|
||||
ColumnPtr mapped = array->getDataPtr();
|
||||
const ColumnArray::Offsets & offsets = array->getOffsets();
|
||||
const ColumnVector<T> * column = checkAndGetColumn<ColumnVector<T>>(&*mapped);
|
||||
const typename ColumnVector<T>::Container & input_data = column->getData();
|
||||
|
||||
// output data
|
||||
Array params_row;
|
||||
AggregateFunctionPtr bitmap_function
|
||||
= AggregateFunctionFactory::instance().get(AggregateFunctionGroupBitmapData<UInt32>::name(), argument_types, params_row);
|
||||
auto col_to = ColumnAggregateFunction::create(bitmap_function);
|
||||
col_to->reserve(offsets.size());
|
||||
|
||||
size_t pos = 0;
|
||||
for (size_t i = 0; i < offsets.size(); ++i)
|
||||
{
|
||||
col_to->insertDefault();
|
||||
AggregateFunctionGroupBitmapData<T> & bitmap_data
|
||||
= *reinterpret_cast<AggregateFunctionGroupBitmapData<T> *>(col_to->getData()[i]);
|
||||
for (; pos < offsets[i]; ++pos)
|
||||
{
|
||||
bitmap_data.rbs.add(input_data[pos]);
|
||||
}
|
||||
}
|
||||
block.getByPosition(result).column = std::move(col_to);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Name>
|
||||
class FunctionBitmapToArrayImpl : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = Name::name;
|
||||
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionBitmapToArrayImpl>(); }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
bool isVariadic() const override { return false; }
|
||||
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
const DataTypeAggregateFunction * bitmap_type = typeid_cast<const DataTypeAggregateFunction *>(arguments[0].get());
|
||||
if (!(bitmap_type && bitmap_type->getFunctionName() == AggregateFunctionGroupBitmapData<UInt32>::name()))
|
||||
throw Exception(
|
||||
"First argument for function " + getName() + " must be an bitmap but it has type " + arguments[0]->getName() + ".",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
const DataTypePtr data_type = bitmap_type->getArgumentsDataTypes()[0];
|
||||
|
||||
return std::make_shared<DataTypeArray>(data_type);
|
||||
}
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
|
||||
{
|
||||
// input data
|
||||
const auto & return_type = block.getByPosition(result).type;
|
||||
auto res_ptr = return_type->createColumn();
|
||||
ColumnArray & res = static_cast<ColumnArray &>(*res_ptr);
|
||||
|
||||
IColumn & res_data = res.getData();
|
||||
ColumnArray::Offsets & res_offsets = res.getOffsets();
|
||||
|
||||
const IDataType * from_type = block.getByPosition(arguments[0]).type.get();
|
||||
const DataTypeAggregateFunction * aggr_type = typeid_cast<const DataTypeAggregateFunction *>(from_type);
|
||||
WhichDataType which(aggr_type->getArgumentsDataTypes()[0]);
|
||||
if (which.isUInt8())
|
||||
executeIntType<UInt8>(block, arguments, input_rows_count, res_data, res_offsets);
|
||||
else if (which.isUInt16())
|
||||
executeIntType<UInt16>(block, arguments, input_rows_count, res_data, res_offsets);
|
||||
else if (which.isUInt32())
|
||||
executeIntType<UInt32>(block, arguments, input_rows_count, res_data, res_offsets);
|
||||
else if (which.isUInt64())
|
||||
executeIntType<UInt64>(block, arguments, input_rows_count, res_data, res_offsets);
|
||||
else
|
||||
throw Exception(
|
||||
"Unexpected type " + from_type->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
block.getByPosition(result).column = std::move(res_ptr);
|
||||
}
|
||||
|
||||
private:
|
||||
using ToType = UInt64;
|
||||
|
||||
template <typename T>
|
||||
void executeIntType(
|
||||
Block & block, const ColumnNumbers & arguments, size_t input_rows_count, IColumn & res_data_col, ColumnArray::Offsets & res_offsets)
|
||||
const
|
||||
{
|
||||
const ColumnAggregateFunction * column
|
||||
= typeid_cast<const ColumnAggregateFunction *>(block.getByPosition(arguments[0]).column.get());
|
||||
|
||||
PaddedPODArray<T> & res_data = typeid_cast<ColumnVector<T> &>(res_data_col).getData();
|
||||
ColumnArray::Offset res_offset = 0;
|
||||
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
const AggregateFunctionGroupBitmapData<T> & bd1
|
||||
= *reinterpret_cast<const AggregateFunctionGroupBitmapData<T> *>(column->getData()[i]);
|
||||
UInt64 count = bd1.rbs.rb_to_array(res_data);
|
||||
res_offset += count;
|
||||
res_offsets.emplace_back(res_offset);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Name>
|
||||
class FunctionBitmapSelfCardinalityImpl : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = Name::name;
|
||||
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionBitmapSelfCardinalityImpl>(); }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
bool isVariadic() const override { return false; }
|
||||
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
auto bitmap_type = typeid_cast<const DataTypeAggregateFunction *>(arguments[0].get());
|
||||
if (!(bitmap_type && bitmap_type->getFunctionName() == AggregateFunctionGroupBitmapData<UInt32>::name()))
|
||||
throw Exception(
|
||||
"First argument for function " + getName() + " must be an bitmap but it has type " + arguments[0]->getName() + ".",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
return std::make_shared<DataTypeNumber<ToType>>();
|
||||
}
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
|
||||
{
|
||||
auto col_to = ColumnVector<ToType>::create(input_rows_count);
|
||||
typename ColumnVector<ToType>::Container & vec_to = col_to->getData();
|
||||
const IDataType * from_type = block.getByPosition(arguments[0]).type.get();
|
||||
|
||||
const DataTypeAggregateFunction * aggr_type = typeid_cast<const DataTypeAggregateFunction *>(from_type);
|
||||
WhichDataType which(aggr_type->getArgumentsDataTypes()[0]);
|
||||
if (which.isUInt8())
|
||||
executeIntType<UInt8>(block, arguments, input_rows_count, vec_to);
|
||||
else if (which.isUInt16())
|
||||
executeIntType<UInt16>(block, arguments, input_rows_count, vec_to);
|
||||
else if (which.isUInt32())
|
||||
executeIntType<UInt32>(block, arguments, input_rows_count, vec_to);
|
||||
else if (which.isUInt64())
|
||||
executeIntType<UInt64>(block, arguments, input_rows_count, vec_to);
|
||||
else
|
||||
throw Exception(
|
||||
"Unexpected type " + from_type->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
block.getByPosition(result).column = std::move(col_to);
|
||||
}
|
||||
|
||||
private:
|
||||
using ToType = UInt64;
|
||||
|
||||
template <typename T>
|
||||
void executeIntType(
|
||||
Block & block, const ColumnNumbers & arguments, size_t input_rows_count, typename ColumnVector<ToType>::Container & vec_to)
|
||||
{
|
||||
const ColumnAggregateFunction * column
|
||||
= typeid_cast<const ColumnAggregateFunction *>(block.getByPosition(arguments[0]).column.get());
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
const AggregateFunctionGroupBitmapData<T> & bd1
|
||||
= *reinterpret_cast<const AggregateFunctionGroupBitmapData<T> *>(column->getData()[i]);
|
||||
vec_to[i] = bd1.rbs.size();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct BitmapAndCardinalityImpl
|
||||
{
|
||||
using ReturnType = UInt64;
|
||||
static UInt64 apply(const AggregateFunctionGroupBitmapData<T> & bd1, const AggregateFunctionGroupBitmapData<T> & bd2)
|
||||
{
|
||||
// roaring_bitmap_and_cardinality( rb1, rb2 );
|
||||
return bd1.rbs.rb_and_cardinality(bd2.rbs);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template <typename T>
|
||||
struct BitmapOrCardinalityImpl
|
||||
{
|
||||
using ReturnType = UInt64;
|
||||
static UInt64 apply(const AggregateFunctionGroupBitmapData<T> & bd1, const AggregateFunctionGroupBitmapData<T> & bd2)
|
||||
{
|
||||
// return roaring_bitmap_or_cardinality( rb1, rb2 );
|
||||
return bd1.rbs.rb_or_cardinality(bd2.rbs);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct BitmapXorCardinalityImpl
|
||||
{
|
||||
using ReturnType = UInt64;
|
||||
static UInt64 apply(const AggregateFunctionGroupBitmapData<T> & bd1, const AggregateFunctionGroupBitmapData<T> & bd2)
|
||||
{
|
||||
// return roaring_bitmap_xor_cardinality( rb1, rb2 );
|
||||
return bd1.rbs.rb_xor_cardinality(bd2.rbs);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct BitmapAndnotCardinalityImpl
|
||||
{
|
||||
using ReturnType = UInt64;
|
||||
static UInt64 apply(const AggregateFunctionGroupBitmapData<T> & bd1, const AggregateFunctionGroupBitmapData<T> & bd2)
|
||||
{
|
||||
// roaring_bitmap_andnot_cardinality( rb1, rb2 );
|
||||
return bd1.rbs.rb_andnot_cardinality(bd2.rbs);
|
||||
}
|
||||
};
|
||||
|
||||
template <template <typename> class Impl, typename Name>
|
||||
class FunctionBitmapCardinality : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = Name::name;
|
||||
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionBitmapCardinality>(); }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
bool isVariadic() const override { return false; }
|
||||
|
||||
size_t getNumberOfArguments() const override { return 2; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
auto bitmap_type0 = typeid_cast<const DataTypeAggregateFunction *>(arguments[0].get());
|
||||
if (!(bitmap_type0 && bitmap_type0->getFunctionName() == AggregateFunctionGroupBitmapData<UInt32>::name()))
|
||||
throw Exception(
|
||||
"First argument for function " + getName() + " must be an bitmap but it has type " + arguments[0]->getName() + ".",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
auto bitmap_type1 = typeid_cast<const DataTypeAggregateFunction *>(arguments[1].get());
|
||||
if (!(bitmap_type1 && bitmap_type1->getFunctionName() == AggregateFunctionGroupBitmapData<UInt32>::name()))
|
||||
throw Exception(
|
||||
"Second argument for function " + getName() + " must be an bitmap but it has type " + arguments[1]->getName() + ".",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
return std::make_shared<DataTypeNumber<ToType>>();
|
||||
}
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
|
||||
{
|
||||
auto col_to = ColumnVector<ToType>::create(input_rows_count);
|
||||
typename ColumnVector<ToType>::Container & vec_to = col_to->getData();
|
||||
const IDataType * from_type = block.getByPosition(arguments[0]).type.get();
|
||||
|
||||
const DataTypeAggregateFunction * aggr_type = typeid_cast<const DataTypeAggregateFunction *>(from_type);
|
||||
WhichDataType which(aggr_type->getArgumentsDataTypes()[0]);
|
||||
if (which.isUInt8())
|
||||
executeIntType<UInt8>(block, arguments, input_rows_count, vec_to);
|
||||
else if (which.isUInt16())
|
||||
executeIntType<UInt16>(block, arguments, input_rows_count, vec_to);
|
||||
else if (which.isUInt32())
|
||||
executeIntType<UInt32>(block, arguments, input_rows_count, vec_to);
|
||||
else if (which.isUInt64())
|
||||
executeIntType<UInt64>(block, arguments, input_rows_count, vec_to);
|
||||
else
|
||||
throw Exception(
|
||||
"Unexpected type " + from_type->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
block.getByPosition(result).column = std::move(col_to);
|
||||
}
|
||||
|
||||
private:
|
||||
using ToType = UInt64;
|
||||
|
||||
template <typename T>
|
||||
void executeIntType(
|
||||
Block & block, const ColumnNumbers & arguments, size_t input_rows_count, typename ColumnVector<ToType>::Container & vec_to)
|
||||
{
|
||||
const ColumnAggregateFunction * columns[2];
|
||||
for (size_t i = 0; i < 2; ++i)
|
||||
columns[i] = typeid_cast<const ColumnAggregateFunction *>(block.getByPosition(arguments[i]).column.get());
|
||||
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
const AggregateFunctionGroupBitmapData<T> & bd1
|
||||
= *reinterpret_cast<const AggregateFunctionGroupBitmapData<T> *>(columns[0]->getData()[i]);
|
||||
const AggregateFunctionGroupBitmapData<T> & bd2
|
||||
= *reinterpret_cast<const AggregateFunctionGroupBitmapData<T> *>(columns[1]->getData()[i]);
|
||||
vec_to[i] = Impl<T>::apply(bd1, bd2);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct BitmapAndImpl
|
||||
{
|
||||
static void apply(AggregateFunctionGroupBitmapData<T> & toBd, const AggregateFunctionGroupBitmapData<T> & bd2)
|
||||
{
|
||||
toBd.rbs.rb_and(bd2.rbs);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct BitmapOrImpl
|
||||
{
|
||||
static void apply(AggregateFunctionGroupBitmapData<T> & toBd, const AggregateFunctionGroupBitmapData<T> & bd2)
|
||||
{
|
||||
toBd.rbs.rb_or(bd2.rbs);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct BitmapXorImpl
|
||||
{
|
||||
static void apply(AggregateFunctionGroupBitmapData<T> & toBd, const AggregateFunctionGroupBitmapData<T> & bd2)
|
||||
{
|
||||
toBd.rbs.rb_xor(bd2.rbs);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct BitmapAndnotImpl
|
||||
{
|
||||
static void apply(AggregateFunctionGroupBitmapData<T> & toBd, const AggregateFunctionGroupBitmapData<T> & bd2)
|
||||
{
|
||||
toBd.rbs.rb_andnot(bd2.rbs);
|
||||
}
|
||||
};
|
||||
|
||||
template <template <typename> class Impl, typename Name>
|
||||
class FunctionBitmap : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = Name::name;
|
||||
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionBitmap>(); }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
bool isVariadic() const override { return false; }
|
||||
|
||||
size_t getNumberOfArguments() const override { return 2; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
auto bitmap_type0 = typeid_cast<const DataTypeAggregateFunction *>(arguments[0].get());
|
||||
if (!(bitmap_type0 && bitmap_type0->getFunctionName() == AggregateFunctionGroupBitmapData<UInt32>::name()))
|
||||
throw Exception(
|
||||
"First argument for function " + getName() + " must be an bitmap but it has type " + arguments[0]->getName() + ".",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
auto bitmap_type1 = typeid_cast<const DataTypeAggregateFunction *>(arguments[1].get());
|
||||
if (!(bitmap_type1 && bitmap_type1->getFunctionName() == AggregateFunctionGroupBitmapData<UInt32>::name()))
|
||||
throw Exception(
|
||||
"Second argument for function " + getName() + " must be an bitmap but it has type " + arguments[1]->getName() + ".",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
return arguments[0];
|
||||
}
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
|
||||
{
|
||||
const IDataType * from_type = block.getByPosition(arguments[0]).type.get();
|
||||
const DataTypeAggregateFunction * aggr_type = typeid_cast<const DataTypeAggregateFunction *>(from_type);
|
||||
WhichDataType which(aggr_type->getArgumentsDataTypes()[0]);
|
||||
if (which.isUInt8())
|
||||
executeBitmapData<UInt8>(block, arguments, result, input_rows_count);
|
||||
else if (which.isUInt16())
|
||||
executeBitmapData<UInt16>(block, arguments, result, input_rows_count);
|
||||
else if (which.isUInt32())
|
||||
executeBitmapData<UInt32>(block, arguments, result, input_rows_count);
|
||||
else if (which.isUInt64())
|
||||
executeBitmapData<UInt64>(block, arguments, result, input_rows_count);
|
||||
else
|
||||
throw Exception(
|
||||
"Unexpected type " + from_type->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
|
||||
private:
|
||||
template <typename T>
|
||||
void executeBitmapData(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count)
|
||||
{
|
||||
const ColumnAggregateFunction * columns[2];
|
||||
for (size_t i = 0; i < 2; ++i)
|
||||
columns[i] = typeid_cast<const ColumnAggregateFunction *>(block.getByPosition(arguments[i]).column.get());
|
||||
|
||||
auto col_to = ColumnAggregateFunction::create(columns[0]->getAggregateFunction());
|
||||
|
||||
col_to->reserve(input_rows_count);
|
||||
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
col_to->insertFrom(columns[0]->getData()[i]);
|
||||
AggregateFunctionGroupBitmapData<T> & toBd = *reinterpret_cast<AggregateFunctionGroupBitmapData<T> *>(col_to->getData()[i]);
|
||||
const AggregateFunctionGroupBitmapData<T> & bd2
|
||||
= *reinterpret_cast<const AggregateFunctionGroupBitmapData<T> *>(columns[1]->getData()[i]);
|
||||
Impl<T>::apply(toBd, bd2);
|
||||
}
|
||||
block.getByPosition(result).column = std::move(col_to);
|
||||
}
|
||||
};
|
||||
|
||||
struct NameBitmapBuild
|
||||
{
|
||||
static constexpr auto name = "bitmapBuild";
|
||||
};
|
||||
using FunctionBitmapBuild = FunctionBitmapBuildImpl<NameBitmapBuild>;
|
||||
|
||||
struct NameBitmapToArray
|
||||
{
|
||||
static constexpr auto name = "bitmapToArray";
|
||||
};
|
||||
using FunctionBitmapToArray = FunctionBitmapToArrayImpl<NameBitmapToArray>;
|
||||
|
||||
struct NameBitmapCardinality
|
||||
{
|
||||
static constexpr auto name = "bitmapCardinality";
|
||||
};
|
||||
struct NameBitmapAndCardinality
|
||||
{
|
||||
static constexpr auto name = "bitmapAndCardinality";
|
||||
};
|
||||
struct NameBitmapOrCardinality
|
||||
{
|
||||
static constexpr auto name = "bitmapOrCardinality";
|
||||
};
|
||||
struct NameBitmapXorCardinality
|
||||
{
|
||||
static constexpr auto name = "bitmapXorCardinality";
|
||||
};
|
||||
struct NameBitmapAndnotCardinality
|
||||
{
|
||||
static constexpr auto name = "bitmapAndnotCardinality";
|
||||
};
|
||||
|
||||
using FunctionBitmapSelfCardinality = FunctionBitmapSelfCardinalityImpl<NameBitmapCardinality>;
|
||||
using FunctionBitmapAndCardinality = FunctionBitmapCardinality<BitmapAndCardinalityImpl, NameBitmapAndCardinality>;
|
||||
using FunctionBitmapOrCardinality = FunctionBitmapCardinality<BitmapOrCardinalityImpl, NameBitmapOrCardinality>;
|
||||
using FunctionBitmapXorCardinality = FunctionBitmapCardinality<BitmapXorCardinalityImpl, NameBitmapXorCardinality>;
|
||||
using FunctionBitmapAndnotCardinality = FunctionBitmapCardinality<BitmapAndnotCardinalityImpl, NameBitmapAndnotCardinality>;
|
||||
|
||||
struct NameBitmapAnd
|
||||
{
|
||||
static constexpr auto name = "bitmapAnd";
|
||||
};
|
||||
struct NameBitmapOr
|
||||
{
|
||||
static constexpr auto name = "bitmapOr";
|
||||
};
|
||||
struct NameBitmapXor
|
||||
{
|
||||
static constexpr auto name = "bitmapXor";
|
||||
};
|
||||
struct NameBitmapAndnot
|
||||
{
|
||||
static constexpr auto name = "bitmapAndnot";
|
||||
};
|
||||
using FunctionBitmapAnd = FunctionBitmap<BitmapAndImpl, NameBitmapAnd>;
|
||||
using FunctionBitmapOr = FunctionBitmap<BitmapOrImpl, NameBitmapOr>;
|
||||
using FunctionBitmapXor = FunctionBitmap<BitmapXorImpl, NameBitmapXor>;
|
||||
using FunctionBitmapAndnot = FunctionBitmap<BitmapAndnotImpl, NameBitmapAndnot>;
|
||||
|
||||
|
||||
}
|
||||
@ -1,5 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/memcmpSmall.h>
|
||||
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Columns/ColumnConst.h>
|
||||
#include <Columns/ColumnDecimal.h>
|
||||
@ -115,30 +117,6 @@ struct NumComparisonImpl
|
||||
};
|
||||
|
||||
|
||||
inline int memcmp16(const void * a, const void * b)
|
||||
{
|
||||
/// Assuming little endian.
|
||||
|
||||
UInt64 a_hi = __builtin_bswap64(unalignedLoad<UInt64>(a));
|
||||
UInt64 b_hi = __builtin_bswap64(unalignedLoad<UInt64>(b));
|
||||
|
||||
if (a_hi < b_hi)
|
||||
return -1;
|
||||
if (a_hi > b_hi)
|
||||
return 1;
|
||||
|
||||
UInt64 a_lo = __builtin_bswap64(unalignedLoad<UInt64>(reinterpret_cast<const char *>(a) + 8));
|
||||
UInt64 b_lo = __builtin_bswap64(unalignedLoad<UInt64>(reinterpret_cast<const char *>(b) + 8));
|
||||
|
||||
if (a_lo < b_lo)
|
||||
return -1;
|
||||
if (a_lo > b_lo)
|
||||
return 1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
template <typename Op>
|
||||
struct StringComparisonImpl
|
||||
{
|
||||
@ -148,27 +126,17 @@ struct StringComparisonImpl
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
size_t size = a_offsets.size();
|
||||
ColumnString::Offset prev_a_offset = 0;
|
||||
ColumnString::Offset prev_b_offset = 0;
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
/// Trailing zero byte of the smaller string is included in the comparison.
|
||||
size_t a_size;
|
||||
size_t b_size;
|
||||
int res;
|
||||
if (i == 0)
|
||||
{
|
||||
a_size = a_offsets[0];
|
||||
b_size = b_offsets[0];
|
||||
res = memcmp(a_data.data(), b_data.data(), std::min(a_size, b_size));
|
||||
}
|
||||
else
|
||||
{
|
||||
a_size = a_offsets[i] - a_offsets[i - 1];
|
||||
b_size = b_offsets[i] - b_offsets[i - 1];
|
||||
res = memcmp(&a_data[a_offsets[i - 1]], &b_data[b_offsets[i - 1]], std::min(a_size, b_size));
|
||||
}
|
||||
c[i] = Op::apply(memcmpSmallAllowOverflow15(
|
||||
a_data.data() + prev_a_offset, a_offsets[i] - prev_a_offset - 1,
|
||||
b_data.data() + prev_b_offset, b_offsets[i] - prev_b_offset - 1), 0);
|
||||
|
||||
c[i] = Op::apply(res, 0) || (res == 0 && Op::apply(a_size, b_size));
|
||||
prev_a_offset = a_offsets[i];
|
||||
prev_b_offset = b_offsets[i];
|
||||
}
|
||||
}
|
||||
|
||||
@ -178,43 +146,33 @@ struct StringComparisonImpl
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
size_t size = a_offsets.size();
|
||||
ColumnString::Offset prev_a_offset = 0;
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
if (i == 0)
|
||||
{
|
||||
int res = memcmp(a_data.data(), b_data.data(), std::min(a_offsets[0] - 1, b_n));
|
||||
c[i] = Op::apply(res, 0) || (res == 0 && Op::apply(a_offsets[0], b_n + 1));
|
||||
}
|
||||
else
|
||||
{
|
||||
int res = memcmp(&a_data[a_offsets[i - 1]], &b_data[i * b_n],
|
||||
std::min(a_offsets[i] - a_offsets[i - 1] - 1, b_n));
|
||||
c[i] = Op::apply(res, 0) || (res == 0 && Op::apply(a_offsets[i] - a_offsets[i - 1], b_n + 1));
|
||||
}
|
||||
c[i] = Op::apply(memcmpSmallAllowOverflow15(
|
||||
a_data.data() + prev_a_offset, a_offsets[i] - prev_a_offset - 1,
|
||||
b_data.data() + i * b_n, b_n), 0);
|
||||
|
||||
prev_a_offset = a_offsets[i];
|
||||
}
|
||||
}
|
||||
|
||||
static void NO_INLINE string_vector_constant(
|
||||
const ColumnString::Chars & a_data, const ColumnString::Offsets & a_offsets,
|
||||
const std::string & b,
|
||||
const ColumnString::Chars & b_data, ColumnString::Offset b_size,
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
size_t size = a_offsets.size();
|
||||
ColumnString::Offset b_size = b.size() + 1;
|
||||
const UInt8 * b_data = reinterpret_cast<const UInt8 *>(b.data());
|
||||
ColumnString::Offset prev_a_offset = 0;
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
/// Trailing zero byte of the smaller string is included in the comparison.
|
||||
if (i == 0)
|
||||
{
|
||||
int res = memcmp(a_data.data(), b_data, std::min(a_offsets[0], b_size));
|
||||
c[i] = Op::apply(res, 0) || (res == 0 && Op::apply(a_offsets[0], b_size));
|
||||
}
|
||||
else
|
||||
{
|
||||
int res = memcmp(&a_data[a_offsets[i - 1]], b_data, std::min(a_offsets[i] - a_offsets[i - 1], b_size));
|
||||
c[i] = Op::apply(res, 0) || (res == 0 && Op::apply(a_offsets[i] - a_offsets[i - 1], b_size));
|
||||
}
|
||||
c[i] = Op::apply(memcmpSmallAllowOverflow15(
|
||||
a_data.data() + prev_a_offset, a_offsets[i] - prev_a_offset - 1,
|
||||
b_data.data(), b_size), 0);
|
||||
|
||||
prev_a_offset = a_offsets[i];
|
||||
}
|
||||
}
|
||||
|
||||
@ -239,13 +197,13 @@ struct StringComparisonImpl
|
||||
|
||||
static void NO_INLINE fixed_string_vector_constant_16(
|
||||
const ColumnString::Chars & a_data,
|
||||
const std::string & b,
|
||||
const ColumnString::Chars & b_data,
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
size_t size = a_data.size();
|
||||
|
||||
for (size_t i = 0, j = 0; i < size; i += 16, ++j)
|
||||
c[j] = Op::apply(memcmp16(&a_data[i], b.data()), 0);
|
||||
c[j] = Op::apply(memcmp16(&a_data[i], &b_data[0]), 0);
|
||||
}
|
||||
|
||||
static void NO_INLINE fixed_string_vector_fixed_string_vector(
|
||||
@ -253,74 +211,73 @@ struct StringComparisonImpl
|
||||
const ColumnString::Chars & b_data, ColumnString::Offset b_n,
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
/** Specialization if both sizes are 16.
|
||||
* To more efficient comparison of IPv6 addresses stored in FixedString(16).
|
||||
*/
|
||||
if (a_n == 16 && b_n == 16)
|
||||
{
|
||||
/** Specialization if both sizes are 16.
|
||||
* To more efficient comparison of IPv6 addresses stored in FixedString(16).
|
||||
*/
|
||||
fixed_string_vector_fixed_string_vector_16(a_data, b_data, c);
|
||||
}
|
||||
else if (a_n == b_n)
|
||||
{
|
||||
size_t size = a_data.size();
|
||||
for (size_t i = 0, j = 0; i < size; i += a_n, ++j)
|
||||
c[j] = Op::apply(memcmpSmallAllowOverflow15(a_data.data() + i, b_data.data() + i, a_n), 0);
|
||||
}
|
||||
else
|
||||
{
|
||||
/// Generic implementation, less efficient.
|
||||
size_t size = a_data.size();
|
||||
size_t size = a_data.size() / a_n;
|
||||
|
||||
for (size_t i = 0, j = 0; i < size; i += a_n, ++j)
|
||||
{
|
||||
int res = memcmp(&a_data[i], &b_data[i], std::min(a_n, b_n));
|
||||
c[j] = Op::apply(res, 0) || (res == 0 && Op::apply(a_n, b_n));
|
||||
}
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
c[i] = Op::apply(memcmpSmallAllowOverflow15(a_data.data() + i * a_n, a_n, b_data.data() + i * b_n, b_n), 0);
|
||||
}
|
||||
}
|
||||
|
||||
static void NO_INLINE fixed_string_vector_constant(
|
||||
const ColumnString::Chars & a_data, ColumnString::Offset a_n,
|
||||
const std::string & b,
|
||||
const ColumnString::Chars & b_data, ColumnString::Offset b_size,
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
ColumnString::Offset b_n = b.size();
|
||||
if (a_n == 16 && b_n == 16)
|
||||
if (a_n == 16 && b_size == 16)
|
||||
{
|
||||
fixed_string_vector_constant_16(a_data, b, c);
|
||||
fixed_string_vector_constant_16(a_data, b_data, c);
|
||||
}
|
||||
else if (a_n == b_size)
|
||||
{
|
||||
size_t size = a_data.size();
|
||||
for (size_t i = 0, j = 0; i < size; i += a_n, ++j)
|
||||
c[j] = Op::apply(memcmpSmallAllowOverflow15(a_data.data() + i, b_data.data(), a_n), 0);
|
||||
}
|
||||
else
|
||||
{
|
||||
size_t size = a_data.size();
|
||||
const UInt8 * b_data = reinterpret_cast<const UInt8 *>(b.data());
|
||||
for (size_t i = 0, j = 0; i < size; i += a_n, ++j)
|
||||
{
|
||||
int res = memcmp(&a_data[i], b_data, std::min(a_n, b_n));
|
||||
c[j] = Op::apply(res, 0) || (res == 0 && Op::apply(a_n, b_n));
|
||||
}
|
||||
c[j] = Op::apply(memcmpSmallAllowOverflow15(a_data.data() + i, a_n, b_data.data(), b_size), 0);
|
||||
}
|
||||
}
|
||||
|
||||
static void constant_string_vector(
|
||||
const std::string & a,
|
||||
const ColumnString::Chars & a_data, ColumnString::Offset a_size,
|
||||
const ColumnString::Chars & b_data, const ColumnString::Offsets & b_offsets,
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
StringComparisonImpl<typename Op::SymmetricOp>::string_vector_constant(b_data, b_offsets, a, c);
|
||||
StringComparisonImpl<typename Op::SymmetricOp>::string_vector_constant(b_data, b_offsets, a_data, a_size, c);
|
||||
}
|
||||
|
||||
static void constant_fixed_string_vector(
|
||||
const std::string & a,
|
||||
const ColumnString::Chars & a_data, ColumnString::Offset a_size,
|
||||
const ColumnString::Chars & b_data, ColumnString::Offset b_n,
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
StringComparisonImpl<typename Op::SymmetricOp>::fixed_string_vector_constant(b_data, b_n, a, c);
|
||||
StringComparisonImpl<typename Op::SymmetricOp>::fixed_string_vector_constant(b_data, b_n, a_data, a_size, c);
|
||||
}
|
||||
|
||||
static void constant_constant(
|
||||
const std::string & a,
|
||||
const std::string & b,
|
||||
const ColumnString::Chars & a_data, ColumnString::Offset a_size,
|
||||
const ColumnString::Chars & b_data, ColumnString::Offset b_size,
|
||||
UInt8 & c)
|
||||
{
|
||||
size_t a_n = a.size();
|
||||
size_t b_n = b.size();
|
||||
|
||||
int res = memcmp(a.data(), b.data(), std::min(a_n, b_n));
|
||||
c = Op::apply(res, 0) || (res == 0 && Op::apply(a_n, b_n));
|
||||
c = Op::apply(memcmpSmallAllowOverflow15(a_data.data(), a_size, b_data.data(), b_size), 0);
|
||||
}
|
||||
};
|
||||
|
||||
@ -335,11 +292,21 @@ struct StringEqualsImpl
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
size_t size = a_offsets.size();
|
||||
ColumnString::Offset prev_a_offset = 0;
|
||||
ColumnString::Offset prev_b_offset = 0;
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
c[i] = positive == ((i == 0)
|
||||
? (a_offsets[0] == b_offsets[0] && !memcmp(a_data.data(), b_data.data(), a_offsets[0] - 1))
|
||||
: (a_offsets[i] - a_offsets[i - 1] == b_offsets[i] - b_offsets[i - 1]
|
||||
&& !memcmp(&a_data[a_offsets[i - 1]], &b_data[b_offsets[i - 1]], a_offsets[i] - a_offsets[i - 1] - 1)));
|
||||
{
|
||||
auto a_size = a_offsets[i] - prev_a_offset - 1;
|
||||
auto b_size = b_offsets[i] - prev_b_offset - 1;
|
||||
|
||||
c[i] = positive == memequalSmallAllowOverflow15(
|
||||
a_data.data() + prev_a_offset, a_size,
|
||||
b_data.data() + prev_b_offset, b_size);
|
||||
|
||||
prev_a_offset = a_offsets[i];
|
||||
prev_b_offset = b_offsets[i];
|
||||
}
|
||||
}
|
||||
|
||||
static void NO_INLINE string_vector_fixed_string_vector(
|
||||
@ -348,76 +315,65 @@ struct StringEqualsImpl
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
size_t size = a_offsets.size();
|
||||
ColumnString::Offset prev_a_offset = 0;
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
c[i] = positive == ((i == 0)
|
||||
? (a_offsets[0] == b_n + 1 && !memcmp(a_data.data(), b_data.data(), b_n))
|
||||
: (a_offsets[i] - a_offsets[i - 1] == b_n + 1
|
||||
&& !memcmp(&a_data[a_offsets[i - 1]], &b_data[b_n * i], b_n)));
|
||||
{
|
||||
auto a_size = a_offsets[i] - prev_a_offset - 1;
|
||||
|
||||
c[i] = positive == memequalSmallAllowOverflow15(
|
||||
a_data.data() + prev_a_offset, a_size,
|
||||
b_data.data() + b_n * i, b_n);
|
||||
|
||||
prev_a_offset = a_offsets[i];
|
||||
}
|
||||
}
|
||||
|
||||
static void NO_INLINE string_vector_constant(
|
||||
const ColumnString::Chars & a_data, const ColumnString::Offsets & a_offsets,
|
||||
const std::string & b,
|
||||
const ColumnString::Chars & b_data, ColumnString::Offset b_size,
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
size_t size = a_offsets.size();
|
||||
ColumnString::Offset b_n = b.size();
|
||||
const UInt8 * b_data = reinterpret_cast<const UInt8 *>(b.data());
|
||||
ColumnString::Offset prev_a_offset = 0;
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
c[i] = positive == ((i == 0)
|
||||
? (a_offsets[0] == b_n + 1 && !memcmp(a_data.data(), b_data, b_n))
|
||||
: (a_offsets[i] - a_offsets[i - 1] == b_n + 1
|
||||
&& !memcmp(&a_data[a_offsets[i - 1]], b_data, b_n)));
|
||||
{
|
||||
auto a_size = a_offsets[i] - prev_a_offset - 1;
|
||||
|
||||
c[i] = positive == memequalSmallAllowOverflow15(
|
||||
a_data.data() + prev_a_offset, a_size,
|
||||
b_data.data(), b_size);
|
||||
|
||||
prev_a_offset = a_offsets[i];
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef __SSE2__
|
||||
static void NO_INLINE fixed_string_vector_fixed_string_vector_16(
|
||||
const ColumnString::Chars & a_data,
|
||||
const ColumnString::Chars & b_data,
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
size_t size = c.size();
|
||||
size_t size = a_data.size() / 16;
|
||||
|
||||
const __m128i * a_pos = reinterpret_cast<const __m128i *>(a_data.data());
|
||||
const __m128i * b_pos = reinterpret_cast<const __m128i *>(b_data.data());
|
||||
UInt8 * c_pos = c.data();
|
||||
UInt8 * c_end = c_pos + size;
|
||||
|
||||
while (c_pos < c_end)
|
||||
{
|
||||
*c_pos = positive == (0xFFFF == _mm_movemask_epi8(_mm_cmpeq_epi8(
|
||||
_mm_loadu_si128(a_pos),
|
||||
_mm_loadu_si128(b_pos))));
|
||||
|
||||
++a_pos;
|
||||
++b_pos;
|
||||
++c_pos;
|
||||
}
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
c[i] = positive == memequal16(
|
||||
a_data.data() + i * 16,
|
||||
b_data.data() + i * 16);
|
||||
}
|
||||
|
||||
static void NO_INLINE fixed_string_vector_constant_16(
|
||||
const ColumnString::Chars & a_data,
|
||||
const std::string & b,
|
||||
const ColumnString::Chars & b_data,
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
size_t size = c.size();
|
||||
size_t size = a_data.size() / 16;
|
||||
|
||||
const __m128i * a_pos = reinterpret_cast<const __m128i *>(a_data.data());
|
||||
const __m128i b_value = _mm_loadu_si128(reinterpret_cast<const __m128i *>(b.data()));
|
||||
UInt8 * c_pos = c.data();
|
||||
UInt8 * c_end = c_pos + size;
|
||||
|
||||
while (c_pos < c_end)
|
||||
{
|
||||
*c_pos = positive == (0xFFFF == _mm_movemask_epi8(_mm_cmpeq_epi8(
|
||||
_mm_loadu_si128(a_pos),
|
||||
b_value)));
|
||||
|
||||
++a_pos;
|
||||
++c_pos;
|
||||
}
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
c[i] = positive == memequal16(
|
||||
a_data.data() + i * 16,
|
||||
b_data.data());
|
||||
}
|
||||
#endif
|
||||
|
||||
static void NO_INLINE fixed_string_vector_fixed_string_vector(
|
||||
const ColumnString::Chars & a_data, ColumnString::Offset a_n,
|
||||
@ -427,38 +383,32 @@ struct StringEqualsImpl
|
||||
/** Specialization if both sizes are 16.
|
||||
* To more efficient comparison of IPv6 addresses stored in FixedString(16).
|
||||
*/
|
||||
#ifdef __SSE2__
|
||||
if (a_n == 16 && b_n == 16)
|
||||
{
|
||||
fixed_string_vector_fixed_string_vector_16(a_data, b_data, c);
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
size_t size = a_data.size();
|
||||
for (size_t i = 0, j = 0; i < size; i += a_n, ++j)
|
||||
c[j] = positive == (a_n == b_n && !memcmp(&a_data[i], &b_data[i], a_n));
|
||||
size_t size = a_data.size() / a_n;
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
c[i] = positive == memequalSmallAllowOverflow15(a_data.data() + i * a_n, a_n, b_data.data() + i * b_n, b_n);
|
||||
}
|
||||
}
|
||||
|
||||
static void NO_INLINE fixed_string_vector_constant(
|
||||
const ColumnString::Chars & a_data, ColumnString::Offset a_n,
|
||||
const std::string & b,
|
||||
const ColumnString::Chars & b_data, ColumnString::Offset b_size,
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
ColumnString::Offset b_n = b.size();
|
||||
#ifdef __SSE2__
|
||||
if (a_n == 16 && b_n == 16)
|
||||
if (a_n == 16 && b_size == 16)
|
||||
{
|
||||
fixed_string_vector_constant_16(a_data, b, c);
|
||||
fixed_string_vector_constant_16(a_data, b_data, c);
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
size_t size = a_data.size();
|
||||
const UInt8 * b_data = reinterpret_cast<const UInt8 *>(b.data());
|
||||
for (size_t i = 0, j = 0; i < size; i += a_n, ++j)
|
||||
c[j] = positive == (a_n == b_n && !memcmp(&a_data[i], b_data, a_n));
|
||||
size_t size = a_data.size() / a_n;
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
c[i] = positive == memequalSmallAllowOverflow15(a_data.data() + i * a_n, a_n, b_data.data(), b_size);
|
||||
}
|
||||
}
|
||||
|
||||
@ -471,27 +421,27 @@ struct StringEqualsImpl
|
||||
}
|
||||
|
||||
static void constant_string_vector(
|
||||
const std::string & a,
|
||||
const ColumnString::Chars & a_data, ColumnString::Offset a_size,
|
||||
const ColumnString::Chars & b_data, const ColumnString::Offsets & b_offsets,
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
string_vector_constant(b_data, b_offsets, a, c);
|
||||
string_vector_constant(b_data, b_offsets, a_data, a_size, c);
|
||||
}
|
||||
|
||||
static void constant_fixed_string_vector(
|
||||
const std::string & a,
|
||||
const ColumnString::Chars & a_data, ColumnString::Offset a_size,
|
||||
const ColumnString::Chars & b_data, ColumnString::Offset b_n,
|
||||
PaddedPODArray<UInt8> & c)
|
||||
{
|
||||
fixed_string_vector_constant(b_data, b_n, a, c);
|
||||
fixed_string_vector_constant(b_data, b_n, a_data, a_size, c);
|
||||
}
|
||||
|
||||
static void constant_constant(
|
||||
const std::string & a,
|
||||
const std::string & b,
|
||||
const ColumnString::Chars & a_data, ColumnString::Offset a_size,
|
||||
const ColumnString::Chars & b_data, ColumnString::Offset b_size,
|
||||
UInt8 & c)
|
||||
{
|
||||
c = positive == (a == b);
|
||||
c = positive == memequalSmallAllowOverflow15(a_data.data(), a_size, b_data.data(), b_size);
|
||||
}
|
||||
};
|
||||
|
||||
@ -744,18 +694,62 @@ private:
|
||||
const ColumnString * c1_string = checkAndGetColumn<ColumnString>(c1);
|
||||
const ColumnFixedString * c0_fixed_string = checkAndGetColumn<ColumnFixedString>(c0);
|
||||
const ColumnFixedString * c1_fixed_string = checkAndGetColumn<ColumnFixedString>(c1);
|
||||
|
||||
const ColumnConst * c0_const = checkAndGetColumnConstStringOrFixedString(c0);
|
||||
const ColumnConst * c1_const = checkAndGetColumnConstStringOrFixedString(c1);
|
||||
|
||||
if (!((c0_string || c0_fixed_string || c0_const) && (c1_string || c1_fixed_string || c1_const)))
|
||||
return false;
|
||||
|
||||
const ColumnString::Chars * c0_const_chars = nullptr;
|
||||
const ColumnString::Chars * c1_const_chars = nullptr;
|
||||
ColumnString::Offset c0_const_size = 0;
|
||||
ColumnString::Offset c1_const_size = 0;
|
||||
|
||||
if (c0_const)
|
||||
{
|
||||
const ColumnString * c0_const_string = checkAndGetColumn<ColumnString>(&c0_const->getDataColumn());
|
||||
const ColumnFixedString * c0_const_fixed_string = checkAndGetColumn<ColumnFixedString>(&c0_const->getDataColumn());
|
||||
|
||||
if (c0_const_string)
|
||||
{
|
||||
c0_const_chars = &c0_const_string->getChars();
|
||||
c0_const_size = c0_const_string->getDataAt(0).size;
|
||||
}
|
||||
else if (c0_const_fixed_string)
|
||||
{
|
||||
c0_const_chars = &c0_const_fixed_string->getChars();
|
||||
c0_const_size = c0_const_fixed_string->getN();
|
||||
}
|
||||
else
|
||||
throw Exception("Logical error: ColumnConst contains not String nor FixedString column", ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
|
||||
if (c1_const)
|
||||
{
|
||||
const ColumnString * c1_const_string = checkAndGetColumn<ColumnString>(&c1_const->getDataColumn());
|
||||
const ColumnFixedString * c1_const_fixed_string = checkAndGetColumn<ColumnFixedString>(&c1_const->getDataColumn());
|
||||
|
||||
if (c1_const_string)
|
||||
{
|
||||
c1_const_chars = &c1_const_string->getChars();
|
||||
c1_const_size = c1_const_string->getDataAt(0).size;
|
||||
}
|
||||
else if (c1_const_fixed_string)
|
||||
{
|
||||
c1_const_chars = &c1_const_fixed_string->getChars();
|
||||
c1_const_size = c1_const_fixed_string->getN();
|
||||
}
|
||||
else
|
||||
throw Exception("Logical error: ColumnConst contains not String nor FixedString column", ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
|
||||
using StringImpl = StringComparisonImpl<Op<int, int>>;
|
||||
|
||||
if (c0_const && c1_const)
|
||||
{
|
||||
UInt8 res = 0;
|
||||
StringImpl::constant_constant(c0_const->getValue<String>(), c1_const->getValue<String>(), res);
|
||||
StringImpl::constant_constant(*c0_const_chars, c0_const_size, *c1_const_chars, c1_const_size, res);
|
||||
block.getByPosition(result).column = block.getByPosition(result).type->createColumnConst(c0_const->size(), toField(res));
|
||||
return true;
|
||||
}
|
||||
@ -778,7 +772,7 @@ private:
|
||||
else if (c0_string && c1_const)
|
||||
StringImpl::string_vector_constant(
|
||||
c0_string->getChars(), c0_string->getOffsets(),
|
||||
c1_const->getValue<String>(),
|
||||
*c1_const_chars, c1_const_size,
|
||||
c_res->getData());
|
||||
else if (c0_fixed_string && c1_string)
|
||||
StringImpl::fixed_string_vector_string_vector(
|
||||
@ -793,16 +787,16 @@ private:
|
||||
else if (c0_fixed_string && c1_const)
|
||||
StringImpl::fixed_string_vector_constant(
|
||||
c0_fixed_string->getChars(), c0_fixed_string->getN(),
|
||||
c1_const->getValue<String>(),
|
||||
*c1_const_chars, c1_const_size,
|
||||
c_res->getData());
|
||||
else if (c0_const && c1_string)
|
||||
StringImpl::constant_string_vector(
|
||||
c0_const->getValue<String>(),
|
||||
*c0_const_chars, c0_const_size,
|
||||
c1_string->getChars(), c1_string->getOffsets(),
|
||||
c_res->getData());
|
||||
else if (c0_const && c1_fixed_string)
|
||||
StringImpl::constant_fixed_string_vector(
|
||||
c0_const->getValue<String>(),
|
||||
*c0_const_chars, c0_const_size,
|
||||
c1_fixed_string->getChars(), c1_fixed_string->getN(),
|
||||
c_res->getData());
|
||||
else
|
||||
|
||||
@ -6,9 +6,11 @@
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnFixedString.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Columns/ColumnNullable.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Common/memcmpSmall.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -272,8 +274,7 @@ struct ArrayIndexNumNullImpl
|
||||
}
|
||||
};
|
||||
|
||||
/// Implementation for arrays of strings when the 2nd function argument
|
||||
/// is a NULL value.
|
||||
/// Implementation for arrays of strings when the 2nd function argument is a NULL value.
|
||||
template <typename IndexConv>
|
||||
struct ArrayIndexStringNullImpl
|
||||
{
|
||||
@ -311,12 +312,11 @@ struct ArrayIndexStringImpl
|
||||
{
|
||||
static void vector_const(
|
||||
const ColumnString::Chars & data, const ColumnArray::Offsets & offsets, const ColumnString::Offsets & string_offsets,
|
||||
const String & value,
|
||||
const ColumnString::Chars & value, ColumnString::Offset value_size,
|
||||
PaddedPODArray<typename IndexConv::ResultType> & result,
|
||||
const PaddedPODArray<UInt8> * null_map_data)
|
||||
{
|
||||
const auto size = offsets.size();
|
||||
const auto value_size = value.size();
|
||||
result.resize(size);
|
||||
|
||||
ColumnArray::Offset current_offset = 0;
|
||||
@ -331,12 +331,12 @@ struct ArrayIndexStringImpl
|
||||
? 0
|
||||
: string_offsets[current_offset + j - 1];
|
||||
|
||||
ColumnArray::Offset string_size = string_offsets[current_offset + j] - string_pos;
|
||||
ColumnArray::Offset string_size = string_offsets[current_offset + j] - string_pos - 1;
|
||||
|
||||
if (null_map_data && (*null_map_data)[current_offset + j])
|
||||
{
|
||||
}
|
||||
else if (string_size == value_size + 1 && 0 == memcmp(value.data(), &data[string_pos], value_size))
|
||||
else if (memequalSmallAllowOverflow15(value.data(), value_size, &data[string_pos], string_size))
|
||||
{
|
||||
if (!IndexConv::apply(j, current))
|
||||
break;
|
||||
@ -381,7 +381,7 @@ struct ArrayIndexStringImpl
|
||||
if (null_map_item && (*null_map_item)[i])
|
||||
hit = true;
|
||||
}
|
||||
else if (string_size == value_size && 0 == memcmp(&item_values[value_pos], &data[string_pos], value_size))
|
||||
else if (memequalSmallAllowOverflow15(&item_values[value_pos], value_size, &data[string_pos], string_size))
|
||||
hit = true;
|
||||
|
||||
if (hit)
|
||||
@ -708,16 +708,32 @@ private:
|
||||
const auto item_arg = block.getByPosition(arguments[1]).column.get();
|
||||
|
||||
if (item_arg->onlyNull())
|
||||
{
|
||||
ArrayIndexStringNullImpl<IndexConv>::vector_const(col_nested->getChars(), col_array->getOffsets(),
|
||||
col_nested->getOffsets(), col_res->getData(), null_map_data);
|
||||
}
|
||||
else if (const auto item_arg_const = checkAndGetColumnConstStringOrFixedString(item_arg))
|
||||
ArrayIndexStringImpl<IndexConv>::vector_const(col_nested->getChars(), col_array->getOffsets(),
|
||||
col_nested->getOffsets(), item_arg_const->getValue<String>(), col_res->getData(),
|
||||
null_map_data);
|
||||
{
|
||||
const ColumnString * item_const_string = checkAndGetColumn<ColumnString>(&item_arg_const->getDataColumn());
|
||||
const ColumnFixedString * item_const_fixedstring = checkAndGetColumn<ColumnFixedString>(&item_arg_const->getDataColumn());
|
||||
|
||||
if (item_const_string)
|
||||
ArrayIndexStringImpl<IndexConv>::vector_const(col_nested->getChars(), col_array->getOffsets(), col_nested->getOffsets(),
|
||||
item_const_string->getChars(), item_const_string->getDataAt(0).size,
|
||||
col_res->getData(), null_map_data);
|
||||
else if (item_const_fixedstring)
|
||||
ArrayIndexStringImpl<IndexConv>::vector_const(col_nested->getChars(), col_array->getOffsets(), col_nested->getOffsets(),
|
||||
item_const_fixedstring->getChars(), item_const_fixedstring->getN(),
|
||||
col_res->getData(), null_map_data);
|
||||
else
|
||||
throw Exception("Logical error: ColumnConst contains not String nor FixedString column", ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
else if (const auto item_arg_vector = checkAndGetColumn<ColumnString>(item_arg))
|
||||
{
|
||||
ArrayIndexStringImpl<IndexConv>::vector_vector(col_nested->getChars(), col_array->getOffsets(),
|
||||
col_nested->getOffsets(), item_arg_vector->getChars(), item_arg_vector->getOffsets(),
|
||||
col_res->getData(), null_map_data, null_map_item);
|
||||
}
|
||||
else
|
||||
return false;
|
||||
|
||||
|
||||
@ -13,6 +13,7 @@ namespace DB
|
||||
void registerFunctionsArithmetic(FunctionFactory &);
|
||||
void registerFunctionsArray(FunctionFactory &);
|
||||
void registerFunctionsTuple(FunctionFactory &);
|
||||
void registerFunctionsBitmap(FunctionFactory &);
|
||||
void registerFunctionsCoding(FunctionFactory &);
|
||||
void registerFunctionsComparison(FunctionFactory &);
|
||||
void registerFunctionsConditional(FunctionFactory &);
|
||||
@ -53,6 +54,7 @@ void registerFunctions()
|
||||
registerFunctionsArithmetic(factory);
|
||||
registerFunctionsArray(factory);
|
||||
registerFunctionsTuple(factory);
|
||||
registerFunctionsBitmap(factory);
|
||||
registerFunctionsCoding(factory);
|
||||
registerFunctionsComparison(factory);
|
||||
registerFunctionsConditional(factory);
|
||||
|
||||
@ -75,7 +75,7 @@ struct Settings
|
||||
M(SettingFloat, totals_auto_threshold, 0.5, "The threshold for totals_mode = 'auto'.") \
|
||||
\
|
||||
M(SettingBool, compile, false, "Whether query compilation is enabled.") \
|
||||
M(SettingBool, compile_expressions, true, "Compile some scalar functions and operators to native code.") \
|
||||
M(SettingBool, compile_expressions, false, "Compile some scalar functions and operators to native code.") \
|
||||
M(SettingUInt64, min_count_to_compile, 3, "The number of structurally identical queries before they are compiled.") \
|
||||
M(SettingUInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled") \
|
||||
M(SettingUInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.") \
|
||||
|
||||
55
dbms/tests/performance/string_sort/string_sort.xml
Normal file
55
dbms/tests/performance/string_sort/string_sort.xml
Normal file
@ -0,0 +1,55 @@
|
||||
<test>
|
||||
<name>String sorting</name>
|
||||
|
||||
<preconditions>
|
||||
<table_exists>hits_10m_single</table_exists>
|
||||
</preconditions>
|
||||
|
||||
<type>loop</type>
|
||||
|
||||
<stop_conditions>
|
||||
<all_of>
|
||||
<iterations>5</iterations>
|
||||
<min_time_not_changing_for_ms>10000</min_time_not_changing_for_ms>
|
||||
</all_of>
|
||||
<any_of>
|
||||
<iterations>50</iterations>
|
||||
<total_time_ms>60000</total_time_ms>
|
||||
</any_of>
|
||||
</stop_conditions>
|
||||
|
||||
<substitutions>
|
||||
<substitution>
|
||||
<name>str1</name>
|
||||
<values>
|
||||
<value>URL</value>
|
||||
<value>Referer</value>
|
||||
<value>Title</value>
|
||||
<value>SearchPhrase</value>
|
||||
<value>MobilePhoneModel</value>
|
||||
<value>PageCharset</value>
|
||||
</values>
|
||||
</substitution>
|
||||
<substitution>
|
||||
<name>str2</name>
|
||||
<values>
|
||||
<value>URL</value>
|
||||
<value>Referer</value>
|
||||
<value>Title</value>
|
||||
<value>SearchPhrase</value>
|
||||
<value>MobilePhoneModel</value>
|
||||
<value>PageCharset</value>
|
||||
</values>
|
||||
</substitution>
|
||||
</substitutions>
|
||||
|
||||
<query><![CDATA[SELECT {str1} FROM hits_10m_single ORDER BY {str1} LIMIT 10]]></query>
|
||||
<query><![CDATA[SELECT {str1} FROM hits_10m_single ORDER BY {str1} LIMIT 9000000, 10]]></query>
|
||||
|
||||
<query><![CDATA[SELECT {str1}, {str2} FROM hits_10m_single ORDER BY {str1}, {str2} LIMIT 10]]></query>
|
||||
<query><![CDATA[SELECT {str1}, {str2} FROM hits_10m_single ORDER BY {str1}, {str2} LIMIT 9000000, 10]]></query>
|
||||
|
||||
<main_metric>
|
||||
<min_time/>
|
||||
</main_metric>
|
||||
</test>
|
||||
@ -0,0 +1,17 @@
|
||||
[1,2,3,4,5]
|
||||
[3]
|
||||
[1,2,3,4,5]
|
||||
[1,2,4,5]
|
||||
[1,2]
|
||||
5
|
||||
1
|
||||
5
|
||||
4
|
||||
2
|
||||
70
|
||||
2019-01-01 50 [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50]
|
||||
2019-01-02 60 [11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70]
|
||||
60 50 70 40 20 30
|
||||
60 50 70 40 20 30
|
||||
2019-01-01 50
|
||||
2019-01-02 60
|
||||
77
dbms/tests/queries/0_stateless/00829_bitmap_function.sql
Normal file
77
dbms/tests/queries/0_stateless/00829_bitmap_function.sql
Normal file
@ -0,0 +1,77 @@
|
||||
SELECT bitmapToArray(bitmapBuild([1, 2, 3, 4, 5]));
|
||||
SELECT bitmapToArray(bitmapAnd(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])));
|
||||
SELECT bitmapToArray(bitmapOr(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])));
|
||||
SELECT bitmapToArray(bitmapXor(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])));
|
||||
SELECT bitmapToArray(bitmapAndnot(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])));
|
||||
SELECT bitmapCardinality(bitmapBuild([1, 2, 3, 4, 5]));
|
||||
SELECT bitmapAndCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]));
|
||||
SELECT bitmapOrCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]));
|
||||
SELECT bitmapXorCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]));
|
||||
SELECT bitmapAndnotCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]));
|
||||
|
||||
DROP TABLE IF EXISTS test.bitmap_test;
|
||||
CREATE TABLE test.bitmap_test(pickup_date Date, city_id UInt32, uid UInt32)ENGINE = Memory;
|
||||
INSERT INTO test.bitmap_test SELECT '2019-01-01', 1, number FROM numbers(1,50);
|
||||
INSERT INTO test.bitmap_test SELECT '2019-01-02', 1, number FROM numbers(11,60);
|
||||
|
||||
|
||||
SELECT groupBitmap( uid ) AS user_num FROM test.bitmap_test;
|
||||
|
||||
SELECT pickup_date, groupBitmap( uid ) AS user_num, bitmapToArray(groupBitmapState( uid )) AS users FROM test.bitmap_test GROUP BY pickup_date;
|
||||
|
||||
SELECT
|
||||
bitmapCardinality(day_today) AS today_users,
|
||||
bitmapCardinality(day_before) AS before_users,
|
||||
bitmapOrCardinality(day_today, day_before) AS all_users,
|
||||
bitmapAndCardinality(day_today, day_before) AS old_users,
|
||||
bitmapAndnotCardinality(day_today, day_before) AS new_users,
|
||||
bitmapXorCardinality(day_today, day_before) AS diff_users
|
||||
FROM
|
||||
(
|
||||
SELECT city_id, groupBitmapState( uid ) AS day_today FROM test.bitmap_test WHERE pickup_date = '2019-01-02' GROUP BY city_id
|
||||
)
|
||||
ALL LEFT JOIN
|
||||
(
|
||||
SELECT city_id, groupBitmapState( uid ) AS day_before FROM test.bitmap_test WHERE pickup_date = '2019-01-01' GROUP BY city_id
|
||||
)
|
||||
USING city_id;
|
||||
|
||||
SELECT
|
||||
bitmapCardinality(day_today) AS today_users,
|
||||
bitmapCardinality(day_before) AS before_users,
|
||||
bitmapCardinality(bitmapOr(day_today, day_before))ll_users,
|
||||
bitmapCardinality(bitmapAnd(day_today, day_before)) AS old_users,
|
||||
bitmapCardinality(bitmapAndnot(day_today, day_before)) AS new_users,
|
||||
bitmapCardinality(bitmapXor(day_today, day_before)) AS diff_users
|
||||
FROM
|
||||
(
|
||||
SELECT city_id, groupBitmapState( uid ) AS day_today FROM test.bitmap_test WHERE pickup_date = '2019-01-02' GROUP BY city_id
|
||||
)
|
||||
ALL LEFT JOIN
|
||||
(
|
||||
SELECT city_id, groupBitmapState( uid ) AS day_before FROM test.bitmap_test WHERE pickup_date = '2019-01-01' GROUP BY city_id
|
||||
)
|
||||
USING city_id;
|
||||
|
||||
|
||||
DROP TABLE IF EXISTS test.bitmap_state_test;
|
||||
CREATE TABLE test.bitmap_state_test
|
||||
(
|
||||
pickup_date Date,
|
||||
city_id UInt32,
|
||||
uv AggregateFunction( groupBitmap, UInt32 )
|
||||
)
|
||||
ENGINE = AggregatingMergeTree( pickup_date, ( pickup_date, city_id ), 8192);
|
||||
|
||||
INSERT INTO test.bitmap_state_test SELECT
|
||||
pickup_date,
|
||||
city_id,
|
||||
groupBitmapState(uid) AS uv
|
||||
FROM test.bitmap_test
|
||||
GROUP BY pickup_date, city_id;
|
||||
|
||||
SELECT pickup_date, groupBitmapMerge(uv) AS users from test.bitmap_state_test group by pickup_date;
|
||||
|
||||
DROP TABLE IF EXISTS test.bitmap_test;
|
||||
DROP TABLE IF EXISTS test.bitmap_state_test;
|
||||
|
||||
369
dbms/tests/queries/0_stateless/00912_string_comparison.reference
Normal file
369
dbms/tests/queries/0_stateless/00912_string_comparison.reference
Normal file
@ -0,0 +1,369 @@
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
1 0 0 1 1
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 1 0 1 0
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
1 0 0 1 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 0 1 0 1
|
||||
0 1 0 1 0
|
||||
0 0 1 0 1
|
||||
0 1 0 1 0
|
||||
0 0 1 0 1
|
||||
0 1 0 1 0
|
||||
0 0 1 0 1
|
||||
0 1 0 1 0
|
||||
0 0 1 0 1
|
||||
1
|
||||
18
dbms/tests/queries/0_stateless/00912_string_comparison.sql
Normal file
18
dbms/tests/queries/0_stateless/00912_string_comparison.sql
Normal file
@ -0,0 +1,18 @@
|
||||
WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'x' AS a, prefix || 'y' AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
|
||||
WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'y' AS a, prefix || 'x' AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
|
||||
WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'x' AS a, prefix || 'x' AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
|
||||
|
||||
WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'x' || prefix AS a, prefix || 'y' || prefix AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
|
||||
WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'y' || prefix AS a, prefix || 'x' || prefix AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
|
||||
WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'x' || prefix AS a, prefix || 'x' || prefix AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
|
||||
|
||||
WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'x' || prefix AS a, prefix || 'y' AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
|
||||
WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'y' || prefix AS a, prefix || 'x' AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
|
||||
WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'x' || prefix AS a, prefix || 'x' AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
|
||||
|
||||
WITH arrayJoin(['aaa', 'bbb']) AS a, 'aaa\0bbb' AS b SELECT a = b, a < b, a > b, a <= b, a >= b;
|
||||
WITH arrayJoin(['aaa', 'zzz']) AS a, 'aaa\0bbb' AS b SELECT a = b, a < b, a > b, a <= b, a >= b;
|
||||
WITH arrayJoin(['aaa', 'bbb']) AS a, materialize('aaa\0bbb') AS b SELECT a = b, a < b, a > b, a <= b, a >= b;
|
||||
WITH arrayJoin(['aaa', 'zzz']) AS a, materialize('aaa\0bbb') AS b SELECT a = b, a < b, a > b, a <= b, a >= b;
|
||||
|
||||
SELECT empty(toFixedString('', 1 + randConstant() % 100));
|
||||
@ -179,6 +179,48 @@ binary decimal
|
||||
01101000 = 104
|
||||
```
|
||||
|
||||
|
||||
##groupBitmap
|
||||
|
||||
Bitmap or Aggregate calculations from a unsigned integer column, return cardinality of type UInt64, if add suffix -State, then return [bitmap object](../functions/bitmap_functions.md).
|
||||
|
||||
```
|
||||
groupBitmap(expr)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
`expr` – An expression that results in `UInt*` type.
|
||||
|
||||
**Return value**
|
||||
|
||||
Value of the `UInt64` type.
|
||||
|
||||
**Example**
|
||||
|
||||
Test data:
|
||||
|
||||
```
|
||||
userid
|
||||
1
|
||||
1
|
||||
2
|
||||
3
|
||||
```
|
||||
|
||||
Query:
|
||||
|
||||
```
|
||||
SELECT groupBitmap(userid) as num FROM t
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```
|
||||
num
|
||||
3
|
||||
```
|
||||
|
||||
## min(x) {#agg_function-min}
|
||||
|
||||
Calculates the minimum.
|
||||
|
||||
277
docs/en/query_language/functions/bitmap_functions.md
Normal file
277
docs/en/query_language/functions/bitmap_functions.md
Normal file
@ -0,0 +1,277 @@
|
||||
# Bitmap functions
|
||||
|
||||
Bitmap functions work for two bitmaps Object value calculation, it is to return new bitmap or cardinality while using formula calculation, such as and, or, xor, and not, etc.
|
||||
|
||||
There are 2 kinds of construction methods for Bitmap Object. One is to be constructed by aggregation function groupBitmap with -State, the other is to be constructed by Array Object. It is also to convert Bitmap Object to Array Object.
|
||||
|
||||
RoaringBitmap is wrapped into a data structure while actual storage of Bitmap objects. When the cardinality is less than or equal to 32, it uses Set objet. When the cardinality is greater than 32, it uses RoaringBitmap object. That is why storage of low cardinality set is faster.
|
||||
|
||||
For more information on RoaringBitmap, see: [CRoaring](https://github.com/RoaringBitmap/CRoaring).
|
||||
|
||||
|
||||
## bitmapBuild
|
||||
|
||||
Build a bitmap from unsigned integer array.
|
||||
|
||||
```
|
||||
bitmapBuild(array)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `array` – unsigned integer array.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT bitmapBuild([1, 2, 3, 4, 5]) AS res
|
||||
```
|
||||
|
||||
## bitmapToArray
|
||||
|
||||
Convert bitmap to integer array.
|
||||
|
||||
```
|
||||
bitmapToArray(bitmap)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `bitmap` – bitmap object.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT bitmapToArray(bitmapBuild([1, 2, 3, 4, 5])) AS res
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─────────┐
|
||||
│ [1,2,3,4,5] │
|
||||
└─────────────┘
|
||||
```
|
||||
|
||||
|
||||
## bitmapAnd
|
||||
|
||||
Two bitmap and calculation, the result is a new bitmap.
|
||||
|
||||
```
|
||||
bitmapAnd(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `bitmap` – bitmap object.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT bitmapToArray(bitmapAnd(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─┐
|
||||
│ [3] │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
|
||||
## bitmapOr
|
||||
|
||||
Two bitmap or calculation, the result is a new bitmap.
|
||||
|
||||
```
|
||||
bitmapOr(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `bitmap` – bitmap object.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT bitmapToArray(bitmapOr(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─────────┐
|
||||
│ [1,2,3,4,5] │
|
||||
└─────────────┘
|
||||
```
|
||||
|
||||
## bitmapXor
|
||||
|
||||
Two bitmap xor calculation, the result is a new bitmap.
|
||||
|
||||
```
|
||||
bitmapXor(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `bitmap` – bitmap object.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT bitmapToArray(bitmapXor(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res
|
||||
```
|
||||
|
||||
```
|
||||
┌─res───────┐
|
||||
│ [1,2,4,5] │
|
||||
└───────────┘
|
||||
```
|
||||
|
||||
## bitmapAndnot
|
||||
|
||||
Two bitmap andnot calculation, the result is a new bitmap.
|
||||
|
||||
```
|
||||
bitmapAndnot(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `bitmap` – bitmap object.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT bitmapToArray(bitmapAndnot(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res
|
||||
```
|
||||
|
||||
```
|
||||
┌─res───┐
|
||||
│ [1,2] │
|
||||
└───────┘
|
||||
```
|
||||
|
||||
## bitmapCardinality
|
||||
|
||||
Retrun bitmap cardinality of type UInt64.
|
||||
|
||||
|
||||
```
|
||||
bitmapCardinality(bitmap)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `bitmap` – bitmap object.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT bitmapCardinality(bitmapBuild([1, 2, 3, 4, 5])) AS res
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─┐
|
||||
│ 5 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
## bitmapAndCardinality
|
||||
|
||||
Two bitmap and calculation, return cardinality of type UInt64.
|
||||
|
||||
|
||||
```
|
||||
bitmapAndCardinality(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `bitmap` – bitmap object.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT bitmapAndCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res;
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─┐
|
||||
│ 1 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
|
||||
## bitmapOrCardinality
|
||||
|
||||
Two bitmap or calculation, return cardinality of type UInt64.
|
||||
|
||||
```
|
||||
bitmapOrCardinality(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `bitmap` – bitmap object.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT bitmapOrCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res;
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─┐
|
||||
│ 5 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
## bitmapXorCardinality
|
||||
|
||||
Two bitmap xor calculation, return cardinality of type UInt64.
|
||||
|
||||
```
|
||||
bitmapXorCardinality(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `bitmap` – bitmap object.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT bitmapXorCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res;
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─┐
|
||||
│ 4 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
|
||||
## bitmapAndnotCardinality
|
||||
|
||||
Two bitmap andnot calculation, return cardinality of type UInt64.
|
||||
|
||||
```
|
||||
bitmapAndnotCardinality(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `bitmap` – bitmap object.
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT bitmapAndnotCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res;
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─┐
|
||||
│ 2 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
|
||||
[Original article](https://clickhouse.yandex/docs/en/query_language/functions/bitmap_functions/) <!--hide-->
|
||||
@ -86,6 +86,7 @@ functions/arithmetic_functions.md query_language/functions/arithmetic_functions.
|
||||
functions/array_functions.md query_language/functions/array_functions.md
|
||||
functions/array_join.md query_language/functions/array_join.md
|
||||
functions/bit_functions.md query_language/functions/bit_functions.md
|
||||
functions/bitmap_functions.md query_language/functions/bitmap_functions.md
|
||||
functions/comparison_functions.md query_language/functions/comparison_functions.md
|
||||
functions/conditional_functions.md query_language/functions/conditional_functions.md
|
||||
functions/date_time_functions.md query_language/functions/date_time_functions.md
|
||||
|
||||
@ -115,6 +115,7 @@ nav:
|
||||
- 'Working with Arrays': 'query_language/functions/array_functions.md'
|
||||
- 'Splitting and Merging Strings and Arrays': 'query_language/functions/splitting_merging_functions.md'
|
||||
- 'Bit': 'query_language/functions/bit_functions.md'
|
||||
- 'Bitmap functions': 'query_language/functions/bitmap_functions.md'
|
||||
- 'Hash': 'query_language/functions/hash_functions.md'
|
||||
- 'Generating Pseudo-Random Numbers': 'query_language/functions/random_functions.md'
|
||||
- 'Encoding': 'query_language/functions/encoding_functions.md'
|
||||
|
||||
@ -1,11 +1,5 @@
|
||||
find_package (Threads)
|
||||
|
||||
add_executable (util-clickhouse-compressor main.cpp)
|
||||
target_link_libraries (util-clickhouse-compressor PRIVATE clickhouse-compressor-lib)
|
||||
set_target_properties(util-clickhouse-compressor PROPERTIES OUTPUT_NAME "clickhouse-compressor")
|
||||
|
||||
#install (TARGETS util-clickhouse-compressor RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse-compressor)
|
||||
|
||||
add_executable (zstd_test zstd_test.cpp)
|
||||
target_link_libraries (zstd_test PRIVATE ${ZSTD_LIBRARY} common Threads::Threads)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user