thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 09:32:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into pufit/fix_s3_threads

Browse Source 
# Conflicts:
#	src/Storages/StorageS3.cpp
#	src/Storages/StorageS3.h
#	src/Storages/StorageURL.cpp
#	src/Storages/StorageURL.h
This commit is contained in:
pufit 2023-08-21 21:32:15 -04:00
commit 9d454d9afc
95 changed files with 3437 additions and 303 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.gitmodules vendored
View File

@ -347,3 +347,15 @@
[submodule "contrib/incbin"]
path = contrib/incbin
url = https://github.com/graphitemaster/incbin.git
[submodule "contrib/usearch"]
path = contrib/usearch
url = https://github.com/unum-cloud/usearch.git
[submodule "contrib/SimSIMD"]
path = contrib/SimSIMD
url = https://github.com/ashvardanian/SimSIMD.git
[submodule "contrib/FP16"]
path = contrib/FP16
url = https://github.com/Maratyszcza/FP16.git
[submodule "contrib/robin-map"]
path = contrib/robin-map
url = https://github.com/Tessil/robin-map.git

11
contrib/CMakeLists.txt vendored
View File

@ -196,6 +196,17 @@ if (ARCH_S390X)
add_contrib(crc32-s390x-cmake crc32-s390x)
endif()
add_contrib (annoy-cmake annoy)
option(ENABLE_USEARCH "Enable USearch (Approximate Neighborhood Search, HNSW) support" ${ENABLE_LIBRARIES})
if (ENABLE_USEARCH)
add_contrib (FP16-cmake FP16)
add_contrib (robin-map-cmake robin-map)
add_contrib (SimSIMD-cmake SimSIMD)
add_contrib (usearch-cmake usearch) # requires: FP16, robin-map, SimdSIMD
else ()
message(STATUS "Not using USearch")
endif ()
add_contrib (xxHash-cmake xxHash)
add_contrib (libbcrypt-cmake libbcrypt)

1
contrib/FP16 vendored Submodule

@ -0,0 +1 @@
Subproject commit 0a92994d729ff76a58f692d3028ca1b64b145d91

1
contrib/FP16-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1 @@
# See contrib/usearch-cmake/CMakeLists.txt

1
contrib/SimSIMD vendored Submodule

@ -0,0 +1 @@
Subproject commit de2cb75b9e9e3389d5e1e51fd9f8ed151f3c17cf

1
contrib/SimSIMD-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1 @@
# See contrib/usearch-cmake/CMakeLists.txt

2
contrib/boost vendored

@ -1 +1 @@
Subproject commit bb179652862b528d94a9032a784796c4db846c3f
Subproject commit 063a9372b4ae304e869a5c5724971d0501552731

6
contrib/boost-cmake/CMakeLists.txt
View File

@ -19,6 +19,12 @@ add_library (_boost_filesystem ${SRCS_FILESYSTEM})
add_library (boost::filesystem ALIAS _boost_filesystem)
target_include_directories (_boost_filesystem SYSTEM BEFORE PUBLIC ${LIBRARY_DIR})
if (OS_LINUX)
target_compile_definitions (_boost_filesystem PRIVATE
BOOST_FILESYSTEM_HAS_POSIX_AT_APIS=1
)
endif ()
# headers-only
add_library (_boost_headers_only INTERFACE)

2
contrib/orc vendored

@ -1 +1 @@
Subproject commit 568d1d60c250af1890f226c182bc15bd8cc94cf1
Subproject commit a20d1d9d7ad4a4be7b7ba97588e16ca8b9abb2b6

1
contrib/robin-map vendored Submodule

@ -0,0 +1 @@
Subproject commit 851a59e0e3063ee0e23089062090a73fd3de482d

1
contrib/robin-map-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1 @@
# See contrib/usearch-cmake/CMakeLists.txt

1
contrib/usearch vendored Submodule

@ -0,0 +1 @@
Subproject commit 387b78b28b17b8954024ffc81e97cbcfa10d1f30

17
contrib/usearch-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,17 @@
set(USEARCH_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/usearch")
set(USEARCH_SOURCE_DIR "${USEARCH_PROJECT_DIR}/include")
set(FP16_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/FP16")
set(ROBIN_MAP_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/robin-map")
set(SIMSIMD_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/SimSIMD-map")
add_library(_usearch INTERFACE)
target_include_directories(_usearch SYSTEM INTERFACE
${FP16_PROJECT_DIR}/include
${ROBIN_MAP_PROJECT_DIR}/include
${SIMSIMD_PROJECT_DIR}/include
${USEARCH_SOURCE_DIR})
add_library(ch_contrib::usearch ALIAS _usearch)
target_compile_definitions(_usearch INTERFACE ENABLE_USEARCH)

63
docs/en/engines/table-engines/mergetree-family/annindexes.md
View File

@ -142,13 +142,15 @@ was specified for ANN indexes, the default value is 100 million.
- [Annoy](/docs/en/engines/table-engines/mergetree-family/annindexes.md#annoy-annoy)
- [USearch](/docs/en/engines/table-engines/mergetree-family/annindexes.md#usearch-usearch)
## Annoy {#annoy}
Annoy indexes are currently experimental, to use them you first need to `SET allow_experimental_annoy_index = 1`. They are also currently
disabled on ARM due to memory safety problems with the algorithm.
This type of ANN index implements [the Annoy algorithm](https://github.com/spotify/annoy) which is based on a recursive division of the
space in random linear surfaces (lines in 2D, planes in 3D etc.).
This type of ANN index is based on the [Annoy library](https://github.com/spotify/annoy) which recursively divides the space into random
linear surfaces (lines in 2D, planes in 3D etc.).
<div class='vimeo-container'>
<iframe src="//www.youtube.com/embed/QkCCyLW0ehU"
@ -216,3 +218,60 @@ ORDER BY L2Distance(vectors, Point)
LIMIT N
SETTINGS annoy_index_search_k_nodes=100;
```
## USearch {#usearch}
This type of ANN index is based on the [the USearch library](https://github.com/unum-cloud/usearch), which implements the [HNSW
algorithm](https://arxiv.org/abs/1603.09320), i.e., builds a hierarchical graph where each point represents a vector and the edges represent
similarity. Such hierarchical structures can be very efficient on large collections. They may often fetch 0.05% or less data from the
overall dataset, while still providing 99% recall. This is especially useful when working with high-dimensional vectors,
that are expensive to load and compare. The library also has several hardware-specific SIMD optimizations to accelerate further
distance computations on modern Arm (NEON and SVE) and x86 (AVX2 and AVX-512) CPUs and OS-specific optimizations to allow efficient
navigation around immutable persistent files, without loading them into RAM.
<div class='vimeo-container'>
<iframe src="//www.youtube.com/embed/UMrhB3icP9w"
width="640"
height="360"
frameborder="0"
allow="autoplay;
fullscreen;
picture-in-picture"
allowfullscreen>
</iframe>
</div>
Syntax to create an USearch index over an [Array](../../../sql-reference/data-types/array.md) column:
```sql
CREATE TABLE table_with_usearch_index
(
id Int64,
vectors Array(Float32),
INDEX [ann_index_name] vectors TYPE usearch([Distance]) [GRANULARITY N]
)
ENGINE = MergeTree
ORDER BY id;
```
Syntax to create an ANN index over a [Tuple](../../../sql-reference/data-types/tuple.md) column:
```sql
CREATE TABLE table_with_usearch_index
(
id Int64,
vectors Tuple(Float32[, Float32[, ...]]),
INDEX [ann_index_name] vectors TYPE usearch([Distance]) [GRANULARITY N]
)
ENGINE = MergeTree
ORDER BY id;
```
USearch currently supports two distance functions:
- `L2Distance`, also called Euclidean distance, is the length of a line segment between two points in Euclidean space
([Wikipedia](https://en.wikipedia.org/wiki/Euclidean_distance)).
- `cosineDistance`, also called cosine similarity, is the cosine of the angle between two (non-zero) vectors
([Wikipedia](https://en.wikipedia.org/wiki/Cosine_similarity)).
For normalized data, `L2Distance` is usually a better choice, otherwise `cosineDistance` is recommended to compensate for scale. If no
distance function was specified during index creation, `L2Distance` is used as default.

26
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -221,6 +221,10 @@ Default: 1024
Size of cache for index marks. Zero means disabled.
:::note
This setting can be modified at runtime and will take effect immediately.
:::
Type: UInt64
Default: 0
@ -230,6 +234,10 @@ Default: 0
Size of cache for uncompressed blocks of MergeTree indices. Zero means disabled.
:::note
This setting can be modified at runtime and will take effect immediately.
:::
Type: UInt64
Default: 0
@ -255,6 +263,10 @@ Default: SLRU
Size of cache for marks (index of MergeTree family of tables).
:::note
This setting can be modified at runtime and will take effect immediately.
:::
Type: UInt64
Default: 5368709120
@ -288,7 +300,7 @@ Default: 1000
Limit on total number of concurrently executed queries. Zero means Unlimited. Note that limits on insert and select queries, and on the maximum number of queries for users must also be considered. See also max_concurrent_insert_queries, max_concurrent_select_queries, max_concurrent_queries_for_all_users. Zero means unlimited.
:::note
These settings can be modified at runtime and will take effect immediately. Queries that are already running will remain unchanged.
This setting can be modified at runtime and will take effect immediately. Queries that are already running will remain unchanged.
:::
Type: UInt64
@ -300,7 +312,7 @@ Default: 0
Limit on total number of concurrent insert queries. Zero means Unlimited.
:::note
These settings can be modified at runtime and will take effect immediately. Queries that are already running will remain unchanged.
This setting can be modified at runtime and will take effect immediately. Queries that are already running will remain unchanged.
:::
Type: UInt64
@ -312,7 +324,7 @@ Default: 0
Limit on total number of concurrently select queries. Zero means Unlimited.
:::note
These settings can be modified at runtime and will take effect immediately. Queries that are already running will remain unchanged.
This setting can be modified at runtime and will take effect immediately. Queries that are already running will remain unchanged.
:::
Type: UInt64
@ -456,6 +468,10 @@ Sets the cache size (in bytes) for mapped files. This setting allows avoiding fr
Note that the amount of data in mapped files does not consume memory directly and is not accounted for in query or server memory usage — because this memory can be discarded similar to the OS page cache. The cache is dropped (the files are closed) automatically on the removal of old parts in tables of the MergeTree family, also it can be dropped manually by the `SYSTEM DROP MMAP CACHE` query.
:::note
This setting can be modified at runtime and will take effect immediately.
:::
Type: UInt64
Default: 1000
@ -605,6 +621,10 @@ There is one shared cache for the server. Memory is allocated on demand. The cac
The uncompressed cache is advantageous for very short queries in individual cases.
:::note
This setting can be modified at runtime and will take effect immediately.
:::
Type: UInt64
Default: 0

14
docs/en/sql-reference/statements/system.md
View File

@ -66,13 +66,13 @@ RELOAD FUNCTION [ON CLUSTER cluster_name] function_name
## DROP DNS CACHE
Resets ClickHouse’s internal DNS cache. Sometimes (for old ClickHouse versions) it is necessary to use this command when changing the infrastructure (changing the IP address of another ClickHouse server or the server used by dictionaries).
Clears ClickHouse’s internal DNS cache. Sometimes (for old ClickHouse versions) it is necessary to use this command when changing the infrastructure (changing the IP address of another ClickHouse server or the server used by dictionaries).
For more convenient (automatic) cache management, see disable_internal_dns_cache, dns_cache_update_period parameters.
## DROP MARK CACHE
Resets the mark cache.
Clears the mark cache.
## DROP REPLICA
@ -106,22 +106,18 @@ Similar to `SYSTEM DROP REPLICA`, but removes the `Replicated` database replica
## DROP UNCOMPRESSED CACHE
Reset the uncompressed data cache.
Clears the uncompressed data cache.
The uncompressed data cache is enabled/disabled with the query/user/profile-level setting [use_uncompressed_cache](../../operations/settings/settings.md#setting-use_uncompressed_cache).
Its size can be configured using the server-level setting [uncompressed_cache_size](../../operations/server-configuration-parameters/settings.md#server-settings-uncompressed_cache_size).
## DROP COMPILED EXPRESSION CACHE
Reset the compiled expression cache.
Clears the compiled expression cache.
The compiled expression cache is enabled/disabled with the query/user/profile-level setting [compile_expressions](../../operations/settings/settings.md#compile-expressions).
## DROP QUERY CACHE
Resets the [query cache](../../operations/query-cache.md).
```sql
SYSTEM DROP QUERY CACHE [ON CLUSTER cluster_name]
```
Clears the [query cache](../../operations/query-cache.md).
## FLUSH LOGS

19
programs/local/LocalServer.cpp
View File

@ -668,8 +668,7 @@ void LocalServer::processConfig()
uncompressed_cache_size = max_cache_size;
LOG_INFO(log, "Lowered uncompressed cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(uncompressed_cache_size));
}
if (uncompressed_cache_size)
global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size);
global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size);
String mark_cache_policy = config().getString("mark_cache_policy", DEFAULT_MARK_CACHE_POLICY);
size_t mark_cache_size = config().getUInt64("mark_cache_size", DEFAULT_MARK_CACHE_MAX_SIZE);
@ -680,8 +679,7 @@ void LocalServer::processConfig()
mark_cache_size = max_cache_size;
LOG_INFO(log, "Lowered mark cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(mark_cache_size));
}
if (mark_cache_size)
global_context->setMarkCache(mark_cache_policy, mark_cache_size);
global_context->setMarkCache(mark_cache_policy, mark_cache_size);
size_t index_uncompressed_cache_size = config().getUInt64("index_uncompressed_cache_size", DEFAULT_INDEX_UNCOMPRESSED_CACHE_MAX_SIZE);
if (index_uncompressed_cache_size > max_cache_size)
@ -689,8 +687,7 @@ void LocalServer::processConfig()
index_uncompressed_cache_size = max_cache_size;
LOG_INFO(log, "Lowered index uncompressed cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(uncompressed_cache_size));
}
if (index_uncompressed_cache_size)
global_context->setIndexUncompressedCache(index_uncompressed_cache_size);
global_context->setIndexUncompressedCache(index_uncompressed_cache_size);
size_t index_mark_cache_size = config().getUInt64("index_mark_cache_size", DEFAULT_INDEX_MARK_CACHE_MAX_SIZE);
if (index_mark_cache_size > max_cache_size)
@ -698,8 +695,7 @@ void LocalServer::processConfig()
index_mark_cache_size = max_cache_size;
LOG_INFO(log, "Lowered index mark cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(uncompressed_cache_size));
}
if (index_mark_cache_size)
global_context->setIndexMarkCache(index_mark_cache_size);
global_context->setIndexMarkCache(index_mark_cache_size);
size_t mmap_cache_size = config().getUInt64("mmap_cache_size", DEFAULT_MMAP_CACHE_MAX_SIZE);
if (mmap_cache_size > max_cache_size)
@ -707,11 +703,10 @@ void LocalServer::processConfig()
mmap_cache_size = max_cache_size;
LOG_INFO(log, "Lowered mmap file cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(uncompressed_cache_size));
}
if (mmap_cache_size)
global_context->setMMappedFileCache(mmap_cache_size);
global_context->setMMappedFileCache(mmap_cache_size);
/// In Server.cpp (./clickhouse-server), we would initialize the query cache here.
/// Intentionally not doing this in clickhouse-local as it doesn't make sense.
/// Initialize a dummy query cache.
global_context->setQueryCache(0, 0, 0, 0);
#if USE_EMBEDDED_COMPILER
size_t compiled_expression_cache_max_size_in_bytes = config().getUInt64("compiled_expression_cache_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_SIZE);

138
programs/server/Server.cpp
View File

@ -1105,6 +1105,69 @@ try
if (config().has("macros"))
global_context->setMacros(std::make_unique<Macros>(config(), "macros", log));
/// Set up caches.
const size_t max_cache_size = static_cast<size_t>(physical_server_memory * server_settings.cache_size_to_ram_max_ratio);
String uncompressed_cache_policy = server_settings.uncompressed_cache_policy;
size_t uncompressed_cache_size = server_settings.uncompressed_cache_size;
if (uncompressed_cache_size > max_cache_size)
{
uncompressed_cache_size = max_cache_size;
LOG_INFO(log, "Lowered uncompressed cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(uncompressed_cache_size));
}
global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size);
String mark_cache_policy = server_settings.mark_cache_policy;
size_t mark_cache_size = server_settings.mark_cache_size;
if (mark_cache_size > max_cache_size)
{
mark_cache_size = max_cache_size;
LOG_INFO(log, "Lowered mark cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(mark_cache_size));
}
global_context->setMarkCache(mark_cache_policy, mark_cache_size);
size_t index_uncompressed_cache_size = server_settings.index_uncompressed_cache_size;
if (index_uncompressed_cache_size > max_cache_size)
{
index_uncompressed_cache_size = max_cache_size;
LOG_INFO(log, "Lowered index uncompressed cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(uncompressed_cache_size));
}
global_context->setIndexUncompressedCache(index_uncompressed_cache_size);
size_t index_mark_cache_size = server_settings.index_mark_cache_size;
if (index_mark_cache_size > max_cache_size)
{
index_mark_cache_size = max_cache_size;
LOG_INFO(log, "Lowered index mark cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(uncompressed_cache_size));
}
global_context->setIndexMarkCache(index_mark_cache_size);
size_t mmap_cache_size = server_settings.mmap_cache_size;
if (mmap_cache_size > max_cache_size)
{
mmap_cache_size = max_cache_size;
LOG_INFO(log, "Lowered mmap file cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(uncompressed_cache_size));
}
global_context->setMMappedFileCache(mmap_cache_size);
size_t query_cache_max_size_in_bytes = config().getUInt64("query_cache.max_size_in_bytes", DEFAULT_QUERY_CACHE_MAX_SIZE);
size_t query_cache_max_entries = config().getUInt64("query_cache.max_entries", DEFAULT_QUERY_CACHE_MAX_ENTRIES);
size_t query_cache_query_cache_max_entry_size_in_bytes = config().getUInt64("query_cache.max_entry_size_in_bytes", DEFAULT_QUERY_CACHE_MAX_ENTRY_SIZE_IN_BYTES);
size_t query_cache_max_entry_size_in_rows = config().getUInt64("query_cache.max_entry_rows_in_rows", DEFAULT_QUERY_CACHE_MAX_ENTRY_SIZE_IN_ROWS);
if (query_cache_max_size_in_bytes > max_cache_size)
{
query_cache_max_size_in_bytes = max_cache_size;
LOG_INFO(log, "Lowered query cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(uncompressed_cache_size));
}
global_context->setQueryCache(query_cache_max_size_in_bytes, query_cache_max_entries, query_cache_query_cache_max_entry_size_in_bytes, query_cache_max_entry_size_in_rows);
#if USE_EMBEDDED_COMPILER
size_t compiled_expression_cache_max_size_in_bytes = config().getUInt64("compiled_expression_cache_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_SIZE);
size_t compiled_expression_cache_max_elements = config().getUInt64("compiled_expression_cache_elements_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_ENTRIES);
CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_max_size_in_bytes, compiled_expression_cache_max_elements);
#endif
/// Initialize main config reloader.
std::string include_from_path = config().getString("include_from", "/etc/metrika.xml");
@ -1324,7 +1387,14 @@ try
global_context->updateStorageConfiguration(*config);
global_context->updateInterserverCredentials(*config);
global_context->updateUncompressedCacheConfiguration(*config);
global_context->updateMarkCacheConfiguration(*config);
global_context->updateIndexUncompressedCacheConfiguration(*config);
global_context->updateIndexMarkCacheConfiguration(*config);
global_context->updateMMappedFileCacheConfiguration(*config);
global_context->updateQueryCacheConfiguration(*config);
CompressionCodecEncrypted::Configuration::instance().tryLoad(*config, "encryption_codecs");
#if USE_SSL
CertificateReloader::instance().tryLoad(*config);
@ -1484,19 +1554,6 @@ try
/// Limit on total number of concurrently executed queries.
global_context->getProcessList().setMaxSize(server_settings.max_concurrent_queries);
/// Set up caches.
const size_t max_cache_size = static_cast<size_t>(physical_server_memory * server_settings.cache_size_to_ram_max_ratio);
String uncompressed_cache_policy = server_settings.uncompressed_cache_policy;
size_t uncompressed_cache_size = server_settings.uncompressed_cache_size;
if (uncompressed_cache_size > max_cache_size)
{
uncompressed_cache_size = max_cache_size;
LOG_INFO(log, "Lowered uncompressed cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(uncompressed_cache_size));
}
global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size);
/// Load global settings from default_profile and system_profile.
global_context->setDefaultProfiles(config());
@ -1512,61 +1569,6 @@ try
server_settings.async_insert_queue_flush_on_shutdown));
}
String mark_cache_policy = server_settings.mark_cache_policy;
size_t mark_cache_size = server_settings.mark_cache_size;
if (!mark_cache_size)
LOG_ERROR(log, "Too low mark cache size will lead to severe performance degradation.");
if (mark_cache_size > max_cache_size)
{
mark_cache_size = max_cache_size;
LOG_INFO(log, "Lowered mark cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(mark_cache_size));
}
global_context->setMarkCache(mark_cache_policy, mark_cache_size);
size_t index_uncompressed_cache_size = server_settings.index_uncompressed_cache_size;
if (index_uncompressed_cache_size > max_cache_size)
{
index_uncompressed_cache_size = max_cache_size;
LOG_INFO(log, "Lowered index uncompressed cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(uncompressed_cache_size));
}
if (index_uncompressed_cache_size)
global_context->setIndexUncompressedCache(server_settings.index_uncompressed_cache_size);
size_t index_mark_cache_size = server_settings.index_mark_cache_size;
if (index_mark_cache_size > max_cache_size)
{
index_mark_cache_size = max_cache_size;
LOG_INFO(log, "Lowered index mark cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(uncompressed_cache_size));
}
if (index_mark_cache_size)
global_context->setIndexMarkCache(server_settings.index_mark_cache_size);
size_t mmap_cache_size = server_settings.mmap_cache_size;
if (mmap_cache_size > max_cache_size)
{
mmap_cache_size = max_cache_size;
LOG_INFO(log, "Lowered mmap file cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(uncompressed_cache_size));
}
if (mmap_cache_size)
global_context->setMMappedFileCache(server_settings.mmap_cache_size);
size_t query_cache_max_size_in_bytes = config().getUInt64("query_cache.max_size_in_bytes", DEFAULT_QUERY_CACHE_MAX_SIZE);
size_t query_cache_max_entries = config().getUInt64("query_cache.max_entries", DEFAULT_QUERY_CACHE_MAX_ENTRIES);
size_t query_cache_query_cache_max_entry_size_in_bytes = config().getUInt64("query_cache.max_entry_size_in_bytes", DEFAULT_QUERY_CACHE_MAX_ENTRY_SIZE_IN_BYTES);
size_t query_cache_max_entry_size_in_rows = config().getUInt64("query_cache.max_entry_rows_in_rows", DEFAULT_QUERY_CACHE_MAX_ENTRY_SIZE_IN_ROWS);
if (query_cache_max_size_in_bytes > max_cache_size)
{
query_cache_max_size_in_bytes = max_cache_size;
LOG_INFO(log, "Lowered query cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(uncompressed_cache_size));
}
global_context->setQueryCache(query_cache_max_size_in_bytes, query_cache_max_entries, query_cache_query_cache_max_entry_size_in_bytes, query_cache_max_entry_size_in_rows);
#if USE_EMBEDDED_COMPILER
size_t compiled_expression_cache_max_size_in_bytes = config().getUInt64("compiled_expression_cache_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_SIZE);
size_t compiled_expression_cache_max_elements = config().getUInt64("compiled_expression_cache_elements_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_ENTRIES);
CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_max_size_in_bytes, compiled_expression_cache_max_elements);
#endif
/// Set path for format schema files
fs::path format_schema_path(config().getString("format_schema_path", path / "format_schemas/"));
global_context->setFormatSchemaPath(format_schema_path);

4
src/Access/MultipleAccessStorage.cpp
View File

@ -46,7 +46,7 @@ void MultipleAccessStorage::setStorages(const std::vector<StoragePtr> & storages
{
std::lock_guard lock{mutex};
nested_storages = std::make_shared<const Storages>(storages);
ids_cache.reset();
ids_cache.clear();
}
void MultipleAccessStorage::addStorage(const StoragePtr & new_storage)
@ -69,7 +69,7 @@ void MultipleAccessStorage::removeStorage(const StoragePtr & storage_to_remove)
auto new_storages = std::make_shared<Storages>(*nested_storages);
new_storages->erase(new_storages->begin() + index);
nested_storages = new_storages;
ids_cache.reset();
ids_cache.clear();
}
std::vector<StoragePtr> MultipleAccessStorage::getStorages()

4
src/CMakeLists.txt
View File

@ -599,6 +599,10 @@ if (TARGET ch_contrib::annoy)
dbms_target_link_libraries(PUBLIC ch_contrib::annoy)
endif()
if (TARGET ch_contrib::usearch)
dbms_target_link_libraries(PUBLIC ch_contrib::usearch)
endif()
if (TARGET ch_rust::skim)
dbms_target_include_directories(PRIVATE $<TARGET_PROPERTY:ch_rust::skim,INTERFACE_INCLUDE_DIRECTORIES>)
dbms_target_link_libraries(PUBLIC ch_rust::skim)

6
src/Common/CacheBase.h
View File

@ -151,7 +151,7 @@ public:
std::lock_guard cache_lock(mutex);
/// Insert the new value only if the token is still in present in insert_tokens.
/// (The token may be absent because of a concurrent reset() call).
/// (The token may be absent because of a concurrent clear() call).
bool result = false;
auto token_it = insert_tokens.find(key);
if (token_it != insert_tokens.end() && token_it->second.get() == token)
@ -179,13 +179,13 @@ public:
return cache_policy->dump();
}
void reset()
void clear()
{
std::lock_guard lock(mutex);
insert_tokens.clear();
hits = 0;
misses = 0;
cache_policy->reset(lock);
cache_policy->clear(lock);
}
void remove(const Key & key)

4
src/Common/DNSResolver.cpp
View File

@ -270,8 +270,8 @@ std::unordered_set<String> DNSResolver::reverseResolve(const Poco::Net::IPAddres
void DNSResolver::dropCache()
{
impl->cache_host.reset();
impl->cache_address.reset();
impl->cache_host.clear();
impl->cache_address.clear();
std::scoped_lock lock(impl->update_mutex, impl->drop_mutex);

2
src/Common/FieldVisitorDump.cpp
View File

@ -20,7 +20,7 @@ template <typename T>
static inline void writeQuoted(const DecimalField<T> & x, WriteBuffer & buf)
{
writeChar('\'', buf);
writeText(x.getValue(), x.getScale(), buf, {});
writeText(x.getValue(), x.getScale(), buf, /* trailing_zeros */ true);
writeChar('\'', buf);
}

11
src/Common/ICachePolicy.h
View File

@ -10,11 +10,6 @@
namespace DB
{
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
}
template <typename T>
struct EqualWeightFunction
{
@ -46,8 +41,8 @@ public:
virtual size_t count(std::lock_guard<std::mutex> & /*cache_lock*/) const = 0;
virtual size_t maxSize(std::lock_guard<std::mutex>& /*cache_lock*/) const = 0;
virtual void setMaxCount(size_t /*max_count*/, std::lock_guard<std::mutex> & /* cache_lock */) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented for cache policy"); }
virtual void setMaxSize(size_t /*max_size_in_bytes*/, std::lock_guard<std::mutex> & /* cache_lock */) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented for cache policy"); }
virtual void setMaxCount(size_t /*max_count*/, std::lock_guard<std::mutex> & /* cache_lock */) = 0;
virtual void setMaxSize(size_t /*max_size_in_bytes*/, std::lock_guard<std::mutex> & /* cache_lock */) = 0;
virtual void setQuotaForUser(const String & user_name, size_t max_size_in_bytes, size_t max_entries, std::lock_guard<std::mutex> & /*cache_lock*/) { user_quotas->setQuotaForUser(user_name, max_size_in_bytes, max_entries); }
/// HashFunction usually hashes the entire key and the found key will be equal the provided key. In such cases, use get(). It is also
@ -60,7 +55,7 @@ public:
virtual void remove(const Key & key, std::lock_guard<std::mutex> & /*cache_lock*/) = 0;
virtual void reset(std::lock_guard<std::mutex> & /*cache_lock*/) = 0;
virtual void clear(std::lock_guard<std::mutex> & /*cache_lock*/) = 0;
virtual std::vector<KeyMapped> dump() const = 0;
protected:

36
src/Common/LRUCachePolicy.h
View File

@ -7,9 +7,8 @@
namespace DB
{
/// Cache policy LRU evicts entries which are not used for a long time.
/// WeightFunction is a functor that takes Mapped as a parameter and returns "weight" (approximate size)
/// of that value.
/// Cache policy LRU evicts entries which are not used for a long time. Also see cache policy SLRU for reference.
/// WeightFunction is a functor that takes Mapped as a parameter and returns "weight" (approximate size) of that value.
/// Cache starts to evict entries when their total weight exceeds max_size_in_bytes.
/// Value weight should not change after insertion.
/// To work with the thread-safe implementation of this class use a class "CacheBase" with first parameter "LRU"
@ -24,11 +23,12 @@ public:
using typename Base::OnWeightLossFunction;
/** Initialize LRUCachePolicy with max_size_in_bytes and max_count.
* max_size_in_bytes == 0 means the cache accepts no entries.
* max_count == 0 means no elements size restrictions.
*/
LRUCachePolicy(size_t max_size_in_bytes_, size_t max_count_, OnWeightLossFunction on_weight_loss_function_)
: Base(std::make_unique<NoCachePolicyUserQuota>())
, max_size_in_bytes(std::max(1uz, max_size_in_bytes_))
, max_size_in_bytes(max_size_in_bytes_)
, max_count(max_count_)
, on_weight_loss_function(on_weight_loss_function_)
{
@ -49,7 +49,19 @@ public:
return max_size_in_bytes;
}
void reset(std::lock_guard<std::mutex> & /* cache_lock */) override
void setMaxCount(size_t max_count_, std::lock_guard<std::mutex> & /* cache_lock */) override
{
max_count = max_count_;
removeOverflow();
}
void setMaxSize(size_t max_size_in_bytes_, std::lock_guard<std::mutex> & /* cache_lock */) override
{
max_size_in_bytes = max_size_in_bytes_;
removeOverflow();
}
void clear(std::lock_guard<std::mutex> & /* cache_lock */) override
{
queue.clear();
cells.clear();
@ -155,8 +167,8 @@ private:
/// Total weight of values.
size_t current_size_in_bytes = 0;
const size_t max_size_in_bytes;
const size_t max_count;
size_t max_size_in_bytes;
size_t max_count;
WeightFunction weight_function;
OnWeightLossFunction on_weight_loss_function;
@ -172,10 +184,7 @@ private:
auto it = cells.find(key);
if (it == cells.end())
{
// Queue became inconsistent
abort();
}
std::terminate(); // Queue became inconsistent
const auto & cell = it->second;
@ -190,10 +199,7 @@ private:
on_weight_loss_function(current_weight_lost);
if (current_size_in_bytes > (1ull << 63))
{
// Queue became inconsistent
abort();
}
std::terminate(); // Queue became inconsistent
}
};

52
src/Common/SLRUCachePolicy.h
View File

@ -9,9 +9,8 @@ namespace DB
{
/// Cache policy SLRU evicts entries which were used only once and are not used for a long time,
/// this policy protects entries which were used more then once from a sequential scan.
/// WeightFunction is a functor that takes Mapped as a parameter and returns "weight" (approximate size)
/// of that value.
/// this policy protects entries which were used more then once from a sequential scan. Also see cache policy LRU for reference.
/// WeightFunction is a functor that takes Mapped as a parameter and returns "weight" (approximate size) of that value.
/// Cache starts to evict entries when their total weight exceeds max_size_in_bytes.
/// Value weight should not change after insertion.
/// To work with the thread-safe implementation of this class use a class "CacheBase" with first parameter "SLRU"
@ -30,8 +29,9 @@ public:
* max_protected_size == 0 means that the default protected size is equal to half of the total max size.
*/
/// TODO: construct from special struct with cache policy parameters (also with max_protected_size).
SLRUCachePolicy(size_t max_size_in_bytes_, size_t max_count_, double size_ratio, OnWeightLossFunction on_weight_loss_function_)
SLRUCachePolicy(size_t max_size_in_bytes_, size_t max_count_, double size_ratio_, OnWeightLossFunction on_weight_loss_function_)
: Base(std::make_unique<NoCachePolicyUserQuota>())
, size_ratio(size_ratio_)
, max_protected_size(static_cast<size_t>(max_size_in_bytes_ * std::min(1.0, size_ratio)))
, max_size_in_bytes(max_size_in_bytes_)
, max_count(max_count_)
@ -54,7 +54,22 @@ public:
return max_size_in_bytes;
}
void reset(std::lock_guard<std::mutex> & /* cache_lock */) override
void setMaxCount(size_t max_count_, std::lock_guard<std::mutex> & /* cache_lock */) override
{
max_count = max_count_;
removeOverflow(protected_queue, max_protected_size, current_protected_size, /*is_protected=*/true);
removeOverflow(probationary_queue, max_size_in_bytes, current_size_in_bytes, /*is_protected=*/false);
}
void setMaxSize(size_t max_size_in_bytes_, std::lock_guard<std::mutex> & /* cache_lock */) override
{
max_protected_size = static_cast<size_t>(max_size_in_bytes_ * std::min(1.0, size_ratio));
max_size_in_bytes = max_size_in_bytes_;
removeOverflow(protected_queue, max_protected_size, current_protected_size, /*is_protected=*/true);
removeOverflow(probationary_queue, max_size_in_bytes, current_size_in_bytes, /*is_protected=*/false);
}
void clear(std::lock_guard<std::mutex> & /* cache_lock */) override
{
cells.clear();
probationary_queue.clear();
@ -68,12 +83,13 @@ public:
auto it = cells.find(key);
if (it == cells.end())
return;
auto & cell = it->second;
current_size_in_bytes -= cell.size;
if (cell.is_protected)
{
current_protected_size -= cell.size;
}
auto & queue = cell.is_protected ? protected_queue : probationary_queue;
queue.erase(cell.queue_iterator);
cells.erase(it);
@ -192,16 +208,17 @@ private:
Cells cells;
const double size_ratio;
size_t current_protected_size = 0;
size_t current_size_in_bytes = 0;
const size_t max_protected_size;
const size_t max_size_in_bytes;
const size_t max_count;
size_t max_protected_size;
size_t max_size_in_bytes;
size_t max_count;
WeightFunction weight_function;
OnWeightLossFunction on_weight_loss_function;
void removeOverflow(SLRUQueue & queue, const size_t max_weight_size, size_t & current_weight_size, bool is_protected)
void removeOverflow(SLRUQueue & queue, size_t max_weight_size, size_t & current_weight_size, bool is_protected)
{
size_t current_weight_lost = 0;
size_t queue_size = queue.size();
@ -223,8 +240,7 @@ private:
{
need_remove = [&]()
{
return ((max_count != 0 && cells.size() > max_count)
|| (current_weight_size > max_weight_size)) && (queue_size > 0);
return ((max_count != 0 && cells.size() > max_count) || (current_weight_size > max_weight_size)) && (queue_size > 0);
};
}
@ -234,10 +250,7 @@ private:
auto it = cells.find(key);
if (it == cells.end())
{
// Queue became inconsistent
abort();
}
std::terminate(); // Queue became inconsistent
auto & cell = it->second;
@ -262,10 +275,7 @@ private:
on_weight_loss_function(current_weight_lost);
if (current_size_in_bytes > (1ull << 63))
{
// Queue became inconsistent
abort();
}
std::terminate(); // Queue became inconsistent
}
};

2
src/Common/TTLCachePolicy.h
View File

@ -121,7 +121,7 @@ public:
max_size_in_bytes = max_size_in_bytes_;
}
void reset(std::lock_guard<std::mutex> & /* cache_lock */) override
void clear(std::lock_guard<std::mutex> & /* cache_lock */) override
{
cache.clear();
}

4
src/Common/tests/gtest_slru_cache.cpp
View File

@ -92,7 +92,7 @@ TEST(SLRUCache, removeFromProtected)
ASSERT_TRUE(value == nullptr);
}
TEST(SLRUCache, reset)
TEST(SLRUCache, clear)
{
using SimpleCacheBase = DB::CacheBase<int, int>;
auto slru_cache = SimpleCacheBase("SLRU", /*max_size_in_bytes=*/10, /*max_count=*/0, /*size_ratio*/0.5);
@ -101,7 +101,7 @@ TEST(SLRUCache, reset)
slru_cache.set(2, std::make_shared<int>(4)); /// add to protected_queue
slru_cache.reset();
slru_cache.clear();
auto value = slru_cache.get(1);
ASSERT_TRUE(value == nullptr);

8
src/Compression/CompressionCodecDelta.cpp
View File

@ -73,8 +73,8 @@ void compressDataForType(const char * source, UInt32 source_size, char * dest)
const char * const source_end = source + source_size;
while (source < source_end)
{
T curr_src = unalignedLoad<T>(source);
unalignedStore<T>(dest, curr_src - prev_src);
T curr_src = unalignedLoadLittleEndian<T>(source);
unalignedStoreLittleEndian<T>(dest, curr_src - prev_src);
prev_src = curr_src;
source += sizeof(T);
@ -94,10 +94,10 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest,
const char * const source_end = source + source_size;
while (source < source_end)
{
accumulator += unalignedLoad<T>(source);
accumulator += unalignedLoadLittleEndian<T>(source);
if (dest + sizeof(accumulator) > output_end) [[unlikely]]
throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress the data");
unalignedStore<T>(dest, accumulator);
unalignedStoreLittleEndian<T>(dest, accumulator);
source += sizeof(T);
dest += sizeof(T);

4
src/Core/Field.h
View File

@ -138,7 +138,7 @@ template <typename T> bool decimalEqual(T x, T y, UInt32 x_scale, UInt32 y_scale
template <typename T> bool decimalLess(T x, T y, UInt32 x_scale, UInt32 y_scale);
template <typename T> bool decimalLessOrEqual(T x, T y, UInt32 x_scale, UInt32 y_scale);
template <typename T>
template <is_decimal T>
class DecimalField
{
public:
@ -838,7 +838,7 @@ template <> struct Field::EnumToType<Field::Types::Decimal32> { using Type = Dec
template <> struct Field::EnumToType<Field::Types::Decimal64> { using Type = DecimalField<Decimal64>; };
template <> struct Field::EnumToType<Field::Types::Decimal128> { using Type = DecimalField<Decimal128>; };
template <> struct Field::EnumToType<Field::Types::Decimal256> { using Type = DecimalField<Decimal256>; };
template <> struct Field::EnumToType<Field::Types::AggregateFunctionState> { using Type = DecimalField<AggregateFunctionStateData>; };
template <> struct Field::EnumToType<Field::Types::AggregateFunctionState> { using Type = AggregateFunctionStateData; };
template <> struct Field::EnumToType<Field::Types::CustomType> { using Type = CustomType; };
template <> struct Field::EnumToType<Field::Types::Bool> { using Type = UInt64; };

2
src/Core/ServerSettings.h
View File

@ -39,7 +39,7 @@ namespace DB
M(UInt64, restore_threads, 16, "The maximum number of threads to execute RESTORE requests.", 0) \
M(Int32, max_connections, 1024, "Max server connections.", 0) \
M(UInt32, asynchronous_metrics_update_period_s, 1, "Period in seconds for updating asynchronous metrics.", 0) \
M(UInt32, asynchronous_heavy_metrics_update_period_s, 120, "Period in seconds for updating asynchronous metrics.", 0) \
M(UInt32, asynchronous_heavy_metrics_update_period_s, 120, "Period in seconds for updating heavy asynchronous metrics.", 0) \
M(String, default_database, "default", "Default database name.", 0) \
M(String, tmp_policy, "", "Policy for storage with temporary data.", 0) \
M(UInt64, max_temporary_data_on_disk_size, 0, "The maximum amount of storage that could be used for external aggregation, joins or sorting., ", 0) \

3
src/Core/Settings.h
View File

@ -779,6 +779,7 @@ class IColumn;
M(Bool, allow_experimental_hash_functions, false, "Enable experimental hash functions", 0) \
M(Bool, allow_experimental_object_type, false, "Allow Object and JSON data types", 0) \
M(Bool, allow_experimental_annoy_index, false, "Allows to use Annoy index. Disabled by default because this feature is experimental", 0) \
M(Bool, allow_experimental_usearch_index, false, "Allows to use USearch index. Disabled by default because this feature is experimental", 0) \
M(UInt64, max_limit_for_ann_queries, 1'000'000, "SELECT queries with LIMIT bigger than this setting cannot use ANN indexes. Helps to prevent memory overflows in ANN search indexes.", 0) \
M(Int64, annoy_index_search_k_nodes, -1, "SELECT queries search up to this many nodes in Annoy indexes.", 0) \
M(Bool, throw_on_unsupported_query_inside_transaction, true, "Throw exception if unsupported query is used inside transaction", 0) \
@ -876,8 +877,10 @@ class IColumn;
M(Bool, input_format_orc_case_insensitive_column_matching, false, "Ignore case when matching ORC columns with CH columns.", 0) \
M(Bool, input_format_parquet_case_insensitive_column_matching, false, "Ignore case when matching Parquet columns with CH columns.", 0) \
M(Bool, input_format_parquet_preserve_order, false, "Avoid reordering rows when reading from Parquet files. Usually makes it much slower.", 0) \
M(Bool, input_format_parquet_filter_push_down, true, "When reading Parquet files, skip whole row groups based on the WHERE/PREWHERE expressions and min/max statistics in the Parquet metadata.", 0) \
M(Bool, input_format_allow_seeks, true, "Allow seeks while reading in ORC/Parquet/Arrow input formats", 0) \
M(Bool, input_format_orc_allow_missing_columns, false, "Allow missing columns while reading ORC input formats", 0) \
M(Bool, input_format_orc_use_fast_decoder, true, "Use a faster ORC decoder implementation.", 0) \
M(Bool, input_format_parquet_allow_missing_columns, false, "Allow missing columns while reading Parquet input formats", 0) \
M(UInt64, input_format_parquet_local_file_min_bytes_for_seek, 8192, "Min bytes required for local read (file) to do seek, instead of read with ignore in Parquet input format", 0) \
M(Bool, input_format_arrow_allow_missing_columns, false, "Allow missing columns while reading Arrow input formats", 0) \

1
src/DataTypes/DataTypeLowCardinality.h
View File

@ -46,6 +46,7 @@ public:
bool canBeUsedInBooleanContext() const override { return dictionary_type->canBeUsedInBooleanContext(); }
bool isValueRepresentedByNumber() const override { return dictionary_type->isValueRepresentedByNumber(); }
bool isValueRepresentedByInteger() const override { return dictionary_type->isValueRepresentedByInteger(); }
bool isValueRepresentedByUnsignedInteger() const override { return dictionary_type->isValueRepresentedByUnsignedInteger(); }
bool isValueUnambiguouslyRepresentedInContiguousMemoryRegion() const override { return true; }
bool haveMaximumSizeOfValue() const override { return dictionary_type->haveMaximumSizeOfValue(); }
size_t getMaximumSizeOfValueInMemory() const override { return dictionary_type->getMaximumSizeOfValueInMemory(); }

1
src/Databases/DatabaseReplicated.cpp
View File

@ -830,6 +830,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep
query_context->setSetting("allow_experimental_hash_functions", 1);
query_context->setSetting("allow_experimental_object_type", 1);
query_context->setSetting("allow_experimental_annoy_index", 1);
query_context->setSetting("allow_experimental_usearch_index", 1);
query_context->setSetting("allow_experimental_bigint_types", 1);
query_context->setSetting("allow_experimental_window_functions", 1);
query_context->setSetting("allow_experimental_geo_types", 1);

2
src/Formats/FormatFactory.cpp
View File

@ -124,6 +124,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings)
format_settings.parquet.output_version = settings.output_format_parquet_version;
format_settings.parquet.case_insensitive_column_matching = settings.input_format_parquet_case_insensitive_column_matching;
format_settings.parquet.preserve_order = settings.input_format_parquet_preserve_order;
format_settings.parquet.filter_push_down = settings.input_format_parquet_filter_push_down;
format_settings.parquet.allow_missing_columns = settings.input_format_parquet_allow_missing_columns;
format_settings.parquet.skip_columns_with_unsupported_types_in_schema_inference = settings.input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference;
format_settings.parquet.output_string_as_string = settings.output_format_parquet_string_as_string;
@ -189,6 +190,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings)
format_settings.orc.case_insensitive_column_matching = settings.input_format_orc_case_insensitive_column_matching;
format_settings.orc.output_string_as_string = settings.output_format_orc_string_as_string;
format_settings.orc.output_compression_method = settings.output_format_orc_compression_method;
format_settings.orc.use_fast_decoder = settings.input_format_orc_use_fast_decoder;
format_settings.defaults_for_omitted_fields = settings.input_format_defaults_for_omitted_fields;
format_settings.capn_proto.enum_comparing_mode = settings.format_capn_proto_enum_comparising_mode;
format_settings.capn_proto.skip_fields_with_unsupported_types_in_schema_inference = settings.input_format_capn_proto_skip_fields_with_unsupported_types_in_schema_inference;

3
src/Formats/FormatFactory.h
View File

@ -90,9 +90,6 @@ private:
const FormatSettings & settings)>;
// Incompatible with FileSegmentationEngine.
//
// In future we may also want to pass some information about WHERE conditions (SelectQueryInfo?)
// and get some information about projections (min/max/count per column per row group).
using RandomAccessInputCreator = std::function<InputFormatPtr(
ReadBuffer & buf,
const Block & header,

2
src/Formats/FormatSettings.h
View File

@ -231,6 +231,7 @@ struct FormatSettings
bool allow_missing_columns = false;
bool skip_columns_with_unsupported_types_in_schema_inference = false;
bool case_insensitive_column_matching = false;
bool filter_push_down = true;
std::unordered_set<int> skip_row_groups = {};
bool output_string_as_string = false;
bool output_fixed_string_as_fixed_byte_array = true;
@ -347,6 +348,7 @@ struct FormatSettings
std::unordered_set<int> skip_stripes = {};
bool output_string_as_string = false;
ORCCompression output_compression_method = ORCCompression::NONE;
bool use_fast_decoder = true;
} orc;
/// For capnProto format we should determine how to

5
src/IO/ReadBufferFromString.h
View File

@ -19,7 +19,10 @@ public:
class ReadBufferFromOwnString : public String, public ReadBufferFromString
{
public:
explicit ReadBufferFromOwnString(const String & s_): String(s_), ReadBufferFromString(*this) {}
template <typename S>
explicit ReadBufferFromOwnString(S && s_) : String(std::forward<S>(s_)), ReadBufferFromString(*this)
{
}
};
}

34
src/Interpreters/Cache/QueryCache.cpp
View File

@ -471,6 +471,21 @@ std::unique_ptr<SourceFromChunks> QueryCache::Reader::getSourceExtremes()
return std::move(source_from_chunks_extremes);
}
QueryCache::QueryCache(size_t max_size_in_bytes, size_t max_entries, size_t max_entry_size_in_bytes_, size_t max_entry_size_in_rows_)
: cache(std::make_unique<TTLCachePolicy<Key, Entry, KeyHasher, QueryCacheEntryWeight, IsStale>>(std::make_unique<PerUserTTLCachePolicyUserQuota>()))
{
updateConfiguration(max_size_in_bytes, max_entries, max_entry_size_in_bytes_, max_entry_size_in_rows_);
}
void QueryCache::updateConfiguration(size_t max_size_in_bytes, size_t max_entries, size_t max_entry_size_in_bytes_, size_t max_entry_size_in_rows_)
{
std::lock_guard lock(mutex);
cache.setMaxSize(max_size_in_bytes);
cache.setMaxCount(max_entries);
max_entry_size_in_bytes = max_entry_size_in_bytes_;
max_entry_size_in_rows = max_entry_size_in_rows_;
}
QueryCache::Reader QueryCache::createReader(const Key & key)
{
std::lock_guard lock(mutex);
@ -488,9 +503,9 @@ QueryCache::Writer QueryCache::createWriter(const Key & key, std::chrono::millis
return Writer(cache, key, max_entry_size_in_bytes, max_entry_size_in_rows, min_query_runtime, squash_partial_results, max_block_size);
}
void QueryCache::reset()
void QueryCache::clear()
{
cache.reset();
cache.clear();
std::lock_guard lock(mutex);
times_executed.clear();
}
@ -521,19 +536,4 @@ std::vector<QueryCache::Cache::KeyMapped> QueryCache::dump() const
return cache.dump();
}
QueryCache::QueryCache(size_t max_size_in_bytes, size_t max_entries, size_t max_entry_size_in_bytes_, size_t max_entry_size_in_rows_)
: cache(std::make_unique<TTLCachePolicy<Key, Entry, KeyHasher, QueryCacheEntryWeight, IsStale>>(std::make_unique<PerUserTTLCachePolicyUserQuota>()))
{
updateConfiguration(max_size_in_bytes, max_entries, max_entry_size_in_bytes_, max_entry_size_in_rows_);
}
void QueryCache::updateConfiguration(size_t max_size_in_bytes, size_t max_entries, size_t max_entry_size_in_bytes_, size_t max_entry_size_in_rows_)
{
std::lock_guard lock(mutex);
cache.setMaxSize(max_size_in_bytes);
cache.setMaxCount(max_entries);
max_entry_size_in_bytes = max_entry_size_in_bytes_;
max_entry_size_in_rows = max_entry_size_in_rows_;
}
}

2
src/Interpreters/Cache/QueryCache.h
View File

@ -180,7 +180,7 @@ public:
Reader createReader(const Key & key);
Writer createWriter(const Key & key, std::chrono::milliseconds min_query_runtime, bool squash_partial_results, size_t max_block_size, size_t max_query_cache_size_in_bytes_quota, size_t max_query_cache_entries_quota);
void reset();
void clear();
size_t weight() const;
size_t count() const;

126
src/Interpreters/Context.cpp
View File

@ -548,7 +548,7 @@ struct ContextSharedPart : boost::noncopyable
*/
#if USE_EMBEDDED_COMPILER
if (auto * cache = CompiledExpressionCacheFactory::instance().tryGetCache())
cache->reset();
cache->clear();
#endif
/// Preemptive destruction is important, because these objects may have a refcount to ContextShared (cyclic reference).
@ -2278,6 +2278,16 @@ void Context::setUncompressedCache(const String & uncompressed_cache_policy, siz
shared->uncompressed_cache = std::make_shared<UncompressedCache>(uncompressed_cache_policy, max_size_in_bytes);
}
void Context::updateUncompressedCacheConfiguration(const Poco::Util::AbstractConfiguration & config)
{
auto lock = getLock();
if (!shared->uncompressed_cache)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Uncompressed cache was not created yet.");
size_t max_size_in_bytes = config.getUInt64("uncompressed_cache_size", DEFAULT_UNCOMPRESSED_CACHE_MAX_SIZE);
shared->uncompressed_cache->setMaxSize(max_size_in_bytes);
}
UncompressedCachePtr Context::getUncompressedCache() const
{
@ -2285,14 +2295,13 @@ UncompressedCachePtr Context::getUncompressedCache() const
return shared->uncompressed_cache;
}
void Context::clearUncompressedCache() const
{
auto lock = getLock();
if (shared->uncompressed_cache)
shared->uncompressed_cache->reset();
}
if (shared->uncompressed_cache)
shared->uncompressed_cache->clear();
}
void Context::setMarkCache(const String & mark_cache_policy, size_t cache_size_in_bytes)
{
@ -2304,6 +2313,17 @@ void Context::setMarkCache(const String & mark_cache_policy, size_t cache_size_i
shared->mark_cache = std::make_shared<MarkCache>(mark_cache_policy, cache_size_in_bytes);
}
void Context::updateMarkCacheConfiguration(const Poco::Util::AbstractConfiguration & config)
{
auto lock = getLock();
if (!shared->mark_cache)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Mark cache was not created yet.");
size_t max_size_in_bytes = config.getUInt64("mark_cache_size", DEFAULT_MARK_CACHE_MAX_SIZE);
shared->mark_cache->setMaxSize(max_size_in_bytes);
}
MarkCachePtr Context::getMarkCache() const
{
auto lock = getLock();
@ -2313,8 +2333,9 @@ MarkCachePtr Context::getMarkCache() const
void Context::clearMarkCache() const
{
auto lock = getLock();
if (shared->mark_cache)
shared->mark_cache->reset();
shared->mark_cache->clear();
}
ThreadPool & Context::getLoadMarksThreadpool() const
@ -2342,20 +2363,30 @@ void Context::setIndexUncompressedCache(size_t max_size_in_bytes)
shared->index_uncompressed_cache = std::make_shared<UncompressedCache>(max_size_in_bytes);
}
void Context::updateIndexUncompressedCacheConfiguration(const Poco::Util::AbstractConfiguration & config)
{
auto lock = getLock();
if (!shared->index_uncompressed_cache)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Index uncompressed cache was not created yet.");
size_t max_size_in_bytes = config.getUInt64("index_uncompressed_cache_size", DEFAULT_INDEX_UNCOMPRESSED_CACHE_MAX_SIZE);
shared->index_uncompressed_cache->setMaxSize(max_size_in_bytes);
}
UncompressedCachePtr Context::getIndexUncompressedCache() const
{
auto lock = getLock();
return shared->index_uncompressed_cache;
}
void Context::clearIndexUncompressedCache() const
{
auto lock = getLock();
if (shared->index_uncompressed_cache)
shared->index_uncompressed_cache->reset();
}
if (shared->index_uncompressed_cache)
shared->index_uncompressed_cache->clear();
}
void Context::setIndexMarkCache(size_t cache_size_in_bytes)
{
@ -2367,6 +2398,17 @@ void Context::setIndexMarkCache(size_t cache_size_in_bytes)
shared->index_mark_cache = std::make_shared<MarkCache>(cache_size_in_bytes);
}
void Context::updateIndexMarkCacheConfiguration(const Poco::Util::AbstractConfiguration & config)
{
auto lock = getLock();
if (!shared->index_mark_cache)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Index mark cache was not created yet.");
size_t max_size_in_bytes = config.getUInt64("index_mark_cache_size", DEFAULT_INDEX_MARK_CACHE_MAX_SIZE);
shared->index_mark_cache->setMaxSize(max_size_in_bytes);
}
MarkCachePtr Context::getIndexMarkCache() const
{
auto lock = getLock();
@ -2376,8 +2418,9 @@ MarkCachePtr Context::getIndexMarkCache() const
void Context::clearIndexMarkCache() const
{
auto lock = getLock();
if (shared->index_mark_cache)
shared->index_mark_cache->reset();
shared->index_mark_cache->clear();
}
void Context::setMMappedFileCache(size_t cache_size_in_num_entries)
@ -2390,6 +2433,17 @@ void Context::setMMappedFileCache(size_t cache_size_in_num_entries)
shared->mmap_cache = std::make_shared<MMappedFileCache>(cache_size_in_num_entries);
}
void Context::updateMMappedFileCacheConfiguration(const Poco::Util::AbstractConfiguration & config)
{
auto lock = getLock();
if (!shared->mmap_cache)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapped file cache was not created yet.");
size_t max_size_in_bytes = config.getUInt64("mmap_cache_size", DEFAULT_MMAP_CACHE_MAX_SIZE);
shared->mmap_cache->setMaxSize(max_size_in_bytes);
}
MMappedFileCachePtr Context::getMMappedFileCache() const
{
auto lock = getLock();
@ -2399,8 +2453,9 @@ MMappedFileCachePtr Context::getMMappedFileCache() const
void Context::clearMMappedFileCache() const
{
auto lock = getLock();
if (shared->mmap_cache)
shared->mmap_cache->reset();
shared->mmap_cache->clear();
}
void Context::setQueryCache(size_t max_size_in_bytes, size_t max_entries, size_t max_entry_size_in_bytes, size_t max_entry_size_in_rows)
@ -2416,14 +2471,15 @@ void Context::setQueryCache(size_t max_size_in_bytes, size_t max_entries, size_t
void Context::updateQueryCacheConfiguration(const Poco::Util::AbstractConfiguration & config)
{
auto lock = getLock();
if (shared->query_cache)
{
size_t max_size_in_bytes = config.getUInt64("query_cache.max_size_in_bytes", DEFAULT_QUERY_CACHE_MAX_SIZE);
size_t max_entries = config.getUInt64("query_cache.max_entries", DEFAULT_QUERY_CACHE_MAX_ENTRIES);
size_t max_entry_size_in_bytes = config.getUInt64("query_cache.max_entry_size_in_bytes", DEFAULT_QUERY_CACHE_MAX_ENTRY_SIZE_IN_BYTES);
size_t max_entry_size_in_rows = config.getUInt64("query_cache.max_entry_rows_in_rows", DEFAULT_QUERY_CACHE_MAX_ENTRY_SIZE_IN_ROWS);
shared->query_cache->updateConfiguration(max_size_in_bytes, max_entries, max_entry_size_in_bytes, max_entry_size_in_rows);
}
if (!shared->query_cache)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Query cache was not created yet.");
size_t max_size_in_bytes = config.getUInt64("query_cache.max_size_in_bytes", DEFAULT_QUERY_CACHE_MAX_SIZE);
size_t max_entries = config.getUInt64("query_cache.max_entries", DEFAULT_QUERY_CACHE_MAX_ENTRIES);
size_t max_entry_size_in_bytes = config.getUInt64("query_cache.max_entry_size_in_bytes", DEFAULT_QUERY_CACHE_MAX_ENTRY_SIZE_IN_BYTES);
size_t max_entry_size_in_rows = config.getUInt64("query_cache.max_entry_rows_in_rows", DEFAULT_QUERY_CACHE_MAX_ENTRY_SIZE_IN_ROWS);
shared->query_cache->updateConfiguration(max_size_in_bytes, max_entries, max_entry_size_in_bytes, max_entry_size_in_rows);
}
QueryCachePtr Context::getQueryCache() const
@ -2435,30 +2491,36 @@ QueryCachePtr Context::getQueryCache() const
void Context::clearQueryCache() const
{
auto lock = getLock();
if (shared->query_cache)
shared->query_cache->reset();
shared->query_cache->clear();
}
void Context::clearCaches() const
{
auto lock = getLock();
if (shared->uncompressed_cache)
shared->uncompressed_cache->reset();
if (!shared->uncompressed_cache)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Uncompressed cache was not created yet.");
shared->uncompressed_cache->clear();
if (shared->mark_cache)
shared->mark_cache->reset();
if (!shared->mark_cache)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Mark cache was not created yet.");
shared->mark_cache->clear();
if (shared->index_uncompressed_cache)
shared->index_uncompressed_cache->reset();
if (!shared->index_uncompressed_cache)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Index uncompressed cache was not created yet.");
shared->index_uncompressed_cache->clear();
if (shared->index_mark_cache)
shared->index_mark_cache->reset();
if (!shared->index_mark_cache)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Index mark cache was not created yet.");
shared->index_mark_cache->clear();
if (shared->mmap_cache)
shared->mmap_cache->reset();
if (!shared->mmap_cache)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Mmapped file cache was not created yet.");
shared->mmap_cache->clear();
/// Intentionally not dropping the query cache which is transactionally inconsistent by design.
/// Intentionally not clearing the query cache which is transactionally inconsistent by design.
}
ThreadPool & Context::getPrefetchThreadpool() const

11
src/Interpreters/Context.h
View File

@ -922,33 +922,32 @@ public:
/// --- Caches ------------------------------------------------------------------------------------------
/// Create a cache of uncompressed blocks of specified size. This can be done only once.
void setUncompressedCache(const String & uncompressed_cache_policy, size_t max_size_in_bytes);
void updateUncompressedCacheConfiguration(const Poco::Util::AbstractConfiguration & config);
std::shared_ptr<UncompressedCache> getUncompressedCache() const;
void clearUncompressedCache() const;
/// Create a cache of marks of specified size. This can be done only once.
void setMarkCache(const String & mark_cache_policy, size_t cache_size_in_bytes);
void updateMarkCacheConfiguration(const Poco::Util::AbstractConfiguration & config);
std::shared_ptr<MarkCache> getMarkCache() const;
void clearMarkCache() const;
ThreadPool & getLoadMarksThreadpool() const;
/// Create a cache of index uncompressed blocks of specified size. This can be done only once.
void setIndexUncompressedCache(size_t max_size_in_bytes);
void updateIndexUncompressedCacheConfiguration(const Poco::Util::AbstractConfiguration & config);
std::shared_ptr<UncompressedCache> getIndexUncompressedCache() const;
void clearIndexUncompressedCache() const;
/// Create a cache of index marks of specified size. This can be done only once.
void setIndexMarkCache(size_t cache_size_in_bytes);
void updateIndexMarkCacheConfiguration(const Poco::Util::AbstractConfiguration & config);
std::shared_ptr<MarkCache> getIndexMarkCache() const;
void clearIndexMarkCache() const;
/// Create a cache of mapped files to avoid frequent open/map/unmap/close and to reuse from several threads.
void setMMappedFileCache(size_t cache_size_in_num_entries);
void updateMMappedFileCacheConfiguration(const Poco::Util::AbstractConfiguration & config);
std::shared_ptr<MMappedFileCache> getMMappedFileCache() const;
void clearMMappedFileCache() const;
/// Create a cache of query results for statements which run repeatedly.
void setQueryCache(size_t max_size_in_bytes, size_t max_entries, size_t max_entry_size_in_bytes, size_t max_entry_size_in_rows);
void updateQueryCacheConfiguration(const Poco::Util::AbstractConfiguration & config);
std::shared_ptr<QueryCache> getQueryCache() const;

3
src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -704,6 +704,9 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti
if (index_desc.type == "annoy" && !settings.allow_experimental_annoy_index)
throw Exception(ErrorCodes::INCORRECT_QUERY, "Annoy index is disabled. Turn on allow_experimental_annoy_index");
if (index_desc.type == "usearch" && !settings.allow_experimental_usearch_index)
throw Exception(ErrorCodes::INCORRECT_QUERY, "USearch index is disabled. Turn on allow_experimental_usearch_index");
properties.indices.push_back(index_desc);
}
if (create.columns_list->projections)

2
src/Interpreters/InterpreterSystemQuery.cpp
View File

@ -345,7 +345,7 @@ BlockIO InterpreterSystemQuery::execute()
case Type::DROP_COMPILED_EXPRESSION_CACHE:
getContext()->checkAccess(AccessType::SYSTEM_DROP_COMPILED_EXPRESSION_CACHE);
if (auto * cache = CompiledExpressionCacheFactory::instance().tryGetCache())
cache->reset();
cache->clear();
break;
#endif
#if USE_AWS_S3

1
src/Interpreters/ServerAsynchronousMetrics.cpp
View File

@ -6,6 +6,7 @@
#include <Interpreters/Cache/FileCache.h>
#include <Interpreters/Cache/FileCacheFactory.h>
#include <Interpreters/Context.h>
#include <Interpreters/Cache/QueryCache.h>
#include <Interpreters/JIT/CompiledExpressionCache.h>
#include <Databases/IDatabase.h>

1
src/Parsers/ASTIndexDeclaration.h
View File

@ -14,6 +14,7 @@ class ASTIndexDeclaration : public IAST
public:
static const auto DEFAULT_INDEX_GRANULARITY = 1uz;
static const auto DEFAULT_ANNOY_INDEX_GRANULARITY = 100'000'000uz;
static const auto DEFAULT_USEARCH_INDEX_GRANULARITY = 100'000'000uz;
String name;
IAST * expr;

2
src/Parsers/ParserCreateIndexQuery.cpp
View File

@ -66,6 +66,8 @@ bool ParserCreateIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected
{
if (index->type && index->type->name == "annoy")
index->granularity = ASTIndexDeclaration::DEFAULT_ANNOY_INDEX_GRANULARITY;
else if (index->type && index->type->name == "usearch")
index->granularity = ASTIndexDeclaration::DEFAULT_USEARCH_INDEX_GRANULARITY;
else
index->granularity = ASTIndexDeclaration::DEFAULT_INDEX_GRANULARITY;
}

2
src/Parsers/ParserCreateQuery.cpp
View File

@ -148,6 +148,8 @@ bool ParserIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
{
if (index->type->name == "annoy")
index->granularity = ASTIndexDeclaration::DEFAULT_ANNOY_INDEX_GRANULARITY;
else if (index->type->name == "usearch")
index->granularity = ASTIndexDeclaration::DEFAULT_USEARCH_INDEX_GRANULARITY;
else
index->granularity = ASTIndexDeclaration::DEFAULT_INDEX_GRANULARITY;
}

8
src/Processors/Formats/IInputFormat.h
View File

@ -10,6 +10,8 @@
namespace DB
{
struct SelectQueryInfo;
using ColumnMappingPtr = std::shared_ptr<ColumnMapping>;
/** Input format is a source, that reads data from ReadBuffer.
@ -21,9 +23,13 @@ protected:
ReadBuffer * in [[maybe_unused]] = nullptr;
public:
// ReadBuffer can be nullptr for random-access formats.
/// ReadBuffer can be nullptr for random-access formats.
IInputFormat(Block header, ReadBuffer * in_);
/// If the format is used by a SELECT query, this method may be called.
/// The format may use it for filter pushdown.
virtual void setQueryInfo(const SelectQueryInfo &, ContextPtr) {}
/** In some usecase (hello Kafka) we need to read a lot of tiny streams in exactly the same format.
* The recreating of parser for each small stream takes too long, so we introduce a method
* resetParser() which allow to reset the state of parser to continue reading of

1019
src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

129
src/Processors/Formats/Impl/NativeORCBlockInputFormat.h Normal file
View File

@ -0,0 +1,129 @@
#pragma once
#include "config.h"
#if USE_ORC
# include <Formats/FormatSettings.h>
# include <IO/ReadBufferFromString.h>
# include <Processors/Formats/IInputFormat.h>
# include <Processors/Formats/ISchemaReader.h>
# include <orc/OrcFile.hh>
namespace DB
{
class ORCInputStream : public orc::InputStream
{
public:
ORCInputStream(SeekableReadBuffer & in_, size_t file_size_);
uint64_t getLength() const override;
uint64_t getNaturalReadSize() const override;
void read(void * buf, uint64_t length, uint64_t offset) override;
const std::string & getName() const override { return name; }
protected:
SeekableReadBuffer & in;
size_t file_size;
std::string name = "ORCInputStream";
};
class ORCInputStreamFromString : public ReadBufferFromOwnString, public ORCInputStream
{
public:
template <typename S>
ORCInputStreamFromString(S && s_, size_t file_size_)
: ReadBufferFromOwnString(std::forward<S>(s_)), ORCInputStream(dynamic_cast<SeekableReadBuffer &>(*this), file_size_)
{
}
};
std::unique_ptr<orc::InputStream> asORCInputStream(ReadBuffer & in, const FormatSettings & settings, std::atomic<int> & is_cancelled);
// Reads the whole file into a memory buffer, owned by the returned RandomAccessFile.
std::unique_ptr<orc::InputStream> asORCInputStreamLoadIntoMemory(ReadBuffer & in, std::atomic<int> & is_cancelled);
class ORCColumnToCHColumn;
class NativeORCBlockInputFormat : public IInputFormat
{
public:
NativeORCBlockInputFormat(ReadBuffer & in_, Block header_, const FormatSettings & format_settings_);
String getName() const override { return "ORCBlockInputFormat"; }
void resetParser() override;
const BlockMissingValues & getMissingValues() const override;
size_t getApproxBytesReadForChunk() const override { return approx_bytes_read_for_chunk; }
protected:
Chunk generate() override;
void onCancel() override { is_stopped = 1; }
private:
void prepareFileReader();
bool prepareStripeReader();
std::unique_ptr<orc::Reader> file_reader;
std::unique_ptr<orc::RowReader> stripe_reader;
std::unique_ptr<ORCColumnToCHColumn> orc_column_to_ch_column;
std::unique_ptr<orc::ColumnVectorBatch> batch;
// indices of columns to read from ORC file
std::list<UInt64> include_indices;
BlockMissingValues block_missing_values;
size_t approx_bytes_read_for_chunk;
const FormatSettings format_settings;
const std::unordered_set<int> & skip_stripes;
int total_stripes = 0;
int current_stripe = -1;
std::unique_ptr<orc::StripeInformation> current_stripe_info;
std::atomic<int> is_stopped{0};
};
class NativeORCSchemaReader : public ISchemaReader
{
public:
NativeORCSchemaReader(ReadBuffer & in_, const FormatSettings & format_settings_);
NamesAndTypesList readSchema() override;
private:
const FormatSettings format_settings;
};
class ORCColumnToCHColumn
{
public:
using ORCColumnPtr = const orc::ColumnVectorBatch *;
using ORCTypePtr = const orc::Type *;
using ORCColumnWithType = std::pair<ORCColumnPtr, ORCTypePtr>;
using NameToColumnPtr = std::unordered_map<std::string, ORCColumnWithType>;
ORCColumnToCHColumn(const Block & header_, bool allow_missing_columns_, bool null_as_default_, bool case_insensitive_matching_ = false);
void orcTableToCHChunk(
Chunk & res,
const orc::Type * schema,
const orc::ColumnVectorBatch * table,
size_t num_rows,
BlockMissingValues * block_missing_values = nullptr);
void orcColumnsToCHChunk(
Chunk & res, NameToColumnPtr & name_to_column_ptr, size_t num_rows, BlockMissingValues * block_missing_values = nullptr);
private:
const Block & header;
/// If false, throw exception if some columns in header not exists in arrow table.
bool allow_missing_columns;
bool null_as_default;
bool case_insensitive_matching;
};
}
#endif

54
src/Processors/Formats/Impl/ORCBlockInputFormat.cpp
View File

@ -1,16 +1,17 @@
#include "ORCBlockInputFormat.h"
#include <boost/algorithm/string/case_conv.hpp>
#if USE_ORC
#include <Formats/FormatFactory.h>
#include <Formats/SchemaInferenceUtils.h>
#include <IO/ReadBufferFromMemory.h>
#include <IO/WriteHelpers.h>
#include <IO/copyData.h>
#include "ArrowBufferedStreams.h"
#include "ArrowColumnToCHColumn.h"
#include "ArrowFieldIndexUtil.h"
#include <DataTypes/NestedUtils.h>
#if USE_ORC
# include <DataTypes/NestedUtils.h>
# include <Formats/FormatFactory.h>
# include <Formats/SchemaInferenceUtils.h>
# include <IO/ReadBufferFromMemory.h>
# include <IO/WriteHelpers.h>
# include <IO/copyData.h>
# include <boost/algorithm/string/case_conv.hpp>
# include "ArrowBufferedStreams.h"
# include "ArrowColumnToCHColumn.h"
# include "ArrowFieldIndexUtil.h"
# include "NativeORCBlockInputFormat.h"
namespace DB
{
@ -154,19 +155,24 @@ NamesAndTypesList ORCSchemaReader::readSchema()
*schema, "ORC", format_settings.orc.skip_columns_with_unsupported_types_in_schema_inference);
if (format_settings.schema_inference_make_columns_nullable)
return getNamesAndRecursivelyNullableTypes(header);
return header.getNamesAndTypesList();}
return header.getNamesAndTypesList();
}
void registerInputFormatORC(FormatFactory & factory)
{
factory.registerInputFormat(
"ORC",
[](ReadBuffer &buf,
const Block &sample,
const RowInputFormatParams &,
const FormatSettings & settings)
{
return std::make_shared<ORCBlockInputFormat>(buf, sample, settings);
});
"ORC",
[](ReadBuffer & buf, const Block & sample, const RowInputFormatParams &, const FormatSettings & settings)
{
InputFormatPtr res;
if (settings.orc.use_fast_decoder)
res = std::make_shared<NativeORCBlockInputFormat>(buf, sample, settings);
else
res = std::make_shared<ORCBlockInputFormat>(buf, sample, settings);
return res;
});
factory.markFormatSupportsSubsetOfColumns("ORC");
}
@ -176,7 +182,13 @@ void registerORCSchemaReader(FormatFactory & factory)
"ORC",
[](ReadBuffer & buf, const FormatSettings & settings)
{
return std::make_shared<ORCSchemaReader>(buf, settings);
SchemaReaderPtr res;
if (settings.orc.use_fast_decoder)
res = std::make_shared<NativeORCSchemaReader>(buf, settings);
else
res = std::make_shared<ORCSchemaReader>(buf, settings);
return res;
}
);

56
src/Processors/Formats/Impl/Parquet/PrepareForWrite.cpp
View File

@ -279,6 +279,8 @@ void preparePrimitiveColumn(ColumnPtr column, DataTypePtr type, const std::strin
auto decimal = [&](Int32 bytes, UInt32 precision, UInt32 scale)
{
/// Currently we encode all decimals as byte arrays, even though Decimal32 and Decimal64
/// could be INT32 and INT64 instead. There doesn't seem to be much difference.
state.column_chunk.meta_data.__set_type(parq::Type::FIXED_LEN_BYTE_ARRAY);
schema.__set_type(parq::Type::FIXED_LEN_BYTE_ARRAY);
schema.__set_type_length(bytes);
@ -335,32 +337,42 @@ void preparePrimitiveColumn(ColumnPtr column, DataTypePtr type, const std::strin
case TypeIndex::DateTime64:
{
std::optional<parq::ConvertedType::type> converted;
std::optional<parq::TimeUnit> unit;
switch (assert_cast<const DataTypeDateTime64 &>(*type).getScale())
parq::ConvertedType::type converted;
parq::TimeUnit unit;
const auto & dt = assert_cast<const DataTypeDateTime64 &>(*type);
UInt32 scale = dt.getScale();
UInt32 converted_scale;
if (scale <= 3)
{
case 3:
converted = parq::ConvertedType::TIMESTAMP_MILLIS;
unit.emplace().__set_MILLIS({});
break;
case 6:
converted = parq::ConvertedType::TIMESTAMP_MICROS;
unit.emplace().__set_MICROS({});
break;
case 9:
unit.emplace().__set_NANOS({});
break;
converted = parq::ConvertedType::TIMESTAMP_MILLIS;
unit.__set_MILLIS({});
converted_scale = 3;
}
else if (scale <= 6)
{
converted = parq::ConvertedType::TIMESTAMP_MICROS;
unit.__set_MICROS({});
converted_scale = 6;
}
else if (scale <= 9)
{
unit.__set_NANOS({});
converted_scale = 9;
}
else
{
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected DateTime64 scale: {}", scale);
}
std::optional<parq::LogicalType> t;
if (unit)
{
parq::TimestampType tt;
tt.__set_isAdjustedToUTC(true);
tt.__set_unit(*unit);
t.emplace().__set_TIMESTAMP(tt);
}
parq::TimestampType tt;
/// (Shouldn't we check the DateTime64's timezone parameter here? No, the actual number
/// in DateTime64 column is always in UTC, regardless of the timezone parameter.)
tt.__set_isAdjustedToUTC(true);
tt.__set_unit(unit);
parq::LogicalType t;
t.__set_TIMESTAMP(tt);
types(T::INT64, converted, t);
state.datetime64_multiplier = DataTypeDateTime64::getScaleMultiplier(converted_scale - scale);
break;
}

33
src/Processors/Formats/Impl/Parquet/Write.cpp
View File

@ -256,6 +256,28 @@ struct ConverterNumeric
}
};
struct ConverterDateTime64WithMultiplier
{
using Statistics = StatisticsNumeric<Int64, Int64>;
using Col = ColumnDecimal<DateTime64>;
const Col & column;
Int64 multiplier;
PODArray<Int64> buf;
ConverterDateTime64WithMultiplier(const ColumnPtr & c, Int64 multiplier_) : column(assert_cast<const Col &>(*c)), multiplier(multiplier_) {}
const Int64 * getBatch(size_t offset, size_t count)
{
buf.resize(count);
for (size_t i = 0; i < count; ++i)
/// Not checking overflow because DateTime64 values should already be in the range where
/// they fit in Int64 at any allowed scale (i.e. up to nanoseconds).
buf[i] = column.getData()[offset + i].value * multiplier;
return buf.data();
}
};
struct ConverterString
{
using Statistics = StatisticsStringRef;
@ -788,9 +810,14 @@ void writeColumnChunkBody(ColumnChunkWriteState & s, const WriteOptions & option
break;
case TypeIndex::DateTime64:
writeColumnImpl<parquet::Int64Type>(
s, options, out, ConverterNumeric<ColumnDecimal<DateTime64>, Int64, Int64>(
s.primitive_column));
if (s.datetime64_multiplier == 1)
writeColumnImpl<parquet::Int64Type>(
s, options, out, ConverterNumeric<ColumnDecimal<DateTime64>, Int64, Int64>(
s.primitive_column));
else
writeColumnImpl<parquet::Int64Type>(
s, options, out, ConverterDateTime64WithMultiplier(
s.primitive_column, s.datetime64_multiplier));
break;
case TypeIndex::IPv4:

3
src/Processors/Formats/Impl/Parquet/Write.h
View File

@ -42,7 +42,8 @@ struct ColumnChunkWriteState
ColumnPtr primitive_column;
CompressionMethod compression; // must match what's inside column_chunk
bool is_bool = false;
Int64 datetime64_multiplier = 1; // for converting e.g. seconds to milliseconds
bool is_bool = false; // bool vs UInt8 have the same column type but are encoded differently
/// Repetition and definition levels. Produced by prepareColumnForWrite().
/// def is empty iff max_def == 0, which means no arrays or nullables.

351
src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp
View File

@ -14,11 +14,15 @@
#include <parquet/arrow/reader.h>
#include <parquet/arrow/schema.h>
#include <parquet/file_reader.h>
#include <parquet/statistics.h>
#include "ArrowBufferedStreams.h"
#include "ArrowColumnToCHColumn.h"
#include "ArrowFieldIndexUtil.h"
#include <base/scope_guard.h>
#include <DataTypes/NestedUtils.h>
#include <DataTypes/DataTypeLowCardinality.h>
#include <DataTypes/DataTypeNullable.h>
#include <Common/FieldVisitorsAccurateComparison.h>
namespace CurrentMetrics
{
@ -33,6 +37,7 @@ namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int CANNOT_READ_ALL_DATA;
extern const int CANNOT_PARSE_NUMBER;
}
#define THROW_ARROW_NOT_OK(status) \
@ -42,6 +47,322 @@ namespace ErrorCodes
throw Exception::createDeprecated(_s.ToString(), ErrorCodes::BAD_ARGUMENTS); \
} while (false)
/// Decode min/max value from column chunk statistics.
///
/// There are two questionable decisions in this implementation:
/// * We parse the value from the encoded byte string instead of casting the parquet::Statistics
/// to parquet::TypedStatistics and taking the value from there.
/// * We dispatch based on the parquet logical+converted+physical type instead of the ClickHouse type.
/// The idea is that this is similar to what we'll have to do when reimplementing Parquet parsing in
/// ClickHouse instead of using Arrow (for speed). So, this is an exercise in parsing Parquet manually.
static std::optional<Field> decodePlainParquetValueSlow(const std::string & data, parquet::Type::type physical_type, const parquet::ColumnDescriptor & descr)
{
using namespace parquet;
auto decode_integer = [&](bool signed_) -> UInt64 {
size_t size;
switch (physical_type)
{
case parquet::Type::type::BOOLEAN: size = 1; break;
case parquet::Type::type::INT32: size = 4; break;
case parquet::Type::type::INT64: size = 8; break;
default: throw Exception(ErrorCodes::CANNOT_PARSE_NUMBER, "Unexpected physical type for number");
}
if (data.size() != size)
throw Exception(ErrorCodes::CANNOT_PARSE_NUMBER, "Unexpected size: {}", data.size());
UInt64 val = 0;
memcpy(&val, data.data(), size);
/// Sign-extend.
if (signed_ && size < 8 && (val >> (size * 8 - 1)) != 0)
val |= 0 - (1ul << (size * 8));
return val;
};
/// Decimal.
do // while (false)
{
Int32 scale;
if (descr.logical_type() && descr.logical_type()->is_decimal())
scale = assert_cast<const DecimalLogicalType &>(*descr.logical_type()).scale();
else if (descr.converted_type() == ConvertedType::type::DECIMAL)
scale = descr.type_scale();
else
break;
size_t size;
bool big_endian = false;
switch (physical_type)
{
case Type::type::BOOLEAN: size = 1; break;
case Type::type::INT32: size = 4; break;
case Type::type::INT64: size = 8; break;
case Type::type::FIXED_LEN_BYTE_ARRAY:
big_endian = true;
size = data.size();
break;
default: throw Exception(ErrorCodes::CANNOT_PARSE_NUMBER, "Unexpected decimal physical type");
}
/// Note that size is not necessarily a power of two.
/// E.g. spark turns 8-byte unsigned integers into 9-byte signed decimals.
if (data.size() != size || size < 1 || size > 32)
throw Exception(ErrorCodes::CANNOT_PARSE_NUMBER, "Unexpected decimal size: {} (actual {})", size, data.size());
/// For simplicity, widen all decimals to 256-bit. It should compare correctly with values
/// of different bitness.
Int256 val = 0;
memcpy(&val, data.data(), size);
if (big_endian)
std::reverse(reinterpret_cast<char *>(&val), reinterpret_cast<char *>(&val) + size);
/// Sign-extend.
if (size < 32 && (val >> (size * 8 - 1)) != 0)
val |= ~((Int256(1) << (size * 8)) - 1);
return Field(DecimalField<Decimal256>(Decimal256(val), static_cast<UInt32>(scale)));
}
while (false);
/// Timestamp (decimal).
{
Int32 scale = -1;
bool is_timestamp = true;
if (descr.logical_type() && (descr.logical_type()->is_time() || descr.logical_type()->is_timestamp()))
{
LogicalType::TimeUnit::unit unit = descr.logical_type()->is_time()
? assert_cast<const TimeLogicalType &>(*descr.logical_type()).time_unit()
: assert_cast<const TimestampLogicalType &>(*descr.logical_type()).time_unit();
switch (unit)
{
case LogicalType::TimeUnit::unit::MILLIS: scale = 3; break;
case LogicalType::TimeUnit::unit::MICROS: scale = 6; break;
case LogicalType::TimeUnit::unit::NANOS: scale = 9; break;
default: throw Exception(ErrorCodes::CANNOT_PARSE_NUMBER, "Unknown time unit");
}
}
else switch (descr.converted_type())
{
case ConvertedType::type::TIME_MILLIS: scale = 3; break;
case ConvertedType::type::TIME_MICROS: scale = 6; break;
case ConvertedType::type::TIMESTAMP_MILLIS: scale = 3; break;
case ConvertedType::type::TIMESTAMP_MICROS: scale = 6; break;
default: is_timestamp = false;
}
if (is_timestamp)
{
Int64 val = static_cast<Int64>(decode_integer(/* signed */ true));
return Field(DecimalField<Decimal64>(Decimal64(val), scale));
}
}
/// Floats.
if (physical_type == Type::type::FLOAT)
{
if (data.size() != 4)
throw Exception(ErrorCodes::CANNOT_PARSE_NUMBER, "Unexpected float size");
Float32 val;
memcpy(&val, data.data(), data.size());
return Field(val);
}
if (physical_type == Type::type::DOUBLE)
{
if (data.size() != 8)
throw Exception(ErrorCodes::CANNOT_PARSE_NUMBER, "Unexpected float size");
Float64 val;
memcpy(&val, data.data(), data.size());
return Field(val);
}
/// Strings.
if (physical_type == Type::type::BYTE_ARRAY || physical_type == Type::type::FIXED_LEN_BYTE_ARRAY)
{
/// Arrow's parquet decoder handles missing min/max values slightly incorrectly.
/// In a parquet file, min and max have separate is_set flags, i.e. one may be missing even
/// if the other is set. Arrow decoder ORs (!) these two flags together into one: HasMinMax().
/// So, if exactly one of {min, max} is missing, Arrow reports it as empty string, with no
/// indication that it's actually missing.
///
/// How can exactly one of {min, max} be missing? This happens if one of the two strings
/// exceeds the length limit for stats. Repro:
///
/// insert into function file('t.parquet') select arrayStringConcat(range(number*1000000)) from numbers(2) settings output_format_parquet_use_custom_encoder=0
/// select tupleElement(tupleElement(row_groups[1], 'columns')[1], 'statistics') from file('t.parquet', ParquetMetadata)
///
/// Here the row group contains two strings: one empty, one very long. But the statistics
/// reported by arrow are indistinguishable from statistics if all strings were empty.
/// (Min and max are the last two tuple elements in the output of the second query. Notice
/// how they're empty strings instead of NULLs.)
///
/// So we have to be conservative and treat empty string as unknown.
/// This is unfortunate because it's probably common for string columns to have lots of empty
/// values, and filter pushdown would probably often be useful in that case.
///
/// TODO: Remove this workaround either when we implement our own Parquet decoder that
/// doesn't have this bug, or if it's fixed in Arrow.
if (data.empty())
return std::nullopt;
return Field(data);
}
/// This one's deprecated in Parquet.
if (physical_type == Type::type::INT96)
throw Exception(ErrorCodes::CANNOT_PARSE_NUMBER, "Parquet INT96 type is deprecated and not supported");
/// Integers.
bool signed_ = true;
if (descr.logical_type() && descr.logical_type()->is_int())
signed_ = assert_cast<const IntLogicalType &>(*descr.logical_type()).is_signed();
else
signed_ = descr.converted_type() != ConvertedType::type::UINT_8 &&
descr.converted_type() != ConvertedType::type::UINT_16 &&
descr.converted_type() != ConvertedType::type::UINT_32 &&
descr.converted_type() != ConvertedType::type::UINT_64;
UInt64 val = decode_integer(signed_);
Field field = signed_ ? Field(static_cast<Int64>(val)) : Field(val);
return field;
}
/// Range of values for each column, based on statistics in the Parquet metadata.
/// This is lower/upper bounds, not necessarily exact min and max, e.g. the min/max can be just
/// missing in the metadata.
static std::vector<Range> getHyperrectangleForRowGroup(const parquet::FileMetaData & file, int row_group_idx, const Block & header, const FormatSettings & format_settings)
{
auto column_name_for_lookup = [&](std::string column_name) -> std::string
{
if (format_settings.parquet.case_insensitive_column_matching)
boost::to_lower(column_name);
return column_name;
};
std::unique_ptr<parquet::RowGroupMetaData> row_group = file.RowGroup(row_group_idx);
std::unordered_map<std::string, std::shared_ptr<parquet::Statistics>> name_to_statistics;
for (int i = 0; i < row_group->num_columns(); ++i)
{
auto c = row_group->ColumnChunk(i);
auto s = c->statistics();
if (!s)
continue;
auto path = c->path_in_schema()->ToDotVector();
if (path.size() != 1)
continue; // compound types not supported
name_to_statistics.emplace(column_name_for_lookup(path[0]), s);
}
/// +-----+
/// / /|
/// +-----+ |
/// | | +
/// | |/
/// +-----+
std::vector<Range> hyperrectangle(header.columns(), Range::createWholeUniverse());
for (size_t idx = 0; idx < header.columns(); ++idx)
{
const std::string & name = header.getByPosition(idx).name;
auto it = name_to_statistics.find(column_name_for_lookup(name));
if (it == name_to_statistics.end())
continue;
auto stats = it->second;
auto default_value = [&]() -> Field
{
DataTypePtr type = header.getByPosition(idx).type;
if (type->lowCardinality())
type = assert_cast<const DataTypeLowCardinality &>(*type).getDictionaryType();
if (type->isNullable())
type = assert_cast<const DataTypeNullable &>(*type).getNestedType();
return type->getDefault();
};
/// Only primitive fields are supported, not arrays, maps, tuples, or Nested.
/// Arrays, maps, and Nested can't be meaningfully supported because Parquet only has min/max
/// across all *elements* of the array, not min/max array itself.
/// Same limitation for tuples, but maybe it would make sense to have some kind of tuple
/// expansion in KeyCondition to accept ranges per element instead of whole tuple.
std::optional<Field> min;
std::optional<Field> max;
if (stats->HasMinMax())
{
try
{
min = decodePlainParquetValueSlow(stats->EncodeMin(), stats->physical_type(), *stats->descr());
max = decodePlainParquetValueSlow(stats->EncodeMax(), stats->physical_type(), *stats->descr());
}
catch (Exception & e)
{
e.addMessage(" (When parsing Parquet statistics for column {}, physical type {}, {}. Please report an issue and use input_format_parquet_filter_push_down = false to work around.)", name, static_cast<int>(stats->physical_type()), stats->descr()->ToString());
throw;
}
}
/// In Range, NULL is represented as positive or negative infinity (represented by a special
/// kind of Field, different from floating-point infinities).
bool always_null = stats->descr()->max_definition_level() != 0 &&
stats->HasNullCount() && stats->num_values() == 0;
bool can_be_null = stats->descr()->max_definition_level() != 0 &&
(!stats->HasNullCount() || stats->null_count() != 0);
bool null_as_default = format_settings.null_as_default && !isNullableOrLowCardinalityNullable(header.getByPosition(idx).type);
if (always_null)
{
/// Single-point range containing either the default value of one of the infinities.
if (null_as_default)
hyperrectangle[idx].right = hyperrectangle[idx].left = default_value();
else
hyperrectangle[idx].right = hyperrectangle[idx].left;
continue;
}
if (can_be_null)
{
if (null_as_default)
{
/// Make sure the range contains the default value.
Field def = default_value();
if (min.has_value() && applyVisitor(FieldVisitorAccurateLess(), def, *min))
min = def;
if (max.has_value() && applyVisitor(FieldVisitorAccurateLess(), *max, def))
max = def;
}
else
{
/// Make sure the range reaches infinity on at least one side.
if (min.has_value() && max.has_value())
min.reset();
}
}
else
{
/// If the column doesn't have nulls, exclude both infinities.
if (!min.has_value())
hyperrectangle[idx].left_included = false;
if (!max.has_value())
hyperrectangle[idx].right_included = false;
}
if (min.has_value())
hyperrectangle[idx].left = std::move(min.value());
if (max.has_value())
hyperrectangle[idx].right = std::move(max.value());
}
return hyperrectangle;
}
ParquetBlockInputFormat::ParquetBlockInputFormat(
ReadBuffer & buf,
const Block & header_,
@ -66,6 +387,16 @@ ParquetBlockInputFormat::~ParquetBlockInputFormat()
pool->wait();
}
void ParquetBlockInputFormat::setQueryInfo(const SelectQueryInfo & query_info, ContextPtr context)
{
/// When analyzer is enabled, query_info.filter_asts is missing sets and maybe some type casts,
/// so don't use it. I'm not sure how to support analyzer here: https://github.com/ClickHouse/ClickHouse/issues/53536
if (format_settings.parquet.filter_push_down && !context->getSettingsRef().allow_experimental_analyzer)
key_condition.emplace(query_info, context, getPort().getHeader().getNames(),
std::make_shared<ExpressionActions>(std::make_shared<ActionsDAG>(
getPort().getHeader().getColumnsWithTypeAndName())));
}
void ParquetBlockInputFormat::initializeIfNeeded()
{
if (std::exchange(is_initialized, true))
@ -84,10 +415,12 @@ void ParquetBlockInputFormat::initializeIfNeeded()
std::shared_ptr<arrow::Schema> schema;
THROW_ARROW_NOT_OK(parquet::arrow::FromParquetSchema(metadata->schema(), &schema));
int num_row_groups = metadata->num_row_groups();
if (num_row_groups == 0)
return;
ArrowFieldIndexUtil field_util(
format_settings.parquet.case_insensitive_column_matching,
format_settings.parquet.allow_missing_columns);
column_indices = field_util.findRequiredIndices(getPort().getHeader(), *schema);
int num_row_groups = metadata->num_row_groups();
row_group_batches.reserve(num_row_groups);
for (int row_group = 0; row_group < num_row_groups; ++row_group)
@ -95,6 +428,12 @@ void ParquetBlockInputFormat::initializeIfNeeded()
if (skip_row_groups.contains(row_group))
continue;
if (key_condition.has_value() &&
!key_condition->checkInHyperrectangle(
getHyperrectangleForRowGroup(*metadata, row_group, getPort().getHeader(), format_settings),
getPort().getHeader().getDataTypes()).can_be_true)
continue;
if (row_group_batches.empty() || row_group_batches.back().total_bytes_compressed >= min_bytes_for_seek)
row_group_batches.emplace_back();
@ -102,11 +441,6 @@ void ParquetBlockInputFormat::initializeIfNeeded()
row_group_batches.back().total_rows += metadata->RowGroup(row_group)->num_rows();
row_group_batches.back().total_bytes_compressed += metadata->RowGroup(row_group)->total_compressed_size();
}
ArrowFieldIndexUtil field_util(
format_settings.parquet.case_insensitive_column_matching,
format_settings.parquet.allow_missing_columns);
column_indices = field_util.findRequiredIndices(getPort().getHeader(), *schema);
}
void ParquetBlockInputFormat::initializeRowGroupBatchReader(size_t row_group_batch_idx)
@ -249,7 +583,6 @@ void ParquetBlockInputFormat::decodeOneChunk(size_t row_group_batch_idx, std::un
if (!row_group_batch.record_batch_reader)
initializeRowGroupBatchReader(row_group_batch_idx);
auto batch = row_group_batch.record_batch_reader->Next();
if (!batch.ok())
throw ParsingException(ErrorCodes::CANNOT_READ_ALL_DATA, "Error while reading Parquet data: {}", batch.status().ToString());

12
src/Processors/Formats/Impl/ParquetBlockInputFormat.h
View File

@ -5,6 +5,7 @@
#include <Processors/Formats/IInputFormat.h>
#include <Processors/Formats/ISchemaReader.h>
#include <Formats/FormatSettings.h>
#include <Storages/MergeTree/KeyCondition.h>
namespace parquet { class FileMetaData; }
namespace parquet::arrow { class FileReader; }
@ -55,6 +56,8 @@ public:
~ParquetBlockInputFormat() override;
void setQueryInfo(const SelectQueryInfo & query_info, ContextPtr context) override;
void resetParser() override;
String getName() const override { return "ParquetBlockInputFormat"; }
@ -246,12 +249,15 @@ private:
size_t min_bytes_for_seek;
const size_t max_pending_chunks_per_row_group_batch = 2;
// RandomAccessFile is thread safe, so we share it among threads.
// FileReader is not, so each thread creates its own.
/// RandomAccessFile is thread safe, so we share it among threads.
/// FileReader is not, so each thread creates its own.
std::shared_ptr<arrow::io::RandomAccessFile> arrow_file;
std::shared_ptr<parquet::FileMetaData> metadata;
// indices of columns to read from Parquet file
/// Indices of columns to read from Parquet file.
std::vector<int> column_indices;
/// Pushed-down filter that we'll use to skip row groups.
std::optional<KeyCondition> key_condition;
// Window of active row groups:
//

6
src/Processors/examples/CMakeLists.txt
View File

@ -2,3 +2,9 @@ if (TARGET ch_contrib::hivemetastore)
clickhouse_add_executable (comma_separated_streams comma_separated_streams.cpp)
target_link_libraries (comma_separated_streams PRIVATE dbms)
endif()
if (USE_ORC)
clickhouse_add_executable (native_orc native_orc.cpp)
target_link_libraries (native_orc PRIVATE dbms)
target_include_directories (native_orc PRIVATE ${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include)
endif ()

36
src/Processors/examples/native_orc.cpp Normal file
View File

@ -0,0 +1,36 @@
#include <string>
#include <IO/ReadBufferFromFile.h>
#include <Processors/Formats/Impl/NativeORCBlockInputFormat.h>
#include <IO/copyData.h>
using namespace DB;
int main()
{
/// Read schema from orc file
String path = "/path/to/orc/file";
// String path = "/data1/clickhouse_official/data/user_files/bigolive_audience_stats_orc.orc";
{
ReadBufferFromFile in(path);
NativeORCSchemaReader schema_reader(in, {});
auto schema = schema_reader.readSchema();
std::cout << "schema:" << schema.toString() << std::endl;
}
/// Read schema from string with orc data
{
ReadBufferFromFile in(path);
String content;
WriteBufferFromString out(content);
copyData(in, out);
content.resize(out.count());
ReadBufferFromString in2(content);
NativeORCSchemaReader schema_reader(in2, {});
auto schema = schema_reader.readSchema();
std::cout << "schema:" << schema.toString() << std::endl;
}
return 0;
}

10
src/Storages/HDFS/StorageHDFS.cpp
View File

@ -467,7 +467,8 @@ HDFSSource::HDFSSource(
StorageHDFSPtr storage_,
ContextPtr context_,
UInt64 max_block_size_,
std::shared_ptr<IteratorWrapper> file_iterator_)
std::shared_ptr<IteratorWrapper> file_iterator_,
const SelectQueryInfo & query_info_)
: ISource(info.source_header, false)
, WithContext(context_)
, storage(std::move(storage_))
@ -477,6 +478,7 @@ HDFSSource::HDFSSource(
, max_block_size(max_block_size_)
, file_iterator(file_iterator_)
, columns_description(info.columns_description)
, query_info(query_info_)
{
initialize();
}
@ -515,6 +517,7 @@ bool HDFSSource::initialize()
current_path = path_with_info.path;
input_format = getContext()->getInputFormat(storage->format_name, *read_buf, block_for_format, max_block_size);
input_format->setQueryInfo(query_info, getContext());
QueryPipelineBuilder builder;
builder.init(Pipe(input_format));
@ -727,7 +730,7 @@ bool StorageHDFS::supportsSubsetOfColumns() const
Pipe StorageHDFS::read(
const Names & column_names,
const StorageSnapshotPtr & storage_snapshot,
SelectQueryInfo & /*query_info*/,
SelectQueryInfo & query_info,
ContextPtr context_,
QueryProcessingStage::Enum /*processed_stage*/,
size_t max_block_size,
@ -769,7 +772,8 @@ Pipe StorageHDFS::read(
this_ptr,
context_,
max_block_size,
iterator_wrapper));
iterator_wrapper,
query_info));
}
return Pipe::unitePipes(std::move(pipes));
}

5
src/Storages/HDFS/StorageHDFS.h
View File

@ -8,6 +8,7 @@
#include <Storages/IStorage.h>
#include <Storages/Cache/SchemaCache.h>
#include <Storages/prepareReadingFromFormat.h>
#include <Storages/SelectQueryInfo.h>
#include <Poco/URI.h>
namespace DB
@ -150,7 +151,8 @@ public:
StorageHDFSPtr storage_,
ContextPtr context_,
UInt64 max_block_size_,
std::shared_ptr<IteratorWrapper> file_iterator_);
std::shared_ptr<IteratorWrapper> file_iterator_,
const SelectQueryInfo & query_info_);
String getName() const override;
@ -164,6 +166,7 @@ private:
UInt64 max_block_size;
std::shared_ptr<IteratorWrapper> file_iterator;
ColumnsDescription columns_description;
SelectQueryInfo query_info;
std::unique_ptr<ReadBuffer> read_buf;
std::shared_ptr<IInputFormat> input_format;

5
src/Storages/Hive/StorageHive.cpp
View File

@ -122,6 +122,7 @@ public:
String compression_method_,
Block sample_block_,
ContextPtr context_,
const SelectQueryInfo & query_info_,
UInt64 max_block_size_,
const StorageHive & storage_,
const Names & text_input_field_names_ = {})
@ -138,6 +139,7 @@ public:
, text_input_field_names(text_input_field_names_)
, format_settings(getFormatSettings(getContext()))
, read_settings(getContext()->getReadSettings())
, query_info(query_info_)
{
to_read_block = sample_block;
@ -278,6 +280,7 @@ public:
auto input_format = FormatFactory::instance().getInput(
format, *read_buf, to_read_block, getContext(), max_block_size, updateFormatSettings(current_file), /* max_parsing_threads */ 1);
input_format->setQueryInfo(query_info, getContext());
Pipe pipe(input_format);
if (columns_description.hasDefaults())
@ -392,6 +395,7 @@ private:
const Names & text_input_field_names;
FormatSettings format_settings;
ReadSettings read_settings;
SelectQueryInfo query_info;
HiveFilePtr current_file;
String current_path;
@ -831,6 +835,7 @@ Pipe StorageHive::read(
compression_method,
sample_block,
context_,
query_info,
max_block_size,
*this,
text_input_field_names));

3
src/Storages/MergeTree/ApproximateNearestNeighborIndexesCommon.h
View File

@ -9,6 +9,9 @@
namespace DB
{
static constexpr auto DISTANCE_FUNCTION_L2 = "L2Distance";
static constexpr auto DISTANCE_FUNCTION_COSINE = "cosineDistance";
/// Approximate Nearest Neighbour queries have a similar structure:
/// - reference vector from which all distances are calculated
/// - metric name (e.g L2Distance, LpDistance, etc.)

4
src/Storages/MergeTree/KeyCondition.cpp
View File

@ -764,7 +764,9 @@ KeyCondition::KeyCondition(
++key_index;
}
auto filter_node = buildFilterNode(query, additional_filter_asts);
ASTPtr filter_node;
if (query)
filter_node = buildFilterNode(query, additional_filter_asts);
if (!filter_node)
{

8
src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp
View File

@ -25,12 +25,6 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
}
static constexpr auto DISTANCE_FUNCTION_L2 = "L2Distance";
static constexpr auto DISTANCE_FUNCTION_COSINE = "cosineDistance";
static constexpr auto DEFAULT_TREES = 100uz;
static constexpr auto DEFAULT_DISTANCE_FUNCTION = DISTANCE_FUNCTION_L2;
template <typename Distance>
AnnoyIndexWithSerialization<Distance>::AnnoyIndexWithSerialization(size_t dimensions)
: Base::AnnoyIndex(dimensions)
@ -318,10 +312,12 @@ MergeTreeIndexConditionPtr MergeTreeIndexAnnoy::createIndexCondition(const Selec
MergeTreeIndexPtr annoyIndexCreator(const IndexDescription & index)
{
static constexpr auto DEFAULT_DISTANCE_FUNCTION = DISTANCE_FUNCTION_L2;
String distance_function = DEFAULT_DISTANCE_FUNCTION;
if (!index.arguments.empty())
distance_function = index.arguments[0].get<String>();
static constexpr auto DEFAULT_TREES = 100uz;
UInt64 trees = DEFAULT_TREES;
if (index.arguments.size() > 1)
trees = index.arguments[1].get<UInt64>();

377
src/Storages/MergeTree/MergeTreeIndexUSearch.cpp Normal file
View File

@ -0,0 +1,377 @@
#ifdef ENABLE_USEARCH
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wpass-failed"
#include <Storages/MergeTree/MergeTreeIndexUSearch.h>
#include <Columns/ColumnArray.h>
#include <Common/typeid_cast.h>
#include <Core/Field.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeTuple.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <Interpreters/Context.h>
#include <Interpreters/castColumn.h>
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_ALLOCATE_MEMORY;
extern const int ILLEGAL_COLUMN;
extern const int INCORRECT_DATA;
extern const int INCORRECT_NUMBER_OF_COLUMNS;
extern const int INCORRECT_QUERY;
extern const int LOGICAL_ERROR;
}
template <unum::usearch::metric_kind_t Metric>
USearchIndexWithSerialization<Metric>::USearchIndexWithSerialization(size_t dimensions)
: Base(Base::make(unum::usearch::metric_punned_t(dimensions, Metric)))
{
}
template <unum::usearch::metric_kind_t Metric>
void USearchIndexWithSerialization<Metric>::serialize([[maybe_unused]] WriteBuffer & ostr) const
{
auto callback = [&ostr](void * from, size_t n)
{
ostr.write(reinterpret_cast<const char *>(from), n);
return true;
};
Base::stream(callback);
}
template <unum::usearch::metric_kind_t Metric>
void USearchIndexWithSerialization<Metric>::deserialize([[maybe_unused]] ReadBuffer & istr)
{
BufferBase::Position & pos = istr.position();
unum::usearch::memory_mapped_file_t memory_map(pos, istr.buffer().size() - istr.count());
Base::view(std::move(memory_map));
pos += Base::stream_length();
auto copy = Base::copy();
if (!copy)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not copy usearch index");
Base::swap(copy.index);
}
template <unum::usearch::metric_kind_t Metric>
size_t USearchIndexWithSerialization<Metric>::getDimensions() const
{
return Base::dimensions();
}
template <unum::usearch::metric_kind_t Metric>
MergeTreeIndexGranuleUSearch<Metric>::MergeTreeIndexGranuleUSearch(
const String & index_name_,
const Block & index_sample_block_)
: index_name(index_name_)
, index_sample_block(index_sample_block_)
, index(nullptr)
{
}
template <unum::usearch::metric_kind_t Metric>
MergeTreeIndexGranuleUSearch<Metric>::MergeTreeIndexGranuleUSearch(
const String & index_name_,
const Block & index_sample_block_,
USearchIndexWithSerializationPtr<Metric> index_)
: index_name(index_name_)
, index_sample_block(index_sample_block_)
, index(std::move(index_))
{
}
template <unum::usearch::metric_kind_t Metric>
void MergeTreeIndexGranuleUSearch<Metric>::serializeBinary(WriteBuffer & ostr) const
{
/// Number of dimensions is required in the index constructor,
/// so it must be written and read separately from the other part
writeIntBinary(static_cast<UInt64>(index->getDimensions()), ostr); // write dimension
index->serialize(ostr);
}
template <unum::usearch::metric_kind_t Metric>
void MergeTreeIndexGranuleUSearch<Metric>::deserializeBinary(ReadBuffer & istr, MergeTreeIndexVersion /*version*/)
{
UInt64 dimension;
readIntBinary(dimension, istr);
index = std::make_shared<USearchIndexWithSerialization<Metric>>(dimension);
index->deserialize(istr);
}
template <unum::usearch::metric_kind_t Metric>
MergeTreeIndexAggregatorUSearch<Metric>::MergeTreeIndexAggregatorUSearch(
const String & index_name_,
const Block & index_sample_block_)
: index_name(index_name_)
, index_sample_block(index_sample_block_)
{
}
template <unum::usearch::metric_kind_t Metric>
MergeTreeIndexGranulePtr MergeTreeIndexAggregatorUSearch<Metric>::getGranuleAndReset()
{
auto granule = std::make_shared<MergeTreeIndexGranuleUSearch<Metric>>(index_name, index_sample_block, index);
index = nullptr;
return granule;
}
template <unum::usearch::metric_kind_t Metric>
void MergeTreeIndexAggregatorUSearch<Metric>::update(const Block & block, size_t * pos, size_t limit)
{
if (*pos >= block.rows())
throw Exception(
ErrorCodes::LOGICAL_ERROR,
"The provided position is not less than the number of block rows. Position: {}, Block rows: {}.",
*pos,
block.rows());
size_t rows_read = std::min(limit, block.rows() - *pos);
if (rows_read == 0)
return;
if (index_sample_block.columns() > 1)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected block with single column");
const String & index_column_name = index_sample_block.getByPosition(0).name;
ColumnPtr column_cut = block.getByName(index_column_name).column->cut(*pos, rows_read);
if (const auto & column_array = typeid_cast<const ColumnArray *>(column_cut.get()))
{
const auto & data = column_array->getData();
const auto & array = typeid_cast<const ColumnFloat32 &>(data).getData();
if (array.empty())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Array has 0 rows, {} rows expected", rows_read);
const auto & offsets = column_array->getOffsets();
const size_t num_rows = offsets.size();
/// Check all sizes are the same
size_t size = offsets[0];
for (size_t i = 0; i < num_rows - 1; ++i)
if (offsets[i + 1] - offsets[i] != size)
throw Exception(ErrorCodes::INCORRECT_DATA, "All arrays in column {} must have equal length", index_column_name);
index = std::make_shared<USearchIndexWithSerialization<Metric>>(size);
/// Add all rows of block
if (!index->reserve(unum::usearch::ceil2(index->size() + num_rows)))
throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Could not reserve memory for usearch index");
if (auto rc = index->add(index->size(), array.data()); !rc)
throw Exception(ErrorCodes::INCORRECT_DATA, rc.error.release());
for (size_t current_row = 1; current_row < num_rows; ++current_row)
if (auto rc = index->add(index->size(), &array[offsets[current_row - 1]]); !rc)
throw Exception(ErrorCodes::INCORRECT_DATA, rc.error.release());
}
else if (const auto & column_tuple = typeid_cast<const ColumnTuple *>(column_cut.get()))
{
const auto & columns = column_tuple->getColumns();
std::vector<std::vector<Float32>> data{column_tuple->size(), std::vector<Float32>()};
for (const auto & column : columns)
{
const auto & pod_array = typeid_cast<const ColumnFloat32 *>(column.get())->getData();
for (size_t i = 0; i < pod_array.size(); ++i)
data[i].push_back(pod_array[i]);
}
if (data.empty())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Tuple has 0 rows, {} rows expected", rows_read);
index = std::make_shared<USearchIndexWithSerialization<Metric>>(data[0].size());
if (!index->reserve(unum::usearch::ceil2(index->size() + data.size())))
throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Could not reserve memory for usearch index");
for (const auto & item : data)
if (auto rc = index->add(index->size(), item.data()); !rc)
throw Exception(ErrorCodes::INCORRECT_DATA, rc.error.release());
}
else
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected Array or Tuple column");
*pos += rows_read;
}
MergeTreeIndexConditionUSearch::MergeTreeIndexConditionUSearch(
const IndexDescription & /*index_description*/,
const SelectQueryInfo & query,
const String & distance_function_,
ContextPtr context)
: ann_condition(query, context)
, distance_function(distance_function_)
{
}
bool MergeTreeIndexConditionUSearch::mayBeTrueOnGranule(MergeTreeIndexGranulePtr /*idx_granule*/) const
{
throw Exception(ErrorCodes::LOGICAL_ERROR, "mayBeTrueOnGranule is not supported for ANN skip indexes");
}
bool MergeTreeIndexConditionUSearch::alwaysUnknownOrTrue() const
{
return ann_condition.alwaysUnknownOrTrue(distance_function);
}
std::vector<size_t> MergeTreeIndexConditionUSearch::getUsefulRanges(MergeTreeIndexGranulePtr idx_granule) const
{
if (distance_function == DISTANCE_FUNCTION_L2)
return getUsefulRangesImpl<unum::usearch::metric_kind_t::l2sq_k>(idx_granule);
else if (distance_function == DISTANCE_FUNCTION_COSINE)
return getUsefulRangesImpl<unum::usearch::metric_kind_t::cos_k>(idx_granule);
std::unreachable();
}
template <unum::usearch::metric_kind_t Metric>
std::vector<size_t> MergeTreeIndexConditionUSearch::getUsefulRangesImpl(MergeTreeIndexGranulePtr idx_granule) const
{
const UInt64 limit = ann_condition.getLimit();
const UInt64 index_granularity = ann_condition.getIndexGranularity();
const std::optional<float> comparison_distance = ann_condition.getQueryType() == ApproximateNearestNeighborInformation::Type::Where
? std::optional<float>(ann_condition.getComparisonDistanceForWhereQuery())
: std::nullopt;
if (comparison_distance && comparison_distance.value() < 0)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to optimize query with where without distance");
const std::vector<float> reference_vector = ann_condition.getReferenceVector();
const auto granule = std::dynamic_pointer_cast<MergeTreeIndexGranuleUSearch<Metric>>(idx_granule);
if (granule == nullptr)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Granule has the wrong type");
const USearchIndexWithSerializationPtr<Metric> index = granule->index;
if (ann_condition.getDimensions() != index->dimensions())
throw Exception(
ErrorCodes::INCORRECT_QUERY,
"The dimension of the space in the request ({}) "
"does not match the dimension in the index ({})",
ann_condition.getDimensions(),
index->dimensions());
auto result = index->search(reference_vector.data(), limit);
std::vector<UInt64> neighbors(result.size()); /// indexes of dots which were closest to the reference vector
std::vector<Float32> distances(result.size());
result.dump_to(neighbors.data(), distances.data());
std::vector<size_t> granule_numbers;
granule_numbers.reserve(neighbors.size());
for (size_t i = 0; i < neighbors.size(); ++i)
{
if (comparison_distance && distances[i] > comparison_distance)
continue;
granule_numbers.push_back(neighbors[i] / index_granularity);
}
/// make unique
std::sort(granule_numbers.begin(), granule_numbers.end());
granule_numbers.erase(std::unique(granule_numbers.begin(), granule_numbers.end()), granule_numbers.end());
return granule_numbers;
}
MergeTreeIndexUSearch::MergeTreeIndexUSearch(const IndexDescription & index_, const String & distance_function_)
: IMergeTreeIndex(index_)
, distance_function(distance_function_)
{
}
MergeTreeIndexGranulePtr MergeTreeIndexUSearch::createIndexGranule() const
{
if (distance_function == DISTANCE_FUNCTION_L2)
return std::make_shared<MergeTreeIndexGranuleUSearch<unum::usearch::metric_kind_t::l2sq_k>>(index.name, index.sample_block);
else if (distance_function == DISTANCE_FUNCTION_COSINE)
return std::make_shared<MergeTreeIndexGranuleUSearch<unum::usearch::metric_kind_t::cos_k>>(index.name, index.sample_block);
std::unreachable();
}
MergeTreeIndexAggregatorPtr MergeTreeIndexUSearch::createIndexAggregator() const
{
if (distance_function == DISTANCE_FUNCTION_L2)
return std::make_shared<MergeTreeIndexAggregatorUSearch<unum::usearch::metric_kind_t::l2sq_k>>(index.name, index.sample_block);
else if (distance_function == DISTANCE_FUNCTION_COSINE)
return std::make_shared<MergeTreeIndexAggregatorUSearch<unum::usearch::metric_kind_t::cos_k>>(index.name, index.sample_block);
std::unreachable();
}
MergeTreeIndexConditionPtr MergeTreeIndexUSearch::createIndexCondition(const SelectQueryInfo & query, ContextPtr context) const
{
return std::make_shared<MergeTreeIndexConditionUSearch>(index, query, distance_function, context);
};
MergeTreeIndexPtr usearchIndexCreator(const IndexDescription & index)
{
static constexpr auto default_distance_function = DISTANCE_FUNCTION_L2;
String distance_function = default_distance_function;
if (!index.arguments.empty())
distance_function = index.arguments[0].get<String>();
return std::make_shared<MergeTreeIndexUSearch>(index, distance_function);
}
void usearchIndexValidator(const IndexDescription & index, bool /* attach */)
{
/// Check number and type of USearch index arguments:
if (index.arguments.size() > 1)
throw Exception(ErrorCodes::INCORRECT_QUERY, "USearch index must not have more than one parameters");
if (!index.arguments.empty() && index.arguments[0].getType() != Field::Types::String)
throw Exception(ErrorCodes::INCORRECT_QUERY, "Distance function argument of USearch index must be of type String");
/// Check that the index is created on a single column
if (index.column_names.size() != 1 || index.data_types.size() != 1)
throw Exception(ErrorCodes::INCORRECT_NUMBER_OF_COLUMNS, "USearch indexes must be created on a single column");
/// Check that a supported metric was passed as first argument
if (!index.arguments.empty())
{
String distance_name = index.arguments[0].get<String>();
if (distance_name != DISTANCE_FUNCTION_L2 && distance_name != DISTANCE_FUNCTION_COSINE)
throw Exception(ErrorCodes::INCORRECT_DATA, "USearch index only supports distance functions '{}' and '{}'", DISTANCE_FUNCTION_L2, DISTANCE_FUNCTION_COSINE);
}
/// Check data type of indexed column:
auto throw_unsupported_underlying_column_exception = []()
{
throw Exception(
ErrorCodes::ILLEGAL_COLUMN, "USearch indexes can only be created on columns of type Array(Float32) and Tuple(Float32)");
};
DataTypePtr data_type = index.sample_block.getDataTypes()[0];
if (const auto * data_type_array = typeid_cast<const DataTypeArray *>(data_type.get()))
{
TypeIndex nested_type_index = data_type_array->getNestedType()->getTypeId();
if (!WhichDataType(nested_type_index).isFloat32())
throw_unsupported_underlying_column_exception();
}
else if (const auto * data_type_tuple = typeid_cast<const DataTypeTuple *>(data_type.get()))
{
const DataTypes & inner_types = data_type_tuple->getElements();
for (const auto & inner_type : inner_types)
{
TypeIndex nested_type_index = inner_type->getTypeId();
if (!WhichDataType(nested_type_index).isFloat32())
throw_unsupported_underlying_column_exception();
}
}
else
throw_unsupported_underlying_column_exception();
}
}
#endif

106
src/Storages/MergeTree/MergeTreeIndexUSearch.h Normal file
View File

@ -0,0 +1,106 @@
#pragma once
#ifdef ENABLE_USEARCH
#include <Storages/MergeTree/ApproximateNearestNeighborIndexesCommon.h>
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wpass-failed"
#include <usearch/index_dense.hpp>
#pragma clang diagnostic pop
namespace DB
{
template <unum::usearch::metric_kind_t Metric>
class USearchIndexWithSerialization : public unum::usearch::index_dense_t
{
using Base = unum::usearch::index_dense_t;
public:
explicit USearchIndexWithSerialization(size_t dimensions);
void serialize(WriteBuffer & ostr) const;
void deserialize(ReadBuffer & istr);
size_t getDimensions() const;
};
template <unum::usearch::metric_kind_t Metric>
using USearchIndexWithSerializationPtr = std::shared_ptr<USearchIndexWithSerialization<Metric>>;
template <unum::usearch::metric_kind_t Metric>
struct MergeTreeIndexGranuleUSearch final : public IMergeTreeIndexGranule
{
MergeTreeIndexGranuleUSearch(const String & index_name_, const Block & index_sample_block_);
MergeTreeIndexGranuleUSearch(const String & index_name_, const Block & index_sample_block_, USearchIndexWithSerializationPtr<Metric> index_);
~MergeTreeIndexGranuleUSearch() override = default;
void serializeBinary(WriteBuffer & ostr) const override;
void deserializeBinary(ReadBuffer & istr, MergeTreeIndexVersion version) override;
bool empty() const override { return !index.get(); }
const String index_name;
const Block index_sample_block;
USearchIndexWithSerializationPtr<Metric> index;
};
template <unum::usearch::metric_kind_t Metric>
struct MergeTreeIndexAggregatorUSearch final : IMergeTreeIndexAggregator
{
MergeTreeIndexAggregatorUSearch(const String & index_name_, const Block & index_sample_block);
~MergeTreeIndexAggregatorUSearch() override = default;
bool empty() const override { return !index || index->size() == 0; }
MergeTreeIndexGranulePtr getGranuleAndReset() override;
void update(const Block & block, size_t * pos, size_t limit) override;
const String index_name;
const Block index_sample_block;
USearchIndexWithSerializationPtr<Metric> index;
};
class MergeTreeIndexConditionUSearch final : public IMergeTreeIndexConditionApproximateNearestNeighbor
{
public:
MergeTreeIndexConditionUSearch(const IndexDescription & index_description, const SelectQueryInfo & query, const String & distance_function, ContextPtr context);
~MergeTreeIndexConditionUSearch() override = default;
bool alwaysUnknownOrTrue() const override;
bool mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx_granule) const override;
std::vector<size_t> getUsefulRanges(MergeTreeIndexGranulePtr idx_granule) const override;
private:
template <unum::usearch::metric_kind_t Metric>
std::vector<size_t> getUsefulRangesImpl(MergeTreeIndexGranulePtr idx_granule) const;
const ApproximateNearestNeighborCondition ann_condition;
const String distance_function;
};
class MergeTreeIndexUSearch : public IMergeTreeIndex
{
public:
MergeTreeIndexUSearch(const IndexDescription & index_, const String & distance_function_);
~MergeTreeIndexUSearch() override = default;
MergeTreeIndexGranulePtr createIndexGranule() const override;
MergeTreeIndexAggregatorPtr createIndexAggregator() const override;
MergeTreeIndexConditionPtr createIndexCondition(const SelectQueryInfo & query, ContextPtr context) const override;
bool mayBenefitFromIndexForIn(const ASTPtr & /*node*/) const override { return false; }
private:
const String distance_function;
};
}
#endif

5
src/Storages/MergeTree/MergeTreeIndices.cpp
View File

@ -132,6 +132,11 @@ MergeTreeIndexFactory::MergeTreeIndexFactory()
registerValidator("annoy", annoyIndexValidator);
#endif
#ifdef ENABLE_USEARCH
registerCreator("usearch", usearchIndexCreator);
registerValidator("usearch", usearchIndexValidator);
#endif
registerCreator("inverted", invertedIndexCreator);
registerValidator("inverted", invertedIndexValidator);

5
src/Storages/MergeTree/MergeTreeIndices.h
View File

@ -238,6 +238,11 @@ MergeTreeIndexPtr annoyIndexCreator(const IndexDescription & index);
void annoyIndexValidator(const IndexDescription & index, bool attach);
#endif
#ifdef ENABLE_USEARCH
MergeTreeIndexPtr usearchIndexCreator(const IndexDescription& index);
void usearchIndexValidator(const IndexDescription& index, bool attach);
#endif
MergeTreeIndexPtr invertedIndexCreator(const IndexDescription& index);
void invertedIndexValidator(const IndexDescription& index, bool attach);

3
src/Storages/S3Queue/S3QueueSource.cpp
View File

@ -192,7 +192,8 @@ StorageS3QueueSource::StorageS3QueueSource(
bucket_,
version_id_,
file_iterator,
download_thread_num_);
download_thread_num_,
/* query_info */ std::nullopt);
reader = std::move(internal_source->reader);
if (reader)
reader_future = std::move(internal_source->reader_future);

8
src/Storages/StorageAzureBlob.cpp
View File

@ -706,7 +706,8 @@ Pipe StorageAzureBlob::read(
configuration.compression_method,
object_storage.get(),
configuration.container,
iterator_wrapper));
iterator_wrapper,
query_info));
}
return Pipe::unitePipes(std::move(pipes));
@ -1094,7 +1095,8 @@ StorageAzureBlobSource::StorageAzureBlobSource(
String compression_hint_,
AzureObjectStorage * object_storage_,
const String & container_,
std::shared_ptr<IIterator> file_iterator_)
std::shared_ptr<IIterator> file_iterator_,
const SelectQueryInfo & query_info_)
:ISource(info.source_header, false)
, WithContext(context_)
, requested_columns(info.requested_columns)
@ -1109,6 +1111,7 @@ StorageAzureBlobSource::StorageAzureBlobSource(
, object_storage(std::move(object_storage_))
, container(container_)
, file_iterator(file_iterator_)
, query_info(query_info_)
, create_reader_pool(CurrentMetrics::ObjectStorageAzureThreads, CurrentMetrics::ObjectStorageAzureThreadsActive, 1)
, create_reader_scheduler(threadPoolCallbackRunner<ReaderHolder>(create_reader_pool, "AzureReader"))
{
@ -1142,6 +1145,7 @@ StorageAzureBlobSource::ReaderHolder StorageAzureBlobSource::createReader()
format, *read_buf, sample_block, getContext(), max_block_size,
format_settings, std::nullopt, std::nullopt,
/* is_remote_fs */ true, compression_method);
input_format->setQueryInfo(query_info, getContext());
QueryPipelineBuilder builder;
builder.init(Pipe(input_format));

5
src/Storages/StorageAzureBlob.h
View File

@ -12,6 +12,7 @@
#include <Processors/Executors/PullingPipelineExecutor.h>
#include <Storages/NamedCollectionsHelpers.h>
#include <Storages/prepareReadingFromFormat.h>
#include <Storages/SelectQueryInfo.h>
namespace DB
{
@ -248,7 +249,8 @@ public:
String compression_hint_,
AzureObjectStorage * object_storage_,
const String & container_,
std::shared_ptr<IIterator> file_iterator_);
std::shared_ptr<IIterator> file_iterator_,
const SelectQueryInfo & query_info_);
~StorageAzureBlobSource() override;
@ -269,6 +271,7 @@ private:
AzureObjectStorage * object_storage;
String container;
std::shared_ptr<IIterator> file_iterator;
SelectQueryInfo query_info;
struct ReaderHolder
{

7
src/Storages/StorageFile.cpp
View File

@ -777,6 +777,7 @@ public:
std::shared_ptr<StorageFile> storage_,
const StorageSnapshotPtr & storage_snapshot_,
ContextPtr context_,
const SelectQueryInfo & query_info_,
UInt64 max_block_size_,
FilesIteratorPtr files_iterator_,
std::unique_ptr<ReadBuffer> read_buf_)
@ -790,6 +791,7 @@ public:
, requested_virtual_columns(info.requested_virtual_columns)
, block_for_format(info.format_header)
, context(context_)
, query_info(query_info_)
, max_block_size(max_block_size_)
{
if (!storage->use_table_fd)
@ -965,6 +967,7 @@ public:
chassert(!storage->paths.empty());
const auto max_parsing_threads = std::max<size_t>(settings.max_threads/ storage->paths.size(), 1UL);
input_format = context->getInputFormat(storage->format_name, *read_buf, block_for_format, max_block_size, storage->format_settings, max_parsing_threads);
input_format->setQueryInfo(query_info, context);
QueryPipelineBuilder builder;
builder.init(Pipe(input_format));
@ -1056,6 +1059,7 @@ private:
Block block_for_format;
ContextPtr context; /// TODO Untangle potential issues with context lifetime.
SelectQueryInfo query_info;
UInt64 max_block_size;
bool finished_generate = false;
@ -1067,7 +1071,7 @@ private:
Pipe StorageFile::read(
const Names & column_names,
const StorageSnapshotPtr & storage_snapshot,
SelectQueryInfo & /*query_info*/,
SelectQueryInfo & query_info,
ContextPtr context,
QueryProcessingStage::Enum /*processed_stage*/,
size_t max_block_size,
@ -1146,6 +1150,7 @@ Pipe StorageFile::read(
this_ptr,
storage_snapshot,
context,
query_info,
max_block_size,
files_iterator,
std::move(read_buffer)));

10
src/Storages/StorageS3.cpp
View File

@ -527,7 +527,8 @@ StorageS3Source::StorageS3Source(
const String & bucket_,
const String & version_id_,
std::shared_ptr<IIterator> file_iterator_,
const size_t max_parsing_threads_)
const size_t max_parsing_threads_,
std::optional<SelectQueryInfo> query_info_)
: ISource(info.source_header, false)
, WithContext(context_)
, name(std::move(name_))
@ -542,6 +543,7 @@ StorageS3Source::StorageS3Source(
, client(client_)
, sample_block(info.format_header)
, format_settings(format_settings_)
, query_info(std::move(query_info_))
, requested_virtual_columns(info.requested_virtual_columns)
, file_iterator(file_iterator_)
, max_parsing_threads(max_parsing_threads_)
@ -582,6 +584,9 @@ StorageS3Source::ReaderHolder StorageS3Source::createReader()
/* is_remote_fs */ true,
compression_method);
if (query_info.has_value())
input_format->setQueryInfo(query_info.value(), getContext());
QueryPipelineBuilder builder;
builder.init(Pipe(input_format));
@ -1056,7 +1061,8 @@ Pipe StorageS3::read(
query_configuration.url.bucket,
query_configuration.url.version_id,
iterator_wrapper,
parsing_threads));
parsing_threads,
query_info));
}
return Pipe::unitePipes(std::move(pipes));

5
src/Storages/StorageS3.h
View File

@ -20,6 +20,7 @@
#include <Interpreters/Context.h>
#include <Interpreters/threadPoolCallbackRunner.h>
#include <Storages/Cache/SchemaCache.h>
#include <Storages/SelectQueryInfo.h>
#include <Storages/StorageConfiguration.h>
#include <Storages/prepareReadingFromFormat.h>
@ -129,7 +130,8 @@ public:
const String & bucket,
const String & version_id,
std::shared_ptr<IIterator> file_iterator_,
size_t max_parsing_threads);
size_t max_parsing_threads,
std::optional<SelectQueryInfo> query_info);
~StorageS3Source() override;
@ -152,6 +154,7 @@ private:
std::shared_ptr<const S3::Client> client;
Block sample_block;
std::optional<FormatSettings> format_settings;
std::optional<SelectQueryInfo> query_info;
struct ReaderHolder
{

4
src/Storages/StorageURL.cpp
View File

@ -222,6 +222,7 @@ StorageURLSource::StorageURLSource(
const ConnectionTimeouts & timeouts,
CompressionMethod compression_method,
size_t max_parsing_threads,
const SelectQueryInfo & query_info,
const HTTPHeaderEntries & headers_,
const URIParams & params,
bool glob_url)
@ -285,6 +286,7 @@ StorageURLSource::StorageURLSource(
/* max_download_threads= */ std::nullopt,
/* is_remote_fs= */ true,
compression_method);
input_format->setQueryInfo(query_info, context);
QueryPipelineBuilder builder;
builder.init(Pipe(input_format));
@ -773,6 +775,7 @@ Pipe IStorageURLBase::read(
getHTTPTimeouts(local_context),
compression_method,
parsing_threads,
query_info,
headers,
params,
is_url_with_globs));
@ -819,6 +822,7 @@ Pipe StorageURLWithFailover::read(
getHTTPTimeouts(local_context),
compression_method,
parsing_threads,
query_info,
headers,
params));
std::shuffle(uri_options.begin(), uri_options.end(), thread_local_rng);

1
src/Storages/StorageURL.h
View File

@ -171,6 +171,7 @@ public:
const ConnectionTimeouts & timeouts,
CompressionMethod compression_method,
size_t max_parsing_threads,
const SelectQueryInfo & query_info,
const HTTPHeaderEntries & headers_ = {},
const URIParams & params = {},
bool glob_url = false);

4
tests/integration/README.md
View File

@ -52,6 +52,8 @@ sudo -H pip install \
(highly not recommended) If you really want to use OS packages on modern debian/ubuntu instead of "pip": `sudo apt install -y docker docker-compose python3-pytest python3-dicttoxml python3-docker python3-pymysql python3-protobuf python3-pymongo python3-tzlocal python3-kazoo python3-psycopg2 kafka-python python3-pytest-timeout python3-minio`
Some tests have other dependencies, e.g. spark. See docker/test/integration/runner/Dockerfile for how to install those. See docker/test/integration/runner/dockerd-entrypoint.sh for environment variables that need to be set (e.g. JAVA_PATH).
If you want to run the tests under a non-privileged user, you must add this user to `docker` group: `sudo usermod -aG docker $USER` and re-login.
(You must close all your sessions (for example, restart your computer))
To check, that you have access to Docker, run `docker ps`.
@ -90,7 +92,7 @@ plugins: repeat-0.9.1, xdist-2.5.0, forked-1.4.0, order-1.0.0, timeout-2.1.0
timeout: 900.0s
timeout method: signal
timeout func_only: False
collected 4 items
collected 4 items
test_ssl_cert_authentication/test.py::test_https Copy common default production configuration from /clickhouse-config. Files: config.xml, users.xml
PASSED

0
tests/integration/test_runtime_configurable_cache_size/__init__.py Normal file
View File

9
tests/integration/test_runtime_configurable_cache_size/configs/default.xml Normal file
View File

@ -0,0 +1,9 @@
<clickhouse>
<query_cache>
<max_entries>2</max_entries>
</query_cache>
<mark_cache_size>496</mark_cache_size>
</clickhouse>

5
tests/integration/test_runtime_configurable_cache_size/configs/smaller_mark_cache.xml Normal file
View File

@ -0,0 +1,5 @@
<clickhouse>
<mark_cache_size>248</mark_cache_size>
</clickhouse>

7
tests/integration/test_runtime_configurable_cache_size/configs/smaller_query_cache.xml Normal file
View File

@ -0,0 +1,7 @@
<clickhouse>
<query_cache>
<max_entries>1</max_entries>
</query_cache>
</clickhouse>

144
tests/integration/test_runtime_configurable_cache_size/test.py Normal file
View File

@ -0,0 +1,144 @@
import os
import pytest
import shutil
import time
from helpers.cluster import ClickHouseCluster
# Tests that sizes of in-memory caches (mark / uncompressed / index mark / index uncompressed / mmapped file / query cache) can be changed
# at runtime (issue #51085). This file tests only the mark cache (which uses the SLRU cache policy) and the query cache (which uses the TTL
# cache policy). As such, both tests are representative for the other caches.
cluster = ClickHouseCluster(__file__)
node = cluster.add_instance(
"node",
main_configs=["configs/default.xml"],
stay_alive=True,
)
@pytest.fixture(scope="module")
def start_cluster():
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
CONFIG_DIR = os.path.join(SCRIPT_DIR, "configs")
# temporarily disabled due to https://github.com/ClickHouse/ClickHouse/pull/51446#issuecomment-1687066351
# def test_mark_cache_size_is_runtime_configurable(start_cluster):
# # the initial config specifies the mark cache size as 496 bytes, just enough to hold two marks
# node.query("SYSTEM DROP MARK CACHE")
#
# node.query("CREATE TABLE test1 (val String) ENGINE=MergeTree ORDER BY val")
# node.query("INSERT INTO test1 VALUES ('abc') ('def') ('ghi')")
# node.query("SELECT * FROM test1 WHERE val = 'def'") # cache 1st mark
#
# node.query("CREATE TABLE test2 (val String) ENGINE=MergeTree ORDER BY val")
# node.query("INSERT INTO test2 VALUES ('abc') ('def') ('ghi')")
# node.query("SELECT * FROM test2 WHERE val = 'def'") # cache 2nd mark
#
# # Result checking is based on asynchronous metrics. These are calculated by default every 1.0 sec, and this is also the
# # smallest possible value. Found no statement to force-recalculate them, therefore waaaaait...
# time.sleep(2.0)
# res = node.query(
# "SELECT value FROM system.asynchronous_metrics WHERE metric LIKE 'MarkCacheFiles'"
# )
# assert res == "2\n"
# res = node.query(
# "SELECT value FROM system.asynchronous_metrics WHERE metric LIKE 'MarkCacheBytes'"
# )
# assert res == "496\n"
#
# # switch to a config with a mark cache size of 248 bytes
# node.copy_file_to_container(
# os.path.join(CONFIG_DIR, "smaller_mark_cache.xml"),
# "/etc/clickhouse-server/config.d/default.xml",
# )
#
# node.query("SYSTEM RELOAD CONFIG")
#
# # check that eviction worked as expected
# time.sleep(2.0)
# res = node.query(
# "SELECT value FROM system.asynchronous_metrics WHERE metric LIKE 'MarkCacheFiles'"
# )
# assert res == "1\n"
# res = node.query(
# "SELECT value FROM system.asynchronous_metrics WHERE metric LIKE 'MarkCacheBytes'"
# )
# assert res == "248\n"
#
# # check that the new mark cache maximum size is respected when more marks are cached
# node.query("CREATE TABLE test3 (val String) ENGINE=MergeTree ORDER BY val")
# node.query("INSERT INTO test3 VALUES ('abc') ('def') ('ghi')")
# node.query("SELECT * FROM test3 WHERE val = 'def'")
# time.sleep(2.0)
# res = node.query(
# "SELECT value FROM system.asynchronous_metrics WHERE metric LIKE 'MarkCacheFiles'"
# )
# assert res == "1\n"
# res = node.query(
# "SELECT value FROM system.asynchronous_metrics WHERE metric LIKE 'MarkCacheBytes'"
# )
# assert res == "248\n"
#
# # restore the original config
# node.copy_file_to_container(
# os.path.join(CONFIG_DIR, "default.xml"),
# "/etc/clickhouse-server/config.d/default.xml",
# )
def test_query_cache_size_is_runtime_configurable(start_cluster):
# the inital config specifies the maximum query cache size as 2, run 3 queries, expect 2 cache entries
node.query("SYSTEM DROP QUERY CACHE")
node.query("SELECT 1 SETTINGS use_query_cache = 1, query_cache_ttl = 1")
node.query("SELECT 2 SETTINGS use_query_cache = 1, query_cache_ttl = 1")
node.query("SELECT 3 SETTINGS use_query_cache = 1, query_cache_ttl = 1")
time.sleep(2.0)
res = node.query(
"SELECT value FROM system.asynchronous_metrics WHERE metric = 'QueryCacheEntries'"
)
assert res == "2\n"
# switch to a config with a maximum query cache size of 1
node.copy_file_to_container(
os.path.join(CONFIG_DIR, "smaller_query_cache.xml"),
"/etc/clickhouse-server/config.d/default.xml",
)
node.query("SYSTEM RELOAD CONFIG")
# check that eviction worked as expected
time.sleep(2.0)
res = node.query(
"SELECT value FROM system.asynchronous_metrics WHERE metric = 'QueryCacheEntries'"
)
assert (
res == "2\n"
) # "Why not 1?", you think. Reason is that QC uses the TTLCachePolicy that evicts lazily only upon insert.
# Not a real issue, can be changed later, at least there's a test now.
# Also, you may also wonder "why query_cache_ttl = 1"? Reason is that TTLCachePolicy only removes *stale* entries. With the default TTL
# (60 sec), no entries would be removed at all. Again: not a real issue, can be changed later and there's at least a test now.
# check that the new query cache maximum size is respected when more queries run
node.query("SELECT 4 SETTINGS use_query_cache = 1, query_cache_ttl = 1")
node.query("SELECT 5 SETTINGS use_query_cache = 1, query_cache_ttl = 1")
time.sleep(2.0)
res = node.query(
"SELECT value FROM system.asynchronous_metrics WHERE metric = 'QueryCacheEntries'"
)
assert res == "1\n"
# restore the original config
node.copy_file_to_container(
os.path.join(CONFIG_DIR, "default.xml"),
"/etc/clickhouse-server/config.d/default.xml",
)

4
tests/performance/README.md
View File

@ -4,11 +4,11 @@ This directory contains `.xml`-files with performance tests for @akuzm tool.
### How to write performance test
First of all you should check existing tests don't cover your case. If there are no such tests than you should write your own.
First of all you should check existing tests don't cover your case. If there are no such tests then you should write your own.
You can use `substitions`, `create`, `fill` and `drop` queries to prepare test. You can find examples in this folder.
If your test continued more than 10 minutes, please, add tag `long` to have an opportunity to run all tests and skip long ones.
If your test takes more than 10 minutes, please, add tag `long` to have an opportunity to run all tests and skip long ones.
### How to run performance test

9
tests/performance/parquet_filter.xml Normal file
View File

@ -0,0 +1,9 @@
<test>
<create_query>create table if not exists t (key UInt64, value String) engine = File(Parquet) settings output_format_parquet_use_custom_encoder=1, output_format_parquet_row_group_size=100000</create_query>
<fill_query>insert into t select number, toString(number) from numbers(2000000) settings max_threads=16, max_insert_threads=16, max_insert_block_size=100000, max_block_size=100000</fill_query>
<query>select sum(cityHash64(*)) from t where key between 1050000 and 1150000 settings max_threads=1</query>
<drop_query>drop table if exists t</drop_query>
</test>

143
tests/queries/0_stateless/02354_usearch_index.reference Normal file
View File

@ -0,0 +1,143 @@
--- Negative tests ---
--- Test default GRANULARITY (should be 100 mio. for usearch)---
CREATE TABLE default.tab\n(\n `id` Int32,\n `vector` Array(Float32),\n INDEX usearch_index vector TYPE usearch GRANULARITY 100000000\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192
CREATE TABLE default.tab\n(\n `id` Int32,\n `vector` Array(Float32),\n INDEX usearch_index vector TYPE usearch GRANULARITY 100000000\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192
--- Test with Array, GRANULARITY = 1, index_granularity = 5 ---
WHERE type, L2Distance, check that index is used
Expression ((Projection + Before ORDER BY))
Limit (preliminary LIMIT (without OFFSET))
ReadFromMergeTree (default.tab)
Indexes:
PrimaryKey
Condition: true
Parts: 1/1
Granules: 3/3
Skip
Name: usearch_index
Description: usearch GRANULARITY 1
Parts: 1/1
Granules: 1/3
ORDER BY type, L2Distance, check that index is used
Expression (Projection)
Limit (preliminary LIMIT (without OFFSET))
Sorting (Sorting for ORDER BY)
Expression (Before ORDER BY)
ReadFromMergeTree (default.tab)
Indexes:
PrimaryKey
Condition: true
Parts: 1/1
Granules: 3/3
Skip
Name: usearch_index
Description: usearch GRANULARITY 1
Parts: 1/1
Granules: 3/3
Reference ARRAYs with non-matching dimension are rejected
Special case: MaximumDistance is negative
WHERE type, L2Distance
Special case: setting max_limit_for_ann_queries
Expression (Projection)
Limit (preliminary LIMIT (without OFFSET))
Sorting (Sorting for ORDER BY)
Expression (Before ORDER BY)
ReadFromMergeTree (default.tab)
Indexes:
PrimaryKey
Condition: true
Parts: 1/1
Granules: 3/3
--- Test with Tuple, GRANULARITY = 1, index_granularity = 5 ---
WHERE type, L2Distance, check that index is used
Expression ((Projection + Before ORDER BY))
Limit (preliminary LIMIT (without OFFSET))
ReadFromMergeTree (default.tab)
Indexes:
PrimaryKey
Condition: true
Parts: 1/1
Granules: 3/3
Skip
Name: usearch_index
Description: usearch GRANULARITY 1
Parts: 1/1
Granules: 1/3
ORDER BY type, L2Distance, check that index is used
Expression (Projection)
Limit (preliminary LIMIT (without OFFSET))
Sorting (Sorting for ORDER BY)
Expression (Before ORDER BY)
ReadFromMergeTree (default.tab)
Indexes:
PrimaryKey
Condition: true
Parts: 1/1
Granules: 3/3
Skip
Name: usearch_index
Description: usearch GRANULARITY 1
Parts: 1/1
Granules: 3/3
--- Test non-default metric (cosine distance) ---
--- Test with Array, GRANULARITY = 2, index_granularity = 4 ---
WHERE type, L2Distance, check that index is used
Expression ((Projection + Before ORDER BY))
Limit (preliminary LIMIT (without OFFSET))
ReadFromMergeTree (default.tab)
Indexes:
PrimaryKey
Condition: true
Parts: 1/1
Granules: 4/4
Skip
Name: usearch_index
Description: usearch GRANULARITY 2
Parts: 0/1
Granules: 2/4
ORDER BY type, L2Distance, check that index is used
Expression (Projection)
Limit (preliminary LIMIT (without OFFSET))
Sorting (Sorting for ORDER BY)
Expression (Before ORDER BY)
ReadFromMergeTree (default.tab)
Indexes:
PrimaryKey
Condition: true
Parts: 1/1
Granules: 4/4
Skip
Name: usearch_index
Description: usearch GRANULARITY 2
Parts: 1/1
Granules: 4/4
--- Test with Array, GRANULARITY = 4, index_granularity = 4 ---
WHERE type, L2Distance, check that index is used
Expression ((Projection + Before ORDER BY))
Limit (preliminary LIMIT (without OFFSET))
ReadFromMergeTree (default.tab)
Indexes:
PrimaryKey
Condition: true
Parts: 1/1
Granules: 4/4
Skip
Name: usearch_index
Description: usearch GRANULARITY 4
Parts: 0/1
Granules: 3/4
ORDER BY type, L2Distance, check that index is used
Expression (Projection)
Limit (preliminary LIMIT (without OFFSET))
Sorting (Sorting for ORDER BY)
Expression (Before ORDER BY)
ReadFromMergeTree (default.tab)
Indexes:
PrimaryKey
Condition: true
Parts: 1/1
Granules: 4/4
Skip
Name: usearch_index
Description: usearch GRANULARITY 4
Parts: 1/1
Granules: 4/4

230
tests/queries/0_stateless/02354_usearch_index.sql Normal file
View File

@ -0,0 +1,230 @@
-- Tags: no-fasttest
-- no-fasttest because needs usearch lib
SET allow_experimental_usearch_index = 1;
SET allow_experimental_analyzer = 0;
SELECT '--- Negative tests ---';
DROP TABLE IF EXISTS tab;
-- must have at most 1 arguments
CREATE TABLE tab(id Int32, vector Array(Float32), INDEX usearch_index vector TYPE usearch('too', 'many')) ENGINE = MergeTree ORDER BY id; -- { serverError INCORRECT_QUERY }
-- first argument (distance_function) must be String
CREATE TABLE tab(id Int32, vector Array(Float32), INDEX usearch_index vector TYPE usearch(3)) ENGINE = MergeTree ORDER BY id; -- { serverError INCORRECT_QUERY }
-- must be created on single column
CREATE TABLE tab(id Int32, vector Array(Float32), INDEX usearch_index (vector, id) TYPE usearch()) ENGINE = MergeTree ORDER BY id; -- { serverError INCORRECT_NUMBER_OF_COLUMNS }
-- reject unsupported distance functions
CREATE TABLE tab(id Int32, vector Array(Float32), INDEX usearch_index vector TYPE usearch('wormholeDistance')) ENGINE = MergeTree ORDER BY id; -- { serverError INCORRECT_DATA }
-- must be created on Array/Tuple(Float32) columns
SET allow_suspicious_low_cardinality_types = 1;
CREATE TABLE tab(id Int32, vector Float32, INDEX usearch_index vector TYPE usearch()) ENGINE = MergeTree ORDER BY id; -- { serverError ILLEGAL_COLUMN }
CREATE TABLE tab(id Int32, vector Array(Float64), INDEX usearch_index vector TYPE usearch()) ENGINE = MergeTree ORDER BY id; -- { serverError ILLEGAL_COLUMN }
CREATE TABLE tab(id Int32, vector Tuple(Float64), INDEX usearch_index vector TYPE usearch()) ENGINE = MergeTree ORDER BY id; -- { serverError ILLEGAL_COLUMN }
CREATE TABLE tab(id Int32, vector LowCardinality(Float32), INDEX usearch_index vector TYPE usearch()) ENGINE = MergeTree ORDER BY id; -- { serverError ILLEGAL_COLUMN }
CREATE TABLE tab(id Int32, vector Nullable(Float32), INDEX usearch_index vector TYPE usearch()) ENGINE = MergeTree ORDER BY id; -- { serverError ILLEGAL_COLUMN }
SELECT '--- Test default GRANULARITY (should be 100 mio. for usearch)---';
CREATE TABLE tab (id Int32, vector Array(Float32), INDEX usearch_index(vector) TYPE usearch) ENGINE=MergeTree ORDER BY id;
SHOW CREATE TABLE tab;
DROP TABLE tab;
CREATE TABLE tab (id Int32, vector Array(Float32)) ENGINE=MergeTree ORDER BY id;
ALTER TABLE tab ADD INDEX usearch_index(vector) TYPE usearch;
SHOW CREATE TABLE tab;
DROP TABLE tab;
SELECT '--- Test with Array, GRANULARITY = 1, index_granularity = 5 ---';
DROP TABLE IF EXISTS tab;
CREATE TABLE tab(id Int32, vector Array(Float32), INDEX usearch_index vector TYPE usearch() GRANULARITY 1) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 5;
INSERT INTO tab VALUES (1, [0.0, 0.0, 10.0]), (2, [0.0, 0.0, 10.5]), (3, [0.0, 0.0, 9.5]), (4, [0.0, 0.0, 9.7]), (5, [0.0, 0.0, 10.2]), (6, [10.0, 0.0, 0.0]), (7, [9.5, 0.0, 0.0]), (8, [9.7, 0.0, 0.0]), (9, [10.2, 0.0, 0.0]), (10, [10.5, 0.0, 0.0]), (11, [0.0, 10.0, 0.0]), (12, [0.0, 9.5, 0.0]), (13, [0.0, 9.7, 0.0]), (14, [0.0, 10.2, 0.0]), (15, [0.0, 10.5, 0.0]);
-- rows = 15, index_granularity = 5, GRANULARITY = 1 gives 3 usearch-indexed blocks (each comprising a single granule)
-- condition 'L2Distance(vector, reference_vector) < 1.0' ensures that only one usearch-indexed block produces results --> "Granules: 1/3"
-- SELECT 'WHERE type, L2Distance';
-- SELECT *
-- FROM tab
-- WHERE L2Distance(vector, [0.0, 0.0, 10.0]) < 1.0
-- LIMIT 3;
SELECT 'WHERE type, L2Distance, check that index is used';
EXPLAIN indexes=1
SELECT *
FROM tab
WHERE L2Distance(vector, [0.0, 0.0, 10.0]) < 1.0
LIMIT 3;
-- SELECT 'ORDER BY type, L2Distance';
-- SELECT *
-- FROM tab
-- ORDER BY L2Distance(vector, [0.0, 0.0, 10.0])
-- LIMIT 3;
SELECT 'ORDER BY type, L2Distance, check that index is used';
EXPLAIN indexes=1
SELECT *
FROM tab
ORDER BY L2Distance(vector, [0.0, 0.0, 10.0])
LIMIT 3;
-- Test special cases. Corresponding special case tests are omitted from later tests.
SELECT 'Reference ARRAYs with non-matching dimension are rejected';
SELECT *
FROM tab
ORDER BY L2Distance(vector, [0.0, 0.0])
LIMIT 3; -- { serverError INCORRECT_QUERY }
SELECT 'Special case: MaximumDistance is negative';
SELECT 'WHERE type, L2Distance';
SELECT *
FROM tab
WHERE L2Distance(vector, [0.0, 0.0, 10.0]) < -1.0
LIMIT 3; -- { serverError INCORRECT_QUERY }
SELECT 'Special case: setting max_limit_for_ann_queries';
EXPLAIN indexes=1
SELECT *
FROM tab
ORDER BY L2Distance(vector, [5.3, 7.3, 2.1])
LIMIT 3
SETTINGS max_limit_for_ann_queries=2; -- doesn't use the ann index
DROP TABLE tab;
-- Test Tuple embeddings. Triggers different logic than Array inside MergeTreeIndexUSearch but the same logic as Array above MergeTreeIndexusearch.
-- Therefore test Tuple case just once.
SELECT '--- Test with Tuple, GRANULARITY = 1, index_granularity = 5 ---';
CREATE TABLE tab(id Int32, vector Tuple(Float32, Float32, Float32), INDEX usearch_index vector TYPE usearch() GRANULARITY 1) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 5;
INSERT INTO tab VALUES (1, (0.0, 0.0, 10.0)), (2, (0.0, 0.0, 10.5)), (3, (0.0, 0.0, 9.5)), (4, (0.0, 0.0, 9.7)), (5, (0.0, 0.0, 10.2)), (6, (10.0, 0.0, 0.0)), (7, (9.5, 0.0, 0.0)), (8, (9.7, 0.0, 0.0)), (9, (10.2, 0.0, 0.0)), (10, (10.5, 0.0, 0.0)), (11, (0.0, 10.0, 0.0)), (12, (0.0, 9.5, 0.0)), (13, (0.0, 9.7, 0.0)), (14, (0.0, 10.2, 0.0)), (15, (0.0, 10.5, 0.0));
-- SELECT 'WHERE type, L2Distance';
-- SELECT *
-- FROM tab
-- WHERE L2Distance(vector, (0.0, 0.0, 10.0)) < 1.0
-- LIMIT 3;
SELECT 'WHERE type, L2Distance, check that index is used';
EXPLAIN indexes=1
SELECT *
FROM tab
WHERE L2Distance(vector, (0.0, 0.0, 10.0)) < 1.0
LIMIT 3;
-- SELECT 'ORDER BY type, L2Distance';
-- SELECT *
-- FROM tab
-- ORDER BY L2Distance(vector, (0.0, 0.0, 10.0))
-- LIMIT 3;
SELECT 'ORDER BY type, L2Distance, check that index is used';
EXPLAIN indexes=1
SELECT *
FROM tab
ORDER BY L2Distance(vector, (0.0, 0.0, 10.0))
LIMIT 3;
DROP TABLE tab;
-- Not a systematic test, just to make sure no bad things happen
SELECT '--- Test non-default metric (cosine distance) ---';
CREATE TABLE tab(id Int32, vector Array(Float32), INDEX usearch_index vector TYPE usearch('cosineDistance') GRANULARITY 1) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 5;
INSERT INTO tab VALUES (1, [0.0, 0.0, 10.0]), (2, [0.0, 0.0, 10.5]), (3, [0.0, 0.0, 9.5]), (4, [0.0, 0.0, 9.7]), (5, [0.0, 0.0, 10.2]), (6, [10.0, 0.0, 0.0]), (7, [9.5, 0.0, 0.0]), (8, [9.7, 0.0, 0.0]), (9, [10.2, 0.0, 0.0]), (10, [10.5, 0.0, 0.0]), (11, [0.0, 10.0, 0.0]), (12, [0.0, 9.5, 0.0]), (13, [0.0, 9.7, 0.0]), (14, [0.0, 10.2, 0.0]), (15, [0.0, 10.5, 0.0]);
-- SELECT 'WHERE type, L2Distance';
-- SELECT *
-- FROM tab
-- WHERE L2Distance(vector, [0.0, 0.0, 10.0]) < 1.0
-- LIMIT 3;
-- SELECT 'ORDER BY type, L2Distance';
-- SELECT *
-- FROM tab
-- ORDER BY L2Distance(vector, [0.0, 0.0, 10.0])
-- LIMIT 3;
DROP TABLE tab;
SELECT '--- Test with Array, GRANULARITY = 2, index_granularity = 4 ---';
CREATE TABLE tab(id Int32, vector Array(Float32), INDEX usearch_index vector TYPE usearch() GRANULARITY 2) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 4;
INSERT INTO tab VALUES (1, [0.0, 0.0, 10.0, 0.0]), (2, [0.0, 0.0, 10.5, 0.0]), (3, [0.0, 0.0, 9.5, 0.0]), (4, [0.0, 0.0, 9.7, 0.0]), (5, [10.0, 0.0, 0.0, 0.0]), (6, [9.5, 0.0, 0.0, 0.0]), (7, [9.7, 0.0, 0.0, 0.0]), (8, [10.2, 0.0, 0.0, 0.0]), (9, [0.0, 10.0, 0.0, 0.0]), (10, [0.0, 9.5, 0.0, 0.0]), (11, [0.0, 9.7, 0.0, 0.0]), (12, [0.0, 9.7, 0.0, 0.0]), (13, [0.0, 0.0, 0.0, 10.3]), (14, [0.0, 0.0, 0.0, 9.5]), (15, [0.0, 0.0, 0.0, 10.0]), (16, [0.0, 0.0, 0.0, 10.5]);
-- rows = 16, index_granularity = 4, GRANULARITY = 2 gives 2 usearch-indexed blocks (each comprising two granules)
-- condition 'L2Distance(vector, reference_vector) < 1.0' ensures that only one usearch-indexed block produces results --> "Granules: 2/4"
-- SELECT 'WHERE type, L2Distance';
-- SELECT *
-- FROM tab
-- WHERE L2Distance(vector, [10.0, 0.0, 10.0, 0.0]) < 5.0
-- LIMIT 3;
SELECT 'WHERE type, L2Distance, check that index is used';
EXPLAIN indexes=1
SELECT *
FROM tab
WHERE L2Distance(vector, [10.0, 0.0, 10.0, 0.0]) < 5.0
LIMIT 3;
-- SELECT 'ORDER BY type, L2Distance';
-- SELECT *
-- FROM tab
-- ORDER BY L2Distance(vector, [10.0, 0.0, 10.0, 0.0])
-- LIMIT 3;
SELECT 'ORDER BY type, L2Distance, check that index is used';
EXPLAIN indexes=1
SELECT *
FROM tab
ORDER BY L2Distance(vector, [10.0, 0.0, 10.0, 0.0])
LIMIT 3;
DROP TABLE tab;
SELECT '--- Test with Array, GRANULARITY = 4, index_granularity = 4 ---';
CREATE TABLE tab(id Int32, vector Array(Float32), INDEX usearch_index vector TYPE usearch() GRANULARITY 4) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 4;
INSERT INTO tab VALUES (1, [0.0, 0.0, 10.0, 0.0]), (2, [0.0, 0.0, 10.5, 0.0]), (3, [0.0, 0.0, 9.5, 0.0]), (4, [0.0, 0.0, 9.7, 0.0]), (5, [10.0, 0.0, 0.0, 0.0]), (6, [9.5, 0.0, 0.0, 0.0]), (7, [9.7, 0.0, 0.0, 0.0]), (8, [10.2, 0.0, 0.0, 0.0]), (9, [0.0, 10.0, 0.0, 0.0]), (10, [0.0, 9.5, 0.0, 0.0]), (11, [0.0, 9.7, 0.0, 0.0]), (12, [0.0, 9.7, 0.0, 0.0]), (13, [0.0, 0.0, 0.0, 10.3]), (14, [0.0, 0.0, 0.0, 9.5]), (15, [0.0, 0.0, 0.0, 10.0]), (16, [0.0, 0.0, 0.0, 10.5]);
-- rows = 16, index_granularity = 4, GRANULARITY = 4 gives a single usearch-indexed block (comprising all granules)
-- no two matches happen to be located in the same granule, so with LIMIT = 3, we'll get "Granules: 2/4"
-- SELECT 'WHERE type, L2Distance';
-- SELECT *
-- FROM tab
-- WHERE L2Distance(vector, [10.0, 0.0, 10.0, 0.0]) < 5.0
-- LIMIT 3;
SELECT 'WHERE type, L2Distance, check that index is used';
EXPLAIN indexes=1
SELECT *
FROM tab
WHERE L2Distance(vector, [10.0, 0.0, 10.0, 0.0]) < 5.0
LIMIT 3;
-- SELECT 'ORDER BY type, L2Distance';
-- SELECT *
-- FROM tab
-- ORDER BY L2Distance(vector, [10.0, 0.0, 10.0, 0.0])
-- LIMIT 3;
SELECT 'ORDER BY type, L2Distance, check that index is used';
EXPLAIN indexes=1
SELECT *
FROM tab
ORDER BY L2Distance(vector, [10.0, 0.0, 10.0, 0.0])
LIMIT 3;
DROP TABLE tab;

7
tests/queries/0_stateless/02735_parquet_encoder.reference
View File

@ -54,3 +54,10 @@ BYTE_ARRAY String
never gonna
give you
up
ms Nullable(DateTime64(3, \'UTC\'))
us Nullable(DateTime64(6, \'UTC\'))
ns Nullable(DateTime64(9, \'UTC\'))
cs Nullable(DateTime64(3, \'UTC\'))
s Nullable(DateTime64(3, \'UTC\'))
dus Nullable(DateTime64(9, \'UTC\'))
12670676506515577395

12
tests/queries/0_stateless/02735_parquet_encoder.sql
View File

@ -168,3 +168,15 @@ select columns.5, columns.6 from file(strings3_02735.parquet, ParquetMetadata) a
select * from file(strings1_02735.parquet);
select * from file(strings2_02735.parquet);
select * from file(strings3_02735.parquet);
-- DateTime64 with different units.
insert into function file(datetime64_02735.parquet) select
toDateTime64(number / 1e3, 3) as ms,
toDateTime64(number / 1e6, 6) as us,
toDateTime64(number / 1e9, 9) as ns,
toDateTime64(number / 1e2, 2) as cs,
toDateTime64(number, 0) as s,
toDateTime64(number / 1e7, 7) as dus
from numbers(2000);
desc file(datetime64_02735.parquet);
select sum(cityHash64(*)) from file(datetime64_02735.parquet);

73
tests/queries/0_stateless/02841_parquet_filter_pushdown.reference Normal file
View File

@ -0,0 +1,73 @@
number Nullable(UInt64)
u8 Nullable(UInt8)
u16 Nullable(UInt16)
u32 Nullable(UInt32)
u64 Nullable(UInt64)
i8 Nullable(Int8)
i16 Nullable(Int16)
i32 Nullable(Int32)
i64 Nullable(Int64)
date32 Nullable(Date32)
dt64_ms Nullable(DateTime64(3, \'UTC\'))
dt64_us Nullable(DateTime64(6, \'UTC\'))
dt64_ns Nullable(DateTime64(9, \'UTC\'))
dt64_s Nullable(DateTime64(3, \'UTC\'))
dt64_cs Nullable(DateTime64(3, \'UTC\'))
f32 Nullable(Float32)
f64 Nullable(Float64)
s Nullable(String)
fs Nullable(FixedString(9))
d32 Nullable(Decimal(9, 3))
d64 Nullable(Decimal(18, 10))
d128 Nullable(Decimal(38, 20))
d256 Nullable(Decimal(76, 40))
800 3959600
1000 4999500
1800 6479100
500 2474750
300 1604850
500 2474750
300 1604850
500 2474750
2100 5563950
300 1184850
400 1599800
300 1184850
500 2524750
500 2524750
300 1514850
300 1514850
300 1594850
300 1594850
200 999900
200 999900
200 999900
200 999900
0 \N
400 1709800
0 \N
10000 49995000
0 \N
200 909900
10000 49995000
0 \N
2
500 244750
500 244750
300 104850
300 104850
200 179900
200 179900
200 179900
200 179900
200 19900
200 19900
600 259700
600 259700
500 244750
500 244750
0 \N
500 244750
500 244750
500 244750
500 244750

137
tests/queries/0_stateless/02841_parquet_filter_pushdown.sql Normal file
View File

@ -0,0 +1,137 @@
-- Tags: no-fasttest, no-parallel
set output_format_parquet_row_group_size = 100;
set input_format_null_as_default = 1;
set engine_file_truncate_on_insert = 1;
set optimize_or_like_chain = 0;
set max_block_size = 100000;
set max_insert_threads = 1;
-- Analyzer breaks the queries with IN and some queries with BETWEEN.
-- TODO: Figure out why.
set allow_experimental_analyzer=0;
-- Try all the types.
insert into function file('02841.parquet')
-- Use negative numbers to test sign extension for signed types and lack of sign extension for
-- unsigned types.
with 5000 - number as n select
number,
intDiv(n, 11)::UInt8 as u8,
n::UInt16 u16,
n::UInt32 as u32,
n::UInt64 as u64,
intDiv(n, 11)::Int8 as i8,
n::Int16 i16,
n::Int32 as i32,
n::Int64 as i64,
toDate32(n*500000) as date32,
toDateTime64(n*1e6, 3) as dt64_ms,
toDateTime64(n*1e6, 6) as dt64_us,
toDateTime64(n*1e6, 9) as dt64_ns,
toDateTime64(n*1e6, 0) as dt64_s,
toDateTime64(n*1e6, 2) as dt64_cs,
(n/1000)::Float32 as f32,
(n/1000)::Float64 as f64,
n::String as s,
n::String::FixedString(9) as fs,
n::Decimal32(3)/1234 as d32,
n::Decimal64(10)/12345678 as d64,
n::Decimal128(20)/123456789012345 as d128,
n::Decimal256(40)/123456789012345/678901234567890 as d256
from numbers(10000);
desc file('02841.parquet');
-- To generate reference results, use a temporary table and GROUP BYs to simulate row group filtering:
-- create temporary table t as with [as above] select intDiv(number, 100) as group, [as above];
-- then e.g. for a query that filters by `x BETWEEN a AND b`:
-- select sum(c), sum(h) from (select count() as c, sum(number) as h, min(x) as mn, max(x) as mx from t group by group) where a <= mx and b >= mn;
-- Go over all types individually.
select count(), sum(number) from file('02841.parquet') where indexHint(u8 in (10, 15, 250));
select count(), sum(number) from file('02841.parquet') where indexHint(i8 between -3 and 2);
select count(), sum(number) from file('02841.parquet') where indexHint(u16 between 4000 and 61000 or u16 == 42);
select count(), sum(number) from file('02841.parquet') where indexHint(i16 between -150 and 250);
select count(), sum(number) from file('02841.parquet') where indexHint(u32 in (42, 4294966296));
select count(), sum(number) from file('02841.parquet') where indexHint(i32 between -150 and 250);
select count(), sum(number) from file('02841.parquet') where indexHint(u64 in (42, 18446744073709550616));
select count(), sum(number) from file('02841.parquet') where indexHint(i64 between -150 and 250);
select count(), sum(number) from file('02841.parquet') where indexHint(date32 between '1992-01-01' and '2023-08-02');
select count(), sum(number) from file('02841.parquet') where indexHint(dt64_ms between '2000-01-01' and '2005-01-01');
select count(), sum(number) from file('02841.parquet') where indexHint(dt64_us between toDateTime64(900000000, 2) and '2005-01-01');
select count(), sum(number) from file('02841.parquet') where indexHint(dt64_ns between '2000-01-01' and '2005-01-01');
select count(), sum(number) from file('02841.parquet') where indexHint(dt64_s between toDateTime64('-2.01e8'::Decimal64(0), 0) and toDateTime64(1.5e8::Decimal64(0), 0));
select count(), sum(number) from file('02841.parquet') where indexHint(dt64_cs between toDateTime64('-2.01e8'::Decimal64(1), 1) and toDateTime64(1.5e8::Decimal64(2), 2));
select count(), sum(number) from file('02841.parquet') where indexHint(f32 between -0.11::Float32 and 0.06::Float32);
select count(), sum(number) from file('02841.parquet') where indexHint(f64 between -0.11 and 0.06);
select count(), sum(number) from file('02841.parquet') where indexHint(s between '-9' and '1!!!');
select count(), sum(number) from file('02841.parquet') where indexHint(fs between '-9' and '1!!!');
select count(), sum(number) from file('02841.parquet') where indexHint(d32 between '-0.011'::Decimal32(3) and 0.006::Decimal32(3));
select count(), sum(number) from file('02841.parquet') where indexHint(d64 between '-0.0000011'::Decimal64(7) and 0.0000006::Decimal64(9));
select count(), sum(number) from file('02841.parquet') where indexHint(d128 between '-0.00000000000011'::Decimal128(20) and 0.00000000000006::Decimal128(20));
select count(), sum(number) from file('02841.parquet') where indexHint(d256 between '-0.00000000000000000000000000011'::Decimal256(40) and 0.00000000000000000000000000006::Decimal256(35));
-- Some random other cases.
select count(), sum(number) from file('02841.parquet') where indexHint(0);
select count(), sum(number) from file('02841.parquet') where indexHint(s like '99%' or u64 == 2000);
select count(), sum(number) from file('02841.parquet') where indexHint(s like 'z%');
select count(), sum(number) from file('02841.parquet') where indexHint(u8 == 10 or 1 == 1);
select count(), sum(number) from file('02841.parquet') where indexHint(u8 < 0);
select count(), sum(number) from file('02841.parquet') where indexHint(u64 + 1000000 == 1001000);
select count(), sum(number) from file('02841.parquet') where indexHint(u64 + 1000000 == 1001000) settings input_format_parquet_filter_push_down = 0;
select count(), sum(number) from file('02841.parquet') where indexHint(u32 + 1000000 == 999000);
-- Very long string, which makes the Parquet encoder omit the corresponding min/max stat.
insert into function file('02841.parquet')
select arrayStringConcat(range(number*1000000)) as s from numbers(2);
select count() from file('02841.parquet') where indexHint(s > '');
-- Nullable and LowCardinality.
insert into function file('02841.parquet') select
number,
if(number%234 == 0, NULL, number) as sometimes_null,
toNullable(number) as never_null,
if(number%345 == 0, number::String, NULL) as mostly_null,
toLowCardinality(if(number%234 == 0, NULL, number)) as sometimes_null_lc,
toLowCardinality(toNullable(number)) as never_null_lc,
toLowCardinality(if(number%345 == 0, number::String, NULL)) as mostly_null_lc
from numbers(1000);
select count(), sum(number) from file('02841.parquet') where indexHint(sometimes_null is NULL);
select count(), sum(number) from file('02841.parquet') where indexHint(sometimes_null_lc is NULL);
select count(), sum(number) from file('02841.parquet') where indexHint(mostly_null is not NULL);
select count(), sum(number) from file('02841.parquet') where indexHint(mostly_null_lc is not NULL);
select count(), sum(number) from file('02841.parquet') where indexHint(sometimes_null > 850);
select count(), sum(number) from file('02841.parquet') where indexHint(sometimes_null_lc > 850);
select count(), sum(number) from file('02841.parquet') where indexHint(never_null > 850);
select count(), sum(number) from file('02841.parquet') where indexHint(never_null_lc > 850);
select count(), sum(number) from file('02841.parquet') where indexHint(never_null < 150);
select count(), sum(number) from file('02841.parquet') where indexHint(never_null_lc < 150);
-- Quirk with infinities: this reads too much because KeyCondition represents NULLs as infinities.
select count(), sum(number) from file('02841.parquet') where indexHint(sometimes_null < 150);
select count(), sum(number) from file('02841.parquet') where indexHint(sometimes_null_lc < 150);
-- Settings that affect the table schema or contents.
insert into function file('02841.parquet') select
number,
if(number%234 == 0, NULL, number + 100) as positive_or_null,
if(number%234 == 0, NULL, -number - 100) as negative_or_null,
if(number%234 == 0, NULL, 'I am a string') as string_or_null
from numbers(1000);
select count(), sum(number) from file('02841.parquet') where indexHint(positive_or_null < 50); -- quirk with infinities
select count(), sum(number) from file('02841.parquet', Parquet, 'number UInt64, positive_or_null UInt64') where indexHint(positive_or_null < 50);
select count(), sum(number) from file('02841.parquet') where indexHint(negative_or_null > -50);
select count(), sum(number) from file('02841.parquet', Parquet, 'number UInt64, negative_or_null Int64') where indexHint(negative_or_null > -50);
select count(), sum(number) from file('02841.parquet') where indexHint(string_or_null == ''); -- quirk with infinities
select count(), sum(number) from file('02841.parquet', Parquet, 'number UInt64, string_or_null String') where indexHint(string_or_null == '');
select count(), sum(number) from file('02841.parquet', Parquet, 'number UInt64, nEgAtIvE_oR_nUlL Int64') where indexHint(nEgAtIvE_oR_nUlL > -50) settings input_format_parquet_case_insensitive_column_matching = 1;

1
tests/queries/0_stateless/02845_parquet_odd_decimals.reference Normal file
View File

@ -0,0 +1 @@
100

13
tests/queries/0_stateless/02845_parquet_odd_decimals.sh Executable file
View File

@ -0,0 +1,13 @@
#!/usr/bin/env bash
# Tags: no-fasttest
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
# 9-byte decimals produced by spark in integration test test_storage_delta/test.py::test_single_log_file
${CLICKHOUSE_CLIENT} --query="drop table if exists 02845_parquet_odd_decimals"
${CLICKHOUSE_CLIENT} --query="create table 02845_parquet_odd_decimals (\`col-1de12c05-5dd5-4fa7-9f93-33c43c9a4028\` Decimal(20, 0), \`col-5e1b97f1-dade-4c7d-b71b-e31d789e01a4\` String) engine Memory"
${CLICKHOUSE_CLIENT} --query="insert into 02845_parquet_odd_decimals from infile '$CUR_DIR/data_parquet/nine_byte_decimals_from_spark.parquet'"
${CLICKHOUSE_CLIENT} --query="select count() from 02845_parquet_odd_decimals"

BIN
tests/queries/0_stateless/data_parquet/nine_byte_decimals_from_spark.parquet Normal file
View File

Binary file not shown.

7
utils/check-style/aspell-ignore/en/aspell-dict.txt
View File

@ -315,6 +315,7 @@ Greenwald
HDDs
HHMM
HMAC
HNSW
HSTS
HTTPConnection
HTTPThreads
@ -697,6 +698,7 @@ Promtail
Protobuf
ProtobufSingle
ProxySQL
PyArrow
PyCharm
QEMU
QTCreator
@ -921,6 +923,7 @@ URL's
URLHash
URLHierarchy
URLPathHierarchy
USearch
UUIDNumToString
UUIDStringToNum
UUIDs
@ -1086,8 +1089,8 @@ authenticators
autocompletion
autodetect
autodetected
autogenerated
autogenerate
autogenerated
autogeneration
autostart
avgWeighted
@ -2001,7 +2004,6 @@ ptrs
pushdown
pwrite
py
PyArrow
qryn
quantile
quantileBFloat
@ -2501,6 +2503,7 @@ uring
url
urlCluster
urls
usearch
userspace
userver
utils