thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-29 02:52:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into support_orc_reader_timezone

Browse Source
This commit is contained in:
kevinyhzou 2024-07-05 10:27:39 +08:00
commit a92d6b0761
321 changed files with 11294 additions and 3473 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/icu-cmake/CMakeLists.txt
View File

@ -5,7 +5,7 @@ else ()
endif ()
if (NOT ENABLE_ICU)
message(STATUS "Not using icu")
message(STATUS "Not using ICU")
return()
endif()

4
contrib/jemalloc-cmake/CMakeLists.txt
View File

@ -34,7 +34,11 @@ if (OS_LINUX)
# avoid spurious latencies and additional work associated with
# MADV_DONTNEED. See
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
else()
set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true")
endif()
else()
set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
endif()

2
contrib/llvm-project vendored

@ -1 +1 @@
Subproject commit d2142eed98046a47ff7112e3cc1e197c8a5cd80f
Subproject commit 2a8967b60cbe5bc2df253712bac343cc5263c5fc

2
contrib/openssl vendored

@ -1 +1 @@
Subproject commit 5d81fa7068fc8c07f4d0997d5b703f3c541a637c
Subproject commit ee2bb8513b28bf86b35404dd17a0e29305ca9e08

2
contrib/sysroot vendored

@ -1 +1 @@
Subproject commit 39c4713334f9f156dbf508f548d510d9129a657c
Subproject commit cc385041b226d1fc28ead14dbab5d40a5f821dd8

2
contrib/vectorscan vendored

@ -1 +1 @@
Subproject commit 38431d111781843741a781a57a6381a527d900a4
Subproject commit d29730e1cb9daaa66bda63426cdce83505d2c809

27
contrib/vectorscan-cmake/CMakeLists.txt
View File

@ -1,11 +1,8 @@
# We use vectorscan, a portable and API/ABI-compatible drop-in replacement for hyperscan.
# Vectorscan is drop-in replacement for Hyperscan.
if ((ARCH_AMD64 AND NOT NO_SSE3_OR_HIGHER) OR ARCH_AARCH64)
option (ENABLE_VECTORSCAN "Enable vectorscan library" ${ENABLE_LIBRARIES})
option (ENABLE_VECTORSCAN "Enable vectorscan" ${ENABLE_LIBRARIES})
endif()
# TODO PPC should generally work but needs manual generation of ppc/config.h file on a PPC machine
if (NOT ENABLE_VECTORSCAN)
message (STATUS "Not using vectorscan")
return()
@ -272,34 +269,24 @@ if (ARCH_AARCH64)
)
endif()
# TODO
# if (ARCH_PPC64LE)
# list(APPEND SRCS
# "${LIBRARY_DIR}/src/util/supervector/arch/ppc64el/impl.cpp"
# )
# endif()
add_library (_vectorscan ${SRCS})
target_compile_options (_vectorscan PRIVATE
-fno-sanitize=undefined # assume the library takes care of itself
-O2 -fno-strict-aliasing -fno-omit-frame-pointer -fvisibility=hidden # options from original build system
)
# library has too much debug information
if (OMIT_HEAVY_DEBUG_SYMBOLS)
target_compile_options (_vectorscan PRIVATE -g0)
endif()
# Include version header manually generated by running the original build system
target_include_directories (_vectorscan SYSTEM PRIVATE common)
target_include_directories (_vectorscan SYSTEM PUBLIC "${LIBRARY_DIR}/src")
# Makes the version header visible. It was generated by running the native build system manually.
# Please update whenever you update vectorscan.
target_include_directories (_vectorscan SYSTEM PUBLIC common)
# vectorscan inherited some patched in-source versions of boost headers to fix a bug in
# boost 1.69. This bug has been solved long ago but vectorscan's source code still
# points to the patched versions, so include it here.
target_include_directories (_vectorscan SYSTEM PRIVATE "${LIBRARY_DIR}/include")
target_include_directories (_vectorscan SYSTEM PUBLIC "${LIBRARY_DIR}/src")
# Include platform-specific config header generated by manually running the original build system
# Please regenerate these files if you update vectorscan.

6
contrib/vectorscan-cmake/common/hs_version.h
View File

@ -32,8 +32,12 @@
/**
* A version string to identify this release of Hyperscan.
*/
#define HS_VERSION_STRING "5.4.7 2022-06-20"
#define HS_VERSION_STRING "5.4.11 2024-07-04"
#define HS_VERSION_32BIT ((5 << 24) | (1 << 16) | (7 << 8) | 0)
#define HS_MAJOR 5
#define HS_MINOR 4
#define HS_PATCH 11
#endif /* HS_VERSION_H_C6428FAF8E3713 */

1
docker/packager/binary-builder/build.sh
View File

@ -111,6 +111,7 @@ fi
mv ./programs/clickhouse* /output || mv ./programs/*_fuzzer /output
[ -x ./programs/self-extracting/clickhouse ] && mv ./programs/self-extracting/clickhouse /output
[ -x ./programs/self-extracting/clickhouse-stripped ] && mv ./programs/self-extracting/clickhouse-stripped /output
[ -x ./programs/self-extracting/clickhouse-keeper ] && mv ./programs/self-extracting/clickhouse-keeper /output
mv ./src/unit_tests_dbms /output ||: # may not exist for some binary builds
mv ./programs/*.dict ./programs/*.options ./programs/*_seed_corpus.zip /output ||: # libFuzzer oss-fuzz compatible infrastructure

3
docker/packager/packager
View File

@ -276,10 +276,7 @@ def parse_env_variables(
if is_release_build(debug_build, package_type, sanitizer, coverage):
cmake_flags.append("-DSPLIT_DEBUG_SYMBOLS=ON")
result.append("WITH_PERFORMANCE=1")
if is_cross_arm:
cmake_flags.append("-DBUILD_STANDALONE_KEEPER=1")
else:
result.append("BUILD_MUSL_KEEPER=1")
elif package_type == "fuzzers":
cmake_flags.append("-DENABLE_FUZZING=1")
cmake_flags.append("-DENABLE_PROTOBUF=1")

4
docker/test/stateful/run.sh
View File

@ -213,6 +213,10 @@ function run_tests()
ADDITIONAL_OPTIONS+=('--s3-storage')
fi
if [[ -n "$USE_AZURE_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_AZURE_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then
ADDITIONAL_OPTIONS+=('--azure-blob-storage')
fi
if [[ -n "$USE_DATABASE_ORDINARY" ]] && [[ "$USE_DATABASE_ORDINARY" -eq 1 ]]; then
ADDITIONAL_OPTIONS+=('--db-engine=Ordinary')
fi

2
docker/test/stateless/run.sh
View File

@ -207,7 +207,7 @@ function run_tests()
if [[ -n "$USE_AZURE_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_AZURE_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then
# to disable the same tests
ADDITIONAL_OPTIONS+=('--s3-storage')
ADDITIONAL_OPTIONS+=('--azure-blob-storage')
# azurite is slow, but with these two settings it can be super slow
ADDITIONAL_OPTIONS+=('--no-random-settings')
ADDITIONAL_OPTIONS+=('--no-random-merge-tree-settings')

27
docker/test/stress/run.sh
View File

@ -110,6 +110,15 @@ start_server
clickhouse-client --query "SHOW TABLES FROM datasets"
clickhouse-client --query "SHOW TABLES FROM test"
if [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" == "1" ]]; then
TEMP_POLICY="s3_cache"
elif [[ "$USE_AZURE_STORAGE_FOR_MERGE_TREE" == "1" ]]; then
TEMP_POLICY="azure_cache"
else
TEMP_POLICY="default"
fi
clickhouse-client --query "CREATE TABLE test.hits_s3 (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16,
EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32,
UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String,
@ -135,7 +144,7 @@ clickhouse-client --query "CREATE TABLE test.hits_s3 (WatchID UInt64, JavaEnabl
URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String,
ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64),
IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate)
ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='$TEMP_POLICY'"
clickhouse-client --query "CREATE TABLE test.hits (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16,
EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32,
UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String,
@ -161,7 +170,7 @@ clickhouse-client --query "CREATE TABLE test.hits (WatchID UInt64, JavaEnable U
URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String,
ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64),
IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate)
ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='$TEMP_POLICY'"
clickhouse-client --query "CREATE TABLE test.visits (CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8,
VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32,
Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String,
@ -195,7 +204,7 @@ clickhouse-client --query "CREATE TABLE test.visits (CounterID UInt32, StartDat
Market Nested(Type UInt8, GoalID UInt32, OrderID String, OrderPrice Int64, PP UInt32, DirectPlaceID UInt32, DirectOrderID UInt32,
DirectBannerID UInt32, GoodID String, GoodName String, GoodQuantity Int32, GoodPrice Int64), IslandID FixedString(16))
ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='$TEMP_POLICY'"
clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
clickhouse-client --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
@ -211,12 +220,12 @@ clickhouse-client --query "SYSTEM STOP THREAD FUZZER"
stop_server
# Let's enable S3 storage by default
export USE_S3_STORAGE_FOR_MERGE_TREE=1
export RANDOMIZE_OBJECT_KEY_TYPE=1
export ZOOKEEPER_FAULT_INJECTION=1
export THREAD_POOL_FAULT_INJECTION=1
configure
if [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" == "1" ]]; then
# But we still need default disk because some tables loaded only into it
sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \
| sed "s|<main><disk>s3</disk></main>|<main><disk>s3</disk></main><default><disk>default</disk></default>|" \
@ -224,6 +233,16 @@ sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \
mv /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
sudo chown clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
elif [[ "$USE_AZURE_STORAGE_FOR_MERGE_TREE" == "1" ]]; then
# But we still need default disk because some tables loaded only into it
sudo cat /etc/clickhouse-server/config.d/azure_storage_policy_by_default.xml \
| sed "s|<main><disk>azure</disk></main>|<main><disk>azure</disk></main><default><disk>default</disk></default>|" \
> /etc/clickhouse-server/config.d/azure_storage_policy_by_default.xml.tmp
mv /etc/clickhouse-server/config.d/azure_storage_policy_by_default.xml.tmp /etc/clickhouse-server/config.d/azure_storage_policy_by_default.xml
sudo chown clickhouse /etc/clickhouse-server/config.d/azure_storage_policy_by_default.xml
sudo chgrp clickhouse /etc/clickhouse-server/config.d/azure_storage_policy_by_default.xml
fi
sudo cat /etc/clickhouse-server/config.d/logger_trace.xml \
| sed "s|<level>trace</level>|<level>test</level>|" \

36
docs/en/sql-reference/table-functions/fuzzQuery.md Normal file
View File

@ -0,0 +1,36 @@
---
slug: /en/sql-reference/table-functions/fuzzQuery
sidebar_position: 75
sidebar_label: fuzzQuery
---
# fuzzQuery
Perturbs the given query string with random variations.
``` sql
fuzzQuery(query[, max_query_length[, random_seed]])
```
**Arguments**
- `query` (String) - The source query to perform the fuzzing on.
- `max_query_length` (UInt64) - A maximum length the query can get during the fuzzing process.
- `random_seed` (UInt64) - A random seed for producing stable results.
**Returned Value**
A table object with a single column containing perturbed query strings.
## Usage Example
``` sql
SELECT * FROM fuzzQuery('SELECT materialize(\'a\' AS key) GROUP BY key') LIMIT 2;
```
```
┌─query──────────────────────────────────────────────────────────┐
1. │ SELECT 'a' AS key GROUP BY key │
2. │ EXPLAIN PIPELINE compact = true SELECT 'a' AS key GROUP BY key │
└────────────────────────────────────────────────────────────────┘
```

20
programs/CMakeLists.txt
View File

@ -66,18 +66,18 @@ else()
message(STATUS "Library bridge mode: OFF")
endif()
if (ENABLE_CLICKHOUSE_KEEPER)
message(STATUS "ClickHouse keeper mode: ON")
else()
message(STATUS "ClickHouse keeper mode: OFF")
endif()
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
message(STATUS "ClickHouse keeper-converter mode: ON")
else()
message(STATUS "ClickHouse keeper-converter mode: OFF")
endif()
if (ENABLE_CLICKHOUSE_KEEPER)
message(STATUS "ClickHouse Keeper: ON")
else()
message(STATUS "ClickHouse Keeper: OFF")
endif()
if (ENABLE_CLICKHOUSE_KEEPER_CLIENT)
message(STATUS "ClickHouse keeper-client mode: ON")
else()
@ -131,10 +131,6 @@ add_subdirectory (static-files-disk-uploader)
add_subdirectory (su)
add_subdirectory (disks)
if (ENABLE_CLICKHOUSE_KEEPER)
add_subdirectory (keeper)
endif()
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
add_subdirectory (keeper-converter)
endif()
@ -143,6 +139,10 @@ if (ENABLE_CLICKHOUSE_KEEPER_CLIENT)
add_subdirectory (keeper-client)
endif()
if (ENABLE_CLICKHOUSE_KEEPER)
add_subdirectory (keeper)
endif()
if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
add_subdirectory (odbc-bridge)
endif ()

5
programs/client/Client.h
View File

@ -9,7 +9,10 @@ namespace DB
class Client : public ClientBase
{
public:
Client() = default;
Client()
{
fuzzer = QueryFuzzer(randomSeed(), &std::cout, &std::cerr);
}
void initialize(Poco::Util::Application & self) override;

196
programs/keeper/CMakeLists.txt
View File

@ -1,4 +1,5 @@
set(CLICKHOUSE_KEEPER_SOURCES
keeper_main.cpp
Keeper.cpp
)
@ -8,6 +9,9 @@ set (CLICKHOUSE_KEEPER_LINK
clickhouse_common_io
clickhouse_common_zookeeper
daemon
clickhouse-keeper-converter-lib
clickhouse-keeper-client-lib
clickhouse_functions
dbms
)
@ -17,199 +21,11 @@ install(FILES keeper_config.xml DESTINATION "${CLICKHOUSE_ETC_DIR}/clickhouse-ke
if (BUILD_STANDALONE_KEEPER)
# Straight list of all required sources
set(CLICKHOUSE_KEEPER_STANDALONE_SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperReconfiguration.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/RaftServerConfig.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/ACLMap.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/Changelog.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/CoordinationSettings.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/FourLetterCommand.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/InMemoryLogStore.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperConnectionStats.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperDispatcher.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperLogStore.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperServer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperContext.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperFeatureFlags.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperSnapshotManager.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperSnapshotManagerS3.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperStateMachine.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperContext.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperStateManager.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperStorage.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperConstants.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperAsynchronousMetrics.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperCommon.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/SessionExpiryQueue.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/SummingStateMachine.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/WriteBufferFromNuraftBuffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/ZooKeeperDataReader.cpp
clickhouse_add_executable(clickhouse-keeper ${CLICKHOUSE_KEEPER_SOURCES})
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Core/SettingsFields.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Core/BaseSettings.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Core/ServerSettings.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Core/Field.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Core/SettingsEnums.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Core/ServerUUID.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Core/UUID.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Core/BackgroundSchedulePool.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/IO/ReadBuffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/HTTPPathHints.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/KeeperTCPHandler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/TCPServer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/NotFoundHandler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/ProtocolServerAdapter.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/CertificateReloader.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/PrometheusRequestHandler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/PrometheusMetricsWriter.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/waitServersToFinish.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/ServerType.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/HTTPRequestHandlerFactoryMain.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/KeeperReadinessHandler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/CloudPlacementInfo.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/HTTP/HTTPServer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/HTTP/ReadHeaders.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/HTTP/HTTPServerConnection.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/HTTP/HTTPServerRequest.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/HTTP/HTTPServerResponse.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/HTTP/HTTPServerConnectionFactory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CachedCompressedReadBuffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CheckingCompressedReadBuffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressedReadBufferBase.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressedReadBuffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressedReadBufferFromFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressedWriteBuffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecEncrypted.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecLZ4.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecMultiple.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecNone.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionCodecZSTD.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/CompressionFactory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/ICompressionCodec.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Compression/LZ4_decompress_faster.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/CurrentThread.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/NamedCollections/NamedCollections.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/NamedCollections/NamedCollectionConfiguration.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/Jemalloc.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/IKeeper.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/TestKeeper.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeperCommon.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeperConstants.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeper.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeperImpl.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeperIO.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeperLock.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/ZooKeeperNodeCache.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/registerDisks.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/IDisk.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/DiskFactory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/DiskSelector.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/DiskLocal.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/DiskLocalCheckThread.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/LocalDirectorySyncGuard.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/TemporaryFileOnDisk.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/loadLocalDiskConfig.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/DiskType.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/IObjectStorage.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/MetadataOperationsHolder.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/MetadataStorageTransactionState.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/MetadataStorageFromDiskTransactionOperations.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/DiskObjectStorage.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/ObjectStorageIteratorAsync.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/ObjectStorageIterator.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/StoredObject.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/S3/S3Capabilities.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/S3/diskSettings.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/S3/DiskS3Utils.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/ObjectStorageFactory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/MetadataStorageFactory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/ObjectStorages/RegisterDiskObjectStorage.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/IO/createReadBufferFromFileBase.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/IO/IOUringReader.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/IO/getIOUringReader.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/IO/WriteBufferFromTemporaryFile.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/IO/WriteBufferWithFinalizeCallback.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/IO/AsynchronousBoundedReadBuffer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/IO/getThreadPoolReader.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/IO/ThreadPoolRemoteFSReader.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Disks/IO/ThreadPoolReader.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Daemon/BaseDaemon.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Daemon/SentryWriter.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Daemon/GraphiteWriter.cpp
${CMAKE_CURRENT_BINARY_DIR}/../../src/Daemon/GitHash.generated.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/Standalone/Context.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/Standalone/Settings.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/Standalone/ThreadStatusExt.cpp
Keeper.cpp
clickhouse-keeper.cpp
)
# List of resources for clickhouse-keeper client
if (ENABLE_CLICKHOUSE_KEEPER_CLIENT)
list(APPEND CLICKHOUSE_KEEPER_STANDALONE_SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/../../programs/keeper-client/KeeperClient.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../programs/keeper-client/Commands.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../programs/keeper-client/Parser.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Client/LineReader.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../src/Client/ReplxxLineReader.cpp
)
endif()
clickhouse_add_executable(clickhouse-keeper ${CLICKHOUSE_KEEPER_STANDALONE_SOURCES})
# Remove some redundant dependencies
target_compile_definitions (clickhouse-keeper PRIVATE -DCLICKHOUSE_KEEPER_STANDALONE_BUILD)
target_compile_definitions (clickhouse-keeper PUBLIC -DWITHOUT_TEXT_LOG)
if (ENABLE_CLICKHOUSE_KEEPER_CLIENT AND TARGET ch_rust::skim)
target_link_libraries(clickhouse-keeper PRIVATE ch_rust::skim)
endif()
target_link_libraries(clickhouse-keeper
PRIVATE
ch_contrib::abseil_swiss_tables
ch_contrib::nuraft
ch_contrib::lz4
ch_contrib::zstd
ch_contrib::cityhash
ch_contrib::jemalloc
common ch_contrib::double_conversion
ch_contrib::dragonbox_to_chars
pcg_random
ch_contrib::pdqsort
ch_contrib::miniselect
clickhouse_common_config_no_zookeeper_log
loggers_no_text_log
clickhouse_common_io
clickhouse_parsers # Otherwise compression will not built. FIXME.
)
target_link_libraries(clickhouse-keeper PUBLIC ${CLICKHOUSE_KEEPER_LINK})
set_target_properties(clickhouse-keeper PROPERTIES RUNTIME_OUTPUT_DIRECTORY ../)
if (SPLIT_DEBUG_SYMBOLS)
clickhouse_split_debug_symbols(TARGET clickhouse-keeper DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH ../clickhouse-keeper)
else()

15
programs/keeper/Keeper.cpp
View File

@ -27,6 +27,8 @@
#include <sys/stat.h>
#include <pwd.h>
#include <Common/Jemalloc.h>
#include <Interpreters/Context.h>
#include <Coordination/FourLetterCommand.h>
@ -75,16 +77,6 @@ int mainEntryClickHouseKeeper(int argc, char ** argv)
}
}
#ifdef CLICKHOUSE_KEEPER_STANDALONE_BUILD
// Weak symbols don't work correctly on Darwin
// so we have a stub implementation to avoid linker errors
void collectCrashLog(
Int32, UInt64, const String &, const StackTrace &)
{}
#endif
namespace DB
{
@ -277,6 +269,9 @@ HTTPContextPtr httpContext()
int Keeper::main(const std::vector<std::string> & /*args*/)
try
{
#if USE_JEMALLOC
setJemallocBackgroundThreads(true);
#endif
Poco::Logger * log = &logger();
UseSSL use_ssl;

30
programs/keeper/clickhouse-keeper.cpp
View File

@ -1,30 +0,0 @@
#include <Common/StringUtils.h>
#include "config_tools.h"
int mainEntryClickHouseKeeper(int argc, char ** argv);
#if ENABLE_CLICKHOUSE_KEEPER_CLIENT
int mainEntryClickHouseKeeperClient(int argc, char ** argv);
#endif
int main(int argc_, char ** argv_)
{
#if ENABLE_CLICKHOUSE_KEEPER_CLIENT
if (argc_ >= 2)
{
/// 'clickhouse-keeper --client ...' and 'clickhouse-keeper client ...' are OK
if (strcmp(argv_[1], "--client") == 0 || strcmp(argv_[1], "client") == 0)
{
argv_[1] = argv_[0];
return mainEntryClickHouseKeeperClient(--argc_, argv_ + 1);
}
}
if (argc_ > 0 && (strcmp(argv_[0], "clickhouse-keeper-client") == 0 || endsWith(argv_[0], "/clickhouse-keeper-client")))
return mainEntryClickHouseKeeperClient(argc_, argv_);
#endif
return mainEntryClickHouseKeeper(argc_, argv_);
}

189
programs/keeper/keeper_main.cpp Normal file
View File

@ -0,0 +1,189 @@
#include <unistd.h>
#include <fcntl.h>
#include <new>
#include <iostream>
#include <vector>
#include <string_view>
#include <utility> /// pair
#include <fmt/format.h>
#include "config.h"
#include "config_tools.h"
#include <Common/EnvironmentChecks.h>
#include <Common/Coverage.h>
#include <Common/StringUtils.h>
#include <Common/getHashOfLoadedBinary.h>
#include <Common/IO.h>
#include <base/phdr_cache.h>
#include <base/coverage.h>
int mainEntryClickHouseKeeper(int argc, char ** argv);
#if ENABLE_CLICKHOUSE_KEEPER_CONVERTER
int mainEntryClickHouseKeeperConverter(int argc, char ** argv);
#endif
#if ENABLE_CLICKHOUSE_KEEPER_CLIENT
int mainEntryClickHouseKeeperClient(int argc, char ** argv);
#endif
namespace
{
using MainFunc = int (*)(int, char**);
/// Add an item here to register new application
std::pair<std::string_view, MainFunc> clickhouse_applications[] =
{
// keeper
{"keeper", mainEntryClickHouseKeeper},
#if ENABLE_CLICKHOUSE_KEEPER_CONVERTER
{"converter", mainEntryClickHouseKeeperConverter},
{"keeper-converter", mainEntryClickHouseKeeperConverter},
#endif
#if ENABLE_CLICKHOUSE_KEEPER_CLIENT
{"client", mainEntryClickHouseKeeperClient},
{"keeper-client", mainEntryClickHouseKeeperClient},
#endif
};
int printHelp(int, char **)
{
std::cerr << "Use one of the following commands:" << std::endl;
for (auto & application : clickhouse_applications)
std::cerr << "clickhouse " << application.first << " [args] " << std::endl;
return -1;
}
}
bool isClickhouseApp(std::string_view app_suffix, std::vector<char *> & argv)
{
/// Use app if the first arg 'app' is passed (the arg should be quietly removed)
if (argv.size() >= 2)
{
auto first_arg = argv.begin() + 1;
/// 'clickhouse --client ...' and 'clickhouse client ...' are Ok
if (*first_arg == app_suffix
|| (std::string_view(*first_arg).starts_with("--") && std::string_view(*first_arg).substr(2) == app_suffix))
{
argv.erase(first_arg);
return true;
}
}
/// keeper suffix is default which will be used if no other app is detected
if (app_suffix == "keeper")
return false;
/// Use app if clickhouse binary is run through symbolic link with name clickhouse-app
std::string app_name = "clickhouse-" + std::string(app_suffix);
return !argv.empty() && (app_name == argv[0] || endsWith(argv[0], "/" + app_name));
}
/// Don't allow dlopen in the main ClickHouse binary, because it is harmful and insecure.
/// We don't use it. But it can be used by some libraries for implementation of "plugins".
/// We absolutely discourage the ancient technique of loading
/// 3rd-party uncontrolled dangerous libraries into the process address space,
/// because it is insane.
#if !defined(USE_MUSL)
extern "C"
{
void * dlopen(const char *, int)
{
return nullptr;
}
void * dlmopen(long, const char *, int) // NOLINT
{
return nullptr;
}
int dlclose(void *)
{
return 0;
}
const char * dlerror()
{
return "ClickHouse does not allow dynamic library loading";
}
}
#endif
/// Prevent messages from JeMalloc in the release build.
/// Some of these messages are non-actionable for the users, such as:
/// <jemalloc>: Number of CPUs detected is not deterministic. Per-CPU arena disabled.
#if USE_JEMALLOC && defined(NDEBUG) && !defined(SANITIZER)
extern "C" void (*malloc_message)(void *, const char *s);
__attribute__((constructor(0))) void init_je_malloc_message() { malloc_message = [](void *, const char *){}; }
#endif
/// This allows to implement assert to forbid initialization of a class in static constructors.
/// Usage:
///
/// extern bool inside_main;
/// class C { C() { assert(inside_main); } };
bool inside_main = false;
int main(int argc_, char ** argv_)
{
inside_main = true;
SCOPE_EXIT({ inside_main = false; });
/// PHDR cache is required for query profiler to work reliably
/// It also speed up exception handling, but exceptions from dynamically loaded libraries (dlopen)
/// will work only after additional call of this function.
/// Note: we forbid dlopen in our code.
updatePHDRCache();
#if !defined(USE_MUSL)
checkHarmfulEnvironmentVariables(argv_);
#endif
/// This is used for testing. For example,
/// clickhouse-local should be able to run a simple query without throw/catch.
if (getenv("CLICKHOUSE_TERMINATE_ON_ANY_EXCEPTION")) // NOLINT(concurrency-mt-unsafe)
DB::terminate_on_any_exception = true;
/// Reset new handler to default (that throws std::bad_alloc)
/// It is needed because LLVM library clobbers it.
std::set_new_handler(nullptr);
std::vector<char *> argv(argv_, argv_ + argc_);
/// Print a basic help if nothing was matched
MainFunc main_func = mainEntryClickHouseKeeper;
if (isClickhouseApp("help", argv))
{
main_func = printHelp;
}
else
{
for (auto & application : clickhouse_applications)
{
if (isClickhouseApp(application.first, argv))
{
main_func = application.second;
break;
}
}
}
int exit_code = main_func(static_cast<int>(argv.size()), argv.data());
#if defined(SANITIZE_COVERAGE)
dumpCoverage();
#endif
return exit_code;
}

267
programs/main.cpp
View File

@ -1,5 +1,3 @@
#include <csignal>
#include <csetjmp>
#include <unistd.h>
#include <fcntl.h>
@ -7,7 +5,6 @@
#include <iostream>
#include <vector>
#include <string>
#include <tuple>
#include <string_view>
#include <utility> /// pair
@ -16,6 +13,8 @@
#include "config.h"
#include "config_tools.h"
#include <Common/EnvironmentChecks.h>
#include <Common/Coverage.h>
#include <Common/StringUtils.h>
#include <Common/getHashOfLoadedBinary.h>
#include <Common/IO.h>
@ -119,268 +118,6 @@ std::pair<std::string_view, std::string_view> clickhouse_short_names[] =
{"chc", "client"},
};
enum class InstructionFail : uint8_t
{
NONE = 0,
SSE3 = 1,
SSSE3 = 2,
SSE4_1 = 3,
SSE4_2 = 4,
POPCNT = 5,
AVX = 6,
AVX2 = 7,
AVX512 = 8
};
auto instructionFailToString(InstructionFail fail)
{
switch (fail)
{
#define ret(x) return std::make_tuple(STDERR_FILENO, x, sizeof(x) - 1)
case InstructionFail::NONE:
ret("NONE");
case InstructionFail::SSE3:
ret("SSE3");
case InstructionFail::SSSE3:
ret("SSSE3");
case InstructionFail::SSE4_1:
ret("SSE4.1");
case InstructionFail::SSE4_2:
ret("SSE4.2");
case InstructionFail::POPCNT:
ret("POPCNT");
case InstructionFail::AVX:
ret("AVX");
case InstructionFail::AVX2:
ret("AVX2");
case InstructionFail::AVX512:
ret("AVX512");
#undef ret
}
}
sigjmp_buf jmpbuf;
[[noreturn]] void sigIllCheckHandler(int, siginfo_t *, void *)
{
siglongjmp(jmpbuf, 1);
}
/// Check if necessary SSE extensions are available by trying to execute some sse instructions.
/// If instruction is unavailable, SIGILL will be sent by kernel.
void checkRequiredInstructionsImpl(volatile InstructionFail & fail)
{
#if defined(__SSE3__)
fail = InstructionFail::SSE3;
__asm__ volatile ("addsubpd %%xmm0, %%xmm0" : : : "xmm0");
#endif
#if defined(__SSSE3__)
fail = InstructionFail::SSSE3;
__asm__ volatile ("pabsw %%xmm0, %%xmm0" : : : "xmm0");
#endif
#if defined(__SSE4_1__)
fail = InstructionFail::SSE4_1;
__asm__ volatile ("pmaxud %%xmm0, %%xmm0" : : : "xmm0");
#endif
#if defined(__SSE4_2__)
fail = InstructionFail::SSE4_2;
__asm__ volatile ("pcmpgtq %%xmm0, %%xmm0" : : : "xmm0");
#endif
/// Defined by -msse4.2
#if defined(__POPCNT__)
fail = InstructionFail::POPCNT;
{
uint64_t a = 0;
uint64_t b = 0;
__asm__ volatile ("popcnt %1, %0" : "=r"(a) :"r"(b) :);
}
#endif
#if defined(__AVX__)
fail = InstructionFail::AVX;
__asm__ volatile ("vaddpd %%ymm0, %%ymm0, %%ymm0" : : : "ymm0");
#endif
#if defined(__AVX2__)
fail = InstructionFail::AVX2;
__asm__ volatile ("vpabsw %%ymm0, %%ymm0" : : : "ymm0");
#endif
#if defined(__AVX512__)
fail = InstructionFail::AVX512;
__asm__ volatile ("vpabsw %%zmm0, %%zmm0" : : : "zmm0");
#endif
fail = InstructionFail::NONE;
}
/// Macros to avoid using strlen(), since it may fail if SSE is not supported.
#define writeError(data) do \
{ \
static_assert(__builtin_constant_p(data)); \
if (!writeRetry(STDERR_FILENO, data, sizeof(data) - 1)) \
_Exit(1); \
} while (false)
/// Check SSE and others instructions availability. Calls exit on fail.
/// This function must be called as early as possible, even before main, because static initializers may use unavailable instructions.
void checkRequiredInstructions()
{
struct sigaction sa{};
struct sigaction sa_old{};
sa.sa_sigaction = sigIllCheckHandler;
sa.sa_flags = SA_SIGINFO;
auto signal = SIGILL;
if (sigemptyset(&sa.sa_mask) != 0
|| sigaddset(&sa.sa_mask, signal) != 0
|| sigaction(signal, &sa, &sa_old) != 0)
{
/// You may wonder about strlen.
/// Typical implementation of strlen is using SSE4.2 or AVX2.
/// But this is not the case because it's compiler builtin and is executed at compile time.
writeError("Can not set signal handler\n");
_Exit(1);
}
volatile InstructionFail fail = InstructionFail::NONE;
if (sigsetjmp(jmpbuf, 1))
{
writeError("Instruction check fail. The CPU does not support ");
if (!std::apply(writeRetry, instructionFailToString(fail)))
_Exit(1);
writeError(" instruction set.\n");
_Exit(1);
}
checkRequiredInstructionsImpl(fail);
if (sigaction(signal, &sa_old, nullptr))
{
writeError("Can not set signal handler\n");
_Exit(1);
}
}
struct Checker
{
Checker()
{
checkRequiredInstructions();
}
} checker
#ifndef OS_DARWIN
__attribute__((init_priority(101))) /// Run before other static initializers.
#endif
;
#if !defined(USE_MUSL)
/// NOTE: We will migrate to full static linking or our own dynamic loader to make this code obsolete.
void checkHarmfulEnvironmentVariables(char ** argv)
{
std::initializer_list<const char *> harmful_env_variables = {
/// The list is a selection from "man ld-linux".
"LD_PRELOAD",
"LD_LIBRARY_PATH",
"LD_ORIGIN_PATH",
"LD_AUDIT",
"LD_DYNAMIC_WEAK",
/// The list is a selection from "man dyld" (osx).
"DYLD_LIBRARY_PATH",
"DYLD_FALLBACK_LIBRARY_PATH",
"DYLD_VERSIONED_LIBRARY_PATH",
"DYLD_INSERT_LIBRARIES",
};
bool require_reexec = false;
for (const auto * var : harmful_env_variables)
{
if (const char * value = getenv(var); value && value[0]) // NOLINT(concurrency-mt-unsafe)
{
/// NOTE: setenv() is used over unsetenv() since unsetenv() marked as harmful
if (setenv(var, "", true)) // NOLINT(concurrency-mt-unsafe) // this is safe if not called concurrently
{
fmt::print(stderr, "Cannot override {} environment variable", var);
_exit(1);
}
require_reexec = true;
}
}
if (require_reexec)
{
/// Use execvp() over execv() to search in PATH.
///
/// This should be safe, since:
/// - if argv[0] is relative path - it is OK
/// - if argv[0] has only basename, the it will search in PATH, like shell will do.
///
/// Also note, that this (search in PATH) because there is no easy and
/// portable way to get absolute path of argv[0].
/// - on linux there is /proc/self/exec and AT_EXECFN
/// - but on other OSes there is no such thing (especially on OSX).
///
/// And since static linking will be done someday anyway,
/// let's not pollute the code base with special cases.
int error = execvp(argv[0], argv);
_exit(error);
}
}
#endif
#if defined(SANITIZE_COVERAGE)
__attribute__((no_sanitize("coverage")))
void dumpCoverage()
{
/// A user can request to dump the coverage information into files at exit.
/// This is useful for non-server applications such as clickhouse-format or clickhouse-client,
/// that cannot introspect it with SQL functions at runtime.
/// The CLICKHOUSE_WRITE_COVERAGE environment variable defines a prefix for a filename 'prefix.pid'
/// containing the list of addresses of covered .
/// The format is even simpler than Clang's "sancov": an array of 64-bit addresses, native byte order, no header.
if (const char * coverage_filename_prefix = getenv("CLICKHOUSE_WRITE_COVERAGE")) // NOLINT(concurrency-mt-unsafe)
{
auto dump = [](const std::string & name, auto span)
{
/// Write only non-zeros.
std::vector<uintptr_t> data;
data.reserve(span.size());
for (auto addr : span)
if (addr)
data.push_back(addr);
int fd = ::open(name.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0400);
if (-1 == fd)
{
writeError("Cannot open a file to write the coverage data\n");
}
else
{
if (!writeRetry(fd, reinterpret_cast<const char *>(data.data()), data.size() * sizeof(data[0])))
writeError("Cannot write the coverage data to a file\n");
if (0 != ::close(fd))
writeError("Cannot close the file with coverage data\n");
}
};
dump(fmt::format("{}.{}", coverage_filename_prefix, getpid()), getCumulativeCoverage());
}
}
#endif
}
bool isClickhouseApp(std::string_view app_suffix, std::vector<char *> & argv)

17
programs/self-extracting/CMakeLists.txt
View File

@ -10,9 +10,24 @@ else ()
set (COMPRESSOR "${PROJECT_BINARY_DIR}/utils/self-extracting-executable/compressor")
endif ()
add_custom_target (self-extracting ALL
add_custom_target (self-extracting-server ALL
${CMAKE_COMMAND} -E remove clickhouse clickhouse-stripped
COMMAND ${COMPRESSOR} ${DECOMPRESSOR} clickhouse ../clickhouse
COMMAND ${COMPRESSOR} ${DECOMPRESSOR} clickhouse-stripped ../clickhouse-stripped
DEPENDS clickhouse clickhouse-stripped compressor
)
set(self_extracting_deps "self-extracting-server")
if (BUILD_STANDALONE_KEEPER)
add_custom_target (self-extracting-keeper ALL
${CMAKE_COMMAND} -E remove clickhouse-keeper
COMMAND ${COMPRESSOR} ${DECOMPRESSOR} clickhouse-keeper ../clickhouse-keeper
DEPENDS compressor clickhouse-keeper
)
list(APPEND self_extracting_deps "self-extracting-keeper")
endif()
add_custom_target (self-extracting ALL
DEPENDS ${self_extracting_deps}
)

5
programs/server/Server.cpp
View File

@ -11,6 +11,7 @@
#include <Poco/Util/HelpFormatter.h>
#include <Poco/Environment.h>
#include <Poco/Config.h>
#include <Common/Jemalloc.h>
#include <Common/scope_guard_safe.h>
#include <Common/logger_useful.h>
#include <base/phdr_cache.h>
@ -628,6 +629,10 @@ static void initializeAzureSDKLogger(
int Server::main(const std::vector<std::string> & /*args*/)
try
{
#if USE_JEMALLOC
setJemallocBackgroundThreads(true);
#endif
Stopwatch startup_watch;
Poco::Logger * log = &logger();

116
src/Backups/BackupIO_AzureBlobStorage.cpp
View File

@ -29,48 +29,49 @@ namespace ErrorCodes
}
BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage(
const StorageAzureConfiguration & configuration_,
const AzureBlobStorage::ConnectionParams & connection_params_,
const String & blob_path_,
bool allow_azure_native_copy,
const ReadSettings & read_settings_,
const WriteSettings & write_settings_,
const ContextPtr & context_)
: BackupReaderDefault(read_settings_, write_settings_, getLogger("BackupReaderAzureBlobStorage"))
, data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, configuration_.getConnectionURL().toString(), false, false}
, configuration(configuration_)
, data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, connection_params_.getConnectionURL(), false, false}
, connection_params(connection_params_)
, blob_path(blob_path_)
{
auto client_ptr = configuration.createClient(/* is_readonly */false, /* attempt_to_create_container */true);
client_ptr->SetClickhouseOptions(Azure::Storage::Blobs::ClickhouseClientOptions{.IsClientForDisk=true});
auto client_ptr = AzureBlobStorage::getContainerClient(connection_params, /*readonly=*/ false);
auto settings_ptr = AzureBlobStorage::getRequestSettingsForBackup(context_->getSettingsRef(), allow_azure_native_copy);
object_storage = std::make_unique<AzureObjectStorage>("BackupReaderAzureBlobStorage",
object_storage = std::make_unique<AzureObjectStorage>(
"BackupReaderAzureBlobStorage",
std::move(client_ptr),
configuration.createSettings(context_),
configuration_.container,
configuration.getConnectionURL().toString());
std::move(settings_ptr),
connection_params.getContainer(),
connection_params.getConnectionURL());
client = object_storage->getAzureBlobStorageClient();
auto settings_copy = *object_storage->getSettings();
settings_copy.use_native_copy = allow_azure_native_copy;
settings = std::make_unique<const AzureObjectStorageSettings>(settings_copy);
settings = object_storage->getSettings();
}
BackupReaderAzureBlobStorage::~BackupReaderAzureBlobStorage() = default;
bool BackupReaderAzureBlobStorage::fileExists(const String & file_name)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
return object_storage->exists(StoredObject(key));
}
UInt64 BackupReaderAzureBlobStorage::getFileSize(const String & file_name)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
ObjectMetadata object_metadata = object_storage->getObjectMetadata(key);
return object_metadata.size_bytes;
}
std::unique_ptr<SeekableReadBuffer> BackupReaderAzureBlobStorage::readFile(const String & file_name)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
return std::make_unique<ReadBufferFromAzureBlobStorage>(
client, key, read_settings, settings->max_single_read_retries,
settings->max_single_download_retries);
@ -85,23 +86,23 @@ void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup,
&& destination_data_source_description.is_encrypted == encrypted_in_backup)
{
LOG_TRACE(log, "Copying {} from AzureBlobStorage to disk {}", path_in_backup, destination_disk->getName());
auto write_blob_function = [&](const Strings & blob_path, WriteMode mode, const std::optional<ObjectAttributes> &) -> size_t
auto write_blob_function = [&](const Strings & dst_blob_path, WriteMode mode, const std::optional<ObjectAttributes> &) -> size_t
{
/// Object storage always uses mode `Rewrite` because it simulates append using metadata and different files.
if (blob_path.size() != 2 || mode != WriteMode::Rewrite)
if (dst_blob_path.size() != 2 || mode != WriteMode::Rewrite)
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Blob writing function called with unexpected blob_path.size={} or mode={}",
blob_path.size(), mode);
dst_blob_path.size(), mode);
copyAzureBlobStorageFile(
client,
destination_disk->getObjectStorage()->getAzureBlobStorageClient(),
configuration.container,
fs::path(configuration.blob_path) / path_in_backup,
connection_params.getContainer(),
fs::path(blob_path) / path_in_backup,
0,
file_size,
/* dest_container */ blob_path[1],
/* dest_path */ blob_path[0],
/* dest_container */ dst_blob_path[1],
/* dest_path */ dst_blob_path[0],
settings,
read_settings,
threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(), "BackupRDAzure"));
@ -119,28 +120,33 @@ void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup,
BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage(
const StorageAzureConfiguration & configuration_,
const AzureBlobStorage::ConnectionParams & connection_params_,
const String & blob_path_,
bool allow_azure_native_copy,
const ReadSettings & read_settings_,
const WriteSettings & write_settings_,
const ContextPtr & context_,
bool attempt_to_create_container)
: BackupWriterDefault(read_settings_, write_settings_, getLogger("BackupWriterAzureBlobStorage"))
, data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, configuration_.getConnectionURL().toString(), false, false}
, configuration(configuration_)
, data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, connection_params_.getConnectionURL(), false, false}
, connection_params(connection_params_)
, blob_path(blob_path_)
{
auto client_ptr = configuration.createClient(/* is_readonly */false, attempt_to_create_container);
client_ptr->SetClickhouseOptions(Azure::Storage::Blobs::ClickhouseClientOptions{.IsClientForDisk=true});
if (!attempt_to_create_container)
connection_params.endpoint.container_already_exists = true;
object_storage = std::make_unique<AzureObjectStorage>("BackupWriterAzureBlobStorage",
auto client_ptr = AzureBlobStorage::getContainerClient(connection_params, /*readonly=*/ false);
auto settings_ptr = AzureBlobStorage::getRequestSettingsForBackup(context_->getSettingsRef(), allow_azure_native_copy);
object_storage = std::make_unique<AzureObjectStorage>(
"BackupWriterAzureBlobStorage",
std::move(client_ptr),
configuration.createSettings(context_),
configuration.container,
configuration_.getConnectionURL().toString());
std::move(settings_ptr),
connection_params.getContainer(),
connection_params.getConnectionURL());
client = object_storage->getAzureBlobStorageClient();
auto settings_copy = *object_storage->getSettings();
settings_copy.use_native_copy = allow_azure_native_copy;
settings = std::make_unique<const AzureObjectStorageSettings>(settings_copy);
settings = object_storage->getSettings();
}
void BackupWriterAzureBlobStorage::copyFileFromDisk(
@ -159,18 +165,18 @@ void BackupWriterAzureBlobStorage::copyFileFromDisk(
{
/// getBlobPath() can return more than 3 elements if the file is stored as multiple objects in AzureBlobStorage container.
/// In this case we can't use the native copy.
if (auto blob_path = src_disk->getBlobPath(src_path); blob_path.size() == 2)
if (auto src_blob_path = src_disk->getBlobPath(src_path); src_blob_path.size() == 2)
{
LOG_TRACE(log, "Copying file {} from disk {} to AzureBlobStorag", src_path, src_disk->getName());
copyAzureBlobStorageFile(
src_disk->getObjectStorage()->getAzureBlobStorageClient(),
client,
/* src_container */ blob_path[1],
/* src_path */ blob_path[0],
/* src_container */ src_blob_path[1],
/* src_path */ src_blob_path[0],
start_pos,
length,
configuration.container,
fs::path(configuration.blob_path) / path_in_backup,
connection_params.getContainer(),
fs::path(blob_path) / path_in_backup,
settings,
read_settings,
threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(), "BackupWRAzure"));
@ -188,11 +194,11 @@ void BackupWriterAzureBlobStorage::copyFile(const String & destination, const St
copyAzureBlobStorageFile(
client,
client,
configuration.container,
fs::path(configuration.blob_path)/ source,
connection_params.getContainer(),
fs::path(blob_path)/ source,
0,
size,
/* dest_container */ configuration.container,
/* dest_container */ connection_params.getContainer(),
/* dest_path */ destination,
settings,
read_settings,
@ -206,22 +212,28 @@ void BackupWriterAzureBlobStorage::copyDataToFile(
UInt64 length)
{
copyDataToAzureBlobStorageFile(
create_read_buffer, start_pos, length, client, configuration.container,
fs::path(configuration.blob_path) / path_in_backup, settings,
threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(), "BackupWRAzure"));
create_read_buffer,
start_pos,
length,
client,
connection_params.getContainer(),
fs::path(blob_path) / path_in_backup,
settings,
threadPoolCallbackRunnerUnsafe<void>(getBackupsIOThreadPool().get(),
"BackupWRAzure"));
}
BackupWriterAzureBlobStorage::~BackupWriterAzureBlobStorage() = default;
bool BackupWriterAzureBlobStorage::fileExists(const String & file_name)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
return object_storage->exists(StoredObject(key));
}
UInt64 BackupWriterAzureBlobStorage::getFileSize(const String & file_name)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
RelativePathsWithMetadata children;
object_storage->listObjects(key,children,/*max_keys*/0);
if (children.empty())
@ -231,7 +243,7 @@ UInt64 BackupWriterAzureBlobStorage::getFileSize(const String & file_name)
std::unique_ptr<ReadBuffer> BackupWriterAzureBlobStorage::readFile(const String & file_name, size_t /*expected_file_size*/)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
return std::make_unique<ReadBufferFromAzureBlobStorage>(
client, key, read_settings, settings->max_single_read_retries,
settings->max_single_download_retries);
@ -239,7 +251,7 @@ std::unique_ptr<ReadBuffer> BackupWriterAzureBlobStorage::readFile(const String
std::unique_ptr<WriteBuffer> BackupWriterAzureBlobStorage::writeFile(const String & file_name)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
return std::make_unique<WriteBufferFromAzureBlobStorage>(
client,
key,
@ -251,7 +263,7 @@ std::unique_ptr<WriteBuffer> BackupWriterAzureBlobStorage::writeFile(const Strin
void BackupWriterAzureBlobStorage::removeFile(const String & file_name)
{
String key = fs::path(configuration.blob_path) / file_name;
String key = fs::path(blob_path) / file_name;
StoredObject object(key);
object_storage->removeObjectIfExists(object);
}
@ -260,7 +272,7 @@ void BackupWriterAzureBlobStorage::removeFiles(const Strings & file_names)
{
StoredObjects objects;
for (const auto & file_name : file_names)
objects.emplace_back(fs::path(configuration.blob_path) / file_name);
objects.emplace_back(fs::path(blob_path) / file_name);
object_storage->removeObjectsIfExist(objects);
@ -270,7 +282,7 @@ void BackupWriterAzureBlobStorage::removeFilesBatch(const Strings & file_names)
{
StoredObjects objects;
for (const auto & file_name : file_names)
objects.emplace_back(fs::path(configuration.blob_path) / file_name);
objects.emplace_back(fs::path(blob_path) / file_name);
object_storage->removeObjectsIfExist(objects);
}

20
src/Backups/BackupIO_AzureBlobStorage.h
View File

@ -1,12 +1,10 @@
#pragma once
#include "config.h"
#if USE_AZURE_BLOB_STORAGE
#include <Backups/BackupIO_Default.h>
#include <Disks/DiskType.h>
#include <Interpreters/Context_fwd.h>
#include <Storages/ObjectStorage/Azure/Configuration.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h>
namespace DB
@ -17,7 +15,8 @@ class BackupReaderAzureBlobStorage : public BackupReaderDefault
{
public:
BackupReaderAzureBlobStorage(
const StorageAzureConfiguration & configuration_,
const AzureBlobStorage::ConnectionParams & connection_params_,
const String & blob_path_,
bool allow_azure_native_copy,
const ReadSettings & read_settings_,
const WriteSettings & write_settings_,
@ -40,16 +39,18 @@ public:
private:
const DataSourceDescription data_source_description;
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> client;
StorageAzureConfiguration configuration;
AzureBlobStorage::ConnectionParams connection_params;
String blob_path;
std::unique_ptr<AzureObjectStorage> object_storage;
std::shared_ptr<const AzureObjectStorageSettings> settings;
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings;
};
class BackupWriterAzureBlobStorage : public BackupWriterDefault
{
public:
BackupWriterAzureBlobStorage(
const StorageAzureConfiguration & configuration_,
const AzureBlobStorage::ConnectionParams & connection_params_,
const String & blob_path_,
bool allow_azure_native_copy,
const ReadSettings & read_settings_,
const WriteSettings & write_settings_,
@ -87,9 +88,10 @@ private:
const DataSourceDescription data_source_description;
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> client;
StorageAzureConfiguration configuration;
AzureBlobStorage::ConnectionParams connection_params;
String blob_path;
std::unique_ptr<AzureObjectStorage> object_storage;
std::shared_ptr<const AzureObjectStorageSettings> settings;
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings;
};
}

68
src/Backups/registerBackupEngineAzureBlobStorage.cpp
View File

@ -5,6 +5,7 @@
#if USE_AZURE_BLOB_STORAGE
#include <Backups/BackupIO_AzureBlobStorage.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
#include <Backups/BackupImpl.h>
#include <IO/Archives/hasRegisteredArchiveFileExtension.h>
#include <Interpreters/Context.h>
@ -49,7 +50,9 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
const String & id_arg = params.backup_info.id_arg;
const auto & args = params.backup_info.args;
StorageAzureConfiguration configuration;
String blob_path;
AzureBlobStorage::ConnectionParams connection_params;
auto request_settings = AzureBlobStorage::getRequestSettings(params.context->getSettingsRef());
if (!id_arg.empty())
{
@ -59,55 +62,42 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
if (!config.has(config_prefix))
throw Exception(ErrorCodes::BAD_ARGUMENTS, "There is no collection named `{}` in config", id_arg);
if (config.has(config_prefix + ".connection_string"))
connection_params =
{
configuration.connection_url = config.getString(config_prefix + ".connection_string");
configuration.is_connection_string = true;
configuration.container = config.getString(config_prefix + ".container");
}
else
{
configuration.connection_url = config.getString(config_prefix + ".storage_account_url");
configuration.is_connection_string = false;
configuration.container = config.getString(config_prefix + ".container");
configuration.account_name = config.getString(config_prefix + ".account_name");
configuration.account_key = config.getString(config_prefix + ".account_key");
if (config.has(config_prefix + ".account_name") && config.has(config_prefix + ".account_key"))
{
configuration.account_name = config.getString(config_prefix + ".account_name");
configuration.account_key = config.getString(config_prefix + ".account_key");
}
}
.endpoint = AzureBlobStorage::processEndpoint(config, config_prefix),
.auth_method = AzureBlobStorage::getAuthMethod(config, config_prefix),
.client_options = AzureBlobStorage::getClientOptions(*request_settings, /*for_disk=*/ true),
};
if (args.size() > 1)
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Backup AzureBlobStorage requires 1 or 2 arguments: named_collection, [filename]");
if (args.size() == 1)
configuration.setPath(args[0].safeGet<String>());
blob_path = args[0].safeGet<String>();
}
else
{
if (args.size() == 3)
{
configuration.connection_url = args[0].safeGet<String>();
configuration.is_connection_string = !configuration.connection_url.starts_with("http");
auto connection_url = args[0].safeGet<String>();
auto container_name = args[1].safeGet<String>();
blob_path = args[2].safeGet<String>();
configuration.container = args[1].safeGet<String>();
configuration.blob_path = args[2].safeGet<String>();
AzureBlobStorage::processURL(connection_url, container_name, connection_params.endpoint, connection_params.auth_method);
connection_params.client_options = AzureBlobStorage::getClientOptions(*request_settings, /*for_disk=*/ true);
}
else if (args.size() == 5)
{
configuration.connection_url = args[0].safeGet<String>();
configuration.is_connection_string = false;
connection_params.endpoint.storage_account_url = args[0].safeGet<String>();
connection_params.endpoint.container_name = args[1].safeGet<String>();
blob_path = args[2].safeGet<String>();
configuration.container = args[1].safeGet<String>();
configuration.blob_path = args[2].safeGet<String>();
configuration.account_name = args[3].safeGet<String>();
configuration.account_key = args[4].safeGet<String>();
auto account_name = args[3].safeGet<String>();
auto account_key = args[4].safeGet<String>();
connection_params.auth_method = std::make_shared<Azure::Storage::StorageSharedKeyCredential>(account_name, account_key);
connection_params.client_options = AzureBlobStorage::getClientOptions(*request_settings, /*for_disk=*/ true);
}
else
{
@ -117,16 +107,12 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
}
BackupImpl::ArchiveParams archive_params;
if (hasRegisteredArchiveFileExtension(configuration.getPath()))
if (hasRegisteredArchiveFileExtension(blob_path))
{
if (params.is_internal_backup)
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Using archives with backups on clusters is disabled");
auto path = configuration.getPath();
auto filename = removeFileNameFromURL(path);
configuration.setPath(path);
archive_params.archive_name = filename;
archive_params.archive_name = removeFileNameFromURL(blob_path);
archive_params.compression_method = params.compression_method;
archive_params.compression_level = params.compression_level;
archive_params.password = params.password;
@ -141,7 +127,8 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
if (params.open_mode == IBackup::OpenMode::READ)
{
auto reader = std::make_shared<BackupReaderAzureBlobStorage>(
configuration,
connection_params,
blob_path,
params.allow_azure_native_copy,
params.read_settings,
params.write_settings,
@ -159,7 +146,8 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
else
{
auto writer = std::make_shared<BackupWriterAzureBlobStorage>(
configuration,
connection_params,
blob_path,
params.allow_azure_native_copy,
params.read_settings,
params.write_settings,

2
src/Client/ClientBase.h
View File

@ -6,13 +6,13 @@
#include <Common/ProgressIndication.h>
#include <Common/InterruptListener.h>
#include <Common/ShellCommand.h>
#include <Common/QueryFuzzer.h>
#include <Common/Stopwatch.h>
#include <Common/DNSResolver.h>
#include <Core/ExternalTable.h>
#include <Poco/Util/Application.h>
#include <Interpreters/Context.h>
#include <Client/Suggest.h>
#include <Client/QueryFuzzer.h>
#include <boost/program_options.hpp>
#include <Storages/StorageFile.h>
#include <Storages/SelectQueryInfo.h>

6
src/Client/HedgedConnections.cpp
View File

@ -195,6 +195,12 @@ void HedgedConnections::sendQuery(
modified_settings.parallel_replica_offset = fd_to_replica_location[replica.packet_receiver->getFileDescriptor()].offset;
}
/// FIXME: Remove once we will make `allow_experimental_analyzer` obsolete setting.
/// Make the analyzer being set, so it will be effectively applied on the remote server.
/// In other words, the initiator always controls whether the analyzer enabled or not for
/// all servers involved in the distributed query processing.
modified_settings.set("allow_experimental_analyzer", static_cast<bool>(modified_settings.allow_experimental_analyzer));
replica.connection->sendQuery(timeouts, query, /* query_parameters */ {}, query_id, stage, &modified_settings, &client_info, with_pending_data, {});
replica.change_replica_timeout.setRelative(timeouts.receive_data_timeout);
replica.packet_receiver->setTimeout(hedged_connections_factory.getConnectionTimeouts().receive_timeout);

6
src/Client/MultiplexedConnections.cpp
View File

@ -150,6 +150,12 @@ void MultiplexedConnections::sendQuery(
}
}
/// FIXME: Remove once we will make `allow_experimental_analyzer` obsolete setting.
/// Make the analyzer being set, so it will be effectively applied on the remote server.
/// In other words, the initiator always controls whether the analyzer enabled or not for
/// all servers involved in the distributed query processing.
modified_settings.set("allow_experimental_analyzer", static_cast<bool>(modified_settings.allow_experimental_analyzer));
const bool enable_sample_offset_parallel_processing = settings.max_parallel_replicas > 1 && settings.allow_experimental_parallel_reading_from_replicas == 0;
size_t num_replicas = replica_states.size();

6
src/Columns/ColumnObject.cpp
View File

@ -1093,4 +1093,10 @@ void ColumnObject::finalize()
checkObjectHasNoAmbiguosPaths(getKeys());
}
void ColumnObject::updateHashFast(SipHash & hash) const
{
for (const auto & entry : subcolumns)
for (auto & part : entry->data.data)
part->updateHashFast(hash);
}
}

2
src/Columns/ColumnObject.h
View File

@ -242,7 +242,7 @@ public:
const char * skipSerializedInArena(const char *) const override { throwMustBeConcrete(); }
void updateHashWithValue(size_t, SipHash &) const override { throwMustBeConcrete(); }
void updateWeakHash32(WeakHash32 &) const override { throwMustBeConcrete(); }
void updateHashFast(SipHash &) const override { throwMustBeConcrete(); }
void updateHashFast(SipHash & hash) const override;
void expand(const Filter &, bool) override { throwMustBeConcrete(); }
bool hasEqualValues() const override { throwMustBeConcrete(); }
size_t byteSizeAt(size_t) const override { throwMustBeConcrete(); }

313
src/Common/CgroupsMemoryUsageObserver.cpp
View File

@ -12,7 +12,9 @@
#include <base/getMemoryAmount.h>
#include <base/sleep.h>
#include <cstdint>
#include <filesystem>
#include <memory>
#include <optional>
#include "config.h"
@ -22,24 +24,169 @@
#define STRINGIFY(x) STRINGIFY_HELPER(x)
#endif
using namespace DB;
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_CLOSE_FILE;
extern const int CANNOT_OPEN_FILE;
extern const int FILE_DOESNT_EXIST;
extern const int INCORRECT_DATA;
}
CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_)
: log(getLogger("CgroupsMemoryUsageObserver"))
, wait_time(wait_time_)
, memory_usage_file(log)
}
namespace
{
LOG_INFO(log, "Initialized cgroups memory limit observer, wait time is {} sec", wait_time.count());
/// Format is
/// kernel 5
/// rss 15
/// [...]
uint64_t readMetricFromStatFile(ReadBufferFromFile & buf, const std::string & key)
{
while (!buf.eof())
{
std::string current_key;
readStringUntilWhitespace(current_key, buf);
if (current_key != key)
{
std::string dummy;
readStringUntilNewlineInto(dummy, buf);
buf.ignore();
continue;
}
assertChar(' ', buf);
uint64_t value = 0;
readIntText(value, buf);
return value;
}
throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot find '{}' in '{}'", key, buf.getFileName());
}
struct CgroupsV1Reader : ICgroupsReader
{
explicit CgroupsV1Reader(const std::filesystem::path & stat_file_dir) : buf(stat_file_dir / "memory.stat") { }
uint64_t readMemoryUsage() override
{
std::lock_guard lock(mutex);
buf.rewind();
return readMetricFromStatFile(buf, "rss");
}
private:
std::mutex mutex;
ReadBufferFromFile buf TSA_GUARDED_BY(mutex);
};
struct CgroupsV2Reader : ICgroupsReader
{
explicit CgroupsV2Reader(const std::filesystem::path & stat_file_dir)
: current_buf(stat_file_dir / "memory.current"), stat_buf(stat_file_dir / "memory.stat")
{
}
uint64_t readMemoryUsage() override
{
std::lock_guard lock(mutex);
current_buf.rewind();
stat_buf.rewind();
int64_t mem_usage = 0;
/// memory.current contains a single number
/// the reason why we subtract it described here: https://github.com/ClickHouse/ClickHouse/issues/64652#issuecomment-2149630667
readIntText(mem_usage, current_buf);
mem_usage -= readMetricFromStatFile(stat_buf, "inactive_file");
chassert(mem_usage >= 0, "Negative memory usage");
return mem_usage;
}
private:
std::mutex mutex;
ReadBufferFromFile current_buf TSA_GUARDED_BY(mutex);
ReadBufferFromFile stat_buf TSA_GUARDED_BY(mutex);
};
/// Caveats:
/// - All of the logic in this file assumes that the current process is the only process in the
/// containing cgroup (or more precisely: the only process with significant memory consumption).
/// If this is not the case, then other processe's memory consumption may affect the internal
/// memory tracker ...
/// - Cgroups v1 and v2 allow nested cgroup hierarchies. As v1 is deprecated for over half a
/// decade and will go away at some point, hierarchical detection is only implemented for v2.
/// - I did not test what happens if a host has v1 and v2 simultaneously enabled. I believe such
/// systems existed only for a short transition period.
std::optional<std::string> getCgroupsV2Path()
{
if (!cgroupsV2Enabled())
return {};
if (!cgroupsV2MemoryControllerEnabled())
return {};
String cgroup = cgroupV2OfProcess();
auto current_cgroup = cgroup.empty() ? default_cgroups_mount : (default_cgroups_mount / cgroup);
/// Return the bottom-most nested current memory file. If there is no such file at the current
/// level, try again at the parent level as memory settings are inherited.
while (current_cgroup != default_cgroups_mount.parent_path())
{
const auto current_path = current_cgroup / "memory.current";
const auto stat_path = current_cgroup / "memory.stat";
if (std::filesystem::exists(current_path) && std::filesystem::exists(stat_path))
return {current_cgroup};
current_cgroup = current_cgroup.parent_path();
}
return {};
}
std::optional<std::string> getCgroupsV1Path()
{
auto path = default_cgroups_mount / "memory/memory.stat";
if (!std::filesystem::exists(path))
return {};
return {default_cgroups_mount / "memory"};
}
std::pair<std::string, CgroupsMemoryUsageObserver::CgroupsVersion> getCgroupsPath()
{
auto v2_path = getCgroupsV2Path();
if (v2_path.has_value())
return {*v2_path, CgroupsMemoryUsageObserver::CgroupsVersion::V2};
auto v1_path = getCgroupsV1Path();
if (v1_path.has_value())
return {*v1_path, CgroupsMemoryUsageObserver::CgroupsVersion::V1};
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Cannot find cgroups v1 or v2 current memory file");
}
}
namespace DB
{
CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_)
: log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_)
{
const auto [cgroup_path, version] = getCgroupsPath();
if (version == CgroupsVersion::V2)
cgroup_reader = std::make_unique<CgroupsV2Reader>(cgroup_path);
else
cgroup_reader = std::make_unique<CgroupsV1Reader>(cgroup_path);
LOG_INFO(
log,
"Will read the current memory usage from '{}' (cgroups version: {}), wait time is {} sec",
cgroup_path,
(version == CgroupsVersion::V1) ? "v1" : "v2",
wait_time.count());
}
CgroupsMemoryUsageObserver::~CgroupsMemoryUsageObserver()
@ -84,7 +231,8 @@ void CgroupsMemoryUsageObserver::setMemoryUsageLimits(uint64_t hard_limit_, uint
mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0);
# endif
/// Reset current usage in memory tracker. Expect zero for free_memory_in_allocator_arenas as we just purged them.
uint64_t memory_usage = memory_usage_file.readMemoryUsage();
uint64_t memory_usage = cgroup_reader->readMemoryUsage();
LOG_TRACE(log, "Read current memory usage {} bytes ({}) from cgroups", memory_usage, ReadableSize(memory_usage));
MemoryTracker::setRSS(memory_usage, 0);
LOG_INFO(log, "Purged jemalloc arenas. Current memory usage is {}", ReadableSize(memory_usage));
@ -104,152 +252,6 @@ void CgroupsMemoryUsageObserver::setOnMemoryAmountAvailableChangedFn(OnMemoryAmo
on_memory_amount_available_changed = on_memory_amount_available_changed_;
}
namespace
{
/// Caveats:
/// - All of the logic in this file assumes that the current process is the only process in the
/// containing cgroup (or more precisely: the only process with significant memory consumption).
/// If this is not the case, then other processe's memory consumption may affect the internal
/// memory tracker ...
/// - Cgroups v1 and v2 allow nested cgroup hierarchies. As v1 is deprecated for over half a
/// decade and will go away at some point, hierarchical detection is only implemented for v2.
/// - I did not test what happens if a host has v1 and v2 simultaneously enabled. I believe such
/// systems existed only for a short transition period.
std::optional<std::string> getCgroupsV2FileName()
{
if (!cgroupsV2Enabled())
return {};
if (!cgroupsV2MemoryControllerEnabled())
return {};
String cgroup = cgroupV2OfProcess();
auto current_cgroup = cgroup.empty() ? default_cgroups_mount : (default_cgroups_mount / cgroup);
/// Return the bottom-most nested current memory file. If there is no such file at the current
/// level, try again at the parent level as memory settings are inherited.
while (current_cgroup != default_cgroups_mount.parent_path())
{
auto path = current_cgroup / "memory.current";
if (std::filesystem::exists(path))
return {path};
current_cgroup = current_cgroup.parent_path();
}
return {};
}
std::optional<std::string> getCgroupsV1FileName()
{
auto path = default_cgroups_mount / "memory/memory.stat";
if (!std::filesystem::exists(path))
return {};
return {path};
}
std::pair<std::string, CgroupsMemoryUsageObserver::CgroupsVersion> getCgroupsFileName()
{
auto v2_file_name = getCgroupsV2FileName();
if (v2_file_name.has_value())
return {*v2_file_name, CgroupsMemoryUsageObserver::CgroupsVersion::V2};
auto v1_file_name = getCgroupsV1FileName();
if (v1_file_name.has_value())
return {*v1_file_name, CgroupsMemoryUsageObserver::CgroupsVersion::V1};
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Cannot find cgroups v1 or v2 current memory file");
}
}
CgroupsMemoryUsageObserver::MemoryUsageFile::MemoryUsageFile(LoggerPtr log_)
: log(log_)
{
std::tie(file_name, version) = getCgroupsFileName();
LOG_INFO(log, "Will read the current memory usage from '{}' (cgroups version: {})", file_name, (version == CgroupsVersion::V1) ? "v1" : "v2");
fd = ::open(file_name.data(), O_RDONLY);
if (fd == -1)
ErrnoException::throwFromPath(
(errno == ENOENT) ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE,
file_name, "Cannot open file '{}'", file_name);
}
CgroupsMemoryUsageObserver::MemoryUsageFile::~MemoryUsageFile()
{
assert(fd != -1);
if (::close(fd) != 0)
{
try
{
ErrnoException::throwFromPath(
ErrorCodes::CANNOT_CLOSE_FILE,
file_name, "Cannot close file '{}'", file_name);
}
catch (const ErrnoException &)
{
tryLogCurrentException(log, __PRETTY_FUNCTION__);
}
}
}
uint64_t CgroupsMemoryUsageObserver::MemoryUsageFile::readMemoryUsage() const
{
/// File read is probably not read is thread-safe, just to be sure
std::lock_guard lock(mutex);
ReadBufferFromFileDescriptor buf(fd);
buf.rewind();
uint64_t mem_usage = 0;
switch (version)
{
case CgroupsVersion::V1:
{
/// Format is
/// kernel 5
/// rss 15
/// [...]
std::string key;
bool found_rss = false;
while (!buf.eof())
{
readStringUntilWhitespace(key, buf);
if (key != "rss")
{
std::string dummy;
readStringUntilNewlineInto(dummy, buf);
buf.ignore();
continue;
}
assertChar(' ', buf);
readIntText(mem_usage, buf);
found_rss = true;
break;
}
if (!found_rss)
throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot find 'rss' in '{}'", file_name);
break;
}
case CgroupsVersion::V2:
{
readIntText(mem_usage, buf);
break;
}
}
LOG_TRACE(log, "Read current memory usage {} from cgroups", ReadableSize(mem_usage));
return mem_usage;
}
void CgroupsMemoryUsageObserver::startThread()
{
if (!thread.joinable())
@ -301,7 +303,8 @@ void CgroupsMemoryUsageObserver::runThread()
std::lock_guard<std::mutex> limit_lock(limit_mutex);
if (soft_limit > 0 && hard_limit > 0)
{
uint64_t memory_usage = memory_usage_file.readMemoryUsage();
uint64_t memory_usage = cgroup_reader->readMemoryUsage();
LOG_TRACE(log, "Read current memory usage {} bytes ({}) from cgroups", memory_usage, ReadableSize(memory_usage));
if (memory_usage > hard_limit)
{
if (last_memory_usage <= hard_limit)

27
src/Common/CgroupsMemoryUsageObserver.h
View File

@ -3,11 +3,19 @@
#include <Common/ThreadPool.h>
#include <chrono>
#include <memory>
#include <mutex>
namespace DB
{
struct ICgroupsReader
{
virtual ~ICgroupsReader() = default;
virtual uint64_t readMemoryUsage() = 0;
};
/// Does two things:
/// 1. Periodically reads the memory usage of the process from Linux cgroups.
/// You can specify soft or hard memory limits:
@ -61,27 +69,12 @@ private:
uint64_t last_memory_usage = 0; /// how much memory does the process use
uint64_t last_available_memory_amount; /// how much memory can the process use
/// Represents the cgroup virtual file that shows the memory consumption of the process's cgroup.
struct MemoryUsageFile
{
public:
explicit MemoryUsageFile(LoggerPtr log_);
~MemoryUsageFile();
uint64_t readMemoryUsage() const;
private:
LoggerPtr log;
mutable std::mutex mutex;
int fd TSA_GUARDED_BY(mutex) = -1;
CgroupsVersion version;
std::string file_name;
};
MemoryUsageFile memory_usage_file;
void stopThread();
void runThread();
std::unique_ptr<ICgroupsReader> cgroup_reader;
std::mutex thread_mutex;
std::condition_variable cond;
ThreadFromGlobalPool thread;

184
src/Common/CollectionOfDerived.h Normal file
View File

@ -0,0 +1,184 @@
#pragma once
#include <base/defines.h>
#include <Common/Exception.h>
#include <algorithm>
#include <memory>
#include <typeindex>
#include <vector>
#include <string>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
/* This is a collections of objects derived from ItemBase.
* Collection contains no more than one instance for each derived type.
* The derived type is used to access the instance.
*/
template<class ItemBase>
class CollectionOfDerivedItems
{
public:
using Self = CollectionOfDerivedItems<ItemBase>;
using ItemPtr = std::shared_ptr<ItemBase>;
private:
struct Rec
{
std::type_index type_idx;
ItemPtr ptr;
bool operator<(const Rec & other) const
{
return type_idx < other.type_idx;
}
bool operator<(const std::type_index & value) const
{
return type_idx < value;
}
bool operator==(const Rec & other) const
{
return type_idx == other.type_idx;
}
};
using Records = std::vector<Rec>;
public:
void swap(Self & other) noexcept
{
records.swap(other.records);
}
void clear()
{
records.clear();
}
bool empty() const
{
return records.empty();
}
size_t size() const
{
return records.size();
}
Self clone() const
{
Self result;
result.records.reserve(records.size());
for (const auto & rec : records)
result.records.emplace_back(rec.type_idx, rec.ptr->clone());
return result;
}
void append(Self && other)
{
auto middle_idx = records.size();
std::move(other.records.begin(), other.records.end(), std::back_inserter(records));
std::inplace_merge(records.begin(), records.begin() + middle_idx, records.end());
chassert(isUniqTypes());
}
template <class T>
void add(std::shared_ptr<T> info)
{
static_assert(std::is_base_of_v<ItemBase, T>, "Template parameter must inherit items base class");
return addImpl(std::type_index(typeid(T)), std::move(info));
}
template <class T>
std::shared_ptr<T> get() const
{
static_assert(std::is_base_of_v<ItemBase, T>, "Template parameter must inherit items base class");
auto it = getImpl(std::type_index(typeid(T)));
if (it == records.cend())
return nullptr;
auto cast = std::dynamic_pointer_cast<T>(it->ptr);
chassert(cast);
return cast;
}
template <class T>
std::shared_ptr<T> extract()
{
static_assert(std::is_base_of_v<ItemBase, T>, "Template parameter must inherit items base class");
auto it = getImpl(std::type_index(typeid(T)));
if (it == records.cend())
return nullptr;
auto cast = std::dynamic_pointer_cast<T>(it->ptr);
chassert(cast);
records.erase(it);
return cast;
}
std::string debug() const
{
std::string result;
for (auto & rec : records)
{
result.append(rec.type_idx.name());
result.append(" ");
}
return result;
}
private:
bool isUniqTypes() const
{
auto uniq_it = std::adjacent_find(records.begin(), records.end());
return uniq_it == records.end();
}
void addImpl(std::type_index type_idx, ItemPtr item)
{
auto it = std::lower_bound(records.begin(), records.end(), type_idx);
if (it == records.end())
{
records.emplace_back(type_idx, item);
return;
}
if (it->type_idx == type_idx)
throw Exception(ErrorCodes::LOGICAL_ERROR, "inserted items must be unique by their type, type {} is inserted twice", type_idx.name());
records.emplace(it, type_idx, item);
chassert(isUniqTypes());
}
Records::const_iterator getImpl(std::type_index type_idx) const
{
auto it = std::lower_bound(records.cbegin(), records.cend(), type_idx);
if (it == records.cend())
return records.cend();
if (it->type_idx != type_idx)
return records.cend();
return it;
}
Records records;
};
}

1
src/Common/Config/ConfigProcessor.cpp
View File

@ -316,7 +316,6 @@ void ConfigProcessor::mergeRecursive(XMLDocumentPtr config, Node * config_root,
}
else if (replace)
{
with_element.removeAttribute("replace");
NodePtr new_node = config->importNode(with_node, true);
config_root->replaceChild(new_node, config_node);
}

65
src/Common/Coverage.cpp Normal file
View File

@ -0,0 +1,65 @@
#include <Common/Coverage.h>
#if defined(SANITIZE_COVERAGE)
#include <fcntl.h>
#include <unistd.h>
#include <string>
#include <vector>
#include <Common/IO.h>
#include <base/coverage.h>
#include <fmt/format.h>
/// Macros to avoid using strlen(), since it may fail if SSE is not supported.
#define writeError(data) do \
{ \
static_assert(__builtin_constant_p(data)); \
if (!writeRetry(STDERR_FILENO, data, sizeof(data) - 1)) \
_Exit(1); \
} while (false)
__attribute__((no_sanitize("coverage")))
void dumpCoverage()
{
/// A user can request to dump the coverage information into files at exit.
/// This is useful for non-server applications such as clickhouse-format or clickhouse-client,
/// that cannot introspect it with SQL functions at runtime.
/// The CLICKHOUSE_WRITE_COVERAGE environment variable defines a prefix for a filename 'prefix.pid'
/// containing the list of addresses of covered .
/// The format is even simpler than Clang's "sancov": an array of 64-bit addresses, native byte order, no header.
if (const char * coverage_filename_prefix = getenv("CLICKHOUSE_WRITE_COVERAGE")) // NOLINT(concurrency-mt-unsafe)
{
auto dump = [](const std::string & name, auto span)
{
/// Write only non-zeros.
std::vector<uintptr_t> data;
data.reserve(span.size());
for (auto addr : span)
if (addr)
data.push_back(addr);
int fd = ::open(name.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0400);
if (-1 == fd)
{
writeError("Cannot open a file to write the coverage data\n");
}
else
{
if (!writeRetry(fd, reinterpret_cast<const char *>(data.data()), data.size() * sizeof(data[0])))
writeError("Cannot write the coverage data to a file\n");
if (0 != ::close(fd))
writeError("Cannot close the file with coverage data\n");
}
};
dump(fmt::format("{}.{}", coverage_filename_prefix, getpid()), getCumulativeCoverage());
}
}
#endif

5
src/Common/Coverage.h Normal file
View File

@ -0,0 +1,5 @@
#pragma once
#if defined(SANITIZE_COVERAGE)
void dumpCoverage();
#endif

234
src/Common/EnvironmentChecks.cpp Normal file
View File

@ -0,0 +1,234 @@
#include <Common/EnvironmentChecks.h>
#include <Common/IO.h>
#include <fmt/format.h>
#include <csignal>
#include <csetjmp>
#include <cstdint>
#include <tuple>
#include <unistd.h>
namespace
{
enum class InstructionFail : uint8_t
{
NONE = 0,
SSE3 = 1,
SSSE3 = 2,
SSE4_1 = 3,
SSE4_2 = 4,
POPCNT = 5,
AVX = 6,
AVX2 = 7,
AVX512 = 8
};
auto instructionFailToString(InstructionFail fail)
{
switch (fail)
{
#define ret(x) return std::make_tuple(STDERR_FILENO, x, sizeof(x) - 1)
case InstructionFail::NONE:
ret("NONE");
case InstructionFail::SSE3:
ret("SSE3");
case InstructionFail::SSSE3:
ret("SSSE3");
case InstructionFail::SSE4_1:
ret("SSE4.1");
case InstructionFail::SSE4_2:
ret("SSE4.2");
case InstructionFail::POPCNT:
ret("POPCNT");
case InstructionFail::AVX:
ret("AVX");
case InstructionFail::AVX2:
ret("AVX2");
case InstructionFail::AVX512:
ret("AVX512");
#undef ret
}
}
sigjmp_buf jmpbuf;
[[noreturn]] void sigIllCheckHandler(int, siginfo_t *, void *)
{
siglongjmp(jmpbuf, 1);
}
/// Check if necessary SSE extensions are available by trying to execute some sse instructions.
/// If instruction is unavailable, SIGILL will be sent by kernel.
void checkRequiredInstructionsImpl(volatile InstructionFail & fail)
{
#if defined(__SSE3__)
fail = InstructionFail::SSE3;
__asm__ volatile ("addsubpd %%xmm0, %%xmm0" : : : "xmm0");
#endif
#if defined(__SSSE3__)
fail = InstructionFail::SSSE3;
__asm__ volatile ("pabsw %%xmm0, %%xmm0" : : : "xmm0");
#endif
#if defined(__SSE4_1__)
fail = InstructionFail::SSE4_1;
__asm__ volatile ("pmaxud %%xmm0, %%xmm0" : : : "xmm0");
#endif
#if defined(__SSE4_2__)
fail = InstructionFail::SSE4_2;
__asm__ volatile ("pcmpgtq %%xmm0, %%xmm0" : : : "xmm0");
#endif
/// Defined by -msse4.2
#if defined(__POPCNT__)
fail = InstructionFail::POPCNT;
{
uint64_t a = 0;
uint64_t b = 0;
__asm__ volatile ("popcnt %1, %0" : "=r"(a) :"r"(b) :);
}
#endif
#if defined(__AVX__)
fail = InstructionFail::AVX;
__asm__ volatile ("vaddpd %%ymm0, %%ymm0, %%ymm0" : : : "ymm0");
#endif
#if defined(__AVX2__)
fail = InstructionFail::AVX2;
__asm__ volatile ("vpabsw %%ymm0, %%ymm0" : : : "ymm0");
#endif
#if defined(__AVX512__)
fail = InstructionFail::AVX512;
__asm__ volatile ("vpabsw %%zmm0, %%zmm0" : : : "zmm0");
#endif
fail = InstructionFail::NONE;
}
/// Macros to avoid using strlen(), since it may fail if SSE is not supported.
#define writeError(data) do \
{ \
static_assert(__builtin_constant_p(data)); \
if (!writeRetry(STDERR_FILENO, data, sizeof(data) - 1)) \
_Exit(1); \
} while (false)
/// Check SSE and others instructions availability. Calls exit on fail.
/// This function must be called as early as possible, even before main, because static initializers may use unavailable instructions.
void checkRequiredInstructions()
{
struct sigaction sa{};
struct sigaction sa_old{};
sa.sa_sigaction = sigIllCheckHandler;
sa.sa_flags = SA_SIGINFO;
auto signal = SIGILL;
if (sigemptyset(&sa.sa_mask) != 0
|| sigaddset(&sa.sa_mask, signal) != 0
|| sigaction(signal, &sa, &sa_old) != 0)
{
/// You may wonder about strlen.
/// Typical implementation of strlen is using SSE4.2 or AVX2.
/// But this is not the case because it's compiler builtin and is executed at compile time.
writeError("Can not set signal handler\n");
_Exit(1);
}
volatile InstructionFail fail = InstructionFail::NONE;
if (sigsetjmp(jmpbuf, 1))
{
writeError("Instruction check fail. The CPU does not support ");
if (!std::apply(writeRetry, instructionFailToString(fail)))
_Exit(1);
writeError(" instruction set.\n");
_Exit(1);
}
checkRequiredInstructionsImpl(fail);
if (sigaction(signal, &sa_old, nullptr))
{
writeError("Can not set signal handler\n");
_Exit(1);
}
}
struct Checker
{
Checker()
{
checkRequiredInstructions();
}
} checker
#ifndef OS_DARWIN
__attribute__((init_priority(101))) /// Run before other static initializers.
#endif
;
}
#if !defined(USE_MUSL)
/// NOTE: We will migrate to full static linking or our own dynamic loader to make this code obsolete.
void checkHarmfulEnvironmentVariables(char ** argv)
{
std::initializer_list<const char *> harmful_env_variables = {
/// The list is a selection from "man ld-linux".
"LD_PRELOAD",
"LD_LIBRARY_PATH",
"LD_ORIGIN_PATH",
"LD_AUDIT",
"LD_DYNAMIC_WEAK",
/// The list is a selection from "man dyld" (osx).
"DYLD_LIBRARY_PATH",
"DYLD_FALLBACK_LIBRARY_PATH",
"DYLD_VERSIONED_LIBRARY_PATH",
"DYLD_INSERT_LIBRARIES",
};
bool require_reexec = false;
for (const auto * var : harmful_env_variables)
{
if (const char * value = getenv(var); value && value[0]) // NOLINT(concurrency-mt-unsafe)
{
/// NOTE: setenv() is used over unsetenv() since unsetenv() marked as harmful
if (setenv(var, "", true)) // NOLINT(concurrency-mt-unsafe) // this is safe if not called concurrently
{
fmt::print(stderr, "Cannot override {} environment variable", var);
_exit(1);
}
require_reexec = true;
}
}
if (require_reexec)
{
/// Use execvp() over execv() to search in PATH.
///
/// This should be safe, since:
/// - if argv[0] is relative path - it is OK
/// - if argv[0] has only basename, the it will search in PATH, like shell will do.
///
/// Also note, that this (search in PATH) because there is no easy and
/// portable way to get absolute path of argv[0].
/// - on linux there is /proc/self/exec and AT_EXECFN
/// - but on other OSes there is no such thing (especially on OSX).
///
/// And since static linking will be done someday anyway,
/// let's not pollute the code base with special cases.
int error = execvp(argv[0], argv);
_exit(error);
}
}
#endif

5
src/Common/EnvironmentChecks.h Normal file
View File

@ -0,0 +1,5 @@
#pragma once
#if !defined(USE_MUSL)
void checkHarmfulEnvironmentVariables(char ** argv);
#endif

26
src/Common/Jemalloc.cpp
View File

@ -46,6 +46,20 @@ void checkJemallocProfilingEnabled()
"set: MALLOC_CONF=background_thread:true,prof:true");
}
template <typename T>
void setJemallocValue(const char * name, T value)
{
T old_value;
size_t old_value_size = sizeof(T);
if (mallctl(name, &old_value, &old_value_size, reinterpret_cast<void*>(&value), sizeof(T)))
{
LOG_WARNING(getLogger("Jemalloc"), "mallctl for {} failed", name);
return;
}
LOG_INFO(getLogger("Jemalloc"), "Value for {} set to {} (from {})", name, value, old_value);
}
void setJemallocProfileActive(bool value)
{
checkJemallocProfilingEnabled();
@ -58,7 +72,7 @@ void setJemallocProfileActive(bool value)
return;
}
mallctl("prof.active", nullptr, nullptr, &value, sizeof(bool));
setJemallocValue("prof.active", value);
LOG_TRACE(getLogger("SystemJemalloc"), "Profiling is {}", value ? "enabled" : "disabled");
}
@ -84,6 +98,16 @@ std::string flushJemallocProfile(const std::string & file_prefix)
return profile_dump_path;
}
void setJemallocBackgroundThreads(bool enabled)
{
setJemallocValue("background_thread", enabled);
}
void setJemallocMaxBackgroundThreads(size_t max_threads)
{
setJemallocValue("max_background_threads", max_threads);
}
}
#endif

4
src/Common/Jemalloc.h
View File

@ -17,6 +17,10 @@ void setJemallocProfileActive(bool value);
std::string flushJemallocProfile(const std::string & file_prefix);
void setJemallocBackgroundThreads(bool enabled);
void setJemallocMaxBackgroundThreads(size_t max_threads);
}
#endif

7
src/Common/ProfileEvents.cpp
View File

@ -238,7 +238,12 @@
\
M(CannotRemoveEphemeralNode, "Number of times an error happened while trying to remove ephemeral node. This is not an issue, because our implementation of ZooKeeper library guarantee that the session will expire and the node will be removed.") \
\
M(RegexpCreated, "Compiled regular expressions. Identical regular expressions compiled just once and cached forever.") \
M(RegexpWithMultipleNeedlesCreated, "Regular expressions with multiple needles (VectorScan library) compiled.") \
M(RegexpWithMultipleNeedlesGlobalCacheHit, "Number of times we fetched compiled regular expression with multiple needles (VectorScan library) from the global cache.") \
M(RegexpWithMultipleNeedlesGlobalCacheMiss, "Number of times we failed to fetch compiled regular expression with multiple needles (VectorScan library) from the global cache.") \
M(RegexpLocalCacheHit, "Number of times we fetched compiled regular expression from a local cache.") \
M(RegexpLocalCacheMiss, "Number of times we failed to fetch compiled regular expression from a local cache.") \
\
M(ContextLock, "Number of times the lock of Context was acquired or tried to acquire. This is global lock.") \
M(ContextLockWaitMicroseconds, "Context lock wait time in microseconds") \
\

46
src/Client/QueryFuzzer.cpp → src/Common/QueryFuzzer.cpp
View File

@ -68,22 +68,21 @@ Field QueryFuzzer::getRandomField(int type)
{
case 0:
{
return bad_int64_values[fuzz_rand() % (sizeof(bad_int64_values)
/ sizeof(*bad_int64_values))];
return bad_int64_values[fuzz_rand() % std::size(bad_int64_values)];
}
case 1:
{
static constexpr double values[]
= {NAN, INFINITY, -INFINITY, 0., -0., 0.0001, 0.5, 0.9999,
1., 1.0001, 2., 10.0001, 100.0001, 1000.0001, 1e10, 1e20,
FLT_MIN, FLT_MIN + FLT_EPSILON, FLT_MAX, FLT_MAX + FLT_EPSILON}; return values[fuzz_rand() % (sizeof(values) / sizeof(*values))];
FLT_MIN, FLT_MIN + FLT_EPSILON, FLT_MAX, FLT_MAX + FLT_EPSILON}; return values[fuzz_rand() % std::size(values)];
}
case 2:
{
static constexpr UInt64 scales[] = {0, 1, 2, 10};
return DecimalField<Decimal64>(
bad_int64_values[fuzz_rand() % (sizeof(bad_int64_values) / sizeof(*bad_int64_values))],
static_cast<UInt32>(scales[fuzz_rand() % (sizeof(scales) / sizeof(*scales))])
bad_int64_values[fuzz_rand() % std::size(bad_int64_values)],
static_cast<UInt32>(scales[fuzz_rand() % std::size(scales)])
);
}
default:
@ -165,7 +164,8 @@ Field QueryFuzzer::fuzzField(Field field)
{
size_t pos = fuzz_rand() % arr.size();
arr.erase(arr.begin() + pos);
std::cerr << "erased\n";
if (debug_stream)
*debug_stream << "erased\n";
}
if (fuzz_rand() % 5 == 0)
@ -174,12 +174,14 @@ Field QueryFuzzer::fuzzField(Field field)
{
size_t pos = fuzz_rand() % arr.size();
arr.insert(arr.begin() + pos, fuzzField(arr[pos]));
std::cerr << fmt::format("inserted (pos {})\n", pos);
if (debug_stream)
*debug_stream << fmt::format("inserted (pos {})\n", pos);
}
else
{
arr.insert(arr.begin(), getRandomField(0));
std::cerr << "inserted (0)\n";
if (debug_stream)
*debug_stream << "inserted (0)\n";
}
}
@ -197,7 +199,9 @@ Field QueryFuzzer::fuzzField(Field field)
{
size_t pos = fuzz_rand() % arr.size();
arr.erase(arr.begin() + pos);
std::cerr << "erased\n";
if (debug_stream)
*debug_stream << "erased\n";
}
if (fuzz_rand() % 5 == 0)
@ -206,12 +210,16 @@ Field QueryFuzzer::fuzzField(Field field)
{
size_t pos = fuzz_rand() % arr.size();
arr.insert(arr.begin() + pos, fuzzField(arr[pos]));
std::cerr << fmt::format("inserted (pos {})\n", pos);
if (debug_stream)
*debug_stream << fmt::format("inserted (pos {})\n", pos);
}
else
{
arr.insert(arr.begin(), getRandomField(0));
std::cerr << "inserted (0)\n";
if (debug_stream)
*debug_stream << "inserted (0)\n";
}
}
@ -344,7 +352,8 @@ void QueryFuzzer::fuzzOrderByList(IAST * ast)
}
else
{
std::cerr << "No random column.\n";
if (debug_stream)
*debug_stream << "No random column.\n";
}
}
@ -378,7 +387,8 @@ void QueryFuzzer::fuzzColumnLikeExpressionList(IAST * ast)
if (col)
impl->children.insert(pos, col);
else
std::cerr << "No random column.\n";
if (debug_stream)
*debug_stream << "No random column.\n";
}
// We don't have to recurse here to fuzz the children, this is handled by
@ -1361,11 +1371,15 @@ void QueryFuzzer::fuzzMain(ASTPtr & ast)
collectFuzzInfoMain(ast);
fuzz(ast);
std::cout << std::endl;
WriteBufferFromOStream ast_buf(std::cout, 4096);
if (out_stream)
{
*out_stream << std::endl;
WriteBufferFromOStream ast_buf(*out_stream, 4096);
formatAST(*ast, ast_buf, false /*highlight*/);
ast_buf.finalize();
std::cout << std::endl << std::endl;
*out_stream << std::endl << std::endl;
}
}
}

35
src/Client/QueryFuzzer.h → src/Common/QueryFuzzer.h
View File

@ -35,9 +35,31 @@ struct ASTWindowDefinition;
* queries, so you want to feed it a lot of queries to get some interesting mix
* of them. Normally we feed SQL regression tests to it.
*/
struct QueryFuzzer
class QueryFuzzer
{
pcg64 fuzz_rand{randomSeed()};
public:
explicit QueryFuzzer(pcg64 fuzz_rand_ = randomSeed(), std::ostream * out_stream_ = nullptr, std::ostream * debug_stream_ = nullptr)
: fuzz_rand(fuzz_rand_)
, out_stream(out_stream_)
, debug_stream(debug_stream_)
{
}
// This is the only function you have to call -- it will modify the passed
// ASTPtr to point to new AST with some random changes.
void fuzzMain(ASTPtr & ast);
ASTs getInsertQueriesForFuzzedTables(const String & full_query);
ASTs getDropQueriesForFuzzedTables(const ASTDropQuery & drop_query);
void notifyQueryFailed(ASTPtr ast);
static bool isSuitableForFuzzing(const ASTCreateQuery & create);
private:
pcg64 fuzz_rand;
std::ostream * out_stream = nullptr;
std::ostream * debug_stream = nullptr;
// We add elements to expression lists with fixed probability. Some elements
// are so large, that the expected number of elements we add to them is
@ -66,10 +88,6 @@ struct QueryFuzzer
std::unordered_map<std::string, size_t> index_of_fuzzed_table;
std::set<IAST::Hash> created_tables_hashes;
// This is the only function you have to call -- it will modify the passed
// ASTPtr to point to new AST with some random changes.
void fuzzMain(ASTPtr & ast);
// Various helper functions follow, normally you shouldn't have to call them.
Field getRandomField(int type);
Field fuzzField(Field field);
@ -77,9 +95,6 @@ struct QueryFuzzer
ASTPtr getRandomExpressionList();
DataTypePtr fuzzDataType(DataTypePtr type);
DataTypePtr getRandomType();
ASTs getInsertQueriesForFuzzedTables(const String & full_query);
ASTs getDropQueriesForFuzzedTables(const ASTDropQuery & drop_query);
void notifyQueryFailed(ASTPtr ast);
void replaceWithColumnLike(ASTPtr & ast);
void replaceWithTableLike(ASTPtr & ast);
void fuzzOrderByElement(ASTOrderByElement * elem);
@ -102,8 +117,6 @@ struct QueryFuzzer
void addTableLike(ASTPtr ast);
void addColumnLike(ASTPtr ast);
void collectFuzzInfoRecurse(ASTPtr ast);
static bool isSuitableForFuzzing(const ASTCreateQuery & create);
};
}

4
src/Compression/CompressionFactory.cpp
View File

@ -185,7 +185,6 @@ void registerCodecDeflateQpl(CompressionCodecFactory & factory);
/// Keeper use only general-purpose codecs, so we don't need these special codecs
/// in standalone build
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
void registerCodecDelta(CompressionCodecFactory & factory);
void registerCodecT64(CompressionCodecFactory & factory);
void registerCodecDoubleDelta(CompressionCodecFactory & factory);
@ -193,7 +192,6 @@ void registerCodecGorilla(CompressionCodecFactory & factory);
void registerCodecEncrypted(CompressionCodecFactory & factory);
void registerCodecFPC(CompressionCodecFactory & factory);
void registerCodecGCD(CompressionCodecFactory & factory);
#endif
CompressionCodecFactory::CompressionCodecFactory()
{
@ -205,7 +203,6 @@ CompressionCodecFactory::CompressionCodecFactory()
#endif
registerCodecLZ4HC(*this);
registerCodecMultiple(*this);
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
registerCodecDelta(*this);
registerCodecT64(*this);
registerCodecDoubleDelta(*this);
@ -216,7 +213,6 @@ CompressionCodecFactory::CompressionCodecFactory()
registerCodecDeflateQpl(*this);
#endif
registerCodecGCD(*this);
#endif
default_codec = get("LZ4", {});
}

486
src/Coordination/Standalone/Context.cpp
View File

@ -1,486 +0,0 @@
#include <Interpreters/Context.h>
#include <Common/Config/ConfigProcessor.h>
#include <Common/Macros.h>
#include <Common/ThreadPool.h>
#include <Common/callOnce.h>
#include <Disks/IO/IOUringReader.h>
#include <IO/S3Settings.h>
#include <Disks/IO/getIOUringReader.h>
#include <Core/ServerSettings.h>
#include <boost/noncopyable.hpp>
#include <memory>
#include <cassert>
namespace ProfileEvents
{
extern const Event ContextLock;
extern const Event ContextLockWaitMicroseconds;
}
namespace CurrentMetrics
{
extern const Metric ContextLockWait;
extern const Metric BackgroundSchedulePoolTask;
extern const Metric BackgroundSchedulePoolSize;
extern const Metric IOWriterThreads;
extern const Metric IOWriterThreadsActive;
extern const Metric IOWriterThreadsScheduled;
}
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int UNSUPPORTED_METHOD;
}
struct ContextSharedPart : boost::noncopyable
{
ContextSharedPart()
: macros(std::make_unique<Macros>())
{}
~ContextSharedPart()
{
if (keeper_dispatcher)
{
try
{
keeper_dispatcher->shutdown();
}
catch (...)
{
tryLogCurrentException(__PRETTY_FUNCTION__);
}
}
/// Wait for thread pool for background reads and writes,
/// since it may use per-user MemoryTracker which will be destroyed here.
if (asynchronous_remote_fs_reader)
{
try
{
asynchronous_remote_fs_reader->wait();
asynchronous_remote_fs_reader.reset();
}
catch (...)
{
tryLogCurrentException(__PRETTY_FUNCTION__);
}
}
if (asynchronous_local_fs_reader)
{
try
{
asynchronous_local_fs_reader->wait();
asynchronous_local_fs_reader.reset();
}
catch (...)
{
tryLogCurrentException(__PRETTY_FUNCTION__);
}
}
if (synchronous_local_fs_reader)
{
try
{
synchronous_local_fs_reader->wait();
synchronous_local_fs_reader.reset();
}
catch (...)
{
tryLogCurrentException(__PRETTY_FUNCTION__);
}
}
if (threadpool_writer)
{
try
{
threadpool_writer->wait();
threadpool_writer.reset();
}
catch (...)
{
tryLogCurrentException(__PRETTY_FUNCTION__);
}
}
}
/// For access of most of shared objects.
mutable SharedMutex mutex;
ServerSettings server_settings;
String path; /// Path to the data directory, with a slash at the end.
ConfigurationPtr config; /// Global configuration settings.
MultiVersion<Macros> macros; /// Substitutions extracted from config.
OnceFlag schedule_pool_initialized;
mutable std::unique_ptr<BackgroundSchedulePool> schedule_pool; /// A thread pool that can run different jobs in background
RemoteHostFilter remote_host_filter; /// Allowed URL from config.xml
mutable OnceFlag readers_initialized;
mutable std::unique_ptr<IAsynchronousReader> asynchronous_remote_fs_reader;
mutable std::unique_ptr<IAsynchronousReader> asynchronous_local_fs_reader;
mutable std::unique_ptr<IAsynchronousReader> synchronous_local_fs_reader;
#if USE_LIBURING
mutable OnceFlag io_uring_reader_initialized;
mutable std::unique_ptr<IOUringReader> io_uring_reader;
#endif
mutable OnceFlag threadpool_writer_initialized;
mutable std::unique_ptr<ThreadPool> threadpool_writer;
mutable ThrottlerPtr remote_read_throttler; /// A server-wide throttler for remote IO reads
mutable ThrottlerPtr remote_write_throttler; /// A server-wide throttler for remote IO writes
mutable ThrottlerPtr local_read_throttler; /// A server-wide throttler for local IO reads
mutable ThrottlerPtr local_write_throttler; /// A server-wide throttler for local IO writes
std::optional<S3SettingsByEndpoint> storage_s3_settings TSA_GUARDED_BY(mutex); /// Settings of S3 storage
mutable std::mutex keeper_dispatcher_mutex;
mutable std::shared_ptr<KeeperDispatcher> keeper_dispatcher TSA_GUARDED_BY(keeper_dispatcher_mutex);
};
ContextData::ContextData() = default;
ContextData::ContextData(const ContextData &) = default;
Context::Context() = default;
Context::Context(const Context & rhs) : ContextData(rhs), std::enable_shared_from_this<Context>(rhs) {}
Context::~Context() = default;
SharedContextHolder::SharedContextHolder(SharedContextHolder &&) noexcept = default;
SharedContextHolder & SharedContextHolder::operator=(SharedContextHolder &&) noexcept = default;
SharedContextHolder::SharedContextHolder() = default;
SharedContextHolder::~SharedContextHolder() = default;
SharedContextHolder::SharedContextHolder(std::unique_ptr<ContextSharedPart> shared_context)
: shared(std::move(shared_context)) {}
void SharedContextHolder::reset() { shared.reset(); }
void Context::makeGlobalContext()
{
initGlobal();
global_context = shared_from_this();
}
ContextMutablePtr Context::createGlobal(ContextSharedPart * shared_part)
{
auto res = std::shared_ptr<Context>(new Context);
res->shared = shared_part;
return res;
}
void Context::initGlobal()
{
assert(!global_context_instance);
global_context_instance = shared_from_this();
}
SharedContextHolder Context::createShared()
{
return SharedContextHolder(std::make_unique<ContextSharedPart>());
}
ContextMutablePtr Context::getGlobalContext() const
{
auto ptr = global_context.lock();
if (!ptr) throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no global context or global context has expired");
return ptr;
}
std::unique_lock<SharedMutex> Context::getGlobalLock() const
{
ProfileEvents::increment(ProfileEvents::ContextLock);
CurrentMetrics::Increment increment{CurrentMetrics::ContextLockWait};
Stopwatch watch;
auto lock = std::unique_lock(shared->mutex);
ProfileEvents::increment(ProfileEvents::ContextLockWaitMicroseconds, watch.elapsedMicroseconds());
return lock;
}
std::shared_lock<SharedMutex> Context::getGlobalSharedLock() const
{
ProfileEvents::increment(ProfileEvents::ContextLock);
CurrentMetrics::Increment increment{CurrentMetrics::ContextLockWait};
Stopwatch watch;
auto lock = std::shared_lock(shared->mutex);
ProfileEvents::increment(ProfileEvents::ContextLockWaitMicroseconds, watch.elapsedMicroseconds());
return lock;
}
std::unique_lock<SharedMutex> Context::getLocalLock() const
{
ProfileEvents::increment(ProfileEvents::ContextLock);
CurrentMetrics::Increment increment{CurrentMetrics::ContextLockWait};
Stopwatch watch;
auto lock = std::unique_lock(mutex);
ProfileEvents::increment(ProfileEvents::ContextLockWaitMicroseconds, watch.elapsedMicroseconds());
return lock;
}
std::shared_lock<SharedMutex> Context::getLocalSharedLock() const
{
ProfileEvents::increment(ProfileEvents::ContextLock);
CurrentMetrics::Increment increment{CurrentMetrics::ContextLockWait};
Stopwatch watch;
auto lock = std::shared_lock(mutex);
ProfileEvents::increment(ProfileEvents::ContextLockWaitMicroseconds, watch.elapsedMicroseconds());
return lock;
}
String Context::getPath() const
{
auto lock = getGlobalSharedLock();
return shared->path;
}
void Context::setPath(const String & path)
{
auto lock = getGlobalLock();
shared->path = path;
}
MultiVersion<Macros>::Version Context::getMacros() const
{
return shared->macros.get();
}
void Context::setMacros(std::unique_ptr<Macros> && macros)
{
shared->macros.set(std::move(macros));
}
BackgroundSchedulePool & Context::getSchedulePool() const
{
callOnce(shared->schedule_pool_initialized, [&] {
shared->schedule_pool = std::make_unique<BackgroundSchedulePool>(
shared->server_settings.background_schedule_pool_size,
CurrentMetrics::BackgroundSchedulePoolTask,
CurrentMetrics::BackgroundSchedulePoolSize,
"BgSchPool");
});
return *shared->schedule_pool;
}
void Context::setRemoteHostFilter(const Poco::Util::AbstractConfiguration & config)
{
shared->remote_host_filter.setValuesFromConfig(config);
}
const RemoteHostFilter & Context::getRemoteHostFilter() const
{
return shared->remote_host_filter;
}
IAsynchronousReader & Context::getThreadPoolReader(FilesystemReaderType type) const
{
callOnce(shared->readers_initialized, [&] {
const auto & config = getConfigRef();
shared->asynchronous_remote_fs_reader = createThreadPoolReader(FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER, config);
shared->asynchronous_local_fs_reader = createThreadPoolReader(FilesystemReaderType::ASYNCHRONOUS_LOCAL_FS_READER, config);
shared->synchronous_local_fs_reader = createThreadPoolReader(FilesystemReaderType::SYNCHRONOUS_LOCAL_FS_READER, config);
});
switch (type)
{
case FilesystemReaderType::ASYNCHRONOUS_REMOTE_FS_READER:
return *shared->asynchronous_remote_fs_reader;
case FilesystemReaderType::ASYNCHRONOUS_LOCAL_FS_READER:
return *shared->asynchronous_local_fs_reader;
case FilesystemReaderType::SYNCHRONOUS_LOCAL_FS_READER:
return *shared->synchronous_local_fs_reader;
}
}
#if USE_LIBURING
IOUringReader & Context::getIOUringReader() const
{
callOnce(shared->io_uring_reader_initialized, [&] {
shared->io_uring_reader = createIOUringReader();
});
return *shared->io_uring_reader;
}
#endif
std::shared_ptr<FilesystemCacheLog> Context::getFilesystemCacheLog() const
{
return nullptr;
}
std::shared_ptr<FilesystemReadPrefetchesLog> Context::getFilesystemReadPrefetchesLog() const
{
return nullptr;
}
std::shared_ptr<BlobStorageLog> Context::getBlobStorageLog() const
{
return nullptr;
}
void Context::setConfig(const ConfigurationPtr & config)
{
auto lock = getGlobalLock();
shared->config = config;
}
const Poco::Util::AbstractConfiguration & Context::getConfigRef() const
{
auto lock = getGlobalSharedLock();
return shared->config ? *shared->config : Poco::Util::Application::instance().config();
}
std::shared_ptr<AsyncReadCounters> Context::getAsyncReadCounters() const
{
auto lock = getLocalLock();
if (!async_read_counters)
async_read_counters = std::make_shared<AsyncReadCounters>();
return async_read_counters;
}
ThreadPool & Context::getThreadPoolWriter() const
{
callOnce(shared->threadpool_writer_initialized, [&] {
const auto & config = getConfigRef();
auto pool_size = config.getUInt(".threadpool_writer_pool_size", 100);
auto queue_size = config.getUInt(".threadpool_writer_queue_size", 1000000);
shared->threadpool_writer = std::make_unique<ThreadPool>(
CurrentMetrics::IOWriterThreads, CurrentMetrics::IOWriterThreadsActive, CurrentMetrics::IOWriterThreadsScheduled, pool_size, pool_size, queue_size);
});
return *shared->threadpool_writer;
}
ThrottlerPtr Context::getRemoteReadThrottler() const
{
return nullptr;
}
ThrottlerPtr Context::getRemoteWriteThrottler() const
{
return nullptr;
}
ThrottlerPtr Context::getLocalReadThrottler() const
{
return nullptr;
}
ThrottlerPtr Context::getLocalWriteThrottler() const
{
return nullptr;
}
ReadSettings Context::getReadSettings() const
{
return ReadSettings{};
}
ResourceManagerPtr Context::getResourceManager() const
{
return nullptr;
}
ClassifierPtr Context::getWorkloadClassifier() const
{
return nullptr;
}
void Context::initializeKeeperDispatcher([[maybe_unused]] bool start_async) const
{
const auto & config_ref = getConfigRef();
std::lock_guard lock(shared->keeper_dispatcher_mutex);
if (shared->keeper_dispatcher)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to initialize Keeper multiple times");
if (config_ref.has("keeper_server"))
{
shared->keeper_dispatcher = std::make_shared<KeeperDispatcher>();
shared->keeper_dispatcher->initialize(config_ref, true, start_async, getMacros());
}
}
std::shared_ptr<KeeperDispatcher> Context::getKeeperDispatcher() const
{
std::lock_guard lock(shared->keeper_dispatcher_mutex);
if (!shared->keeper_dispatcher)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Keeper must be initialized before requests");
return shared->keeper_dispatcher;
}
std::shared_ptr<KeeperDispatcher> Context::tryGetKeeperDispatcher() const
{
std::lock_guard lock(shared->keeper_dispatcher_mutex);
return shared->keeper_dispatcher;
}
void Context::shutdownKeeperDispatcher() const
{
std::lock_guard lock(shared->keeper_dispatcher_mutex);
if (shared->keeper_dispatcher)
{
shared->keeper_dispatcher->shutdown();
shared->keeper_dispatcher.reset();
}
}
void Context::updateKeeperConfiguration([[maybe_unused]] const Poco::Util::AbstractConfiguration & config_)
{
std::lock_guard lock(shared->keeper_dispatcher_mutex);
if (!shared->keeper_dispatcher)
return;
shared->keeper_dispatcher->updateConfiguration(config_, getMacros());
}
std::shared_ptr<zkutil::ZooKeeper> Context::getZooKeeper() const
{
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Cannot connect to ZooKeeper from Keeper");
}
const S3SettingsByEndpoint & Context::getStorageS3Settings() const
{
std::lock_guard lock(shared->mutex);
if (!shared->storage_s3_settings)
{
const auto & config = shared->config ? *shared->config : Poco::Util::Application::instance().config();
shared->storage_s3_settings.emplace().loadFromConfig(config, "s3", getSettingsRef());
}
return *shared->storage_s3_settings;
}
const ServerSettings & Context::getServerSettings() const
{
return shared->server_settings;
}
bool Context::hasTraceCollector() const
{
return false;
}
bool Context::isBackgroundOperationContext() const
{
return false;
}
}

178
src/Coordination/Standalone/Context.h
View File

@ -1,178 +0,0 @@
#pragma once
#include <Interpreters/Context_fwd.h>
#include <Coordination/KeeperDispatcher.h>
#include <Common/MultiVersion.h>
#include <Common/RemoteHostFilter.h>
#include <Common/SharedMutex.h>
#include <Disks/IO/getThreadPoolReader.h>
#include <Core/Settings.h>
#include <Core/ServerSettings.h>
#include <Core/BackgroundSchedulePool.h>
#include <IO/AsyncReadCounters.h>
#include <Common/Scheduler/IResourceManager.h>
#include <Poco/Util/Application.h>
#include <memory>
#include "config.h"
namespace zkutil
{
class ZooKeeper;
using ZooKeeperPtr = std::shared_ptr<ZooKeeper>;
}
namespace DB
{
struct ContextSharedPart;
class Macros;
class FilesystemCacheLog;
class FilesystemReadPrefetchesLog;
class BlobStorageLog;
class IOUringReader;
class S3SettingsByEndpoint;
/// A small class which owns ContextShared.
/// We don't use something like unique_ptr directly to allow ContextShared type to be incomplete.
struct SharedContextHolder
{
~SharedContextHolder();
SharedContextHolder();
explicit SharedContextHolder(std::unique_ptr<ContextSharedPart> shared_context);
SharedContextHolder(SharedContextHolder &&) noexcept;
SharedContextHolder & operator=(SharedContextHolder &&) noexcept;
ContextSharedPart * get() const { return shared.get(); }
void reset();
private:
std::unique_ptr<ContextSharedPart> shared;
};
class ContextData
{
protected:
ContextWeakMutablePtr global_context;
inline static ContextPtr global_context_instance;
ContextSharedPart * shared;
/// Query metrics for reading data asynchronously with IAsynchronousReader.
mutable std::shared_ptr<AsyncReadCounters> async_read_counters;
Settings settings; /// Setting for query execution.
public:
/// Use copy constructor or createGlobal() instead
ContextData();
ContextData(const ContextData &);
};
class Context : public ContextData, public std::enable_shared_from_this<Context>
{
private:
/// ContextData mutex
mutable SharedMutex mutex;
Context();
Context(const Context &);
std::unique_lock<SharedMutex> getGlobalLock() const;
std::shared_lock<SharedMutex> getGlobalSharedLock() const;
std::unique_lock<SharedMutex> getLocalLock() const;
std::shared_lock<SharedMutex> getLocalSharedLock() const;
public:
/// Create initial Context with ContextShared and etc.
static ContextMutablePtr createGlobal(ContextSharedPart * shared_part);
static SharedContextHolder createShared();
ContextMutablePtr getGlobalContext() const;
static ContextPtr getGlobalContextInstance() { return global_context_instance; }
void makeGlobalContext();
void initGlobal();
~Context();
using ConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfiguration>;
/// Global application configuration settings.
void setConfig(const ConfigurationPtr & config);
const Poco::Util::AbstractConfiguration & getConfigRef() const;
const Settings & getSettingsRef() const { return settings; }
String getPath() const;
void setPath(const String & path);
MultiVersion<Macros>::Version getMacros() const;
void setMacros(std::unique_ptr<Macros> && macros);
BackgroundSchedulePool & getSchedulePool() const;
/// Storage of allowed hosts from config.xml
void setRemoteHostFilter(const Poco::Util::AbstractConfiguration & config);
const RemoteHostFilter & getRemoteHostFilter() const;
std::shared_ptr<FilesystemCacheLog> getFilesystemCacheLog() const;
std::shared_ptr<FilesystemReadPrefetchesLog> getFilesystemReadPrefetchesLog() const;
std::shared_ptr<BlobStorageLog> getBlobStorageLog() const;
enum class ApplicationType : uint8_t
{
KEEPER,
SERVER,
};
void setApplicationType(ApplicationType) {}
ApplicationType getApplicationType() const { return ApplicationType::KEEPER; }
IAsynchronousReader & getThreadPoolReader(FilesystemReaderType type) const;
#if USE_LIBURING
IOUringReader & getIOUringReader() const;
#endif
std::shared_ptr<AsyncReadCounters> getAsyncReadCounters() const;
ThreadPool & getThreadPoolWriter() const;
ThrottlerPtr getRemoteReadThrottler() const;
ThrottlerPtr getRemoteWriteThrottler() const;
ThrottlerPtr getLocalReadThrottler() const;
ThrottlerPtr getLocalWriteThrottler() const;
ReadSettings getReadSettings() const;
/// Resource management related
ResourceManagerPtr getResourceManager() const;
ClassifierPtr getWorkloadClassifier() const;
std::shared_ptr<KeeperDispatcher> getKeeperDispatcher() const;
std::shared_ptr<KeeperDispatcher> tryGetKeeperDispatcher() const;
void initializeKeeperDispatcher(bool start_async) const;
void shutdownKeeperDispatcher() const;
void updateKeeperConfiguration(const Poco::Util::AbstractConfiguration & config);
zkutil::ZooKeeperPtr getZooKeeper() const;
const S3SettingsByEndpoint & getStorageS3Settings() const;
const String & getUserName() const { static std::string user; return user; }
const ServerSettings & getServerSettings() const;
bool hasTraceCollector() const;
bool isBackgroundOperationContext() const;
};
}

24
src/Coordination/Standalone/Settings.cpp
View File

@ -1,24 +0,0 @@
#include <Core/Settings.h>
namespace DB
{
IMPLEMENT_SETTINGS_TRAITS(SettingsTraits, LIST_OF_SETTINGS)
std::vector<String> Settings::getAllRegisteredNames() const
{
std::vector<String> all_settings;
for (const auto & setting_field : all())
{
all_settings.push_back(setting_field.getName());
}
return all_settings;
}
void Settings::set(std::string_view name, const Field & value)
{
BaseSettings::set(name, value);
}
}

19
src/Coordination/Standalone/ThreadStatusExt.cpp
View File

@ -1,19 +0,0 @@
#include <Common/CurrentThread.h>
#include <Common/ThreadStatus.h>
namespace DB
{
void CurrentThread::detachFromGroupIfNotDetached()
{
}
void CurrentThread::attachToGroup(const ThreadGroupPtr &)
{
}
void ThreadStatus::initGlobalProfiler(UInt64 /*global_profiler_real_time_period*/, UInt64 /*global_profiler_cpu_time_period*/)
{
}
}

12
src/Core/Settings.h
View File

@ -36,7 +36,7 @@ class IColumn;
M(Dialect, dialect, Dialect::clickhouse, "Which dialect will be used to parse query", 0)\
M(UInt64, min_compress_block_size, 65536, "The actual size of the block to compress, if the uncompressed data less than max_compress_block_size is no less than this value and no less than the volume of data for one mark.", 0) \
M(UInt64, max_compress_block_size, 1048576, "The maximum size of blocks of uncompressed data before compressing for writing to a table.", 0) \
M(UInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size for reading", 0) \
M(UInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size in rows for reading", 0) \
M(UInt64, max_insert_block_size, DEFAULT_INSERT_BLOCK_SIZE, "The maximum block size for insertion, if we control the creation of blocks for insertion.", 0) \
M(UInt64, min_insert_block_size_rows, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough.", 0) \
M(UInt64, min_insert_block_size_bytes, (DEFAULT_INSERT_BLOCK_SIZE * 256), "Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough.", 0) \
@ -125,6 +125,9 @@ class IColumn;
M(Bool, s3_ignore_file_doesnt_exist, false, "Return 0 rows when the requested files don't exist, instead of throwing an exception in S3 table engine", 0) \
M(Bool, hdfs_ignore_file_doesnt_exist, false, "Return 0 rows when the requested files don't exist, instead of throwing an exception in HDFS table engine", 0) \
M(Bool, azure_ignore_file_doesnt_exist, false, "Return 0 rows when the requested files don't exist, instead of throwing an exception in AzureBlobStorage table engine", 0) \
M(UInt64, azure_sdk_max_retries, 10, "Maximum number of retries in azure sdk", 0) \
M(UInt64, azure_sdk_retry_initial_backoff_ms, 10, "Minimal backoff between retries in azure sdk", 0) \
M(UInt64, azure_sdk_retry_max_backoff_ms, 1000, "Maximal backoff between retries in azure sdk", 0) \
M(Bool, s3_validate_request_settings, true, "Validate S3 request settings", 0) \
M(Bool, s3_disable_checksum, S3::DEFAULT_DISABLE_CHECKSUM, "Do not calculate a checksum when sending a file to S3. This speeds up writes by avoiding excessive processing passes on a file. It is mostly safe as the data of MergeTree tables is checksummed by ClickHouse anyway, and when S3 is accessed with HTTPS, the TLS layer already provides integrity while transferring through the network. While additional checksums on S3 give defense in depth.", 0) \
M(UInt64, s3_retry_attempts, S3::DEFAULT_RETRY_ATTEMPTS, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries", 0) \
@ -399,7 +402,7 @@ class IColumn;
M(Float, opentelemetry_start_trace_probability, 0., "Probability to start an OpenTelemetry trace for an incoming query.", 0) \
M(Bool, opentelemetry_trace_processors, false, "Collect OpenTelemetry spans for processors.", 0) \
M(Bool, prefer_column_name_to_alias, false, "Prefer using column names instead of aliases if possible.", 0) \
M(Bool, allow_experimental_analyzer, true, "Allow experimental analyzer.", 0) \
M(Bool, allow_experimental_analyzer, true, "Allow experimental analyzer.", IMPORTANT) \
M(Bool, analyzer_compatibility_join_using_top_level_identifier, false, "Force to resolve identifier in JOIN USING from projection (for example, in `SELECT a + 1 AS b FROM t1 JOIN t2 USING (b)` join will be performed by `t1.a + 1 = t2.b`, rather then `t1.b = t2.b`).", 0) \
M(Bool, prefer_global_in_and_join, false, "If enabled, all IN/JOIN operators will be rewritten as GLOBAL IN/JOIN. It's useful when the to-be-joined tables are only available on the initiator and we need to always scatter their data on-the-fly during distributed processing with the GLOBAL keyword. It's also useful to reduce the need to access the external sources joining external tables.", 0) \
M(Bool, enable_vertical_final, true, "If enable, remove duplicated rows during FINAL by marking rows as deleted and filtering them later instead of merging rows", 0) \
@ -612,6 +615,7 @@ class IColumn;
M(UInt64, mutations_sync, 0, "Wait for synchronous execution of ALTER TABLE UPDATE/DELETE queries (mutations). 0 - execute asynchronously. 1 - wait current server. 2 - wait all replicas if they exist.", 0) \
M(Bool, enable_lightweight_delete, true, "Enable lightweight DELETE mutations for mergetree tables.", 0) ALIAS(allow_experimental_lightweight_delete) \
M(UInt64, lightweight_deletes_sync, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes", 0) \
M(LightweightMutationProjectionMode, lightweight_mutation_projection_mode, LightweightMutationProjectionMode::THROW, "When lightweight delete happens on a table with projection(s), the possible operations include throw the exception as projection exists, or drop all projection related to this table then do lightweight delete.", 0) \
M(Bool, apply_deleted_mask, true, "Enables filtering out rows deleted with lightweight DELETE. If disabled, a query will be able to read those rows. This is useful for debugging and \"undelete\" scenarios", 0) \
M(Bool, optimize_normalize_count_variants, true, "Rewrite aggregate functions that semantically equals to count() as count().", 0) \
M(Bool, optimize_injective_functions_inside_uniq, true, "Delete injective functions of one argument inside uniq*() functions.", 0) \
@ -630,9 +634,8 @@ class IColumn;
M(Bool, optimize_time_filter_with_preimage, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')", 0) \
M(Bool, normalize_function_names, true, "Normalize function names to their canonical names", 0) \
M(Bool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there are constants there", 0) \
M(Bool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views if the block is not a duplicate for the table. Use true to always deduplicate in dependent tables.", 0) \
M(Bool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views. Use true to always deduplicate in dependent tables.", 0) \
M(Bool, throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert, true, "Throw exception on INSERT query when the setting `deduplicate_blocks_in_dependent_materialized_views` is enabled along with `async_insert`. It guarantees correctness, because these features can't work together.", 0) \
M(Bool, update_insert_deduplication_token_in_dependent_materialized_views, false, "Should update insert deduplication token with table identifier during insert in dependent materialized views.", 0) \
M(Bool, materialized_views_ignore_errors, false, "Allows to ignore errors for MATERIALIZED VIEW, and deliver original block to the table regardless of MVs", 0) \
M(Bool, ignore_materialized_views_with_dropped_target_table, false, "Ignore MVs with dropped target table during pushing to views", 0) \
M(Bool, allow_experimental_refreshable_materialized_view, false, "Allow refreshable materialized views (CREATE MATERIALIZED VIEW <name> REFRESH ...).", 0) \
@ -949,6 +952,7 @@ class IColumn;
#define OBSOLETE_SETTINGS(M, ALIAS) \
/** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
MAKE_OBSOLETE(M, Bool, update_insert_deduplication_token_in_dependent_materialized_views, 1) \
MAKE_OBSOLETE(M, UInt64, max_memory_usage_for_all_queries, 0) \
MAKE_OBSOLETE(M, UInt64, multiple_joins_rewriter_version, 0) \
MAKE_OBSOLETE(M, Bool, enable_debug_queries, false) \

4
src/Core/SettingsChangesHistory.cpp
View File

@ -62,7 +62,11 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
{"input_format_json_ignore_key_case", false, false, "Ignore json key case while read json field from string."},
{"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
{"input_format_orc_read_use_writer_time_zone", false, false, "Whether use the writer's time zone in ORC stripe for ORC row reader, the default ORC row reader's time zone is GMT."},
{"lightweight_mutation_projection_mode", "throw", "throw", "When lightweight delete happens on a table with projection(s), the possible operations include throw the exception as projection exists, or drop all projection related to this table then do lightweight delete."},
{"database_replicated_allow_heavy_create", true, false, "Long-running DDL queries (CREATE AS SELECT and POPULATE) for Replicated database engine was forbidden"},
{"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"},
{"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"},
{"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"},
}},
{"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
{"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},

4
src/Core/SettingsEnums.cpp
View File

@ -173,6 +173,10 @@ IMPLEMENT_SETTING_ENUM(ParallelReplicasCustomKeyFilterType, ErrorCodes::BAD_ARGU
{{"default", ParallelReplicasCustomKeyFilterType::DEFAULT},
{"range", ParallelReplicasCustomKeyFilterType::RANGE}})
IMPLEMENT_SETTING_ENUM(LightweightMutationProjectionMode, ErrorCodes::BAD_ARGUMENTS,
{{"throw", LightweightMutationProjectionMode::THROW},
{"drop", LightweightMutationProjectionMode::DROP}})
IMPLEMENT_SETTING_AUTO_ENUM(LocalFSReadMethod, ErrorCodes::BAD_ARGUMENTS)
IMPLEMENT_SETTING_ENUM(ParquetVersion, ErrorCodes::BAD_ARGUMENTS,

8
src/Core/SettingsEnums.h
View File

@ -339,6 +339,14 @@ enum class ParallelReplicasCustomKeyFilterType : uint8_t
DECLARE_SETTING_ENUM(ParallelReplicasCustomKeyFilterType)
enum class LightweightMutationProjectionMode : uint8_t
{
THROW,
DROP,
};
DECLARE_SETTING_ENUM(LightweightMutationProjectionMode)
DECLARE_SETTING_ENUM(LocalFSReadMethod)
enum class ObjectStorageQueueMode : uint8_t

45
src/Core/SettingsFields.cpp
View File

@ -380,15 +380,6 @@ void SettingFieldString::readBinary(ReadBuffer & in)
*this = std::move(str);
}
/// Unbeautiful workaround for clickhouse-keeper standalone build ("-DBUILD_STANDALONE_KEEPER=1").
/// In this build, we don't build and link library dbms (to which SettingsField.cpp belongs) but
/// only build SettingsField.cpp. Further dependencies, e.g. DataTypeString and DataTypeMap below,
/// require building of further files for clickhouse-keeper. To keep dependencies slim, we don't do
/// that. The linker does not complain only because clickhouse-keeper does not call any of below
/// functions. A cleaner alternative would be more modular libraries, e.g. one for data types, which
/// could then be linked by the server and the linker.
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
SettingFieldMap::SettingFieldMap(const Field & f) : value(fieldToMap(f)) {}
String SettingFieldMap::toString() const
@ -428,42 +419,6 @@ void SettingFieldMap::readBinary(ReadBuffer & in)
*this = map;
}
#else
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
SettingFieldMap::SettingFieldMap(const Field &) : value(Map()) {}
String SettingFieldMap::toString() const
{
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Setting of type Map not supported");
}
SettingFieldMap & SettingFieldMap::operator =(const Field &)
{
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Setting of type Map not supported");
}
void SettingFieldMap::parseFromString(const String &)
{
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Setting of type Map not supported");
}
void SettingFieldMap::writeBinary(WriteBuffer &) const
{
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Setting of type Map not supported");
}
void SettingFieldMap::readBinary(ReadBuffer &)
{
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Setting of type Map not supported");
}
#endif
namespace
{
char stringToChar(const String & str)

14
src/Core/SettingsFields.h
View File

@ -247,12 +247,6 @@ struct SettingFieldString
void readBinary(ReadBuffer & in);
};
#ifdef CLICKHOUSE_KEEPER_STANDALONE_BUILD
#define NORETURN [[noreturn]]
#else
#define NORETURN
#endif
struct SettingFieldMap
{
public:
@ -269,11 +263,11 @@ public:
operator const Map &() const { return value; } /// NOLINT
explicit operator Field() const { return value; }
NORETURN String toString() const;
NORETURN void parseFromString(const String & str);
String toString() const;
void parseFromString(const String & str);
NORETURN void writeBinary(WriteBuffer & out) const;
NORETURN void readBinary(ReadBuffer & in);
void writeBinary(WriteBuffer & out) const;
void readBinary(ReadBuffer & in);
};
#undef NORETURN

5
src/Daemon/BaseDaemon.cpp
View File

@ -502,9 +502,7 @@ private:
if (collectCrashLog)
collectCrashLog(sig, thread_num, query_id, stack_trace);
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
Context::getGlobalContextInstance()->handleCrash();
#endif
/// Send crash report to developers (if configured)
if (sig != SanitizerTrap)
@ -533,8 +531,6 @@ private:
}
}
/// ClickHouse Keeper does not link to some parts of Settings.
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
/// List changed settings.
if (!query_id.empty())
{
@ -549,7 +545,6 @@ private:
LOG_FATAL(log, "Changed settings: {}", changed_settings);
}
}
#endif
/// When everything is done, we will try to send these error messages to the client.
if (thread_ptr)

2
src/Daemon/SentryWriter.cpp
View File

@ -19,7 +19,7 @@
#include "config.h"
#include <Common/config_version.h>
#if USE_SENTRY && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD)
#if USE_SENTRY
# include <sentry.h>
# include <cstdio>

2
src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
View File

@ -78,7 +78,6 @@ SeekableReadBufferPtr ReadBufferFromRemoteFSGather::createImplementationBuffer(c
std::unique_ptr<ReadBufferFromFileBase> buf;
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
if (with_file_cache)
{
auto cache_key = settings.remote_fs_cache->createKeyForPath(object_path);
@ -96,7 +95,6 @@ SeekableReadBufferPtr ReadBufferFromRemoteFSGather::createImplementationBuffer(c
/* read_until_position */std::nullopt,
cache_log);
}
#endif
/// Can't wrap CachedOnDiskReadBufferFromFile in CachedInMemoryReadBufferFromFile because the
/// former doesn't support seeks.

4
src/Disks/IO/WriteBufferFromAzureBlobStorage.cpp
View File

@ -39,7 +39,7 @@ struct WriteBufferFromAzureBlobStorage::PartData
size_t data_size = 0;
};
BufferAllocationPolicyPtr createBufferAllocationPolicy(const AzureObjectStorageSettings & settings)
BufferAllocationPolicyPtr createBufferAllocationPolicy(const AzureBlobStorage::RequestSettings & settings)
{
BufferAllocationPolicy::Settings allocation_settings;
allocation_settings.strict_size = settings.strict_upload_part_size;
@ -57,7 +57,7 @@ WriteBufferFromAzureBlobStorage::WriteBufferFromAzureBlobStorage(
const String & blob_path_,
size_t buf_size_,
const WriteSettings & write_settings_,
std::shared_ptr<const AzureObjectStorageSettings> settings_,
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings_,
ThreadPoolCallbackRunnerUnsafe<void> schedule_)
: WriteBufferFromFileBase(buf_size_, nullptr, 0)
, log(getLogger("WriteBufferFromAzureBlobStorage"))

2
src/Disks/IO/WriteBufferFromAzureBlobStorage.h
View File

@ -35,7 +35,7 @@ public:
const String & blob_path_,
size_t buf_size_,
const WriteSettings & write_settings_,
std::shared_ptr<const AzureObjectStorageSettings> settings_,
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings_,
ThreadPoolCallbackRunnerUnsafe<void> schedule_ = {});
~WriteBufferFromAzureBlobStorage() override;

272
src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp
View File

@ -1,272 +0,0 @@
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.h>
#if USE_AZURE_BLOB_STORAGE
#include <Common/Exception.h>
#include <Common/re2.h>
#include <azure/identity/managed_identity_credential.hpp>
#include <azure/identity/workload_identity_credential.hpp>
#include <azure/storage/blobs/blob_options.hpp>
#include <azure/core/http/curl_transport.hpp>
#include <Poco/Util/AbstractConfiguration.h>
#include <Interpreters/Context.h>
using namespace Azure::Storage::Blobs;
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
void validateStorageAccountUrl(const String & storage_account_url)
{
const auto * storage_account_url_pattern_str = R"(http(()|s)://[a-z0-9-.:]+(()|/)[a-z0-9]*(()|/))";
static const RE2 storage_account_url_pattern(storage_account_url_pattern_str);
if (!re2::RE2::FullMatch(storage_account_url, storage_account_url_pattern))
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Blob Storage URL is not valid, should follow the format: {}, got: {}", storage_account_url_pattern_str, storage_account_url);
}
void validateContainerName(const String & container_name)
{
auto len = container_name.length();
if (len < 3 || len > 64)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"AzureBlob Storage container name is not valid, should have length between 3 and 64, but has length: {}", len);
const auto * container_name_pattern_str = R"([a-z][a-z0-9-]+)";
static const RE2 container_name_pattern(container_name_pattern_str);
if (!re2::RE2::FullMatch(container_name, container_name_pattern))
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"AzureBlob Storage container name is not valid, should follow the format: {}, got: {}",
container_name_pattern_str, container_name);
}
AzureBlobStorageEndpoint processAzureBlobStorageEndpoint(const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
{
String storage_url;
String account_name;
String container_name;
String prefix;
if (config.has(config_prefix + ".endpoint"))
{
String endpoint = config.getString(config_prefix + ".endpoint");
/// For some authentication methods account name is not present in the endpoint
/// 'endpoint_contains_account_name' bool is used to understand how to split the endpoint (default : true)
bool endpoint_contains_account_name = config.getBool(config_prefix + ".endpoint_contains_account_name", true);
size_t pos = endpoint.find("//");
if (pos == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected '//' in endpoint");
if (endpoint_contains_account_name)
{
size_t acc_pos_begin = endpoint.find('/', pos+2);
if (acc_pos_begin == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected account_name in endpoint");
storage_url = endpoint.substr(0,acc_pos_begin);
size_t acc_pos_end = endpoint.find('/',acc_pos_begin+1);
if (acc_pos_end == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected container_name in endpoint");
account_name = endpoint.substr(acc_pos_begin+1,(acc_pos_end-acc_pos_begin)-1);
size_t cont_pos_end = endpoint.find('/', acc_pos_end+1);
if (cont_pos_end != std::string::npos)
{
container_name = endpoint.substr(acc_pos_end+1,(cont_pos_end-acc_pos_end)-1);
prefix = endpoint.substr(cont_pos_end+1);
}
else
{
container_name = endpoint.substr(acc_pos_end+1);
}
}
else
{
size_t cont_pos_begin = endpoint.find('/', pos+2);
if (cont_pos_begin == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected container_name in endpoint");
storage_url = endpoint.substr(0,cont_pos_begin);
size_t cont_pos_end = endpoint.find('/',cont_pos_begin+1);
if (cont_pos_end != std::string::npos)
{
container_name = endpoint.substr(cont_pos_begin+1,(cont_pos_end-cont_pos_begin)-1);
prefix = endpoint.substr(cont_pos_end+1);
}
else
{
container_name = endpoint.substr(cont_pos_begin+1);
}
}
}
else if (config.has(config_prefix + ".connection_string"))
{
storage_url = config.getString(config_prefix + ".connection_string");
container_name = config.getString(config_prefix + ".container_name");
}
else if (config.has(config_prefix + ".storage_account_url"))
{
storage_url = config.getString(config_prefix + ".storage_account_url");
validateStorageAccountUrl(storage_url);
container_name = config.getString(config_prefix + ".container_name");
}
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected either `storage_account_url` or `connection_string` or `endpoint` in config");
if (!container_name.empty())
validateContainerName(container_name);
std::optional<bool> container_already_exists {};
if (config.has(config_prefix + ".container_already_exists"))
container_already_exists = {config.getBool(config_prefix + ".container_already_exists")};
return {storage_url, account_name, container_name, prefix, container_already_exists};
}
template <class T>
std::unique_ptr<T> getClientWithConnectionString(const String & connection_str, const String & container_name, const BlobClientOptions & client_options) = delete;
template<>
std::unique_ptr<BlobServiceClient> getClientWithConnectionString(const String & connection_str, const String & /*container_name*/, const BlobClientOptions & client_options)
{
return std::make_unique<BlobServiceClient>(BlobServiceClient::CreateFromConnectionString(connection_str, client_options));
}
template<>
std::unique_ptr<BlobContainerClient> getClientWithConnectionString(const String & connection_str, const String & container_name, const BlobClientOptions & client_options)
{
return std::make_unique<BlobContainerClient>(BlobContainerClient::CreateFromConnectionString(connection_str, container_name, client_options));
}
template <class T>
std::unique_ptr<T> getAzureBlobStorageClientWithAuth(
const String & url,
const String & container_name,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
const Azure::Storage::Blobs::BlobClientOptions & client_options)
{
std::string connection_str;
if (config.has(config_prefix + ".connection_string"))
connection_str = config.getString(config_prefix + ".connection_string");
if (!connection_str.empty())
return getClientWithConnectionString<T>(connection_str, container_name, client_options);
if (config.has(config_prefix + ".account_key") && config.has(config_prefix + ".account_name"))
{
auto storage_shared_key_credential = std::make_shared<Azure::Storage::StorageSharedKeyCredential>(
config.getString(config_prefix + ".account_name"),
config.getString(config_prefix + ".account_key")
);
return std::make_unique<T>(url, storage_shared_key_credential, client_options);
}
if (config.getBool(config_prefix + ".use_workload_identity", false))
{
auto workload_identity_credential = std::make_shared<Azure::Identity::WorkloadIdentityCredential>();
return std::make_unique<T>(url, workload_identity_credential, client_options);
}
auto managed_identity_credential = std::make_shared<Azure::Identity::ManagedIdentityCredential>();
return std::make_unique<T>(url, managed_identity_credential, client_options);
}
Azure::Storage::Blobs::BlobClientOptions getAzureBlobClientOptions(const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
{
Azure::Core::Http::Policies::RetryOptions retry_options;
retry_options.MaxRetries = config.getUInt(config_prefix + ".max_tries", 10);
retry_options.RetryDelay = std::chrono::milliseconds(config.getUInt(config_prefix + ".retry_initial_backoff_ms", 10));
retry_options.MaxRetryDelay = std::chrono::milliseconds(config.getUInt(config_prefix + ".retry_max_backoff_ms", 1000));
using CurlOptions = Azure::Core::Http::CurlTransportOptions;
CurlOptions curl_options;
curl_options.NoSignal = true;
if (config.has(config_prefix + ".curl_ip_resolve"))
{
auto value = config.getString(config_prefix + ".curl_ip_resolve");
if (value == "ipv4")
curl_options.IPResolve = CurlOptions::CURL_IPRESOLVE_V4;
else if (value == "ipv6")
curl_options.IPResolve = CurlOptions::CURL_IPRESOLVE_V6;
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected value for option 'curl_ip_resolve': {}. Expected one of 'ipv4' or 'ipv6'", value);
}
Azure::Storage::Blobs::BlobClientOptions client_options;
client_options.Retry = retry_options;
client_options.Transport.Transport = std::make_shared<Azure::Core::Http::CurlTransport>(curl_options);
client_options.ClickhouseOptions = Azure::Storage::Blobs::ClickhouseClientOptions{.IsClientForDisk=true};
return client_options;
}
std::unique_ptr<BlobContainerClient> getAzureBlobContainerClient(const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
{
auto endpoint = processAzureBlobStorageEndpoint(config, config_prefix);
auto container_name = endpoint.container_name;
auto final_url = endpoint.getEndpoint();
auto client_options = getAzureBlobClientOptions(config, config_prefix);
if (endpoint.container_already_exists.value_or(false))
return getAzureBlobStorageClientWithAuth<BlobContainerClient>(final_url, container_name, config, config_prefix, client_options);
auto blob_service_client = getAzureBlobStorageClientWithAuth<BlobServiceClient>(endpoint.getEndpointWithoutContainer(), container_name, config, config_prefix, client_options);
try
{
return std::make_unique<BlobContainerClient>(blob_service_client->CreateBlobContainer(container_name).Value);
}
catch (const Azure::Storage::StorageException & e)
{
/// If container_already_exists is not set (in config), ignore already exists error.
/// (Conflict - The specified container already exists)
if (!endpoint.container_already_exists.has_value() && e.StatusCode == Azure::Core::Http::HttpStatusCode::Conflict)
return getAzureBlobStorageClientWithAuth<BlobContainerClient>(final_url, container_name, config, config_prefix, client_options);
throw;
}
}
std::unique_ptr<AzureObjectStorageSettings> getAzureBlobStorageSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context)
{
std::unique_ptr<AzureObjectStorageSettings> settings = std::make_unique<AzureObjectStorageSettings>();
settings->max_single_part_upload_size = config.getUInt64(config_prefix + ".max_single_part_upload_size", context->getSettings().azure_max_single_part_upload_size);
settings->min_bytes_for_seek = config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024);
settings->max_single_read_retries = config.getInt(config_prefix + ".max_single_read_retries", 3);
settings->max_single_download_retries = config.getInt(config_prefix + ".max_single_download_retries", 3);
settings->list_object_keys_size = config.getInt(config_prefix + ".list_object_keys_size", 1000);
settings->min_upload_part_size = config.getUInt64(config_prefix + ".min_upload_part_size", context->getSettings().azure_min_upload_part_size);
settings->max_upload_part_size = config.getUInt64(config_prefix + ".max_upload_part_size", context->getSettings().azure_max_upload_part_size);
settings->max_single_part_copy_size = config.getUInt64(config_prefix + ".max_single_part_copy_size", context->getSettings().azure_max_single_part_copy_size);
settings->use_native_copy = config.getBool(config_prefix + ".use_native_copy", false);
settings->max_blocks_in_multipart_upload = config.getUInt64(config_prefix + ".max_blocks_in_multipart_upload", 50000);
settings->max_unexpected_write_error_retries = config.getUInt64(config_prefix + ".max_unexpected_write_error_retries", context->getSettings().azure_max_unexpected_write_error_retries);
settings->max_inflight_parts_for_one_file = config.getUInt64(config_prefix + ".max_inflight_parts_for_one_file", context->getSettings().azure_max_inflight_parts_for_one_file);
settings->strict_upload_part_size = config.getUInt64(config_prefix + ".strict_upload_part_size", context->getSettings().azure_strict_upload_part_size);
settings->upload_part_size_multiply_factor = config.getUInt64(config_prefix + ".upload_part_size_multiply_factor", context->getSettings().azure_upload_part_size_multiply_factor);
settings->upload_part_size_multiply_parts_count_threshold = config.getUInt64(config_prefix + ".upload_part_size_multiply_parts_count_threshold", context->getSettings().azure_upload_part_size_multiply_parts_count_threshold);
return settings;
}
}
#endif

58
src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.h
View File

@ -1,58 +0,0 @@
#pragma once
#include "config.h"
#if USE_AZURE_BLOB_STORAGE
#include <azure/storage/blobs.hpp>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h>
namespace DB
{
struct AzureBlobStorageEndpoint
{
const String storage_account_url;
const String account_name;
const String container_name;
const String prefix;
const std::optional<bool> container_already_exists;
String getEndpoint()
{
String url = storage_account_url;
if (url.ends_with('/'))
url.pop_back();
if (!account_name.empty())
url += "/" + account_name;
if (!container_name.empty())
url += "/" + container_name;
if (!prefix.empty())
url += "/" + prefix;
return url;
}
String getEndpointWithoutContainer()
{
String url = storage_account_url;
if (!account_name.empty())
url += "/" + account_name;
return url;
}
};
std::unique_ptr<Azure::Storage::Blobs::BlobContainerClient> getAzureBlobContainerClient(const Poco::Util::AbstractConfiguration & config, const String & config_prefix);
AzureBlobStorageEndpoint processAzureBlobStorageEndpoint(const Poco::Util::AbstractConfiguration & config, const String & config_prefix);
std::unique_ptr<AzureObjectStorageSettings> getAzureBlobStorageSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context);
}
#endif

351
src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp Normal file
View File

@ -0,0 +1,351 @@
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
#if USE_AZURE_BLOB_STORAGE
#include <Common/Exception.h>
#include <Common/re2.h>
#include <azure/identity/managed_identity_credential.hpp>
#include <azure/identity/workload_identity_credential.hpp>
#include <azure/storage/blobs/blob_options.hpp>
#include <Poco/Util/AbstractConfiguration.h>
#include <Interpreters/Context.h>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
namespace AzureBlobStorage
{
static void validateStorageAccountUrl(const String & storage_account_url)
{
const auto * storage_account_url_pattern_str = R"(http(()|s)://[a-z0-9-.:]+(()|/)[a-z0-9]*(()|/))";
static const RE2 storage_account_url_pattern(storage_account_url_pattern_str);
if (!re2::RE2::FullMatch(storage_account_url, storage_account_url_pattern))
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Blob Storage URL is not valid, should follow the format: {}, got: {}", storage_account_url_pattern_str, storage_account_url);
}
static void validateContainerName(const String & container_name)
{
auto len = container_name.length();
if (len < 3 || len > 64)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"AzureBlob Storage container name is not valid, should have length between 3 and 64, but has length: {}", len);
const auto * container_name_pattern_str = R"([a-z][a-z0-9-]+)";
static const RE2 container_name_pattern(container_name_pattern_str);
if (!re2::RE2::FullMatch(container_name, container_name_pattern))
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"AzureBlob Storage container name is not valid, should follow the format: {}, got: {}",
container_name_pattern_str, container_name);
}
static bool isConnectionString(const std::string & candidate)
{
return !candidate.starts_with("http");
}
String ConnectionParams::getConnectionURL() const
{
if (std::holds_alternative<ConnectionString>(auth_method))
{
auto parsed_connection_string = Azure::Storage::_internal::ParseConnectionString(endpoint.storage_account_url);
return parsed_connection_string.BlobServiceUrl.GetAbsoluteUrl();
}
return endpoint.storage_account_url;
}
std::unique_ptr<ServiceClient> ConnectionParams::createForService() const
{
return std::visit([this]<typename T>(const T & auth)
{
if constexpr (std::is_same_v<T, ConnectionString>)
return std::make_unique<ServiceClient>(ServiceClient::CreateFromConnectionString(auth.toUnderType(), client_options));
else
return std::make_unique<ServiceClient>(endpoint.getEndpointWithoutContainer(), auth, client_options);
}, auth_method);
}
std::unique_ptr<ContainerClient> ConnectionParams::createForContainer() const
{
return std::visit([this]<typename T>(const T & auth)
{
if constexpr (std::is_same_v<T, ConnectionString>)
return std::make_unique<ContainerClient>(ContainerClient::CreateFromConnectionString(auth.toUnderType(), endpoint.container_name, client_options));
else
return std::make_unique<ContainerClient>(endpoint.getEndpoint(), auth, client_options);
}, auth_method);
}
Endpoint processEndpoint(const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
{
String storage_url;
String account_name;
String container_name;
String prefix;
auto get_container_name = [&]
{
if (config.has(config_prefix + ".container_name"))
return config.getString(config_prefix + ".container_name");
if (config.has(config_prefix + ".container"))
return config.getString(config_prefix + ".container");
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected either `container` or `container_name` parameter in config");
};
if (config.has(config_prefix + ".endpoint"))
{
String endpoint = config.getString(config_prefix + ".endpoint");
/// For some authentication methods account name is not present in the endpoint
/// 'endpoint_contains_account_name' bool is used to understand how to split the endpoint (default : true)
bool endpoint_contains_account_name = config.getBool(config_prefix + ".endpoint_contains_account_name", true);
size_t pos = endpoint.find("//");
if (pos == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected '//' in endpoint");
if (endpoint_contains_account_name)
{
size_t acc_pos_begin = endpoint.find('/', pos + 2);
if (acc_pos_begin == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected account_name in endpoint");
storage_url = endpoint.substr(0, acc_pos_begin);
size_t acc_pos_end = endpoint.find('/', acc_pos_begin + 1);
if (acc_pos_end == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected container_name in endpoint");
account_name = endpoint.substr(acc_pos_begin + 1, acc_pos_end - acc_pos_begin - 1);
size_t cont_pos_end = endpoint.find('/', acc_pos_end + 1);
if (cont_pos_end != std::string::npos)
{
container_name = endpoint.substr(acc_pos_end + 1, cont_pos_end - acc_pos_end - 1);
prefix = endpoint.substr(cont_pos_end + 1);
}
else
{
container_name = endpoint.substr(acc_pos_end + 1);
}
}
else
{
size_t cont_pos_begin = endpoint.find('/', pos + 2);
if (cont_pos_begin == std::string::npos)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected container_name in endpoint");
storage_url = endpoint.substr(0, cont_pos_begin);
size_t cont_pos_end = endpoint.find('/', cont_pos_begin + 1);
if (cont_pos_end != std::string::npos)
{
container_name = endpoint.substr(cont_pos_begin + 1,cont_pos_end - cont_pos_begin - 1);
prefix = endpoint.substr(cont_pos_end + 1);
}
else
{
container_name = endpoint.substr(cont_pos_begin + 1);
}
}
}
else if (config.has(config_prefix + ".connection_string"))
{
storage_url = config.getString(config_prefix + ".connection_string");
container_name = get_container_name();
}
else if (config.has(config_prefix + ".storage_account_url"))
{
storage_url = config.getString(config_prefix + ".storage_account_url");
validateStorageAccountUrl(storage_url);
container_name = get_container_name();
}
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected either `storage_account_url` or `connection_string` or `endpoint` in config");
if (!container_name.empty())
validateContainerName(container_name);
std::optional<bool> container_already_exists {};
if (config.has(config_prefix + ".container_already_exists"))
container_already_exists = {config.getBool(config_prefix + ".container_already_exists")};
return {storage_url, account_name, container_name, prefix, "", container_already_exists};
}
void processURL(const String & url, const String & container_name, Endpoint & endpoint, AuthMethod & auth_method)
{
endpoint.container_name = container_name;
if (isConnectionString(url))
{
endpoint.storage_account_url = url;
auth_method = ConnectionString{url};
return;
}
auto pos = url.find('?');
/// If conneciton_url does not have '?', then its not SAS
if (pos == std::string::npos)
{
endpoint.storage_account_url = url;
auth_method = std::make_shared<Azure::Identity::WorkloadIdentityCredential>();
}
else
{
endpoint.storage_account_url = url.substr(0, pos);
endpoint.sas_auth = url.substr(pos + 1);
auth_method = std::make_shared<Azure::Identity::ManagedIdentityCredential>();
}
}
std::unique_ptr<ContainerClient> getContainerClient(const ConnectionParams & params, bool readonly)
{
if (params.endpoint.container_already_exists.value_or(false) || readonly)
return params.createForContainer();
try
{
auto service_client = params.createForService();
return std::make_unique<ContainerClient>(service_client->CreateBlobContainer(params.endpoint.container_name).Value);
}
catch (const Azure::Storage::StorageException & e)
{
/// If container_already_exists is not set (in config), ignore already exists error.
/// (Conflict - The specified container already exists)
if (!params.endpoint.container_already_exists.has_value() && e.StatusCode == Azure::Core::Http::HttpStatusCode::Conflict)
return params.createForContainer();
throw;
}
}
AuthMethod getAuthMethod(const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
{
if (config.has(config_prefix + ".account_key") && config.has(config_prefix + ".account_name"))
{
return std::make_shared<Azure::Storage::StorageSharedKeyCredential>(
config.getString(config_prefix + ".account_name"),
config.getString(config_prefix + ".account_key")
);
}
if (config.has(config_prefix + ".connection_string"))
return ConnectionString{config.getString(config_prefix + ".connection_string")};
if (config.getBool(config_prefix + ".use_workload_identity", false))
return std::make_shared<Azure::Identity::WorkloadIdentityCredential>();
return std::make_shared<Azure::Identity::ManagedIdentityCredential>();
}
BlobClientOptions getClientOptions(const RequestSettings & settings, bool for_disk)
{
Azure::Core::Http::Policies::RetryOptions retry_options;
retry_options.MaxRetries = static_cast<Int32>(settings.sdk_max_retries);
retry_options.RetryDelay = std::chrono::milliseconds(settings.sdk_retry_initial_backoff_ms);
retry_options.MaxRetryDelay = std::chrono::milliseconds(settings.sdk_retry_max_backoff_ms);
Azure::Core::Http::CurlTransportOptions curl_options;
curl_options.NoSignal = true;
curl_options.IPResolve = settings.curl_ip_resolve;
Azure::Storage::Blobs::BlobClientOptions client_options;
client_options.Retry = retry_options;
client_options.Transport.Transport = std::make_shared<Azure::Core::Http::CurlTransport>(curl_options);
client_options.ClickhouseOptions = Azure::Storage::Blobs::ClickhouseClientOptions{.IsClientForDisk=for_disk};
return client_options;
}
std::unique_ptr<RequestSettings> getRequestSettings(const Settings & query_settings)
{
auto settings = std::make_unique<RequestSettings>();
settings->max_single_part_upload_size = query_settings.azure_max_single_part_upload_size;
settings->max_single_read_retries = query_settings.azure_max_single_read_retries;
settings->max_single_download_retries = query_settings.azure_max_single_read_retries;
settings->list_object_keys_size = query_settings.azure_list_object_keys_size;
settings->min_upload_part_size = query_settings.azure_min_upload_part_size;
settings->max_upload_part_size = query_settings.azure_max_upload_part_size;
settings->max_single_part_copy_size = query_settings.azure_max_single_part_copy_size;
settings->max_blocks_in_multipart_upload = query_settings.azure_max_blocks_in_multipart_upload;
settings->max_unexpected_write_error_retries = query_settings.azure_max_unexpected_write_error_retries;
settings->max_inflight_parts_for_one_file = query_settings.azure_max_inflight_parts_for_one_file;
settings->strict_upload_part_size = query_settings.azure_strict_upload_part_size;
settings->upload_part_size_multiply_factor = query_settings.azure_upload_part_size_multiply_factor;
settings->upload_part_size_multiply_parts_count_threshold = query_settings.azure_upload_part_size_multiply_parts_count_threshold;
settings->sdk_max_retries = query_settings.azure_sdk_max_retries;
settings->sdk_retry_initial_backoff_ms = query_settings.azure_sdk_retry_initial_backoff_ms;
settings->sdk_retry_max_backoff_ms = query_settings.azure_sdk_retry_max_backoff_ms;
return settings;
}
std::unique_ptr<RequestSettings> getRequestSettingsForBackup(const Settings & query_settings, bool use_native_copy)
{
auto settings = getRequestSettings(query_settings);
settings->use_native_copy = use_native_copy;
return settings;
}
std::unique_ptr<RequestSettings> getRequestSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context)
{
auto settings = std::make_unique<RequestSettings>();
const auto & settings_ref = context->getSettingsRef();
settings->min_bytes_for_seek = config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024);
settings->use_native_copy = config.getBool(config_prefix + ".use_native_copy", false);
settings->max_single_part_upload_size = config.getUInt64(config_prefix + ".max_single_part_upload_size", settings_ref.azure_max_single_part_upload_size);
settings->max_single_read_retries = config.getUInt64(config_prefix + ".max_single_read_retries", settings_ref.azure_max_single_read_retries);
settings->max_single_download_retries = config.getUInt64(config_prefix + ".max_single_download_retries", settings_ref.azure_max_single_read_retries);
settings->list_object_keys_size = config.getUInt64(config_prefix + ".list_object_keys_size", settings_ref.azure_list_object_keys_size);
settings->min_upload_part_size = config.getUInt64(config_prefix + ".min_upload_part_size", settings_ref.azure_min_upload_part_size);
settings->max_upload_part_size = config.getUInt64(config_prefix + ".max_upload_part_size", settings_ref.azure_max_upload_part_size);
settings->max_single_part_copy_size = config.getUInt64(config_prefix + ".max_single_part_copy_size", settings_ref.azure_max_single_part_copy_size);
settings->max_blocks_in_multipart_upload = config.getUInt64(config_prefix + ".max_blocks_in_multipart_upload", settings_ref.azure_max_blocks_in_multipart_upload);
settings->max_unexpected_write_error_retries = config.getUInt64(config_prefix + ".max_unexpected_write_error_retries", settings_ref.azure_max_unexpected_write_error_retries);
settings->max_inflight_parts_for_one_file = config.getUInt64(config_prefix + ".max_inflight_parts_for_one_file", settings_ref.azure_max_inflight_parts_for_one_file);
settings->strict_upload_part_size = config.getUInt64(config_prefix + ".strict_upload_part_size", settings_ref.azure_strict_upload_part_size);
settings->upload_part_size_multiply_factor = config.getUInt64(config_prefix + ".upload_part_size_multiply_factor", settings_ref.azure_upload_part_size_multiply_factor);
settings->upload_part_size_multiply_parts_count_threshold = config.getUInt64(config_prefix + ".upload_part_size_multiply_parts_count_threshold", settings_ref.azure_upload_part_size_multiply_parts_count_threshold);
settings->sdk_max_retries = config.getUInt64(config_prefix + ".max_tries", settings_ref.azure_sdk_max_retries);
settings->sdk_retry_initial_backoff_ms = config.getUInt64(config_prefix + ".retry_initial_backoff_ms", settings_ref.azure_sdk_retry_initial_backoff_ms);
settings->sdk_retry_max_backoff_ms = config.getUInt64(config_prefix + ".retry_max_backoff_ms", settings_ref.azure_sdk_retry_max_backoff_ms);
if (config.has(config_prefix + ".curl_ip_resolve"))
{
using CurlOptions = Azure::Core::Http::CurlTransportOptions;
auto value = config.getString(config_prefix + ".curl_ip_resolve");
if (value == "ipv4")
settings->curl_ip_resolve = CurlOptions::CURL_IPRESOLVE_V4;
else if (value == "ipv6")
settings->curl_ip_resolve = CurlOptions::CURL_IPRESOLVE_V6;
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected value for option 'curl_ip_resolve': {}. Expected one of 'ipv4' or 'ipv6'", value);
}
return settings;
}
}
}
#endif

138
src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h Normal file
View File

@ -0,0 +1,138 @@
#pragma once
#include "config.h"
#if USE_AZURE_BLOB_STORAGE
#include <azure/storage/blobs.hpp>
#include <azure/storage/blobs/blob_client.hpp>
#include <azure/storage/blobs/blob_options.hpp>
#include <azure/storage/blobs/blob_service_client.hpp>
#include <azure/core/http/curl_transport.hpp>
#include <azure/identity/managed_identity_credential.hpp>
#include <azure/identity/workload_identity_credential.hpp>
#include <Core/Settings.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <Interpreters/Context_fwd.h>
#include <base/strong_typedef.h>
namespace DB
{
namespace AzureBlobStorage
{
using ServiceClient = Azure::Storage::Blobs::BlobServiceClient;
using ContainerClient = Azure::Storage::Blobs::BlobContainerClient;
using BlobClient = Azure::Storage::Blobs::BlobClient;
using BlobClientOptions = Azure::Storage::Blobs::BlobClientOptions;
struct RequestSettings
{
RequestSettings() = default;
size_t max_single_part_upload_size = 100 * 1024 * 1024; /// NOTE: on 32-bit machines it will be at most 4GB, but size_t is also used in BufferBase for offset
size_t min_bytes_for_seek = 1024 * 1024;
size_t max_single_read_retries = 3;
size_t max_single_download_retries = 3;
size_t list_object_keys_size = 1000;
size_t min_upload_part_size = 16 * 1024 * 1024;
size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024;
size_t max_single_part_copy_size = 256 * 1024 * 1024;
size_t max_unexpected_write_error_retries = 4;
size_t max_inflight_parts_for_one_file = 20;
size_t max_blocks_in_multipart_upload = 50000;
size_t strict_upload_part_size = 0;
size_t upload_part_size_multiply_factor = 2;
size_t upload_part_size_multiply_parts_count_threshold = 500;
size_t sdk_max_retries = 10;
size_t sdk_retry_initial_backoff_ms = 10;
size_t sdk_retry_max_backoff_ms = 1000;
bool use_native_copy = false;
using CurlOptions = Azure::Core::Http::CurlTransportOptions;
CurlOptions::CurlOptIPResolve curl_ip_resolve = CurlOptions::CURL_IPRESOLVE_WHATEVER;
};
struct Endpoint
{
String storage_account_url;
String account_name;
String container_name;
String prefix;
String sas_auth;
std::optional<bool> container_already_exists;
String getEndpoint() const
{
String url = storage_account_url;
if (url.ends_with('/'))
url.pop_back();
if (!account_name.empty())
url += "/" + account_name;
if (!container_name.empty())
url += "/" + container_name;
if (!prefix.empty())
url += "/" + prefix;
if (!sas_auth.empty())
url += "?" + sas_auth;
return url;
}
String getEndpointWithoutContainer() const
{
String url = storage_account_url;
if (!account_name.empty())
url += "/" + account_name;
if (!sas_auth.empty())
url += "?" + sas_auth;
return url;
}
};
using ConnectionString = StrongTypedef<String, struct ConnectionStringTag>;
using AuthMethod = std::variant<
ConnectionString,
std::shared_ptr<Azure::Storage::StorageSharedKeyCredential>,
std::shared_ptr<Azure::Identity::WorkloadIdentityCredential>,
std::shared_ptr<Azure::Identity::ManagedIdentityCredential>>;
struct ConnectionParams
{
Endpoint endpoint;
AuthMethod auth_method;
BlobClientOptions client_options;
String getContainer() const { return endpoint.container_name; }
String getConnectionURL() const;
std::unique_ptr<ServiceClient> createForService() const;
std::unique_ptr<ContainerClient> createForContainer() const;
};
Endpoint processEndpoint(const Poco::Util::AbstractConfiguration & config, const String & config_prefix);
void processURL(const String & url, const String & container_name, Endpoint & endpoint, AuthMethod & auth_method);
std::unique_ptr<ContainerClient> getContainerClient(const ConnectionParams & params, bool readonly);
BlobClientOptions getClientOptions(const RequestSettings & settings, bool for_disk);
AuthMethod getAuthMethod(const Poco::Util::AbstractConfiguration & config, const String & config_prefix);
std::unique_ptr<RequestSettings> getRequestSettings(const Settings & query_settings);
std::unique_ptr<RequestSettings> getRequestSettingsForBackup(const Settings & query_settings, bool use_native_copy);
std::unique_ptr<RequestSettings> getRequestSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context);
}
}
#endif

53
src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp
View File

@ -9,7 +9,7 @@
#include <Disks/IO/ReadBufferFromRemoteFSGather.h>
#include <Disks/IO/AsynchronousBoundedReadBuffer.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
#include <Disks/ObjectStorages/ObjectStorageIteratorAsync.h>
#include <Interpreters/Context.h>
#include <Common/logger_useful.h>
@ -104,7 +104,7 @@ private:
AzureObjectStorage::AzureObjectStorage(
const String & name_,
AzureClientPtr && client_,
ClientPtr && client_,
SettingsPtr && settings_,
const String & object_namespace_,
const String & description_)
@ -397,24 +397,49 @@ void AzureObjectStorage::copyObject( /// NOLINT
}
void AzureObjectStorage::applyNewSettings(
const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
ContextPtr context, const ApplyNewSettingsOptions &)
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
ContextPtr context,
const ApplyNewSettingsOptions & options)
{
auto new_settings = getAzureBlobStorageSettings(config, config_prefix, context);
auto new_settings = AzureBlobStorage::getRequestSettings(config, config_prefix, context);
settings.set(std::move(new_settings));
/// We don't update client
if (!options.allow_client_change)
return;
bool is_client_for_disk = client.get()->GetClickhouseOptions().IsClientForDisk;
AzureBlobStorage::ConnectionParams params
{
.endpoint = AzureBlobStorage::processEndpoint(config, config_prefix),
.auth_method = AzureBlobStorage::getAuthMethod(config, config_prefix),
.client_options = AzureBlobStorage::getClientOptions(*settings.get(), is_client_for_disk),
};
auto new_client = AzureBlobStorage::getContainerClient(params, /*readonly=*/ true);
client.set(std::move(new_client));
}
std::unique_ptr<IObjectStorage> AzureObjectStorage::cloneObjectStorage(const std::string &, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context)
std::unique_ptr<IObjectStorage> AzureObjectStorage::cloneObjectStorage(
const std::string & new_namespace,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
ContextPtr context)
{
return std::make_unique<AzureObjectStorage>(
name,
getAzureBlobContainerClient(config, config_prefix),
getAzureBlobStorageSettings(config, config_prefix, context),
object_namespace,
description
);
auto new_settings = AzureBlobStorage::getRequestSettings(config, config_prefix, context);
bool is_client_for_disk = client.get()->GetClickhouseOptions().IsClientForDisk;
AzureBlobStorage::ConnectionParams params
{
.endpoint = AzureBlobStorage::processEndpoint(config, config_prefix),
.auth_method = AzureBlobStorage::getAuthMethod(config, config_prefix),
.client_options = AzureBlobStorage::getClientOptions(*new_settings, is_client_for_disk),
};
auto new_client = AzureBlobStorage::getContainerClient(params, /*readonly=*/ true);
return std::make_unique<AzureObjectStorage>(name, std::move(new_client), std::move(new_settings), new_namespace, params.endpoint.getEndpointWithoutContainer());
}
}

76
src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h
View File

@ -7,6 +7,8 @@
#include <Disks/ObjectStorages/IObjectStorage.h>
#include <Common/MultiVersion.h>
#include <azure/storage/blobs.hpp>
#include <azure/core/http/curl_transport.hpp>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
namespace Poco
{
@ -16,71 +18,15 @@ class Logger;
namespace DB
{
struct AzureObjectStorageSettings
{
AzureObjectStorageSettings(
uint64_t max_single_part_upload_size_,
uint64_t min_bytes_for_seek_,
int max_single_read_retries_,
int max_single_download_retries_,
int list_object_keys_size_,
size_t min_upload_part_size_,
size_t max_upload_part_size_,
size_t max_single_part_copy_size_,
bool use_native_copy_,
size_t max_unexpected_write_error_retries_,
size_t max_inflight_parts_for_one_file_,
size_t strict_upload_part_size_,
size_t upload_part_size_multiply_factor_,
size_t upload_part_size_multiply_parts_count_threshold_)
: max_single_part_upload_size(max_single_part_upload_size_)
, min_bytes_for_seek(min_bytes_for_seek_)
, max_single_read_retries(max_single_read_retries_)
, max_single_download_retries(max_single_download_retries_)
, list_object_keys_size(list_object_keys_size_)
, min_upload_part_size(min_upload_part_size_)
, max_upload_part_size(max_upload_part_size_)
, max_single_part_copy_size(max_single_part_copy_size_)
, use_native_copy(use_native_copy_)
, max_unexpected_write_error_retries(max_unexpected_write_error_retries_)
, max_inflight_parts_for_one_file(max_inflight_parts_for_one_file_)
, strict_upload_part_size(strict_upload_part_size_)
, upload_part_size_multiply_factor(upload_part_size_multiply_factor_)
, upload_part_size_multiply_parts_count_threshold(upload_part_size_multiply_parts_count_threshold_)
{
}
AzureObjectStorageSettings() = default;
size_t max_single_part_upload_size = 100 * 1024 * 1024; /// NOTE: on 32-bit machines it will be at most 4GB, but size_t is also used in BufferBase for offset
uint64_t min_bytes_for_seek = 1024 * 1024;
size_t max_single_read_retries = 3;
size_t max_single_download_retries = 3;
int list_object_keys_size = 1000;
size_t min_upload_part_size = 16 * 1024 * 1024;
size_t max_upload_part_size = 5ULL * 1024 * 1024 * 1024;
size_t max_single_part_copy_size = 256 * 1024 * 1024;
bool use_native_copy = false;
size_t max_unexpected_write_error_retries = 4;
size_t max_inflight_parts_for_one_file = 20;
size_t max_blocks_in_multipart_upload = 50000;
size_t strict_upload_part_size = 0;
size_t upload_part_size_multiply_factor = 2;
size_t upload_part_size_multiply_parts_count_threshold = 500;
};
using AzureClient = Azure::Storage::Blobs::BlobContainerClient;
using AzureClientPtr = std::unique_ptr<Azure::Storage::Blobs::BlobContainerClient>;
class AzureObjectStorage : public IObjectStorage
{
public:
using SettingsPtr = std::unique_ptr<AzureObjectStorageSettings>;
using ClientPtr = std::unique_ptr<AzureBlobStorage::ContainerClient>;
using SettingsPtr = std::unique_ptr<AzureBlobStorage::RequestSettings>;
AzureObjectStorage(
const String & name_,
AzureClientPtr && client_,
ClientPtr && client_,
SettingsPtr && settings_,
const String & object_namespace_,
const String & description_);
@ -159,12 +105,8 @@ public:
bool isRemote() const override { return true; }
std::shared_ptr<const AzureObjectStorageSettings> getSettings() { return settings.get(); }
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> getAzureBlobStorageClient() override
{
return client.get();
}
std::shared_ptr<const AzureBlobStorage::RequestSettings> getSettings() const { return settings.get(); }
std::shared_ptr<const AzureBlobStorage::ContainerClient> getAzureBlobStorageClient() const override { return client.get(); }
bool supportParallelWrite() const override { return true; }
@ -174,8 +116,8 @@ private:
const String name;
/// client used to access the files in the Blob Storage cloud
MultiVersion<Azure::Storage::Blobs::BlobContainerClient> client;
MultiVersion<AzureObjectStorageSettings> settings;
MultiVersion<AzureBlobStorage::ContainerClient> client;
MultiVersion<AzureBlobStorage::RequestSettings> settings;
const String object_namespace; /// container + prefix
/// We use source url without container and prefix as description, because in Azure there are no limitations for operations between different containers.

2
src/Disks/ObjectStorages/Cached/CachedObjectStorage.h
View File

@ -127,7 +127,7 @@ public:
const FileCacheSettings & getCacheSettings() const { return cache_settings; }
#if USE_AZURE_BLOB_STORAGE
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> getAzureBlobStorageClient() override
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> getAzureBlobStorageClient() const override
{
return object_storage->getAzureBlobStorageClient();
}

2
src/Disks/ObjectStorages/DiskObjectStorage.h
View File

@ -195,7 +195,6 @@ public:
/// DiskObjectStorage(CachedObjectStorage(CachedObjectStorage(S3ObjectStorage)))
String getStructure() const { return fmt::format("DiskObjectStorage-{}({})", getName(), object_storage->getName()); }
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
/// Add a cache layer.
/// Example: DiskObjectStorage(S3ObjectStorage) -> DiskObjectStorage(CachedObjectStorage(S3ObjectStorage))
/// There can be any number of cache layers:
@ -204,7 +203,6 @@ public:
/// Get names of all cache layers. Name is how cache is defined in configuration file.
NameSet getCacheLayersNames() const override;
#endif
bool supportsStat() const override { return metadata_storage->supportsStat(); }
struct stat stat(const String & path) const override;

4
src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp
View File

@ -222,11 +222,7 @@ ObjectKeyWithMetadata DiskObjectStorageMetadata::popLastObject()
bool DiskObjectStorageMetadata::getWriteFullObjectKeySetting()
{
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
return Context::getGlobalContextInstance()->getServerSettings().storage_metadata_write_full_object_key;
#else
return false;
#endif
}
}

2
src/Disks/ObjectStorages/IObjectStorage.h
View File

@ -261,7 +261,7 @@ public:
virtual void setKeysGenerator(ObjectStorageKeysGeneratorPtr) { }
#if USE_AZURE_BLOB_STORAGE
virtual std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> getAzureBlobStorageClient()
virtual std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> getAzureBlobStorageClient() const
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "This function is only implemented for AzureBlobStorage");
}

6
src/Disks/ObjectStorages/MetadataStorageFactory.cpp
View File

@ -2,9 +2,7 @@
#include <Disks/ObjectStorages/MetadataStorageFromDisk.h>
#include <Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h>
#include <Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h>
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
#include <Disks/ObjectStorages/Web/MetadataStorageFromStaticFilesWebServer.h>
#endif
#include <Disks/DiskLocal.h>
#include <Interpreters/Context.h>
@ -135,7 +133,6 @@ void registerPlainRewritableMetadataStorage(MetadataStorageFactory & factory)
});
}
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
void registerMetadataStorageFromStaticFilesWebServer(MetadataStorageFactory & factory)
{
factory.registerMetadataStorageType("web", [](
@ -147,7 +144,6 @@ void registerMetadataStorageFromStaticFilesWebServer(MetadataStorageFactory & fa
return std::make_shared<MetadataStorageFromStaticFilesWebServer>(assert_cast<const WebObjectStorage &>(*object_storage));
});
}
#endif
void registerMetadataStorages()
{
@ -155,9 +151,7 @@ void registerMetadataStorages()
registerMetadataStorageFromDisk(factory);
registerPlainMetadataStorage(factory);
registerPlainRewritableMetadataStorage(factory);
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
registerMetadataStorageFromStaticFilesWebServer(factory);
#endif
}
}

37
src/Disks/ObjectStorages/ObjectStorageFactory.cpp
View File

@ -7,19 +7,17 @@
#include <Disks/ObjectStorages/S3/S3ObjectStorage.h>
#include <Disks/ObjectStorages/S3/diskSettings.h>
#endif
#if USE_HDFS && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD)
#if USE_HDFS
#include <Disks/ObjectStorages/HDFS/HDFSObjectStorage.h>
#include <Storages/ObjectStorage/HDFS/HDFSCommon.h>
#endif
#if USE_AZURE_BLOB_STORAGE && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD)
#if USE_AZURE_BLOB_STORAGE
#include <Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.h>
#include <Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h>
#endif
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
#include <Disks/ObjectStorages/Web/WebObjectStorage.h>
#include <Disks/ObjectStorages/Local/LocalObjectStorage.h>
#include <Disks/loadLocalDiskConfig.h>
#endif
#include <Disks/ObjectStorages/MetadataStorageFactory.h>
#include <Disks/ObjectStorages/PlainObjectStorage.h>
#include <Disks/ObjectStorages/PlainRewritableObjectStorage.h>
@ -284,7 +282,7 @@ void registerS3PlainRewritableObjectStorage(ObjectStorageFactory & factory)
#endif
#if USE_HDFS && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD)
#if USE_HDFS
void registerHDFSObjectStorage(ObjectStorageFactory & factory)
{
factory.registerObjectStorageType(
@ -309,7 +307,7 @@ void registerHDFSObjectStorage(ObjectStorageFactory & factory)
}
#endif
#if USE_AZURE_BLOB_STORAGE && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD)
#if USE_AZURE_BLOB_STORAGE
void registerAzureObjectStorage(ObjectStorageFactory & factory)
{
auto creator = [](
@ -319,21 +317,27 @@ void registerAzureObjectStorage(ObjectStorageFactory & factory)
const ContextPtr & context,
bool /* skip_access_check */) -> ObjectStoragePtr
{
AzureBlobStorageEndpoint endpoint = processAzureBlobStorageEndpoint(config, config_prefix);
auto azure_settings = AzureBlobStorage::getRequestSettings(config, config_prefix, context);
AzureBlobStorage::ConnectionParams params
{
.endpoint = AzureBlobStorage::processEndpoint(config, config_prefix),
.auth_method = AzureBlobStorage::getAuthMethod(config, config_prefix),
.client_options = AzureBlobStorage::getClientOptions(*azure_settings, /*for_disk=*/ true),
};
return createObjectStorage<AzureObjectStorage>(
ObjectStorageType::Azure, config, config_prefix, name,
getAzureBlobContainerClient(config, config_prefix),
getAzureBlobStorageSettings(config, config_prefix, context),
endpoint.prefix.empty() ? endpoint.container_name : endpoint.container_name + "/" + endpoint.prefix,
endpoint.getEndpointWithoutContainer());
AzureBlobStorage::getContainerClient(params, /*readonly=*/ false), std::move(azure_settings),
params.endpoint.prefix.empty() ? params.endpoint.container_name : params.endpoint.container_name + "/" + params.endpoint.prefix,
params.endpoint.getEndpointWithoutContainer());
};
factory.registerObjectStorageType("azure_blob_storage", creator);
factory.registerObjectStorageType("azure", creator);
}
#endif
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
void registerWebObjectStorage(ObjectStorageFactory & factory)
{
factory.registerObjectStorageType("web", [](
@ -381,7 +385,6 @@ void registerLocalObjectStorage(ObjectStorageFactory & factory)
factory.registerObjectStorageType("local_blob_storage", creator);
factory.registerObjectStorageType("local", creator);
}
#endif
void registerObjectStorages()
{
@ -393,18 +396,16 @@ void registerObjectStorages()
registerS3PlainRewritableObjectStorage(factory);
#endif
#if USE_HDFS && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD)
#if USE_HDFS
registerHDFSObjectStorage(factory);
#endif
#if USE_AZURE_BLOB_STORAGE && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD)
#if USE_AZURE_BLOB_STORAGE
registerAzureObjectStorage(factory);
#endif
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
registerWebObjectStorage(factory);
registerLocalObjectStorage(factory);
#endif
}
}

10
src/Disks/ObjectStorages/createMetadataStorageMetrics.h
View File

@ -1,14 +1,14 @@
#pragma once
#include "config.h"
#if USE_AWS_S3
# include <Disks/ObjectStorages/S3/S3ObjectStorage.h>
#endif
#if USE_AZURE_BLOB_STORAGE && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD)
#if USE_AZURE_BLOB_STORAGE
# include <Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h>
#endif
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
#include <Disks/ObjectStorages/Local/LocalObjectStorage.h>
#endif
#include <Disks/ObjectStorages/MetadataStorageMetrics.h>
namespace ProfileEvents
@ -42,7 +42,7 @@ inline MetadataStorageMetrics MetadataStorageMetrics::create<S3ObjectStorage, Me
}
#endif
#if USE_AZURE_BLOB_STORAGE && !defined(CLICKHOUSE_KEEPER_STANDALONE_BUILD)
#if USE_AZURE_BLOB_STORAGE
template <>
inline MetadataStorageMetrics MetadataStorageMetrics::create<AzureObjectStorage, MetadataStorageType::PlainRewritable>()
{
@ -53,7 +53,6 @@ inline MetadataStorageMetrics MetadataStorageMetrics::create<AzureObjectStorage,
}
#endif
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
template <>
inline MetadataStorageMetrics MetadataStorageMetrics::create<LocalObjectStorage, MetadataStorageType::PlainRewritable>()
{
@ -62,6 +61,5 @@ inline MetadataStorageMetrics MetadataStorageMetrics::create<LocalObjectStorage,
.directory_removed = ProfileEvents::DiskPlainRewritableLocalDirectoryRemoved,
.directory_map_size = CurrentMetrics::DiskPlainRewritableLocalDirectoryMapSize};
}
#endif
}

15
src/Disks/registerDisks.cpp
View File

@ -17,8 +17,6 @@ void registerDiskCache(DiskFactory & factory, bool global_skip_access_check);
void registerDiskObjectStorage(DiskFactory & factory, bool global_skip_access_check);
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
void registerDisks(bool global_skip_access_check)
{
auto & factory = DiskFactory::instance();
@ -34,17 +32,4 @@ void registerDisks(bool global_skip_access_check)
registerDiskObjectStorage(factory, global_skip_access_check);
}
#else
void registerDisks(bool global_skip_access_check)
{
auto & factory = DiskFactory::instance();
registerDiskLocal(factory, global_skip_access_check);
registerDiskObjectStorage(factory, global_skip_access_check);
}
#endif
}

26
src/Functions/Regexps.h
View File

@ -23,7 +23,11 @@
namespace ProfileEvents
{
extern const Event RegexpCreated;
extern const Event RegexpWithMultipleNeedlesCreated;
extern const Event RegexpWithMultipleNeedlesGlobalCacheHit;
extern const Event RegexpWithMultipleNeedlesGlobalCacheMiss;
extern const Event RegexpLocalCacheHit;
extern const Event RegexpLocalCacheMiss;
}
@ -72,18 +76,28 @@ public:
Bucket & bucket = known_regexps[hasher(pattern) % CACHE_SIZE];
if (bucket.regexp == nullptr) [[unlikely]]
{
/// insert new entry
ProfileEvents::increment(ProfileEvents::RegexpLocalCacheMiss);
bucket = {pattern, std::make_shared<OptimizedRegularExpression>(createRegexp<like, no_capture, case_insensitive>(pattern))};
}
else
{
if (pattern != bucket.pattern)
{
/// replace existing entry
ProfileEvents::increment(ProfileEvents::RegexpLocalCacheMiss);
bucket = {pattern, std::make_shared<OptimizedRegularExpression>(createRegexp<like, no_capture, case_insensitive>(pattern))};
}
else
ProfileEvents::increment(ProfileEvents::RegexpLocalCacheHit);
}
return bucket.regexp;
}
private:
constexpr static size_t CACHE_SIZE = 100; /// collision probability
constexpr static size_t CACHE_SIZE = 1'000; /// collision probability
std::hash<String> hasher;
struct Bucket
@ -244,7 +258,7 @@ inline Regexps constructRegexps(const std::vector<String> & str_patterns, [[mayb
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Pattern '{}' failed with error '{}'", str_patterns[error->expression], String(error->message));
}
ProfileEvents::increment(ProfileEvents::RegexpCreated);
ProfileEvents::increment(ProfileEvents::RegexpWithMultipleNeedlesCreated);
/// We allocate the scratch space only once, then copy it across multiple threads with hs_clone_scratch
/// function which is faster than allocating scratch space each time in each thread.
@ -322,9 +336,11 @@ inline DeferredConstructedRegexpsPtr getOrSet(const std::vector<std::string_view
{
return constructRegexps<save_indices, with_edit_distance>(str_patterns, edit_distance);
});
ProfileEvents::increment(ProfileEvents::RegexpWithMultipleNeedlesGlobalCacheMiss);
bucket = {std::move(str_patterns), edit_distance, deferred_constructed_regexps};
}
else
{
if (bucket.patterns != str_patterns || bucket.edit_distance != edit_distance)
{
/// replace existing entry
@ -333,8 +349,12 @@ inline DeferredConstructedRegexpsPtr getOrSet(const std::vector<std::string_view
{
return constructRegexps<save_indices, with_edit_distance>(str_patterns, edit_distance);
});
ProfileEvents::increment(ProfileEvents::RegexpWithMultipleNeedlesGlobalCacheMiss);
bucket = {std::move(str_patterns), edit_distance, deferred_constructed_regexps};
}
else
ProfileEvents::increment(ProfileEvents::RegexpWithMultipleNeedlesGlobalCacheHit);
}
return bucket.regexps;
}

8
src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp
View File

@ -47,7 +47,7 @@ namespace
size_t total_size_,
const String & dest_container_for_logging_,
const String & dest_blob_,
std::shared_ptr<const AzureObjectStorageSettings> settings_,
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings_,
ThreadPoolCallbackRunnerUnsafe<void> schedule_,
const Poco::Logger * log_)
: create_read_buffer(create_read_buffer_)
@ -72,7 +72,7 @@ namespace
size_t total_size;
const String & dest_container_for_logging;
const String & dest_blob;
std::shared_ptr<const AzureObjectStorageSettings> settings;
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings;
ThreadPoolCallbackRunnerUnsafe<void> schedule;
const Poco::Logger * log;
size_t max_single_part_upload_size;
@ -318,7 +318,7 @@ void copyDataToAzureBlobStorageFile(
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> dest_client,
const String & dest_container_for_logging,
const String & dest_blob,
std::shared_ptr<const AzureObjectStorageSettings> settings,
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings,
ThreadPoolCallbackRunnerUnsafe<void> schedule)
{
UploadHelper helper{create_read_buffer, dest_client, offset, size, dest_container_for_logging, dest_blob, settings, schedule, &Poco::Logger::get("copyDataToAzureBlobStorageFile")};
@ -335,7 +335,7 @@ void copyAzureBlobStorageFile(
size_t size,
const String & dest_container_for_logging,
const String & dest_blob,
std::shared_ptr<const AzureObjectStorageSettings> settings,
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings,
const ReadSettings & read_settings,
ThreadPoolCallbackRunnerUnsafe<void> schedule)
{

4
src/IO/AzureBlobStorage/copyAzureBlobStorageFile.h
View File

@ -28,7 +28,7 @@ void copyAzureBlobStorageFile(
size_t src_size,
const String & dest_container_for_logging,
const String & dest_blob,
std::shared_ptr<const AzureObjectStorageSettings> settings,
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings,
const ReadSettings & read_settings,
ThreadPoolCallbackRunnerUnsafe<void> schedule_ = {});
@ -45,7 +45,7 @@ void copyDataToAzureBlobStorageFile(
std::shared_ptr<const Azure::Storage::Blobs::BlobContainerClient> client,
const String & dest_container_for_logging,
const String & dest_blob,
std::shared_ptr<const AzureObjectStorageSettings> settings,
std::shared_ptr<const AzureBlobStorage::RequestSettings> settings,
ThreadPoolCallbackRunnerUnsafe<void> schedule_ = {});
}

2
src/IO/S3/BlobStorageLogWriter.cpp
View File

@ -56,7 +56,6 @@ void BlobStorageLogWriter::addEvent(
BlobStorageLogWriterPtr BlobStorageLogWriter::create(const String & disk_name)
{
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD /// Keeper standalone build doesn't have a context
if (auto blob_storage_log = Context::getGlobalContextInstance()->getBlobStorageLog())
{
auto log_writer = std::make_shared<BlobStorageLogWriter>(std::move(blob_storage_log));
@ -67,7 +66,6 @@ BlobStorageLogWriterPtr BlobStorageLogWriter::create(const String & disk_name)
return log_writer;
}
#endif
return {};
}

19
src/Interpreters/AsynchronousInsertQueue.cpp
View File

@ -301,7 +301,13 @@ void AsynchronousInsertQueue::preprocessInsertQuery(const ASTPtr & query, const
auto & insert_query = query->as<ASTInsertQuery &>();
insert_query.async_insert_flush = true;
InterpreterInsertQuery interpreter(query, query_context, query_context->getSettingsRef().insert_allow_materialized_columns);
InterpreterInsertQuery interpreter(
query,
query_context,
query_context->getSettingsRef().insert_allow_materialized_columns,
/* no_squash */ false,
/* no_destination */ false,
/* async_insert */ false);
auto table = interpreter.getTable(insert_query);
auto sample_block = InterpreterInsertQuery::getSampleBlock(insert_query, table, table->getInMemoryMetadataPtr(), query_context);
@ -781,7 +787,12 @@ try
try
{
interpreter = std::make_unique<InterpreterInsertQuery>(
key.query, insert_context, key.settings.insert_allow_materialized_columns, false, false, true);
key.query,
insert_context,
key.settings.insert_allow_materialized_columns,
false,
false,
true);
pipeline = interpreter->execute().pipeline;
chassert(pipeline.pushing());
@ -1000,7 +1011,7 @@ Chunk AsynchronousInsertQueue::processEntriesWithParsing(
}
Chunk chunk(executor.getResultColumns(), total_rows);
chunk.setChunkInfo(std::move(chunk_info));
chunk.getChunkInfos().add(std::move(chunk_info));
return chunk;
}
@ -1052,7 +1063,7 @@ Chunk AsynchronousInsertQueue::processPreprocessedEntries(
}
Chunk chunk(std::move(result_columns), total_rows);
chunk.setChunkInfo(std::move(chunk_info));
chunk.getChunkInfos().add(std::move(chunk_info));
return chunk;
}

8
src/Interpreters/Context.h
View File

@ -1,7 +1,5 @@
#pragma once
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
#include <base/types.h>
#include <Common/isLocalAddress.h>
#include <Common/MultiVersion.h>
@ -1451,9 +1449,3 @@ struct HTTPContext : public IHTTPContext
};
}
#else
#include <Coordination/Standalone/Context.h>
#endif

18
src/Interpreters/InterpreterCheckQuery.cpp
View File

@ -2,6 +2,7 @@
#include <Interpreters/InterpreterFactory.h>
#include <algorithm>
#include <memory>
#include <Access/Common/AccessFlags.h>
@ -22,6 +23,7 @@
#include <Parsers/ASTCheckQuery.h>
#include <Parsers/ASTSetQuery.h>
#include <Processors/Chunk.h>
#include <Processors/IAccumulatingTransform.h>
#include <Processors/IInflatingTransform.h>
#include <Processors/ISimpleTransform.h>
@ -91,7 +93,7 @@ Chunk getChunkFromCheckResult(const String & database, const String & table, con
return Chunk(std::move(columns), 1);
}
class TableCheckTask : public ChunkInfo
class TableCheckTask : public ChunkInfoCloneable<TableCheckTask>
{
public:
TableCheckTask(StorageID table_id, const std::variant<std::monostate, ASTPtr, String> & partition_or_part, ContextPtr context)
@ -110,6 +112,12 @@ public:
context->checkAccess(AccessType::SHOW_TABLES, table_->getStorageID());
}
TableCheckTask(const TableCheckTask & other)
: table(other.table)
, check_data_tasks(other.check_data_tasks)
, is_finished(other.is_finished.load())
{}
std::optional<CheckResult> checkNext() const
{
if (isFinished())
@ -121,8 +129,8 @@ public:
std::this_thread::sleep_for(sleep_time);
});
IStorage::DataValidationTasksPtr check_data_tasks_ = check_data_tasks;
auto result = table->checkDataNext(check_data_tasks_);
IStorage::DataValidationTasksPtr tmp = check_data_tasks;
auto result = table->checkDataNext(tmp);
is_finished = !result.has_value();
return result;
}
@ -180,7 +188,7 @@ protected:
/// source should return at least one row to start pipeline
result.addColumn(ColumnUInt8::create(1, 1));
/// actual data stored in chunk info
result.setChunkInfo(std::move(current_check_task));
result.getChunkInfos().add(std::move(current_check_task));
return result;
}
@ -280,7 +288,7 @@ public:
protected:
void transform(Chunk & chunk) override
{
auto table_check_task = std::dynamic_pointer_cast<const TableCheckTask>(chunk.getChunkInfo());
auto table_check_task = chunk.getChunkInfos().get<TableCheckTask>();
auto check_result = table_check_task->checkNext();
if (!check_result)
{

9
src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -1776,8 +1776,13 @@ BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create)
else
insert->select = create.select->clone();
return InterpreterInsertQuery(insert, getContext(),
getContext()->getSettingsRef().insert_allow_materialized_columns).execute();
return InterpreterInsertQuery(
insert,
getContext(),
getContext()->getSettingsRef().insert_allow_materialized_columns,
/* no_squash */ false,
/* no_destination */ false,
/* async_isnert */ false).execute();
}
return {};

83
src/Interpreters/InterpreterDeleteQuery.cpp
View File

@ -61,24 +61,7 @@ BlockIO InterpreterDeleteQuery::execute()
auto table_lock = table->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout);
auto metadata_snapshot = table->getInMemoryMetadataPtr();
if (table->supportsDelete())
{
/// Convert to MutationCommand
MutationCommands mutation_commands;
MutationCommand mut_command;
mut_command.type = MutationCommand::Type::DELETE;
mut_command.predicate = delete_query.predicate;
mutation_commands.emplace_back(mut_command);
table->checkMutationIsPossible(mutation_commands, getContext()->getSettingsRef());
MutationsInterpreter::Settings settings(false);
MutationsInterpreter(table, metadata_snapshot, mutation_commands, getContext(), settings).validate();
table->mutate(mutation_commands, getContext());
return {};
}
else if (table->supportsLightweightDelete())
auto lightweightDelete = [&]()
{
if (!getContext()->getSettingsRef().enable_lightweight_delete)
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
@ -105,18 +88,78 @@ BlockIO InterpreterDeleteQuery::execute()
context->setSetting("mutations_sync", Field(context->getSettingsRef().lightweight_deletes_sync));
InterpreterAlterQuery alter_interpreter(alter_ast, context);
return alter_interpreter.execute();
};
if (table->supportsDelete())
{
/// Convert to MutationCommand
MutationCommands mutation_commands;
MutationCommand mut_command;
mut_command.type = MutationCommand::Type::DELETE;
mut_command.predicate = delete_query.predicate;
mutation_commands.emplace_back(mut_command);
table->checkMutationIsPossible(mutation_commands, getContext()->getSettingsRef());
MutationsInterpreter::Settings settings(false);
MutationsInterpreter(table, metadata_snapshot, mutation_commands, getContext(), settings).validate();
table->mutate(mutation_commands, getContext());
return {};
}
else if (table->supportsLightweightDelete())
{
return lightweightDelete();
}
else
{
/// Currently just better exception for the case of a table with projection,
/// can act differently according to the setting.
if (table->hasProjection())
{
auto context = Context::createCopy(getContext());
auto mode = context->getSettingsRef().lightweight_mutation_projection_mode;
if (mode == LightweightMutationProjectionMode::THROW)
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED,
"DELETE query is not supported for table {} as it has projections. "
"User should drop all the projections manually before running the query",
table->getStorageID().getFullTableName());
}
else if (mode == LightweightMutationProjectionMode::DROP)
{
std::vector<String> all_projections = metadata_snapshot->projections.getAllRegisteredNames();
context->setSetting("mutations_sync", Field(context->getSettingsRef().lightweight_deletes_sync));
/// Drop projections first so that lightweight delete can be performed.
for (const auto & projection : all_projections)
{
String alter_query =
"ALTER TABLE " + table->getStorageID().getFullTableName()
+ (delete_query.cluster.empty() ? "" : " ON CLUSTER " + backQuoteIfNeed(delete_query.cluster))
+ " DROP PROJECTION IF EXISTS " + projection;
ParserAlterQuery parser;
ASTPtr alter_ast = parseQuery(
parser,
alter_query.data(),
alter_query.data() + alter_query.size(),
"ALTER query",
0,
DBMS_DEFAULT_MAX_PARSER_DEPTH,
DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
InterpreterAlterQuery alter_interpreter(alter_ast, context);
alter_interpreter.execute();
}
}
else
{
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Unrecognized lightweight_mutation_projection_mode, only throw and drop are allowed.");
}
return lightweightDelete();
}
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"DELETE query is not supported for table {}",

8
src/Interpreters/InterpreterExplainQuery.cpp
View File

@ -534,7 +534,13 @@ QueryPipeline InterpreterExplainQuery::executeImpl()
}
else if (dynamic_cast<const ASTInsertQuery *>(ast.getExplainedQuery().get()))
{
InterpreterInsertQuery insert(ast.getExplainedQuery(), getContext());
InterpreterInsertQuery insert(
ast.getExplainedQuery(),
getContext(),
/* allow_materialized */ false,
/* no_squash */ false,
/* no_destination */ false,
/* async_isnert */ false);
auto io = insert.execute();
printPipeline(io.pipeline.getProcessors(), buf);
}

386
src/Interpreters/InterpreterInsertQuery.cpp
View File

@ -16,6 +16,7 @@
#include <Interpreters/getTableExpressions.h>
#include <Interpreters/processColumnTransformers.h>
#include <Interpreters/InterpreterSelectQueryAnalyzer.h>
#include <Interpreters/Context_fwd.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTInsertQuery.h>
#include <Parsers/ASTSelectQuery.h>
@ -26,6 +27,7 @@
#include <Processors/Transforms/CountingTransform.h>
#include <Processors/Transforms/ExpressionTransform.h>
#include <Processors/Transforms/MaterializingTransform.h>
#include <Processors/Transforms/DeduplicationTokenTransforms.h>
#include <Processors/Transforms/SquashingTransform.h>
#include <Processors/Transforms/PlanSquashingTransform.h>
#include <Processors/Transforms/getSourceFromASTInsertQuery.h>
@ -38,6 +40,7 @@
#include <Common/ThreadStatus.h>
#include <Common/checkStackSize.h>
#include <Common/ProfileEvents.h>
#include "base/defines.h"
namespace ProfileEvents
@ -394,56 +397,45 @@ Chain InterpreterInsertQuery::buildPreSinkChain(
return out;
}
BlockIO InterpreterInsertQuery::execute()
std::pair<std::vector<Chain>, std::vector<Chain>> InterpreterInsertQuery::buildPreAndSinkChains(size_t presink_streams, size_t sink_streams, StoragePtr table, const StorageMetadataPtr & metadata_snapshot, const Block & query_sample_block)
{
chassert(presink_streams > 0);
chassert(sink_streams > 0);
ThreadGroupPtr running_group;
if (current_thread)
running_group = current_thread->getThreadGroup();
if (!running_group)
running_group = std::make_shared<ThreadGroup>(getContext());
std::vector<Chain> sink_chains;
std::vector<Chain> presink_chains;
for (size_t i = 0; i < sink_streams; ++i)
{
auto out = buildSink(table, metadata_snapshot, /* thread_status_holder= */ nullptr,
running_group, /* elapsed_counter_ms= */ nullptr);
sink_chains.emplace_back(std::move(out));
}
for (size_t i = 0; i < presink_streams; ++i)
{
auto out = buildPreSinkChain(sink_chains[0].getInputHeader(), table, metadata_snapshot, query_sample_block);
presink_chains.emplace_back(std::move(out));
}
return {std::move(presink_chains), std::move(sink_chains)};
}
QueryPipeline InterpreterInsertQuery::buildInsertSelectPipeline(ASTInsertQuery & query, StoragePtr table)
{
const Settings & settings = getContext()->getSettingsRef();
auto & query = query_ptr->as<ASTInsertQuery &>();
QueryPipelineBuilder pipeline;
std::optional<QueryPipeline> distributed_pipeline;
QueryPlanResourceHolder resources;
StoragePtr table = getTable(query);
checkStorageSupportsTransactionsIfNeeded(table, getContext());
StoragePtr inner_table;
if (const auto * mv = dynamic_cast<const StorageMaterializedView *>(table.get()))
inner_table = mv->getTargetTable();
if (query.partition_by && !table->supportsPartitionBy())
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "PARTITION BY clause is not supported by storage");
auto table_lock = table->lockForShare(getContext()->getInitialQueryId(), settings.lock_acquire_timeout);
auto metadata_snapshot = table->getInMemoryMetadataPtr();
auto query_sample_block = getSampleBlock(query, table, metadata_snapshot, getContext(), no_destination, allow_materialized);
/// For table functions we check access while executing
/// getTable() -> ITableFunction::execute().
if (!query.table_function)
getContext()->checkAccess(AccessType::INSERT, query.table_id, query_sample_block.getNames());
if (query.select && settings.parallel_distributed_insert_select)
// Distributed INSERT SELECT
distributed_pipeline = table->distributedWrite(query, getContext());
std::vector<Chain> presink_chains;
std::vector<Chain> sink_chains;
if (!distributed_pipeline)
{
/// Number of streams works like this:
/// * For the SELECT, use `max_threads`, or `max_insert_threads`, or whatever
/// InterpreterSelectQuery ends up with.
/// * Use `max_insert_threads` streams for various insert-preparation steps, e.g.
/// materializing and squashing (too slow to do in one thread). That's `presink_chains`.
/// * If the table supports parallel inserts, use the same streams for writing to IStorage.
/// Otherwise ResizeProcessor them down to 1 stream.
/// * If it's not an INSERT SELECT, forget all that and use one stream.
size_t pre_streams_size = 1;
size_t sink_streams_size = 1;
if (query.select)
{
bool is_trivial_insert_select = false;
if (settings.optimize_trivial_insert_select)
@ -460,6 +452,8 @@ BlockIO InterpreterInsertQuery::execute()
&& std::all_of(selects.begin(), selects.end(), isTrivialSelect);
}
ContextPtr select_context = getContext();
if (is_trivial_insert_select)
{
/** When doing trivial INSERT INTO ... SELECT ... FROM table,
@ -468,7 +462,7 @@ BlockIO InterpreterInsertQuery::execute()
* to avoid unnecessary squashing.
*/
Settings new_settings = getContext()->getSettings();
Settings new_settings = select_context->getSettings();
new_settings.max_threads = std::max<UInt64>(1, settings.max_insert_threads);
@ -480,68 +474,32 @@ BlockIO InterpreterInsertQuery::execute()
new_settings.preferred_block_size_bytes = settings.min_insert_block_size_bytes;
}
auto new_context = Context::createCopy(context);
new_context->setSettings(new_settings);
new_context->setInsertionTable(getContext()->getInsertionTable(), getContext()->getInsertionTableColumnNames());
auto context_for_trivial_select = Context::createCopy(context);
context_for_trivial_select->setSettings(new_settings);
context_for_trivial_select->setInsertionTable(getContext()->getInsertionTable(), getContext()->getInsertionTableColumnNames());
select_context = context_for_trivial_select;
}
QueryPipelineBuilder pipeline;
{
auto select_query_options = SelectQueryOptions(QueryProcessingStage::Complete, 1);
if (settings.allow_experimental_analyzer)
{
InterpreterSelectQueryAnalyzer interpreter_select_analyzer(query.select, new_context, select_query_options);
InterpreterSelectQueryAnalyzer interpreter_select_analyzer(query.select, select_context, select_query_options);
pipeline = interpreter_select_analyzer.buildQueryPipeline();
}
else
{
InterpreterSelectWithUnionQuery interpreter_select(query.select, new_context, select_query_options);
pipeline = interpreter_select.buildQueryPipeline();
}
}
else
{
/// Passing 1 as subquery_depth will disable limiting size of intermediate result.
auto select_query_options = SelectQueryOptions(QueryProcessingStage::Complete, 1);
if (settings.allow_experimental_analyzer)
{
InterpreterSelectQueryAnalyzer interpreter_select_analyzer(query.select, getContext(), select_query_options);
pipeline = interpreter_select_analyzer.buildQueryPipeline();
}
else
{
InterpreterSelectWithUnionQuery interpreter_select(query.select, getContext(), select_query_options);
InterpreterSelectWithUnionQuery interpreter_select(query.select, select_context, select_query_options);
pipeline = interpreter_select.buildQueryPipeline();
}
}
pipeline.dropTotalsAndExtremes();
if (settings.max_insert_threads > 1)
{
auto table_id = table->getStorageID();
auto views = DatabaseCatalog::instance().getDependentViews(table_id);
/// It breaks some views-related tests and we have dedicated `parallel_view_processing` for views, so let's just skip them.
/// Also it doesn't make sense to reshuffle data if storage doesn't support parallel inserts.
const bool resize_to_max_insert_threads = !table->isView() && views.empty() && table->supportsParallelInsert();
pre_streams_size = resize_to_max_insert_threads ? settings.max_insert_threads
: std::min<size_t>(settings.max_insert_threads, pipeline.getNumStreams());
/// Deduplication when passing insert_deduplication_token breaks if using more than one thread
if (!settings.insert_deduplication_token.toString().empty())
{
LOG_DEBUG(
getLogger("InsertQuery"),
"Insert-select query using insert_deduplication_token, setting streams to 1 to avoid deduplication issues");
pre_streams_size = 1;
}
if (table->supportsParallelInsert())
sink_streams_size = pre_streams_size;
}
pipeline.resize(pre_streams_size);
/// Allow to insert Nullable into non-Nullable columns, NULL values will be added as defaults values.
if (getContext()->getSettingsRef().insert_null_as_default)
{
@ -560,43 +518,21 @@ BlockIO InterpreterInsertQuery::execute()
&& !isVariant(query_columns[col_idx].type)
&& !isDynamic(query_columns[col_idx].type)
&& output_columns.has(query_columns[col_idx].name))
query_sample_block.setColumn(col_idx, ColumnWithTypeAndName(makeNullableOrLowCardinalityNullable(query_columns[col_idx].column), makeNullableOrLowCardinalityNullable(query_columns[col_idx].type), query_columns[col_idx].name));
{
query_sample_block.setColumn(
col_idx,
ColumnWithTypeAndName(
makeNullableOrLowCardinalityNullable(query_columns[col_idx].column),
makeNullableOrLowCardinalityNullable(query_columns[col_idx].type),
query_columns[col_idx].name));
}
}
}
}
ThreadGroupPtr running_group;
if (current_thread)
running_group = current_thread->getThreadGroup();
if (!running_group)
running_group = std::make_shared<ThreadGroup>(getContext());
for (size_t i = 0; i < sink_streams_size; ++i)
{
auto out = buildSink(table, metadata_snapshot, /* thread_status_holder= */ nullptr,
running_group, /* elapsed_counter_ms= */ nullptr);
sink_chains.emplace_back(std::move(out));
}
for (size_t i = 0; i < pre_streams_size; ++i)
{
auto out = buildPreSinkChain(sink_chains[0].getInputHeader(), table, metadata_snapshot, query_sample_block);
presink_chains.emplace_back(std::move(out));
}
}
BlockIO res;
/// What type of query: INSERT or INSERT SELECT or INSERT WATCH?
if (distributed_pipeline)
{
res.pipeline = std::move(*distributed_pipeline);
}
else if (query.select)
{
const auto & header = presink_chains.at(0).getInputHeader();
auto actions_dag = ActionsDAG::makeConvertingActions(
pipeline.getHeader().getColumnsWithTypeAndName(),
header.getColumnsWithTypeAndName(),
query_sample_block.getColumnsWithTypeAndName(),
ActionsDAG::MatchColumnsMode::Position);
auto actions = std::make_shared<ExpressionActions>(actions_dag, ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes));
@ -622,39 +558,85 @@ BlockIO InterpreterInsertQuery::execute()
return counting;
});
if (shouldAddSquashingFroStorage(table))
{
bool table_prefers_large_blocks = table->prefersLargeBlocks();
size_t threads = presink_chains.size();
size_t num_select_threads = pipeline.getNumThreads();
pipeline.resize(1);
pipeline.addTransform(std::make_shared<PlanSquashingTransform>(
header,
table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL));
if (shouldAddSquashingFroStorage(table))
{
pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
{
return std::make_shared<PlanSquashingTransform>(
in_header,
table->prefersLargeBlocks() ? settings.min_insert_block_size_rows : settings.max_block_size,
table->prefersLargeBlocks() ? settings.min_insert_block_size_bytes : 0ULL);
});
}
pipeline.resize(threads);
pipeline.addSimpleTransform([&](const Block &in_header) -> ProcessorPtr
{
return std::make_shared<DeduplicationToken::AddTokenInfoTransform>(in_header);
});
if (!settings.insert_deduplication_token.value.empty())
{
pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
{
return std::make_shared<DeduplicationToken::SetUserTokenTransform>(settings.insert_deduplication_token.value, in_header);
});
pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
{
return std::make_shared<DeduplicationToken::SetSourceBlockNumberTransform>(in_header);
});
}
/// Number of streams works like this:
/// * For the SELECT, use `max_threads`, or `max_insert_threads`, or whatever
/// InterpreterSelectQuery ends up with.
/// * Use `max_insert_threads` streams for various insert-preparation steps, e.g.
/// materializing and squashing (too slow to do in one thread). That's `presink_chains`.
/// * If the table supports parallel inserts, use max_insert_threads for writing to IStorage.
/// Otherwise ResizeProcessor them down to 1 stream.
size_t presink_streams_size = std::max<size_t>(settings.max_insert_threads, pipeline.getNumStreams());
size_t sink_streams_size = table->supportsParallelInsert() ? std::max<size_t>(1, settings.max_insert_threads) : 1;
if (!settings.parallel_view_processing)
{
auto table_id = table->getStorageID();
auto views = DatabaseCatalog::instance().getDependentViews(table_id);
if (table->isView() || !views.empty())
sink_streams_size = 1;
}
auto [presink_chains, sink_chains] = buildPreAndSinkChains(
presink_streams_size, sink_streams_size,
table, metadata_snapshot, query_sample_block);
pipeline.resize(presink_chains.size());
if (shouldAddSquashingFroStorage(table))
{
pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
{
return std::make_shared<ApplySquashingTransform>(
in_header,
table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL);
table->prefersLargeBlocks() ? settings.min_insert_block_size_rows : settings.max_block_size,
table->prefersLargeBlocks() ? settings.min_insert_block_size_bytes : 0ULL);
});
}
size_t num_select_threads = pipeline.getNumThreads();
for (auto & chain : presink_chains)
resources = chain.detachResources();
for (auto & chain : sink_chains)
resources = chain.detachResources();
pipeline.addResources(chain.detachResources());
pipeline.addChains(std::move(presink_chains));
pipeline.resize(sink_chains.size());
pipeline.resize(sink_streams_size);
for (auto & chain : sink_chains)
pipeline.addResources(chain.detachResources());
pipeline.addChains(std::move(sink_chains));
if (!settings.parallel_view_processing)
@ -677,19 +659,35 @@ BlockIO InterpreterInsertQuery::execute()
return std::make_shared<EmptySink>(cur_header);
});
if (!allow_materialized)
{
for (const auto & column : metadata_snapshot->getColumns())
if (column.default_desc.kind == ColumnDefaultKind::Materialized && header.has(column.name))
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert column {}, because it is MATERIALIZED column.", column.name);
return QueryPipelineBuilder::getPipeline(std::move(pipeline));
}
res.pipeline = QueryPipelineBuilder::getPipeline(std::move(pipeline));
}
else
QueryPipeline InterpreterInsertQuery::buildInsertPipeline(ASTInsertQuery & query, StoragePtr table)
{
auto & chain = presink_chains.at(0);
chain.appendChain(std::move(sink_chains.at(0)));
const Settings & settings = getContext()->getSettingsRef();
auto metadata_snapshot = table->getInMemoryMetadataPtr();
auto query_sample_block = getSampleBlock(query, table, metadata_snapshot, getContext(), no_destination, allow_materialized);
Chain chain;
{
auto [presink_chains, sink_chains] = buildPreAndSinkChains(
/* presink_streams */1, /* sink_streams */1,
table, metadata_snapshot, query_sample_block);
chain = std::move(presink_chains.front());
chain.appendChain(std::move(sink_chains.front()));
}
if (!settings.insert_deduplication_token.value.empty())
{
chain.addSource(std::make_shared<DeduplicationToken::SetSourceBlockNumberTransform>(chain.getInputHeader()));
chain.addSource(std::make_shared<DeduplicationToken::SetUserTokenTransform>(settings.insert_deduplication_token.value, chain.getInputHeader()));
}
chain.addSource(std::make_shared<DeduplicationToken::AddTokenInfoTransform>(chain.getInputHeader()));
if (shouldAddSquashingFroStorage(table))
{
@ -716,9 +714,10 @@ BlockIO InterpreterInsertQuery::execute()
counting->setProgressCallback(context_ptr->getProgressCallback());
chain.addSource(std::move(counting));
res.pipeline = QueryPipeline(std::move(presink_chains[0]));
res.pipeline.setNumThreads(std::min<size_t>(res.pipeline.getNumThreads(), settings.max_threads));
res.pipeline.setConcurrencyControl(settings.use_concurrency_control);
QueryPipeline pipeline = QueryPipeline(std::move(chain));
pipeline.setNumThreads(std::min<size_t>(pipeline.getNumThreads(), settings.max_threads));
pipeline.setConcurrencyControl(settings.use_concurrency_control);
if (query.hasInlinedData() && !async_insert)
{
@ -728,15 +727,72 @@ BlockIO InterpreterInsertQuery::execute()
format->addBuffer(std::move(buffer));
auto pipe = getSourceFromInputFormat(query_ptr, std::move(format), getContext(), nullptr);
res.pipeline.complete(std::move(pipe));
}
pipeline.complete(std::move(pipe));
}
res.pipeline.addResources(std::move(resources));
return pipeline;
}
BlockIO InterpreterInsertQuery::execute()
{
const Settings & settings = getContext()->getSettingsRef();
auto & query = query_ptr->as<ASTInsertQuery &>();
StoragePtr table = getTable(query);
checkStorageSupportsTransactionsIfNeeded(table, getContext());
if (query.partition_by && !table->supportsPartitionBy())
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "PARTITION BY clause is not supported by storage");
auto table_lock = table->lockForShare(getContext()->getInitialQueryId(), settings.lock_acquire_timeout);
auto metadata_snapshot = table->getInMemoryMetadataPtr();
auto query_sample_block = getSampleBlock(query, table, metadata_snapshot, getContext(), no_destination, allow_materialized);
/// For table functions we check access while executing
/// getTable() -> ITableFunction::execute().
if (!query.table_function)
getContext()->checkAccess(AccessType::INSERT, query.table_id, query_sample_block.getNames());
if (!allow_materialized)
{
for (const auto & column : metadata_snapshot->getColumns())
if (column.default_desc.kind == ColumnDefaultKind::Materialized && query_sample_block.has(column.name))
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert column {}, because it is MATERIALIZED column.", column.name);
}
BlockIO res;
if (query.select)
{
if (settings.parallel_distributed_insert_select)
{
auto distributed = table->distributedWrite(query, getContext());
if (distributed)
{
res.pipeline = std::move(*distributed);
}
else
{
res.pipeline = buildInsertSelectPipeline(query, table);
}
}
else
{
res.pipeline = buildInsertSelectPipeline(query, table);
}
}
else
{
res.pipeline = buildInsertPipeline(query, table);
}
res.pipeline.addStorageHolder(table);
if (inner_table)
res.pipeline.addStorageHolder(inner_table);
if (const auto * mv = dynamic_cast<const StorageMaterializedView *>(table.get()))
res.pipeline.addStorageHolder(mv->getTargetTable());
return res;
}
@ -757,17 +813,27 @@ void InterpreterInsertQuery::extendQueryLogElemImpl(QueryLogElement & elem, Cont
}
}
void InterpreterInsertQuery::extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr &, ContextPtr context_) const
{
extendQueryLogElemImpl(elem, context_);
}
void registerInterpreterInsertQuery(InterpreterFactory & factory)
{
auto create_fn = [] (const InterpreterFactory::Arguments & args)
{
return std::make_unique<InterpreterInsertQuery>(args.query, args.context, args.allow_materialized);
return std::make_unique<InterpreterInsertQuery>(
args.query,
args.context,
args.allow_materialized,
/* no_squash */false,
/* no_destination */false,
/* async_insert */false);
};
factory.registerInterpreter("InterpreterInsertQuery", create_fn);
}
}

17
src/Interpreters/InterpreterInsertQuery.h
View File

@ -23,10 +23,10 @@ public:
InterpreterInsertQuery(
const ASTPtr & query_ptr_,
ContextPtr context_,
bool allow_materialized_ = false,
bool no_squash_ = false,
bool no_destination_ = false,
bool async_insert_ = false);
bool allow_materialized_,
bool no_squash_,
bool no_destination,
bool async_insert_);
/** Prepare a request for execution. Return block streams
* - the stream into which you can write data to execute the query, if INSERT;
@ -73,12 +73,17 @@ private:
ASTPtr query_ptr;
const bool allow_materialized;
const bool no_squash;
const bool no_destination;
bool no_squash = false;
bool no_destination = false;
const bool async_insert;
std::vector<std::unique_ptr<ReadBuffer>> owned_buffers;
std::pair<std::vector<Chain>, std::vector<Chain>> buildPreAndSinkChains(size_t presink_streams, size_t sink_streams, StoragePtr table, const StorageMetadataPtr & metadata_snapshot, const Block & query_sample_block);
QueryPipeline buildInsertSelectPipeline(ASTInsertQuery & query, StoragePtr table);
QueryPipeline buildInsertPipeline(ASTInsertQuery & query, StoragePtr table);
Chain buildSink(
const StoragePtr & table,
const StorageMetadataPtr & metadata_snapshot,

124
src/Interpreters/Squashing.cpp
View File

@ -1,6 +1,7 @@
#include <vector>
#include <Interpreters/Squashing.h>
#include <Common/CurrentThread.h>
#include <base/defines.h>
namespace DB
@ -11,24 +12,33 @@ namespace ErrorCodes
}
Squashing::Squashing(Block header_, size_t min_block_size_rows_, size_t min_block_size_bytes_)
: header(header_)
, min_block_size_rows(min_block_size_rows_)
: min_block_size_rows(min_block_size_rows_)
, min_block_size_bytes(min_block_size_bytes_)
, header(header_)
{
}
Chunk Squashing::flush()
{
return convertToChunk(std::move(chunks_to_merge_vec));
if (!accumulated)
return {};
auto result = convertToChunk(accumulated.extract());
chassert(result);
return result;
}
Chunk Squashing::squash(Chunk && input_chunk)
{
if (!input_chunk.hasChunkInfo())
if (!input_chunk)
return Chunk();
const auto *info = getInfoFromChunk(input_chunk);
return squash(info->chunks);
auto squash_info = input_chunk.getChunkInfos().extract<ChunksToSquash>();
if (!squash_info)
throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no ChunksToSquash in ChunkInfoPtr");
return squash(std::move(squash_info->chunks), std::move(input_chunk.getChunkInfos()));
}
Chunk Squashing::add(Chunk && input_chunk)
@ -37,48 +47,37 @@ Chunk Squashing::add(Chunk && input_chunk)
return {};
/// Just read block is already enough.
if (isEnoughSize(input_chunk.getNumRows(), input_chunk.bytes()))
if (isEnoughSize(input_chunk))
{
/// If no accumulated data, return just read block.
if (chunks_to_merge_vec.empty())
if (!accumulated)
{
chunks_to_merge_vec.push_back(std::move(input_chunk));
Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
chunks_to_merge_vec.clear();
return res_chunk;
accumulated.add(std::move(input_chunk));
return convertToChunk(accumulated.extract());
}
/// Return accumulated data (maybe it has small size) and place new block to accumulated data.
Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
chunks_to_merge_vec.clear();
changeCurrentSize(input_chunk.getNumRows(), input_chunk.bytes());
chunks_to_merge_vec.push_back(std::move(input_chunk));
Chunk res_chunk = convertToChunk(accumulated.extract());
accumulated.add(std::move(input_chunk));
return res_chunk;
}
/// Accumulated block is already enough.
if (isEnoughSize(accumulated_size.rows, accumulated_size.bytes))
if (isEnoughSize())
{
/// Return accumulated data and place new block to accumulated data.
Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
chunks_to_merge_vec.clear();
changeCurrentSize(input_chunk.getNumRows(), input_chunk.bytes());
chunks_to_merge_vec.push_back(std::move(input_chunk));
Chunk res_chunk = convertToChunk(accumulated.extract());
accumulated.add(std::move(input_chunk));
return res_chunk;
}
/// Pushing data into accumulating vector
expandCurrentSize(input_chunk.getNumRows(), input_chunk.bytes());
chunks_to_merge_vec.push_back(std::move(input_chunk));
accumulated.add(std::move(input_chunk));
/// If accumulated data is big enough, we send it
if (isEnoughSize(accumulated_size.rows, accumulated_size.bytes))
{
Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
changeCurrentSize(0, 0);
chunks_to_merge_vec.clear();
return res_chunk;
}
if (isEnoughSize())
return convertToChunk(accumulated.extract());
return {};
}
@ -90,14 +89,15 @@ Chunk Squashing::convertToChunk(std::vector<Chunk> && chunks) const
auto info = std::make_shared<ChunksToSquash>();
info->chunks = std::move(chunks);
chunks.clear();
return Chunk(header.cloneEmptyColumns(), 0, info);
// It is imortant that chunk is not empty, it has to have columns even if they are empty
auto aggr_chunk = Chunk(header.getColumns(), 0);
aggr_chunk.getChunkInfos().add(std::move(info));
chassert(aggr_chunk);
return aggr_chunk;
}
Chunk Squashing::squash(std::vector<Chunk> & input_chunks)
Chunk Squashing::squash(std::vector<Chunk> && input_chunks, Chunk::ChunkInfoCollection && infos)
{
Chunk accumulated_chunk;
std::vector<IColumn::MutablePtr> mutable_columns = {};
size_t rows = 0;
for (const Chunk & chunk : input_chunks)
@ -119,35 +119,17 @@ Chunk Squashing::squash(std::vector<Chunk> & input_chunks)
for (size_t j = 0, size = mutable_columns.size(); j < size; ++j)
{
const auto source_column = columns[j];
mutable_columns[j]->insertRangeFrom(*source_column, 0, source_column->size());
}
}
accumulated_chunk.setColumns(std::move(mutable_columns), rows);
return accumulated_chunk;
}
const ChunksToSquash* Squashing::getInfoFromChunk(const Chunk & chunk)
{
const auto& info = chunk.getChunkInfo();
const auto * agg_info = typeid_cast<const ChunksToSquash *>(info.get());
Chunk result;
result.setColumns(std::move(mutable_columns), rows);
result.setChunkInfos(infos);
result.getChunkInfos().append(std::move(input_chunks.back().getChunkInfos()));
if (!agg_info)
throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no ChunksToSquash in ChunkInfoPtr");
return agg_info;
}
void Squashing::expandCurrentSize(size_t rows, size_t bytes)
{
accumulated_size.rows += rows;
accumulated_size.bytes += bytes;
}
void Squashing::changeCurrentSize(size_t rows, size_t bytes)
{
accumulated_size.rows = rows;
accumulated_size.bytes = bytes;
chassert(result);
return result;
}
bool Squashing::isEnoughSize(size_t rows, size_t bytes) const
@ -156,4 +138,28 @@ bool Squashing::isEnoughSize(size_t rows, size_t bytes) const
|| (min_block_size_rows && rows >= min_block_size_rows)
|| (min_block_size_bytes && bytes >= min_block_size_bytes);
}
bool Squashing::isEnoughSize() const
{
return isEnoughSize(accumulated.getRows(), accumulated.getBytes());
};
bool Squashing::isEnoughSize(const Chunk & chunk) const
{
return isEnoughSize(chunk.getNumRows(), chunk.bytes());
}
void Squashing::CurrentSize::add(Chunk && chunk)
{
rows += chunk.getNumRows();
bytes += chunk.bytes();
chunks.push_back(std::move(chunk));
}
std::vector<Chunk> Squashing::CurrentSize::extract()
{
auto result = std::move(chunks);
*this = {};
return result;
}
}

46
src/Interpreters/Squashing.h
View File

@ -8,9 +8,18 @@
namespace DB
{
struct ChunksToSquash : public ChunkInfo
class ChunksToSquash : public ChunkInfoCloneable<ChunksToSquash>
{
mutable std::vector<Chunk> chunks = {};
public:
ChunksToSquash() = default;
ChunksToSquash(const ChunksToSquash & other)
{
chunks.reserve(other.chunks.size());
for (const auto & chunk: other.chunks)
chunks.push_back(chunk.clone());
}
std::vector<Chunk> chunks = {};
};
/** Merging consecutive passed blocks to specified minimum size.
@ -36,32 +45,35 @@ public:
static Chunk squash(Chunk && input_chunk);
Chunk flush();
bool isDataLeft()
{
return !chunks_to_merge_vec.empty();
}
void setHeader(Block header_) { header = std::move(header_); }
const Block & getHeader() const { return header; }
Block header;
private:
struct CurrentSize
class CurrentSize
{
std::vector<Chunk> chunks = {};
size_t rows = 0;
size_t bytes = 0;
public:
explicit operator bool () const { return !chunks.empty(); }
size_t getRows() const { return rows; }
size_t getBytes() const { return bytes; }
void add(Chunk && chunk);
std::vector<Chunk> extract();
};
std::vector<Chunk> chunks_to_merge_vec = {};
size_t min_block_size_rows;
size_t min_block_size_bytes;
const size_t min_block_size_rows;
const size_t min_block_size_bytes;
Block header;
CurrentSize accumulated_size;
CurrentSize accumulated;
static const ChunksToSquash * getInfoFromChunk(const Chunk & chunk);
static Chunk squash(std::vector<Chunk> && input_chunks, Chunk::ChunkInfoCollection && infos);
static Chunk squash(std::vector<Chunk> & input_chunks);
void expandCurrentSize(size_t rows, size_t bytes);
void changeCurrentSize(size_t rows, size_t bytes);
bool isEnoughSize() const;
bool isEnoughSize(size_t rows, size_t bytes) const;
bool isEnoughSize(const Chunk & chunk) const;
Chunk convertToChunk(std::vector<Chunk> && chunks) const;
};

8
src/Interpreters/SystemLog.cpp
View File

@ -538,7 +538,13 @@ void SystemLog<LogElement>::flushImpl(const std::vector<LogElement> & to_flush,
insert_context->makeQueryContext();
addSettingsForQuery(insert_context, IAST::QueryKind::Insert);
InterpreterInsertQuery interpreter(query_ptr, insert_context);
InterpreterInsertQuery interpreter(
query_ptr,
insert_context,
/* allow_materialized */ false,
/* no_squash */ false,
/* no_destination */ false,
/* async_isnert */ false);
BlockIO io = interpreter.execute();
PushingPipelineExecutor executor(io.pipeline);

2
src/Interpreters/TreeRewriter.cpp
View File

@ -1188,7 +1188,7 @@ bool TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select
}
}
/// Check for dynamic subcolums in unknown required columns.
/// Check for dynamic subcolumns in unknown required columns.
if (!unknown_required_source_columns.empty())
{
for (const NameAndTypePair & pair : source_columns_ordinary)

30
src/Interpreters/getColumnFromBlock.cpp
View File

@ -31,6 +31,36 @@ ColumnPtr tryGetColumnFromBlock(const Block & block, const NameAndTypePair & req
return castColumn({elem_column, elem_type, ""}, requested_column.type);
}
ColumnPtr tryGetSubcolumnFromBlock(const Block & block, const DataTypePtr & requested_column_type, const NameAndTypePair & requested_subcolumn)
{
const auto * elem = block.findByName(requested_subcolumn.getNameInStorage());
if (!elem)
return nullptr;
auto subcolumn_name = requested_subcolumn.getSubcolumnName();
/// If requested subcolumn is dynamic, we should first perform cast and then
/// extract the subcolumn, because the data of dynamic subcolumn can change after cast.
if (elem->type->hasDynamicSubcolumns() && !elem->type->equals(*requested_column_type))
{
auto casted_column = castColumn({elem->column, elem->type, ""}, requested_column_type);
auto elem_column = requested_column_type->tryGetSubcolumn(subcolumn_name, casted_column);
auto elem_type = requested_column_type->tryGetSubcolumnType(subcolumn_name);
if (!elem_type || !elem_column)
return nullptr;
return elem_column;
}
auto elem_column = elem->type->tryGetSubcolumn(subcolumn_name, elem->column);
auto elem_type = elem->type->tryGetSubcolumnType(subcolumn_name);
if (!elem_type || !elem_column)
return nullptr;
return castColumn({elem_column, elem_type, ""}, requested_subcolumn.type);
}
ColumnPtr getColumnFromBlock(const Block & block, const NameAndTypePair & requested_column)
{
auto result_column = tryGetColumnFromBlock(block, requested_column);

1
src/Interpreters/getColumnFromBlock.h
View File

@ -9,5 +9,6 @@ namespace DB
ColumnPtr getColumnFromBlock(const Block & block, const NameAndTypePair & requested_column);
ColumnPtr tryGetColumnFromBlock(const Block & block, const NameAndTypePair & requested_column);
ColumnPtr tryGetSubcolumnFromBlock(const Block & block, const DataTypePtr & requested_column_type, const NameAndTypePair & requested_subcolumn);
}

20
src/Processors/Chunk.cpp
View File

@ -19,14 +19,6 @@ Chunk::Chunk(DB::Columns columns_, UInt64 num_rows_) : columns(std::move(columns
checkNumRowsIsConsistent();
}
Chunk::Chunk(Columns columns_, UInt64 num_rows_, ChunkInfoPtr chunk_info_)
: columns(std::move(columns_))
, num_rows(num_rows_)
, chunk_info(std::move(chunk_info_))
{
checkNumRowsIsConsistent();
}
static Columns unmuteColumns(MutableColumns && mutable_columns)
{
Columns columns;
@ -43,17 +35,11 @@ Chunk::Chunk(MutableColumns columns_, UInt64 num_rows_)
checkNumRowsIsConsistent();
}
Chunk::Chunk(MutableColumns columns_, UInt64 num_rows_, ChunkInfoPtr chunk_info_)
: columns(unmuteColumns(std::move(columns_)))
, num_rows(num_rows_)
, chunk_info(std::move(chunk_info_))
{
checkNumRowsIsConsistent();
}
Chunk Chunk::clone() const
{
return Chunk(getColumns(), getNumRows(), chunk_info);
auto tmp = Chunk(getColumns(), getNumRows());
tmp.setChunkInfos(chunk_infos.clone());
return tmp;
}
void Chunk::setColumns(Columns columns_, UInt64 num_rows_)

58
src/Processors/Chunk.h
View File

@ -1,7 +1,9 @@
#pragma once
#include <Common/CollectionOfDerived.h>
#include <Columns/IColumn.h>
#include <unordered_map>
#include <memory>
namespace DB
{
@ -9,11 +11,29 @@ namespace DB
class ChunkInfo
{
public:
virtual ~ChunkInfo() = default;
using Ptr = std::shared_ptr<ChunkInfo>;
ChunkInfo() = default;
ChunkInfo(const ChunkInfo&) = default;
ChunkInfo(ChunkInfo&&) = default;
virtual Ptr clone() const = 0;
virtual ~ChunkInfo() = default;
};
using ChunkInfoPtr = std::shared_ptr<const ChunkInfo>;
template<class Derived>
class ChunkInfoCloneable : public ChunkInfo
{
public:
ChunkInfoCloneable() = default;
ChunkInfoCloneable(const ChunkInfoCloneable & other) = default;
Ptr clone() const override
{
return std::static_pointer_cast<ChunkInfo>(std::make_shared<Derived>(*static_cast<const Derived*>(this)));
}
};
/**
* Chunk is a list of columns with the same length.
@ -32,26 +52,26 @@ using ChunkInfoPtr = std::shared_ptr<const ChunkInfo>;
class Chunk
{
public:
using ChunkInfoCollection = CollectionOfDerivedItems<ChunkInfo>;
Chunk() = default;
Chunk(const Chunk & other) = delete;
Chunk(Chunk && other) noexcept
: columns(std::move(other.columns))
, num_rows(other.num_rows)
, chunk_info(std::move(other.chunk_info))
, chunk_infos(std::move(other.chunk_infos))
{
other.num_rows = 0;
}
Chunk(Columns columns_, UInt64 num_rows_);
Chunk(Columns columns_, UInt64 num_rows_, ChunkInfoPtr chunk_info_);
Chunk(MutableColumns columns_, UInt64 num_rows_);
Chunk(MutableColumns columns_, UInt64 num_rows_, ChunkInfoPtr chunk_info_);
Chunk & operator=(const Chunk & other) = delete;
Chunk & operator=(Chunk && other) noexcept
{
columns = std::move(other.columns);
chunk_info = std::move(other.chunk_info);
chunk_infos = std::move(other.chunk_infos);
num_rows = other.num_rows;
other.num_rows = 0;
return *this;
@ -62,15 +82,15 @@ public:
void swap(Chunk & other) noexcept
{
columns.swap(other.columns);
chunk_info.swap(other.chunk_info);
std::swap(num_rows, other.num_rows);
chunk_infos.swap(other.chunk_infos);
}
void clear()
{
num_rows = 0;
columns.clear();
chunk_info.reset();
chunk_infos.clear();
}
const Columns & getColumns() const { return columns; }
@ -81,9 +101,9 @@ public:
/** Get empty columns with the same types as in block. */
MutableColumns cloneEmptyColumns() const;
const ChunkInfoPtr & getChunkInfo() const { return chunk_info; }
bool hasChunkInfo() const { return chunk_info != nullptr; }
void setChunkInfo(ChunkInfoPtr chunk_info_) { chunk_info = std::move(chunk_info_); }
ChunkInfoCollection & getChunkInfos() { return chunk_infos; }
const ChunkInfoCollection & getChunkInfos() const { return chunk_infos; }
void setChunkInfos(ChunkInfoCollection chunk_infos_) { chunk_infos = std::move(chunk_infos_); }
UInt64 getNumRows() const { return num_rows; }
UInt64 getNumColumns() const { return columns.size(); }
@ -107,7 +127,7 @@ public:
private:
Columns columns;
UInt64 num_rows = 0;
ChunkInfoPtr chunk_info;
ChunkInfoCollection chunk_infos;
void checkNumRowsIsConsistent();
};
@ -117,11 +137,15 @@ using Chunks = std::vector<Chunk>;
/// AsyncInsert needs two kinds of information:
/// - offsets of different sub-chunks
/// - tokens of different sub-chunks, which are assigned by setting `insert_deduplication_token`.
class AsyncInsertInfo : public ChunkInfo
class AsyncInsertInfo : public ChunkInfoCloneable<AsyncInsertInfo>
{
public:
AsyncInsertInfo() = default;
explicit AsyncInsertInfo(const std::vector<size_t> & offsets_, const std::vector<String> & tokens_) : offsets(offsets_), tokens(tokens_) {}
AsyncInsertInfo(const AsyncInsertInfo & other) = default;
AsyncInsertInfo(const std::vector<size_t> & offsets_, const std::vector<String> & tokens_)
: offsets(offsets_)
, tokens(tokens_)
{}
std::vector<size_t> offsets;
std::vector<String> tokens;
@ -130,9 +154,11 @@ public:
using AsyncInsertInfoPtr = std::shared_ptr<AsyncInsertInfo>;
/// Extension to support delayed defaults. AddingDefaultsProcessor uses it to replace missing values with column defaults.
class ChunkMissingValues : public ChunkInfo
class ChunkMissingValues : public ChunkInfoCloneable<ChunkMissingValues>
{
public:
ChunkMissingValues(const ChunkMissingValues & other) = default;
using RowsBitMask = std::vector<bool>; /// a bit per row for a column
const RowsBitMask & getDefaultsBitmask(size_t column_idx) const;

5
src/Processors/Executors/PullingAsyncPipelineExecutor.cpp
View File

@ -147,14 +147,11 @@ bool PullingAsyncPipelineExecutor::pull(Block & block, uint64_t milliseconds)
block = lazy_format->getPort(IOutputFormat::PortKind::Main).getHeader().cloneWithColumns(chunk.detachColumns());
if (auto chunk_info = chunk.getChunkInfo())
{
if (const auto * agg_info = typeid_cast<const AggregatedChunkInfo *>(chunk_info.get()))
if (auto agg_info = chunk.getChunkInfos().get<AggregatedChunkInfo>())
{
block.info.bucket_num = agg_info->bucket_num;
block.info.is_overflows = agg_info->is_overflows;
}
}
return true;
}

5
src/Processors/Executors/PullingPipelineExecutor.cpp
View File

@ -73,14 +73,11 @@ bool PullingPipelineExecutor::pull(Block & block)
}
block = pulling_format->getPort(IOutputFormat::PortKind::Main).getHeader().cloneWithColumns(chunk.detachColumns());
if (auto chunk_info = chunk.getChunkInfo())
{
if (const auto * agg_info = typeid_cast<const AggregatedChunkInfo *>(chunk_info.get()))
if (auto agg_info = chunk.getChunkInfos().get<AggregatedChunkInfo>())
{
block.info.bucket_num = agg_info->bucket_num;
block.info.is_overflows = agg_info->is_overflows;
}
}
return true;
}

4
src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp
View File

@ -179,7 +179,9 @@ void ParquetBlockOutputFormat::consume(Chunk chunk)
columns[i]->insertRangeFrom(*concatenated.getColumns()[i], offset, count);
Chunks piece;
piece.emplace_back(std::move(columns), count, concatenated.getChunkInfo());
piece.emplace_back(std::move(columns), count);
piece.back().setChunkInfos(concatenated.getChunkInfos());
writeRowGroup(std::move(piece));
}
}

5
src/Processors/IAccumulatingTransform.cpp
View File

@ -8,8 +8,9 @@ namespace ErrorCodes
}
IAccumulatingTransform::IAccumulatingTransform(Block input_header, Block output_header)
: IProcessor({std::move(input_header)}, {std::move(output_header)}),
input(inputs.front()), output(outputs.front())
: IProcessor({std::move(input_header)}, {std::move(output_header)})
, input(inputs.front())
, output(outputs.front())
{
}

10
src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.cpp
View File

@ -53,13 +53,11 @@ void FinishAggregatingInOrderAlgorithm::consume(Input & input, size_t source_num
if (!input.chunk.hasRows())
return;
const auto & info = input.chunk.getChunkInfo();
if (!info)
if (input.chunk.getChunkInfos().empty())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk info was not set for chunk in FinishAggregatingInOrderAlgorithm");
Int64 allocated_bytes = 0;
/// Will be set by AggregatingInOrderTransform during local aggregation; will be nullptr during merging on initiator.
if (const auto * arenas_info = typeid_cast<const ChunkInfoWithAllocatedBytes *>(info.get()))
if (auto arenas_info = input.chunk.getChunkInfos().get<ChunkInfoWithAllocatedBytes>())
allocated_bytes = arenas_info->allocated_bytes;
states[source_num] = State{input.chunk, description, allocated_bytes};
@ -136,7 +134,7 @@ Chunk FinishAggregatingInOrderAlgorithm::prepareToMerge()
info->chunk_num = chunk_num++;
Chunk chunk;
chunk.setChunkInfo(std::move(info));
chunk.getChunkInfos().add(std::move(info));
return chunk;
}
@ -163,7 +161,7 @@ void FinishAggregatingInOrderAlgorithm::addToAggregation()
chunks.emplace_back(std::move(new_columns), current_rows);
}
chunks.back().setChunkInfo(std::make_shared<AggregatedChunkInfo>());
chunks.back().getChunkInfos().add(std::make_shared<AggregatedChunkInfo>());
states[i].current_row = states[i].to_row;
/// We assume that sizes in bytes of rows are almost the same.

12
src/Processors/Merges/Algorithms/MergeTreePartLevelInfo.h
View File

@ -6,18 +6,22 @@ namespace DB
{
/// To carry part level if chunk is produced by a merge tree source
class MergeTreePartLevelInfo : public ChunkInfo
class MergeTreePartLevelInfo : public ChunkInfoCloneable<MergeTreePartLevelInfo>
{
public:
MergeTreePartLevelInfo() = delete;
explicit MergeTreePartLevelInfo(ssize_t part_level) : origin_merge_tree_part_level(part_level) { }
explicit MergeTreePartLevelInfo(ssize_t part_level)
: origin_merge_tree_part_level(part_level)
{ }
MergeTreePartLevelInfo(const MergeTreePartLevelInfo & other) = default;
size_t origin_merge_tree_part_level = 0;
};
inline size_t getPartLevelFromChunk(const Chunk & chunk)
{
const auto & info = chunk.getChunkInfo();
if (const auto * part_level_info = typeid_cast<const MergeTreePartLevelInfo *>(info.get()))
const auto part_level_info = chunk.getChunkInfos().get<MergeTreePartLevelInfo>();
if (part_level_info)
return part_level_info->origin_merge_tree_part_level;
return 0;
}

Some files were not shown because too many files have changed in this diff Show More