thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #66479 from rschu1ze/bump-rocksdb2

Browse Source 
Bump RocksDB from v7.10.2 to v8.9.1
This commit is contained in:
Robert Schulze 2024-08-06 09:31:20 +00:00 committed by GitHub
commit 4d44682017
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 34 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/rocksdb vendored

@ -1 +1 @@
Subproject commit 01e43568fa9f3f7bf107b2b66c00b286b456f33e
Subproject commit 49ce8a1064dd1ad89117899839bf136365e49e79

24
contrib/rocksdb-cmake/CMakeLists.txt
View File

@ -1,6 +1,6 @@
option (ENABLE_ROCKSDB "Enable RocksDB" ${ENABLE_LIBRARIES})
if (NOT ENABLE_ROCKSDB)
if (NOT ENABLE_ROCKSDB OR NO_SSE3_OR_HIGHER) # assumes SSE4.2 and PCLMUL
message (STATUS "Not using RocksDB")
return()
endif()
@ -39,13 +39,6 @@ if(WITH_ZSTD)
list(APPEND THIRDPARTY_LIBS ch_contrib::zstd)
endif()
add_definitions(-DROCKSDB_PORTABLE)
if(ENABLE_SSE42 AND ENABLE_PCLMULQDQ)
add_definitions(-DHAVE_SSE42)
add_definitions(-DHAVE_PCLMUL)
endif()
if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64")
set (HAS_ARMV8_CRC 1)
# the original build descriptions set specific flags for ARM. These flags are already subsumed by ClickHouse's general
@ -91,7 +84,9 @@ set(SOURCES
${ROCKSDB_SOURCE_DIR}/cache/compressed_secondary_cache.cc
${ROCKSDB_SOURCE_DIR}/cache/lru_cache.cc
${ROCKSDB_SOURCE_DIR}/cache/secondary_cache.cc
${ROCKSDB_SOURCE_DIR}/cache/secondary_cache_adapter.cc
${ROCKSDB_SOURCE_DIR}/cache/sharded_cache.cc
${ROCKSDB_SOURCE_DIR}/cache/tiered_secondary_cache.cc
${ROCKSDB_SOURCE_DIR}/db/arena_wrapped_db_iter.cc
${ROCKSDB_SOURCE_DIR}/db/blob/blob_contents.cc
${ROCKSDB_SOURCE_DIR}/db/blob/blob_fetcher.cc
@ -174,9 +169,11 @@ set(SOURCES
${ROCKSDB_SOURCE_DIR}/db/wal_manager.cc
${ROCKSDB_SOURCE_DIR}/db/wide/wide_column_serialization.cc
${ROCKSDB_SOURCE_DIR}/db/wide/wide_columns.cc
${ROCKSDB_SOURCE_DIR}/db/wide/wide_columns_helper.cc
${ROCKSDB_SOURCE_DIR}/db/write_batch.cc
${ROCKSDB_SOURCE_DIR}/db/write_batch_base.cc
${ROCKSDB_SOURCE_DIR}/db/write_controller.cc
${ROCKSDB_SOURCE_DIR}/db/write_stall_stats.cc
${ROCKSDB_SOURCE_DIR}/db/write_thread.cc
${ROCKSDB_SOURCE_DIR}/env/composite_env.cc
${ROCKSDB_SOURCE_DIR}/env/env.cc
@ -229,6 +226,7 @@ set(SOURCES
${ROCKSDB_SOURCE_DIR}/options/configurable.cc
${ROCKSDB_SOURCE_DIR}/options/customizable.cc
${ROCKSDB_SOURCE_DIR}/options/db_options.cc
${ROCKSDB_SOURCE_DIR}/options/offpeak_time_info.cc
${ROCKSDB_SOURCE_DIR}/options/options.cc
${ROCKSDB_SOURCE_DIR}/options/options_helper.cc
${ROCKSDB_SOURCE_DIR}/options/options_parser.cc
@ -268,6 +266,7 @@ set(SOURCES
${ROCKSDB_SOURCE_DIR}/table/get_context.cc
${ROCKSDB_SOURCE_DIR}/table/iterator.cc
${ROCKSDB_SOURCE_DIR}/table/merging_iterator.cc
${ROCKSDB_SOURCE_DIR}/table/compaction_merging_iterator.cc
${ROCKSDB_SOURCE_DIR}/table/meta_blocks.cc
${ROCKSDB_SOURCE_DIR}/table/persistent_cache_helper.cc
${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_bloom.cc
@ -309,6 +308,7 @@ set(SOURCES
${ROCKSDB_SOURCE_DIR}/util/compression_context_cache.cc
${ROCKSDB_SOURCE_DIR}/util/concurrent_task_limiter_impl.cc
${ROCKSDB_SOURCE_DIR}/util/crc32c.cc
${ROCKSDB_SOURCE_DIR}/util/data_structure.cc
${ROCKSDB_SOURCE_DIR}/util/dynamic_bloom.cc
${ROCKSDB_SOURCE_DIR}/util/hash.cc
${ROCKSDB_SOURCE_DIR}/util/murmurhash.cc
@ -322,6 +322,8 @@ set(SOURCES
${ROCKSDB_SOURCE_DIR}/util/string_util.cc
${ROCKSDB_SOURCE_DIR}/util/thread_local.cc
${ROCKSDB_SOURCE_DIR}/util/threadpool_imp.cc
${ROCKSDB_SOURCE_DIR}/util/udt_util.cc
${ROCKSDB_SOURCE_DIR}/util/write_batch_util.cc
${ROCKSDB_SOURCE_DIR}/util/xxhash.cc
${ROCKSDB_SOURCE_DIR}/utilities/agg_merge/agg_merge.cc
${ROCKSDB_SOURCE_DIR}/utilities/backup/backup_engine.cc
@ -404,12 +406,6 @@ set(SOURCES
${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/util/memarena.cc
build_version.cc) # generated by hand
if(ENABLE_SSE42 AND ENABLE_PCLMULQDQ)
set_source_files_properties(
"${ROCKSDB_SOURCE_DIR}/util/crc32c.cc"
PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")
endif()
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
list(APPEND SOURCES
"${ROCKSDB_SOURCE_DIR}/util/crc32c_ppc.c"

7
src/Coordination/KeeperContext.cpp
View File

@ -23,6 +23,7 @@
#if USE_ROCKSDB
#include <rocksdb/table.h>
#include <rocksdb/convenience.h>
#include <rocksdb/statistics.h>
#include <rocksdb/utilities/db_ttl.h>
#endif
@ -88,7 +89,7 @@ static rocksdb::Options getRocksDBOptionsFromConfig(const Poco::Util::AbstractCo
if (config.has("keeper_server.rocksdb.options"))
{
auto config_options = getOptionsFromConfig(config, "keeper_server.rocksdb.options");
status = rocksdb::GetDBOptionsFromMap(merged, config_options, &merged);
status = rocksdb::GetDBOptionsFromMap({}, merged, config_options, &merged);
if (!status.ok())
{
throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from 'rocksdb.options' : {}",
@ -98,7 +99,7 @@ static rocksdb::Options getRocksDBOptionsFromConfig(const Poco::Util::AbstractCo
if (config.has("rocksdb.column_family_options"))
{
auto column_family_options = getOptionsFromConfig(config, "rocksdb.column_family_options");
status = rocksdb::GetColumnFamilyOptionsFromMap(merged, column_family_options, &merged);
status = rocksdb::GetColumnFamilyOptionsFromMap({}, merged, column_family_options, &merged);
if (!status.ok())
{
throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from 'rocksdb.column_family_options' at: {}", status.ToString());
@ -107,7 +108,7 @@ static rocksdb::Options getRocksDBOptionsFromConfig(const Poco::Util::AbstractCo
if (config.has("rocksdb.block_based_table_options"))
{
auto block_based_table_options = getOptionsFromConfig(config, "rocksdb.block_based_table_options");
status = rocksdb::GetBlockBasedTableOptionsFromMap(table_options, block_based_table_options, &table_options);
status = rocksdb::GetBlockBasedTableOptionsFromMap({}, table_options, block_based_table_options, &table_options);
if (!status.ok())
{
throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from 'rocksdb.block_based_table_options' at: {}", status.ToString());

11
src/Coordination/tests/gtest_coordination.cpp
View File

@ -111,10 +111,13 @@ public:
}
};
using Implementation = testing::Types<TestParam<DB::KeeperMemoryStorage, true>,
TestParam<DB::KeeperMemoryStorage, false>,
TestParam<DB::KeeperRocksStorage, true>,
TestParam<DB::KeeperRocksStorage, false>>;
using Implementation = testing::Types<TestParam<DB::KeeperMemoryStorage, true>
,TestParam<DB::KeeperMemoryStorage, false>
#if USE_ROCKSDB
,TestParam<DB::KeeperRocksStorage, true>
,TestParam<DB::KeeperRocksStorage, false>
#endif
>;
TYPED_TEST_SUITE(CoordinationTest, Implementation);
TYPED_TEST(CoordinationTest, RaftServerConfigParse)

17
src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp
View File

@ -36,10 +36,12 @@
#include <base/sort.h>
#include <rocksdb/advanced_options.h>
#include <rocksdb/compression_type.h>
#include <rocksdb/convenience.h>
#include <rocksdb/env.h>
#include <rocksdb/options.h>
#include <rocksdb/statistics.h>
#include <rocksdb/table.h>
#include <rocksdb/convenience.h>
#include <rocksdb/utilities/db_ttl.h>
#include <cstddef>
@ -428,6 +430,7 @@ void StorageEmbeddedRocksDB::initDB()
rocksdb::Options base;
base.create_if_missing = true;
base.compression = rocksdb::CompressionType::kZSTD;
base.statistics = rocksdb::CreateDBStatistics();
/// It is too verbose by default, and in fact we don't care about rocksdb logs at all.
base.info_log_level = rocksdb::ERROR_LEVEL;
@ -439,7 +442,7 @@ void StorageEmbeddedRocksDB::initDB()
if (config.has("rocksdb.options"))
{
auto config_options = getOptionsFromConfig(config, "rocksdb.options");
status = rocksdb::GetDBOptionsFromMap(merged, config_options, &merged);
status = rocksdb::GetDBOptionsFromMap({}, merged, config_options, &merged);
if (!status.ok())
{
throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from 'rocksdb.options' at: {}: {}",
@ -449,7 +452,7 @@ void StorageEmbeddedRocksDB::initDB()
if (config.has("rocksdb.column_family_options"))
{
auto column_family_options = getOptionsFromConfig(config, "rocksdb.column_family_options");
status = rocksdb::GetColumnFamilyOptionsFromMap(merged, column_family_options, &merged);
status = rocksdb::GetColumnFamilyOptionsFromMap({}, merged, column_family_options, &merged);
if (!status.ok())
{
throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from 'rocksdb.column_family_options' at: {}: {}",
@ -459,7 +462,7 @@ void StorageEmbeddedRocksDB::initDB()
if (config.has("rocksdb.block_based_table_options"))
{
auto block_based_table_options = getOptionsFromConfig(config, "rocksdb.block_based_table_options");
status = rocksdb::GetBlockBasedTableOptionsFromMap(table_options, block_based_table_options, &table_options);
status = rocksdb::GetBlockBasedTableOptionsFromMap({}, table_options, block_based_table_options, &table_options);
if (!status.ok())
{
throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from 'rocksdb.block_based_table_options' at: {}: {}",
@ -484,7 +487,7 @@ void StorageEmbeddedRocksDB::initDB()
if (config.has(config_key))
{
auto table_config_options = getOptionsFromConfig(config, config_key);
status = rocksdb::GetDBOptionsFromMap(merged, table_config_options, &merged);
status = rocksdb::GetDBOptionsFromMap({}, merged, table_config_options, &merged);
if (!status.ok())
{
throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from '{}' at: {}: {}",
@ -496,7 +499,7 @@ void StorageEmbeddedRocksDB::initDB()
if (config.has(config_key))
{
auto table_column_family_options = getOptionsFromConfig(config, config_key);
status = rocksdb::GetColumnFamilyOptionsFromMap(merged, table_column_family_options, &merged);
status = rocksdb::GetColumnFamilyOptionsFromMap({}, merged, table_column_family_options, &merged);
if (!status.ok())
{
throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from '{}' at: {}: {}",
@ -508,7 +511,7 @@ void StorageEmbeddedRocksDB::initDB()
if (config.has(config_key))
{
auto block_based_table_options = getOptionsFromConfig(config, config_key);
status = rocksdb::GetBlockBasedTableOptionsFromMap(table_options, block_based_table_options, &table_options);
status = rocksdb::GetBlockBasedTableOptionsFromMap({}, table_options, block_based_table_options, &table_options);
if (!status.ok())
{
throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from '{}' at: {}: {}",

3
tests/config/config.d/rocksdb.xml
View File

@ -1,7 +1,8 @@
<clickhouse>
<rocksdb>
<options>
<info_log_level>DEBUG_LEVEL</info_log_level>
<!-- https://github.com/ClickHouse/ClickHouse/pull/67274#issuecomment-2255301116 -->
<info_log_level>ERROR_LEVEL</info_log_level>
</options>
</rocksdb>
</clickhouse>