diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index fd2dc851ae7..7228608c9f1 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -748,7 +748,9 @@ if (ThreadFuzzer::instance().isEffective()) /// Directory with metadata of tables, which was marked as dropped by Atomic database fs::create_directories(path / "metadata_dropped/"); +#if USE_ROCKSDB fs::create_directories(path / "rocksdb/"); +#endif } diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index d214952deae..5c9ba177b78 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -78,7 +78,6 @@ M(SyncDrainedConnections, "Number of connections drained synchronously.") \ M(ActiveSyncDrainedConnections, "Number of active connections drained synchronously.") \ M(AsynchronousReadWait, "Number of threads waiting for asynchronous read.") \ - M(ServerStartupSeconds, "Server start seconds") \ namespace CurrentMetrics { diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index af87ba10a31..cb9c6f594a6 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -277,10 +277,10 @@ \ M(AsynchronousReadWaitMicroseconds, "Time spent in waiting for asynchronous reads.") \ \ - M(RocksdbGet, "Number of rocksdb reads(used for file meta cache)") \ - M(RocksdbPut, "Number of rocksdb puts(used for file meta cache)") \ - M(RocksdbDelete, "Number of rocksdb deletes(used for file meta cache)") \ - M(RocksdbSeek, "Number of rocksdb seeks(used for file meta cache)") \ + M(RocksdbGet, "Number of rocksdb reads(used for merge tree metadata cache)") \ + M(RocksdbPut, "Number of rocksdb puts(used for merge tree metadata cache)") \ + M(RocksdbDelete, "Number of rocksdb deletes(used for merge tree metadata cache)") \ + M(RocksdbSeek, "Number of rocksdb seeks(used for merge tree metadata cache)") \ M(MergeTreeMetaCacheHit, "Number of times the read of meta file was done from MergeTree meta cache") \ M(MergeTreeMetaCacheMiss, "Number of times the read of meta file was not done from MergeTree meta cache") \ \ diff --git a/src/Functions/checkPartMetaCache.cpp b/src/Functions/checkPartMetaCache.cpp index e0479a5fdcc..e77b8ca0d50 100644 --- a/src/Functions/checkPartMetaCache.cpp +++ b/src/Functions/checkPartMetaCache.cpp @@ -1,6 +1,8 @@ +#include "config_core.h" + +#if USE_ROCKSDB + #include -#include -#include #include #include #include @@ -14,8 +16,6 @@ #include #include #include -#include -#include #include #include @@ -155,3 +155,4 @@ void registerFunctionCheckPartMetaCache(FunctionFactory & factory) } } +#endif diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index d613d7c85bd..77e3e109081 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -80,7 +80,10 @@ void registerFunctionInitialQueryID(FunctionFactory & factory); void registerFunctionServerUUID(FunctionFactory &); void registerFunctionZooKeeperSessionUptime(FunctionFactory &); void registerFunctionGetOSKernelVersion(FunctionFactory &); + +#if USE_ROCKSDB void registerFunctionCheckPartMetaCache(FunctionFactory &); +#endif #if USE_ICU void registerFunctionConvertCharset(FunctionFactory &); @@ -167,7 +170,10 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionServerUUID(factory); registerFunctionZooKeeperSessionUptime(factory); registerFunctionGetOSKernelVersion(factory); + +#if USE_ROCKSDB registerFunctionCheckPartMetaCache(factory); +#endif #if USE_ICU registerFunctionConvertCharset(factory); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 81db45c6461..c5a5b3d1d49 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -94,11 +94,13 @@ namespace fs = std::filesystem; namespace ProfileEvents { extern const Event ContextLock; - extern const Event CompiledCacheSizeBytes; + +#if USE_ROCKSDB extern const Event RocksdbPut; extern const Event RocksdbGet; extern const Event RocksdbDelete; extern const Event RocksdbSeek; +#endif } namespace CurrentMetrics @@ -278,8 +280,10 @@ struct ContextSharedPart Context::ConfigReloadCallback config_reload_callback; +#if USE_ROCKSDB /// MergeTree metadata cache stored in rocksdb. MergeTreeMetaCachePtr merge_tree_meta_cache; +#endif ContextSharedPart() : access_control(std::make_unique()), macros(std::make_unique()) @@ -392,12 +396,14 @@ struct ContextSharedPart /// Stop zookeeper connection zookeeper.reset(); - /// Shutdown meta file cache +#if USE_ROCKSDB + /// Shutdown meta file cache if (merge_tree_meta_cache) { merge_tree_meta_cache->shutdown(); merge_tree_meta_cache.reset(); } +#endif } /// Can be removed w/o context lock @@ -441,6 +447,7 @@ SharedContextHolder::SharedContextHolder(std::unique_ptr shar void SharedContextHolder::reset() { shared.reset(); } +#if USE_ROCKSDB MergeTreeMetaCache::Status MergeTreeMetaCache::put(const String & key, const String & value) { auto options = rocksdb::WriteOptions(); @@ -491,6 +498,7 @@ void MergeTreeMetaCache::shutdown() rocksdb->Close(); } } +#endif ContextMutablePtr Context::createGlobal(ContextSharedPart * shared) { @@ -2309,6 +2317,7 @@ void Context::initializeTraceCollector() shared->initializeTraceCollector(getTraceLog()); } +#if USE_ROCKSDB void Context::initializeMergeTreeMetaCache(const String & dir, size_t size) { rocksdb::Options options; @@ -2328,6 +2337,7 @@ void Context::initializeMergeTreeMetaCache(const String & dir, size_t size) } shared->merge_tree_meta_cache = std::make_shared(db); } +#endif bool Context::hasTraceCollector() const { diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index b39e06b0b0f..bc191e80c9a 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1,5 +1,6 @@ #pragma once +#include "config_core.h" #include #include #include @@ -15,8 +16,11 @@ #include #include #include + +#if USE_ROCKSDB #include #include +#endif #include "config_core.h" @@ -179,6 +183,7 @@ private: std::unique_ptr shared; }; +#if USE_ROCKSDB class MergeTreeMetaCache { public: @@ -199,6 +204,7 @@ private: Poco::Logger * log = &Poco::Logger::get("MergeTreeMetaCache"); }; using MergeTreeMetaCachePtr = std::shared_ptr; +#endif /** A set of known objects that can be used in the query. * Consists of a shared part (always common to all sessions and queries) @@ -699,7 +705,9 @@ public: UInt32 getZooKeeperSessionUptime() const; +#if USE_ROCKSDB MergeTreeMetaCachePtr getMergeTreeMetaCache() const; +#endif #if USE_NURAFT @@ -788,7 +796,9 @@ public: /// Call after initialization before using trace collector. void initializeTraceCollector(); +#if USE_ROCKSDB void initializeMergeTreeMetaCache(const String & dir, size_t size); +#endif bool hasTraceCollector() const; diff --git a/src/Processors/CMakeLists.txt b/src/Processors/CMakeLists.txt index 7c9ad405432..7e965188b4c 100644 --- a/src/Processors/CMakeLists.txt +++ b/src/Processors/CMakeLists.txt @@ -1,4 +1,4 @@ -#if (ENABLE_EXAMPLES) +if (ENABLE_EXAMPLES) add_subdirectory(examples) -#endif () +endif () diff --git a/src/Processors/examples/CMakeLists.txt b/src/Processors/examples/CMakeLists.txt index dcb640c383a..ceb022432a1 100644 --- a/src/Processors/examples/CMakeLists.txt +++ b/src/Processors/examples/CMakeLists.txt @@ -1,2 +1,4 @@ -add_executable (merge_tree_meta_cache merge_tree_meta_cache.cpp) -target_link_libraries (merge_tree_meta_cache PRIVATE dbms) +if (USE_ROCKSDB) + add_executable (merge_tree_meta_cache merge_tree_meta_cache.cpp) + target_link_libraries (merge_tree_meta_cache PRIVATE dbms) +endif() diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index ad072af10a5..0d2b2f57fd0 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -50,7 +50,7 @@ public: PUT, // override set DROP, // remove keys }; - + static String modifyCacheTypeToString(ModifyCacheType type) { switch (type) diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index e51b64b3842..e5fbdd5cd19 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -91,7 +91,6 @@ void MergeTreeDataPartCompact::calculateEachColumnSizes(ColumnSizeByName & /*eac total_size.marks += mrk_checksum->second.file_size; } -// load marks from meta cache void MergeTreeDataPartCompact::loadIndexGranularity() { String full_path = getFullRelativePath(); diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 5132177aa5c..f6efdc5f05c 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -107,6 +107,7 @@ void MergeTreeDataPartWide::loadIndexGranularity() String full_path = getFullRelativePath(); index_granularity_info.changeGranularityIfRequired(volume->getDisk(), full_path); + if (columns.empty()) throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); diff --git a/src/Storages/MergeTree/MergeTreePartition.cpp b/src/Storages/MergeTree/MergeTreePartition.cpp index 1d4e14c628b..6b5bebd81f6 100644 --- a/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/src/Storages/MergeTree/MergeTreePartition.cpp @@ -160,13 +160,6 @@ namespace }; } -/* -static std::unique_ptr openForReading(const DiskPtr & disk, const String & path) -{ - size_t file_size = disk->getFileSize(path); - return disk->readFile(path, ReadSettings().adjustBufferSize(file_size), file_size); -} -*/ String MergeTreePartition::getID(const MergeTreeData & storage) const { diff --git a/src/Storages/MergeTree/PartMetaCache.cpp b/src/Storages/MergeTree/PartMetaCache.cpp index 33c95d6963e..fe8475d0dda 100644 --- a/src/Storages/MergeTree/PartMetaCache.cpp +++ b/src/Storages/MergeTree/PartMetaCache.cpp @@ -1,5 +1,6 @@ #include "PartMetaCache.h" +#if USE_ROCKSDB #include #include #include @@ -132,3 +133,4 @@ String PartMetaCache::getFullRelativePath() const } } +#endif diff --git a/src/Storages/MergeTree/PartMetaCache.h b/src/Storages/MergeTree/PartMetaCache.h index d6a86d86ba1..5ffd0413c4b 100644 --- a/src/Storages/MergeTree/PartMetaCache.h +++ b/src/Storages/MergeTree/PartMetaCache.h @@ -1,5 +1,8 @@ #pragma once +#include "config_core.h" + +#if USE_ROCKSDB #include #include #include @@ -43,3 +46,4 @@ private: }; } +#endif diff --git a/src/Storages/System/StorageSystemMergeTreeMetaCache.cpp b/src/Storages/System/StorageSystemMergeTreeMetaCache.cpp index 45cf45edb31..f53c32e5a42 100644 --- a/src/Storages/System/StorageSystemMergeTreeMetaCache.cpp +++ b/src/Storages/System/StorageSystemMergeTreeMetaCache.cpp @@ -1,3 +1,6 @@ +#include + +#if USE_ROCKSDB #include #include #include @@ -7,9 +10,9 @@ #include #include #include -#include #include #include + namespace DB { namespace ErrorCodes @@ -137,3 +140,4 @@ void StorageSystemMergeTreeMetaCache::fillData(MutableColumns & res_columns, Con } } +#endif diff --git a/src/Storages/System/StorageSystemMergeTreeMetaCache.h b/src/Storages/System/StorageSystemMergeTreeMetaCache.h index a5f65862243..c8e0f475cd8 100644 --- a/src/Storages/System/StorageSystemMergeTreeMetaCache.h +++ b/src/Storages/System/StorageSystemMergeTreeMetaCache.h @@ -1,5 +1,8 @@ #pragma once +#include "config_core.h" + +#if USE_ROCKSDB #include #include @@ -27,3 +30,4 @@ protected: }; } +#endif diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 0e7d4b624c5..5f5a17069f3 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -68,7 +68,6 @@ #include #include #include -#include #ifdef OS_LINUX #include @@ -76,6 +75,7 @@ #if USE_ROCKSDB #include +#include #endif diff --git a/tests/queries/0_stateless/01233_check_part_meta_cache_replicated.sql b/tests/queries/0_stateless/01233_check_part_meta_cache_replicated.sql index 6d08bb146a5..cb028e77d54 100644 --- a/tests/queries/0_stateless/01233_check_part_meta_cache_replicated.sql +++ b/tests/queries/0_stateless/01233_check_part_meta_cache_replicated.sql @@ -4,7 +4,7 @@ set replication_alter_partitions_sync = 2; DROP DATABASE IF EXISTS test_meta_cache on cluster preonline_hk5; DROP TABLE IF EXISTS test_meta_cache.check_part_meta_cache on cluster preonline_hk5; CREATE DATABASE test_meta_cache on cluster preonline_hk5 ENGINE = Ordinary; -CREATE TABLE test_meta_cache.check_part_meta_cache on cluster preonline_hk5 ( p Date, k UInt64, v1 UInt64, v2 Int64) ENGINE ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/test_meta_cache/check_part_meta_cache', '{replica}') PARTITION BY toYYYYMM(p) ORDER BY k; +CREATE TABLE test_meta_cache.check_part_meta_cache on cluster preonline_hk5 ( p Date, k UInt64, v1 UInt64, v2 Int64) ENGINE ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/{database}/check_part_meta_cache', '{replica}') PARTITION BY toYYYYMM(p) ORDER BY k; with arrayJoin(checkPartMetaCache('test_meta_cache', 'check_part_meta_cache')) as info select count(1), countIf(info.5 = 0); -- Insert first batch of data. diff --git a/tests/queries/0_stateless/01233_check_part_meta_cache_replicated_in_atomic.sql b/tests/queries/0_stateless/01233_check_part_meta_cache_replicated_in_atomic.sql index c41d036cef1..c56e2cb0a99 100644 --- a/tests/queries/0_stateless/01233_check_part_meta_cache_replicated_in_atomic.sql +++ b/tests/queries/0_stateless/01233_check_part_meta_cache_replicated_in_atomic.sql @@ -4,7 +4,7 @@ set replication_alter_partitions_sync = 2; DROP DATABASE IF EXISTS test_meta_cache on cluster preonline_hk5; DROP TABLE IF EXISTS test_meta_cache.check_part_meta_cache on cluster preonline_hk5; CREATE DATABASE test_meta_cache on cluster preonline_hk5 ENGINE = Atomic; -CREATE TABLE test_meta_cache.check_part_meta_cache on cluster preonline_hk5 ( p Date, k UInt64, v1 UInt64, v2 Int64) ENGINE ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/test_meta_cache/check_part_meta_cache', '{replica}') PARTITION BY toYYYYMM(p) ORDER BY k; +CREATE TABLE test_meta_cache.check_part_meta_cache on cluster preonline_hk5 ( p Date, k UInt64, v1 UInt64, v2 Int64) ENGINE ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/{database}/check_part_meta_cache', '{replica}') PARTITION BY toYYYYMM(p) ORDER BY k; with arrayJoin(checkPartMetaCache('test_meta_cache', 'check_part_meta_cache')) as info select count(1), countIf(info.5 = 0); -- Insert first batch of data.