wrap rocksdb metacache related code with USE_ROCKSDB

programs/server/Server.cpp

@@ -748,7 +748,9 @@ if (ThreadFuzzer::instance().isEffective())
         /// Directory with metadata of tables, which was marked as dropped by Atomic database
         fs::create_directories(path / "metadata_dropped/");
 
+#if USE_ROCKSDB
         fs::create_directories(path / "rocksdb/");
+#endif
     }
 
 

src/Common/CurrentMetrics.cpp

@@ -78,7 +78,6 @@
     M(SyncDrainedConnections, "Number of connections drained synchronously.") \
     M(ActiveSyncDrainedConnections, "Number of active connections drained synchronously.") \
     M(AsynchronousReadWait, "Number of threads waiting for asynchronous read.") \
-    M(ServerStartupSeconds, "Server start seconds") \
 
 namespace CurrentMetrics
 {

src/Common/ProfileEvents.cpp

@@ -277,10 +277,10 @@
     \
     M(AsynchronousReadWaitMicroseconds, "Time spent in waiting for asynchronous reads.") \
     \
-    M(RocksdbGet, "Number of rocksdb reads(used for file meta cache)") \
-    M(RocksdbPut, "Number of rocksdb puts(used for file meta cache)") \
-    M(RocksdbDelete, "Number of rocksdb deletes(used for file meta cache)") \
-    M(RocksdbSeek, "Number of rocksdb seeks(used for file meta cache)") \
+    M(RocksdbGet, "Number of rocksdb reads(used for merge tree metadata cache)") \
+    M(RocksdbPut, "Number of rocksdb puts(used for merge tree metadata cache)") \
+    M(RocksdbDelete, "Number of rocksdb deletes(used for merge tree metadata cache)") \
+    M(RocksdbSeek, "Number of rocksdb seeks(used for merge tree metadata cache)") \
     M(MergeTreeMetaCacheHit, "Number of times the read of meta file was done from MergeTree meta cache") \
     M(MergeTreeMetaCacheMiss, "Number of times the read of meta file was not done from MergeTree meta cache") \
     \

src/Functions/checkPartMetaCache.cpp

@@ -1,6 +1,8 @@
+#include "config_core.h"
+
+#if USE_ROCKSDB
+
 #include <Functions/FunctionFactory.h>
-#include <Functions/FunctionHelpers.h>
-#include <DataTypes/IDataType.h>
 #include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypeTuple.h>
 #include <DataTypes/DataTypeString.h>
@@ -14,8 +16,6 @@
 #include <Columns/ColumnFixedString.h>
 #include <Columns/ColumnsNumber.h>
 #include <Interpreters/Context.h>
-#include <Common/Macros.h>
-#include <Common/TypePromotion.h>
 #include <Common/hex.h>
 #include <Core/Field.h>
 
@@ -155,3 +155,4 @@ void registerFunctionCheckPartMetaCache(FunctionFactory & factory)
 }
 
 }
+#endif

src/Functions/registerFunctionsMiscellaneous.cpp

@@ -80,7 +80,10 @@ void registerFunctionInitialQueryID(FunctionFactory & factory);
 void registerFunctionServerUUID(FunctionFactory &);
 void registerFunctionZooKeeperSessionUptime(FunctionFactory &);
 void registerFunctionGetOSKernelVersion(FunctionFactory &);
+
+#if USE_ROCKSDB
 void registerFunctionCheckPartMetaCache(FunctionFactory &);
+#endif
 
 #if USE_ICU
 void registerFunctionConvertCharset(FunctionFactory &);
@@ -167,7 +170,10 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory)
     registerFunctionServerUUID(factory);
     registerFunctionZooKeeperSessionUptime(factory);
     registerFunctionGetOSKernelVersion(factory);
+
+#if USE_ROCKSDB
     registerFunctionCheckPartMetaCache(factory);
+#endif
 
 #if USE_ICU
     registerFunctionConvertCharset(factory);

src/Interpreters/Context.cpp

@@ -94,11 +94,13 @@ namespace fs = std::filesystem;
 namespace ProfileEvents
 {
     extern const Event ContextLock;
-    extern const Event CompiledCacheSizeBytes;
+
+#if USE_ROCKSDB
     extern const Event RocksdbPut;
     extern const Event RocksdbGet;
     extern const Event RocksdbDelete;
     extern const Event RocksdbSeek;
+#endif
 }
 
 namespace CurrentMetrics
@@ -278,8 +280,10 @@ struct ContextSharedPart
 
     Context::ConfigReloadCallback config_reload_callback;
 
+#if USE_ROCKSDB
     /// MergeTree metadata cache stored in rocksdb.
     MergeTreeMetaCachePtr merge_tree_meta_cache;
+#endif
 
     ContextSharedPart()
         : access_control(std::make_unique<AccessControl>()), macros(std::make_unique<Macros>())
@@ -392,12 +396,14 @@ struct ContextSharedPart
             /// Stop zookeeper connection
             zookeeper.reset();
 
-            /// Shutdown meta file cache 
+#if USE_ROCKSDB
+            /// Shutdown meta file cache
             if (merge_tree_meta_cache)
             {
                 merge_tree_meta_cache->shutdown();
                 merge_tree_meta_cache.reset();
             }
+#endif
         }
 
         /// Can be removed w/o context lock
@@ -441,6 +447,7 @@ SharedContextHolder::SharedContextHolder(std::unique_ptr<ContextSharedPart> shar
 
 void SharedContextHolder::reset() { shared.reset(); }
 
+#if USE_ROCKSDB
 MergeTreeMetaCache::Status MergeTreeMetaCache::put(const String & key, const String & value)
 {
     auto options = rocksdb::WriteOptions();
@@ -491,6 +498,7 @@ void MergeTreeMetaCache::shutdown()
         rocksdb->Close();
     }
 }
+#endif
 
 ContextMutablePtr Context::createGlobal(ContextSharedPart * shared)
 {
@@ -2309,6 +2317,7 @@ void Context::initializeTraceCollector()
     shared->initializeTraceCollector(getTraceLog());
 }
 
+#if USE_ROCKSDB
 void Context::initializeMergeTreeMetaCache(const String & dir, size_t size)
 {
     rocksdb::Options options;
@@ -2328,6 +2337,7 @@ void Context::initializeMergeTreeMetaCache(const String & dir, size_t size)
     }
     shared->merge_tree_meta_cache = std::make_shared<MergeTreeMetaCache>(db);
 }
+#endif
 
 bool Context::hasTraceCollector() const
 {

src/Interpreters/Context.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "config_core.h"
 #include <Core/Block.h>
 #include <Core/NamesAndTypes.h>
 #include <Core/Settings.h>
@@ -15,8 +16,11 @@
 #include <Common/isLocalAddress.h>
 #include <base/types.h>
 #include <Storages/MergeTree/ParallelReplicasReadingCoordinator.h>
+
+#if USE_ROCKSDB
 #include <rocksdb/db.h>
 #include <rocksdb/table.h>
+#endif
 
 #include "config_core.h"
 
@@ -179,6 +183,7 @@ private:
     std::unique_ptr<ContextSharedPart> shared;
 };
 
+#if USE_ROCKSDB
 class MergeTreeMetaCache
 {
 public:
@@ -199,6 +204,7 @@ private:
     Poco::Logger * log = &Poco::Logger::get("MergeTreeMetaCache");
 };
 using MergeTreeMetaCachePtr = std::shared_ptr<MergeTreeMetaCache>;
+#endif
 
 /** A set of known objects that can be used in the query.
   * Consists of a shared part (always common to all sessions and queries)
@@ -699,7 +705,9 @@ public:
 
     UInt32 getZooKeeperSessionUptime() const;
 
+#if USE_ROCKSDB
     MergeTreeMetaCachePtr getMergeTreeMetaCache() const;
+#endif
 
 
 #if USE_NURAFT
@@ -788,7 +796,9 @@ public:
     /// Call after initialization before using trace collector.
     void initializeTraceCollector();
 
+#if USE_ROCKSDB
     void initializeMergeTreeMetaCache(const String & dir, size_t size);
+#endif
 
     bool hasTraceCollector() const;
 

src/Processors/CMakeLists.txt

@@ -1,4 +1,4 @@
-#if (ENABLE_EXAMPLES)
+if (ENABLE_EXAMPLES)
     add_subdirectory(examples)
-#endif ()
+endif ()
 

src/Processors/examples/CMakeLists.txt

@@ -1,2 +1,4 @@
-add_executable (merge_tree_meta_cache merge_tree_meta_cache.cpp)
-target_link_libraries (merge_tree_meta_cache PRIVATE dbms)
+if (USE_ROCKSDB)
+    add_executable (merge_tree_meta_cache merge_tree_meta_cache.cpp)
+    target_link_libraries (merge_tree_meta_cache PRIVATE dbms)
+endif()

src/Storages/MergeTree/IMergeTreeDataPart.h

@@ -50,7 +50,7 @@ public:
         PUT, // override set
         DROP, // remove keys
     };
-    
+
     static String modifyCacheTypeToString(ModifyCacheType type)
     {
         switch (type)

src/Storages/MergeTree/MergeTreeDataPartCompact.cpp

@@ -91,7 +91,6 @@ void MergeTreeDataPartCompact::calculateEachColumnSizes(ColumnSizeByName & /*eac
         total_size.marks += mrk_checksum->second.file_size;
 }
 
-// load marks from meta cache
 void MergeTreeDataPartCompact::loadIndexGranularity()
 {
     String full_path = getFullRelativePath();

src/Storages/MergeTree/MergeTreeDataPartWide.cpp

@@ -107,6 +107,7 @@ void MergeTreeDataPartWide::loadIndexGranularity()
     String full_path = getFullRelativePath();
     index_granularity_info.changeGranularityIfRequired(volume->getDisk(), full_path);
 
+
     if (columns.empty())
         throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART);
 

src/Storages/MergeTree/MergeTreePartition.cpp

@@ -160,13 +160,6 @@ namespace
     };
 }
 
-/*
-static std::unique_ptr<ReadBufferFromFileBase> openForReading(const DiskPtr & disk, const String & path)
-{
-    size_t file_size = disk->getFileSize(path);
-    return disk->readFile(path, ReadSettings().adjustBufferSize(file_size), file_size);
-}
-*/
 
 String MergeTreePartition::getID(const MergeTreeData & storage) const
 {

src/Storages/MergeTree/PartMetaCache.cpp

@@ -1,5 +1,6 @@
 #include "PartMetaCache.h"
 
+#if USE_ROCKSDB
 #include <rocksdb/db.h>
 #include <Disks/IDisk.h>
 #include <IO/HashingReadBuffer.h>
@@ -132,3 +133,4 @@ String PartMetaCache::getFullRelativePath() const
 }
 
 }
+#endif

src/Storages/MergeTree/PartMetaCache.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include "config_core.h"
+
+#if USE_ROCKSDB
 #include <city.h>
 #include <Core/Types.h>
 #include <Interpreters/Context.h>
@@ -43,3 +46,4 @@ private:
 };
 
 }
+#endif

src/Storages/System/StorageSystemMergeTreeMetaCache.cpp

@@ -1,3 +1,6 @@
+#include <Storages/System/StorageSystemMergeTreeMetaCache.h>
+
+#if USE_ROCKSDB
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypesNumber.h>
@@ -7,9 +10,9 @@
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/ASTSelectQuery.h>
-#include <Storages/System/StorageSystemMergeTreeMetaCache.h>
 #include <Storages/MergeTree/KeyCondition.h>
 #include <Common/typeid_cast.h>
+
 namespace DB
 {
 namespace ErrorCodes
@@ -137,3 +140,4 @@ void StorageSystemMergeTreeMetaCache::fillData(MutableColumns & res_columns, Con
 }
 
 }
+#endif

src/Storages/System/StorageSystemMergeTreeMetaCache.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include "config_core.h"
+
+#if USE_ROCKSDB
 #include <base/shared_ptr_helper.h>
 #include <Storages/System/IStorageSystemOneBlock.h>
 
@@ -27,3 +30,4 @@ protected:
 };
 
 }
+#endif

src/Storages/System/attachSystemTables.cpp

@@ -68,7 +68,6 @@
 #include <Storages/System/StorageSystemUserDirectories.h>
 #include <Storages/System/StorageSystemPrivileges.h>
 #include <Storages/System/StorageSystemAsynchronousInserts.h>
-#include <Storages/System/StorageSystemMergeTreeMetaCache.h>
 
 #ifdef OS_LINUX
 #include <Storages/System/StorageSystemStackTrace.h>
@@ -76,6 +75,7 @@
 
 #if USE_ROCKSDB
 #include <Storages/RocksDB/StorageSystemRocksDB.h>
+#include <Storages/System/StorageSystemMergeTreeMetaCache.h>
 #endif
 
 

tests/queries/0_stateless/01233_check_part_meta_cache_replicated.sql

@@ -4,7 +4,7 @@ set replication_alter_partitions_sync = 2;
 DROP DATABASE IF EXISTS test_meta_cache on cluster preonline_hk5;
 DROP TABLE IF EXISTS test_meta_cache.check_part_meta_cache on cluster preonline_hk5;
 CREATE DATABASE test_meta_cache on cluster preonline_hk5 ENGINE = Ordinary;
-CREATE TABLE test_meta_cache.check_part_meta_cache on cluster preonline_hk5 ( p Date, k UInt64, v1 UInt64, v2 Int64) ENGINE ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/test_meta_cache/check_part_meta_cache', '{replica}') PARTITION BY toYYYYMM(p) ORDER BY k;
+CREATE TABLE test_meta_cache.check_part_meta_cache on cluster preonline_hk5 ( p Date, k UInt64, v1 UInt64, v2 Int64) ENGINE ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/{database}/check_part_meta_cache', '{replica}') PARTITION BY toYYYYMM(p) ORDER BY k;
 with arrayJoin(checkPartMetaCache('test_meta_cache', 'check_part_meta_cache')) as info select count(1), countIf(info.5 = 0);
 
 -- Insert first batch of data.

tests/queries/0_stateless/01233_check_part_meta_cache_replicated_in_atomic.sql

@@ -4,7 +4,7 @@ set replication_alter_partitions_sync = 2;
 DROP DATABASE IF EXISTS test_meta_cache on cluster preonline_hk5;
 DROP TABLE IF EXISTS test_meta_cache.check_part_meta_cache on cluster preonline_hk5;
 CREATE DATABASE test_meta_cache on cluster preonline_hk5 ENGINE = Atomic;
-CREATE TABLE test_meta_cache.check_part_meta_cache on cluster preonline_hk5 ( p Date, k UInt64, v1 UInt64, v2 Int64) ENGINE ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/test_meta_cache/check_part_meta_cache', '{replica}') PARTITION BY toYYYYMM(p) ORDER BY k;
+CREATE TABLE test_meta_cache.check_part_meta_cache on cluster preonline_hk5 ( p Date, k UInt64, v1 UInt64, v2 Int64) ENGINE ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/{database}/check_part_meta_cache', '{replica}') PARTITION BY toYYYYMM(p) ORDER BY k;
 with arrayJoin(checkPartMetaCache('test_meta_cache', 'check_part_meta_cache')) as info select count(1), countIf(info.5 = 0);
 
 -- Insert first batch of data.