Better

programs/CMakeLists.txt

@@ -49,11 +49,15 @@ option (ENABLE_CLICKHOUSE_GIT_IMPORT "A tool to analyze Git repositories"
 
 
 option (ENABLE_CLICKHOUSE_KEEPER "ClickHouse alternative to ZooKeeper" ${ENABLE_CLICKHOUSE_ALL})
+
+option (ENABLE_CLICKHOUSE_KEEPER_CONVERTER "Util allows to convert ZooKeeper logs and snapshots into clickhouse-keeper snapshot" ${ENABLE_CLICKHOUSE_ALL})
+
 if (NOT USE_NURAFT)
     # RECONFIGURE_MESSAGE_LEVEL should not be used here,
     # since USE_NURAFT is set to OFF for FreeBSD and Darwin.
-    message (STATUS "clickhouse-keeper will not be built (lack of NuRaft)")
+    message (STATUS "clickhouse-keeper and clickhouse-keeper-converter will not be built (lack of NuRaft)")
     set(ENABLE_CLICKHOUSE_KEEPER OFF)
+    set(ENABLE_CLICKHOUSE_KEEPER_CONVERTER OFF)
 endif()
 
 if (CLICKHOUSE_SPLIT_BINARY)
@@ -149,6 +153,12 @@ else()
     message(STATUS "ClickHouse keeper mode: OFF")
 endif()
 
+if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
+    message(STATUS "ClickHouse keeper-converter mode: ON")
+else()
+    message(STATUS "ClickHouse keeper-converter mode: OFF")
+endif()
+
 if(NOT (MAKE_STATIC_LIBRARIES OR SPLIT_SHARED_LIBRARIES))
     set(CLICKHOUSE_ONE_SHARED ON)
 endif()
@@ -270,6 +280,10 @@ if (ENABLE_CLICKHOUSE_KEEPER)
     add_subdirectory (keeper)
 endif()
 
+if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
+    add_subdirectory (keeper-converter)
+endif()
+
 if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
     add_subdirectory (odbc-bridge)
 endif ()
@@ -279,9 +293,51 @@ if (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE)
 endif ()
 
 if (CLICKHOUSE_ONE_SHARED)
-    add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_GIT_IMPORT_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES} ${CLICKHOUSE_KEEPER_SOURCES})
+  add_library(clickhouse-lib SHARED
-    target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_GIT_IMPORT_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK} ${CLICKHOUSE_KEEPER_LINK})
+    ${CLICKHOUSE_SERVER_SOURCES}
-    target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_GIT_IMPORT_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} ${CLICKHOUSE_KEEPER_INCLUDE})
+    ${CLICKHOUSE_CLIENT_SOURCES}
+    ${CLICKHOUSE_LOCAL_SOURCES}
+    ${CLICKHOUSE_BENCHMARK_SOURCES}
+    ${CLICKHOUSE_COPIER_SOURCES}
+    ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES}
+    ${CLICKHOUSE_COMPRESSOR_SOURCES}
+    ${CLICKHOUSE_FORMAT_SOURCES}
+    ${CLICKHOUSE_OBFUSCATOR_SOURCES}
+    ${CLICKHOUSE_GIT_IMPORT_SOURCES}
+    ${CLICKHOUSE_ODBC_BRIDGE_SOURCES}
+    ${CLICKHOUSE_KEEPER_SOURCES}
+    ${CLICKHOUSE_KEEPER_CONVERTER_SOURCES})
+
+    target_link_libraries(clickhouse-lib
+      ${CLICKHOUSE_SERVER_LINK}
+      ${CLICKHOUSE_CLIENT_LINK}
+      ${CLICKHOUSE_LOCAL_LINK}
+      ${CLICKHOUSE_BENCHMARK_LINK}
+      ${CLICKHOUSE_COPIER_LINK}
+      ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK}
+      ${CLICKHOUSE_COMPRESSOR_LINK}
+      ${CLICKHOUSE_FORMAT_LINK}
+      ${CLICKHOUSE_OBFUSCATOR_LINK}
+      ${CLICKHOUSE_GIT_IMPORT_LINK}
+      ${CLICKHOUSE_ODBC_BRIDGE_LINK}
+      ${CLICKHOUSE_KEEPER_LINK}
+      ${CLICKHOUSE_KEEPER_CONVERTER_LINK})
+
+    target_include_directories(clickhouse-lib
+      ${CLICKHOUSE_SERVER_INCLUDE}
+      ${CLICKHOUSE_CLIENT_INCLUDE}
+      ${CLICKHOUSE_LOCAL_INCLUDE}
+      ${CLICKHOUSE_BENCHMARK_INCLUDE}
+      ${CLICKHOUSE_COPIER_INCLUDE}
+      ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE}
+      ${CLICKHOUSE_COMPRESSOR_INCLUDE}
+      ${CLICKHOUSE_FORMAT_INCLUDE}
+      ${CLICKHOUSE_OBFUSCATOR_INCLUDE}
+      ${CLICKHOUSE_GIT_IMPORT_INCLUDE}
+      ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE}
+      ${CLICKHOUSE_KEEPER_INCLUDE}
+      ${CLICKHOUSE_KEEPER_CONVERTER_INCLUDE})
+
     set_target_properties(clickhouse-lib PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse DEBUG_POSTFIX "")
     install (TARGETS clickhouse-lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse)
 endif()
@@ -312,6 +368,10 @@ if (CLICKHOUSE_SPLIT_BINARY)
         list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper)
     endif ()
 
+    if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
+        list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper-converter)
+    endif ()
+
     set_target_properties(${CLICKHOUSE_ALL_TARGETS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
 
     add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_ALL_TARGETS})
@@ -362,6 +422,9 @@ else ()
     if (ENABLE_CLICKHOUSE_KEEPER)
         clickhouse_target_link_split_lib(clickhouse keeper)
     endif()
+    if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
+        clickhouse_target_link_split_lib(clickhouse keeper-converter)
+    endif()
     if (ENABLE_CLICKHOUSE_INSTALL)
         clickhouse_target_link_split_lib(clickhouse install)
     endif ()
@@ -422,6 +485,11 @@ else ()
         install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
         list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper)
     endif ()
+    if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
+        add_custom_target (clickhouse-keeper-converter ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper-converter DEPENDS clickhouse)
+        install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper-converter" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
+        list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper-converter)
+    endif ()
 
     install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 

programs/config_tools.h.in

@@ -17,3 +17,4 @@
 #cmakedefine01 ENABLE_CLICKHOUSE_ODBC_BRIDGE
 #cmakedefine01 ENABLE_CLICKHOUSE_LIBRARY_BRIDGE
 #cmakedefine01 ENABLE_CLICKHOUSE_KEEPER
+#cmakedefine01 ENABLE_CLICKHOUSE_KEEPER_CONVERTER

programs/keeper-converter/CMakeLists.txt

@@ -0,0 +1,9 @@
+set (CLICKHOUSE_KEEPER_CONVERTER_SOURCES KeeperConverter.cpp)
+
+set (CLICKHOUSE_KEEPER_CONVERTER_LINK
+    PRIVATE
+        boost::program_options
+        dbms
+)
+
+clickhouse_program_add(keeper-converter)

programs/keeper-converter/KeeperConverter.cpp

@@ -0,0 +1,61 @@
+#include <iostream>
+#include <optional>
+#include <boost/program_options.hpp>
+
+#include <Coordination/KeeperSnapshotManager.h>
+#include <Coordination/ZooKeeperDataReader.h>
+#include <Common/TerminalSize.h>
+#include <Poco/ConsoleChannel.h>
+#include <Poco/AutoPtr.h>
+#include <Poco/Logger.h>
+#include <common/logger_useful.h>
+
+
+int mainEntryClickHouseKeeperConverter(int argc, char ** argv)
+{
+    using namespace DB;
+    namespace po = boost::program_options;
+
+    po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
+    desc.add_options()
+        ("help,h", "produce help message")
+        ("zookeeper-logs-dir", po::value<std::string>(), "Path to directory with ZooKeeper logs")
+        ("zookeeper-snapshots-dir", po::value<std::string>(), "Path to directory with ZooKeeper snapshots")
+        ("output-dir", po::value<std::string>(), "Directory to place output clickhouse-keeper snapshot")
+    ;
+    po::variables_map options;
+    po::store(po::command_line_parser(argc, argv).options(desc).run(), options);
+    Poco::AutoPtr<Poco::ConsoleChannel> console_channel(new Poco::ConsoleChannel);
+
+    Poco::Logger * logger = &Poco::Logger::get("KeeperConverter");
+    logger->setChannel(console_channel);
+
+    if (options.count("help"))
+    {
+        std::cout << "Usage: " << argv[0] << " --zookeeper-logs-dir /var/lib/zookeeper/data/version-2 --zookeeper-snapshots-dir /var/lib/zookeeper/data/version-2 --output-dir /var/lib/clickhouse/coordination/snapshots" << std::endl;
+        std::cout << desc << std::endl;
+        return 0;
+    }
+
+    try
+    {
+        DB::KeeperStorage storage(500, "");
+
+        DB::deserializeKeeperStorageFromSnapshotsDir(storage, options["zookeeper-snapshots-dir"].as<std::string>(), logger);
+        DB::deserializeLogsAndApplyToStorage(storage, options["zookeeper-logs-dir"].as<std::string>(), logger);
+        DB::SnapshotMetadataPtr snapshot_meta = std::make_shared<DB::SnapshotMetadata>(storage.getZXID(), 1, std::make_shared<nuraft::cluster_config>());
+        DB::KeeperStorageSnapshot snapshot(&storage, snapshot_meta);
+
+        DB::KeeperSnapshotManager manager(options["output-dir"].as<std::string>(), 1);
+        auto snp = manager.serializeSnapshotToBuffer(snapshot);
+        auto path = manager.serializeSnapshotBufferToDisk(*snp, storage.getZXID());
+        std::cout << "Snapshot serialized to path:" << path << std::endl;
+    }
+    catch (...)
+    {
+        std::cerr << getCurrentExceptionMessage(true) << '\n';
+        return getCurrentExceptionCode();
+    }
+
+    return 0;
+}

programs/keeper-converter/clickhouse-keeper-converter.cpp

@@ -0,0 +1,2 @@
+int mainEntryClickHouseKeeperConverter(int argc, char ** argv);
+int main(int argc_, char ** argv_) { return mainEntryClickHouseKeeperConverter(argc_, argv_); }

programs/main.cpp

@@ -59,6 +59,9 @@ int mainEntryClickHouseGitImport(int argc, char ** argv);
 #if ENABLE_CLICKHOUSE_KEEPER
 int mainEntryClickHouseKeeper(int argc, char ** argv);
 #endif
+#if ENABLE_CLICKHOUSE_KEEPER
+int mainEntryClickHouseKeeperConverter(int argc, char ** argv);
+#endif
 #if ENABLE_CLICKHOUSE_INSTALL
 int mainEntryClickHouseInstall(int argc, char ** argv);
 int mainEntryClickHouseStart(int argc, char ** argv);
@@ -119,6 +122,9 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
 #if ENABLE_CLICKHOUSE_KEEPER
     {"keeper", mainEntryClickHouseKeeper},
 #endif
+#if ENABLE_CLICKHOUSE_KEEPER_CONVERTER
+    {"keeper-converter", mainEntryClickHouseKeeperConverter},
+#endif
 #if ENABLE_CLICKHOUSE_INSTALL
     {"install", mainEntryClickHouseInstall},
     {"start", mainEntryClickHouseStart},

src/Coordination/ZooKeeperDataReader.cpp

@@ -45,6 +45,8 @@ void deserializeSnapshotMagic(ReadBuffer & in)
     int64_t dbid;
     Coordination::read(magic_header, in);
     Coordination::read(version, in);
+    if (version != 2)
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot deserialize ZooKeeper data other than version 2, got version {}", version);
     Coordination::read(dbid, in);
     static constexpr int32_t SNP_HEADER = 1514885966; /// "ZKSN"
     if (magic_header != SNP_HEADER)
@@ -98,7 +100,7 @@ void deserializeACLMap(KeeperStorage & storage, ReadBuffer & in)
     }
 }
 
-int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in)
+int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in, Poco::Logger * log)
 {
     int64_t max_zxid = 0;
     std::string path;
@@ -138,7 +140,7 @@ int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in)
         Coordination::read(path, in);
         count++;
         if (count % 1000 == 0)
-            std::cerr << "Deserialized nodes from snapshot:"  << count << std::endl;
+            LOG_INFO(log, "Deserialized nodes from snapshot: {}", count);
     }
 
     for (const auto & itr : storage.container)
@@ -153,23 +155,31 @@ int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in)
     return max_zxid;
 }
 
-void deserializeKeeperStorageFromSnapshot(KeeperStorage & storage, const std::string & snapshot_path)
+void deserializeKeeperStorageFromSnapshot(KeeperStorage & storage, const std::string & snapshot_path, Poco::Logger * log)
 {
+    LOG_INFO(log, "Deserializing storage snapshot {}", snapshot_path);
     int64_t zxid = getZxidFromName(snapshot_path);
 
     ReadBufferFromFile reader(snapshot_path);
 
     deserializeSnapshotMagic(reader);
+
+    LOG_INFO(log, "Magic deserialized, looks OK");
     auto max_session_id = deserializeSessionAndTimeout(storage, reader);
+    LOG_INFO(log, "Sessions and timeouts deserialized");
 
     storage.session_id_counter = max_session_id;
     deserializeACLMap(storage, reader);
+    LOG_INFO(log, "ACLs deserialized");
 
-    int64_t zxid_from_nodes = deserializeStorageData(storage, reader);
+    LOG_INFO(log, "Deserializing data from snapshot");
+    int64_t zxid_from_nodes = deserializeStorageData(storage, reader, log);
     storage.zxid = std::max(zxid, zxid_from_nodes);
+
+    LOG_INFO(log, "Finished, snapshot ZXID {}", storage.zxid);
 }
 
-void deserializeKeeperStorageFromSnapshotsDir(KeeperStorage & storage, const std::string & path)
+void deserializeKeeperStorageFromSnapshotsDir(KeeperStorage & storage, const std::string & path, Poco::Logger * log)
 {
     namespace fs = std::filesystem;
     std::map<int64_t, std::string> existing_snapshots;
@@ -181,9 +191,13 @@ void deserializeKeeperStorageFromSnapshotsDir(KeeperStorage & storage, const std
         int64_t zxid = getZxidFromName(log_path);
         existing_snapshots[zxid] = p.path();
     }
+
+    LOG_INFO(log, "Totally have {} snapshots, will use latest", existing_snapshots.size());
     /// deserialize only from latest snapshot
     if (!existing_snapshots.empty())
-        deserializeKeeperStorageFromSnapshot(storage, existing_snapshots.rbegin()->second);
+        deserializeKeeperStorageFromSnapshot(storage, existing_snapshots.rbegin()->second, log);
+    else
+        throw Exception(ErrorCodes::CORRUPTED_DATA, "No snapshots found on path {}. At least one snapshot must exist.", path);
 }
 
 void deserializeLogMagic(ReadBuffer & in)
@@ -197,6 +211,9 @@ void deserializeLogMagic(ReadBuffer & in)
     static constexpr int32_t LOG_HEADER = 1514884167; /// "ZKLG"
     if (magic_header != LOG_HEADER)
         throw Exception(ErrorCodes::CORRUPTED_DATA ,"Incorrect magic header in file, expected {}, got {}", LOG_HEADER, magic_header);
+
+    if (version != 2)
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED,"Cannot deserialize ZooKeeper data other than version 2, got version {}", version);
 }
 
 
@@ -435,15 +452,7 @@ bool deserializeTxn(KeeperStorage & storage, ReadBuffer & in)
 
     Coordination::ZooKeeperRequestPtr request = deserializeTxnImpl(in, false);
 
-    /// For Error requests ZooKeeper doesn't store version + tree_digest
+    /// Skip all other bytes
-    if (!isErrorRequest(request))
-    {
-        int32_t version;
-        int64_t tree_digest;
-        Coordination::read(version, in);
-        Coordination::read(tree_digest, in);
-    }
-
     int64_t bytes_read = in.count() - count_before;
     if (bytes_read < txn_len)
         in.ignore(txn_len - bytes_read);
@@ -475,25 +484,31 @@ bool deserializeTxn(KeeperStorage & storage, ReadBuffer & in)
     return true;
 }
 
-void deserializeLogAndApplyToStorage(KeeperStorage & storage, const std::string & log_path)
+void deserializeLogAndApplyToStorage(KeeperStorage & storage, const std::string & log_path, Poco::Logger * log)
 {
     ReadBufferFromFile reader(log_path);
+
+    LOG_INFO(log, "Deserializing log {}", log_path);
     deserializeLogMagic(reader);
+    LOG_INFO(log, "Header looks OK");
+
     size_t counter = 0;
     while (!reader.eof() && deserializeTxn(storage, reader))
     {
         counter++;
         if (counter % 1000 == 0)
-            std::cerr << "Deserialized from log: " << counter << std::endl;
+            LOG_INFO(log, "Deserialized txns log: {}", counter);
 
         int8_t forty_two;
         Coordination::read(forty_two, reader);
         if (forty_two != 0x42)
             throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Forty two check byte ({}) is not equal 0x42", forty_two);
     }
+
+    LOG_INFO(log, "Finished {} deserialization, totally read {} records", log_path, counter);
 }
 
-void deserializeLogsAndApplyToStorage(KeeperStorage & storage, const std::string & path)
+void deserializeLogsAndApplyToStorage(KeeperStorage & storage, const std::string & path, Poco::Logger * log)
 {
     namespace fs = std::filesystem;
     std::map<int64_t, std::string> existing_logs;
@@ -506,10 +521,14 @@ void deserializeLogsAndApplyToStorage(KeeperStorage & storage, const std::string
         existing_logs[zxid] = p.path();
     }
 
+    LOG_INFO(log, "Totally have {} logs", existing_logs.size());
+
     for (auto [zxid, log_path] : existing_logs)
     {
         if (zxid > storage.zxid)
-            deserializeLogAndApplyToStorage(storage, log_path);
+            deserializeLogAndApplyToStorage(storage, log_path, log);
+        else
+            LOG_INFO(log, "Skipping log {}, it's ZXID {} is smaller than storages ZXID {}", log_path, zxid, storage.zxid);
     }
 }
 

src/Coordination/ZooKeeperDataReader.h

@@ -6,12 +6,12 @@
 namespace DB
 {
 
-void deserializeKeeperStorageFromSnapshot(KeeperStorage & storage, const std::string & snapshot_path, Poco::Logger * log = nullptr);
+void deserializeKeeperStorageFromSnapshot(KeeperStorage & storage, const std::string & snapshot_path, Poco::Logger * log);
 
-void deserializeKeeperStorageFromSnapshotsDir(KeeperStorage & storage, const std::string & path, Poco::Logger * log = nullptr);
+void deserializeKeeperStorageFromSnapshotsDir(KeeperStorage & storage, const std::string & path, Poco::Logger * log);
 
-void deserializeLogAndApplyToStorage(KeeperStorage & storage, const std::string & log_path, Poco::Logger * log = nullptr);
+void deserializeLogAndApplyToStorage(KeeperStorage & storage, const std::string & log_path, Poco::Logger * log);
 
-void deserializeLogsAndApplyToStorage(KeeperStorage & storage, const std::string & path, Poco::Logger * log = nullptr);
+void deserializeLogsAndApplyToStorage(KeeperStorage & storage, const std::string & path, Poco::Logger * log);
 
 }