thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Better

Browse Source
This commit is contained in:
alesapin 2021-06-17 19:32:50 +03:00
parent d513d14b2c
commit 1a6abb4db4
8 changed files with 193 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
programs/CMakeLists.txt
View File

@ -49,11 +49,15 @@ option (ENABLE_CLICKHOUSE_GIT_IMPORT "A tool to analyze Git repositories"
option (ENABLE_CLICKHOUSE_KEEPER "ClickHouse alternative to ZooKeeper" ${ENABLE_CLICKHOUSE_ALL})
option (ENABLE_CLICKHOUSE_KEEPER_CONVERTER "Util allows to convert ZooKeeper logs and snapshots into clickhouse-keeper snapshot" ${ENABLE_CLICKHOUSE_ALL})
if (NOT USE_NURAFT)
# RECONFIGURE_MESSAGE_LEVEL should not be used here,
# since USE_NURAFT is set to OFF for FreeBSD and Darwin.
message (STATUS "clickhouse-keeper will not be built (lack of NuRaft)")
message (STATUS "clickhouse-keeper and clickhouse-keeper-converter will not be built (lack of NuRaft)")
set(ENABLE_CLICKHOUSE_KEEPER OFF)
set(ENABLE_CLICKHOUSE_KEEPER_CONVERTER OFF)
endif()
if (CLICKHOUSE_SPLIT_BINARY)
@ -149,6 +153,12 @@ else()
message(STATUS "ClickHouse keeper mode: OFF")
endif()
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
message(STATUS "ClickHouse keeper-converter mode: ON")
else()
message(STATUS "ClickHouse keeper-converter mode: OFF")
endif()
if(NOT (MAKE_STATIC_LIBRARIES OR SPLIT_SHARED_LIBRARIES))
set(CLICKHOUSE_ONE_SHARED ON)
endif()
@ -270,6 +280,10 @@ if (ENABLE_CLICKHOUSE_KEEPER)
add_subdirectory (keeper)
endif()
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
add_subdirectory (keeper-converter)
endif()
if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
add_subdirectory (odbc-bridge)
endif ()
@ -279,9 +293,51 @@ if (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE)
endif ()
if (CLICKHOUSE_ONE_SHARED)
add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_GIT_IMPORT_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES} ${CLICKHOUSE_KEEPER_SOURCES})
target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_GIT_IMPORT_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK} ${CLICKHOUSE_KEEPER_LINK})
target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_GIT_IMPORT_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} ${CLICKHOUSE_KEEPER_INCLUDE})
add_library(clickhouse-lib SHARED
${CLICKHOUSE_SERVER_SOURCES}
${CLICKHOUSE_CLIENT_SOURCES}
${CLICKHOUSE_LOCAL_SOURCES}
${CLICKHOUSE_BENCHMARK_SOURCES}
${CLICKHOUSE_COPIER_SOURCES}
${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES}
${CLICKHOUSE_COMPRESSOR_SOURCES}
${CLICKHOUSE_FORMAT_SOURCES}
${CLICKHOUSE_OBFUSCATOR_SOURCES}
${CLICKHOUSE_GIT_IMPORT_SOURCES}
${CLICKHOUSE_ODBC_BRIDGE_SOURCES}
${CLICKHOUSE_KEEPER_SOURCES}
${CLICKHOUSE_KEEPER_CONVERTER_SOURCES})
target_link_libraries(clickhouse-lib
${CLICKHOUSE_SERVER_LINK}
${CLICKHOUSE_CLIENT_LINK}
${CLICKHOUSE_LOCAL_LINK}
${CLICKHOUSE_BENCHMARK_LINK}
${CLICKHOUSE_COPIER_LINK}
${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK}
${CLICKHOUSE_COMPRESSOR_LINK}
${CLICKHOUSE_FORMAT_LINK}
${CLICKHOUSE_OBFUSCATOR_LINK}
${CLICKHOUSE_GIT_IMPORT_LINK}
${CLICKHOUSE_ODBC_BRIDGE_LINK}
${CLICKHOUSE_KEEPER_LINK}
${CLICKHOUSE_KEEPER_CONVERTER_LINK})
target_include_directories(clickhouse-lib
${CLICKHOUSE_SERVER_INCLUDE}
${CLICKHOUSE_CLIENT_INCLUDE}
${CLICKHOUSE_LOCAL_INCLUDE}
${CLICKHOUSE_BENCHMARK_INCLUDE}
${CLICKHOUSE_COPIER_INCLUDE}
${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE}
${CLICKHOUSE_COMPRESSOR_INCLUDE}
${CLICKHOUSE_FORMAT_INCLUDE}
${CLICKHOUSE_OBFUSCATOR_INCLUDE}
${CLICKHOUSE_GIT_IMPORT_INCLUDE}
${CLICKHOUSE_ODBC_BRIDGE_INCLUDE}
${CLICKHOUSE_KEEPER_INCLUDE}
${CLICKHOUSE_KEEPER_CONVERTER_INCLUDE})
set_target_properties(clickhouse-lib PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse DEBUG_POSTFIX "")
install (TARGETS clickhouse-lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse)
endif()
@ -312,6 +368,10 @@ if (CLICKHOUSE_SPLIT_BINARY)
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper)
endif ()
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper-converter)
endif ()
set_target_properties(${CLICKHOUSE_ALL_TARGETS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_ALL_TARGETS})
@ -362,6 +422,9 @@ else ()
if (ENABLE_CLICKHOUSE_KEEPER)
clickhouse_target_link_split_lib(clickhouse keeper)
endif()
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
clickhouse_target_link_split_lib(clickhouse keeper-converter)
endif()
if (ENABLE_CLICKHOUSE_INSTALL)
clickhouse_target_link_split_lib(clickhouse install)
endif ()
@ -422,6 +485,11 @@ else ()
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper)
endif ()
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
add_custom_target (clickhouse-keeper-converter ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper-converter DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper-converter" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper-converter)
endif ()
install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)

1
programs/config_tools.h.in
View File

@ -17,3 +17,4 @@
#cmakedefine01 ENABLE_CLICKHOUSE_ODBC_BRIDGE
#cmakedefine01 ENABLE_CLICKHOUSE_LIBRARY_BRIDGE
#cmakedefine01 ENABLE_CLICKHOUSE_KEEPER
#cmakedefine01 ENABLE_CLICKHOUSE_KEEPER_CONVERTER

9
programs/keeper-converter/CMakeLists.txt Normal file
View File

@ -0,0 +1,9 @@
set (CLICKHOUSE_KEEPER_CONVERTER_SOURCES KeeperConverter.cpp)
set (CLICKHOUSE_KEEPER_CONVERTER_LINK
PRIVATE
boost::program_options
dbms
)
clickhouse_program_add(keeper-converter)

61
programs/keeper-converter/KeeperConverter.cpp Normal file
View File

@ -0,0 +1,61 @@
#include <iostream>
#include <optional>
#include <boost/program_options.hpp>
#include <Coordination/KeeperSnapshotManager.h>
#include <Coordination/ZooKeeperDataReader.h>
#include <Common/TerminalSize.h>
#include <Poco/ConsoleChannel.h>
#include <Poco/AutoPtr.h>
#include <Poco/Logger.h>
#include <common/logger_useful.h>
int mainEntryClickHouseKeeperConverter(int argc, char ** argv)
{
using namespace DB;
namespace po = boost::program_options;
po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
desc.add_options()
("help,h", "produce help message")
("zookeeper-logs-dir", po::value<std::string>(), "Path to directory with ZooKeeper logs")
("zookeeper-snapshots-dir", po::value<std::string>(), "Path to directory with ZooKeeper snapshots")
("output-dir", po::value<std::string>(), "Directory to place output clickhouse-keeper snapshot")
;
po::variables_map options;
po::store(po::command_line_parser(argc, argv).options(desc).run(), options);
Poco::AutoPtr<Poco::ConsoleChannel> console_channel(new Poco::ConsoleChannel);
Poco::Logger * logger = &Poco::Logger::get("KeeperConverter");
logger->setChannel(console_channel);
if (options.count("help"))
{
std::cout << "Usage: " << argv[0] << " --zookeeper-logs-dir /var/lib/zookeeper/data/version-2 --zookeeper-snapshots-dir /var/lib/zookeeper/data/version-2 --output-dir /var/lib/clickhouse/coordination/snapshots" << std::endl;
std::cout << desc << std::endl;
return 0;
}
try
{
DB::KeeperStorage storage(500, "");
DB::deserializeKeeperStorageFromSnapshotsDir(storage, options["zookeeper-snapshots-dir"].as<std::string>(), logger);
DB::deserializeLogsAndApplyToStorage(storage, options["zookeeper-logs-dir"].as<std::string>(), logger);
DB::SnapshotMetadataPtr snapshot_meta = std::make_shared<DB::SnapshotMetadata>(storage.getZXID(), 1, std::make_shared<nuraft::cluster_config>());
DB::KeeperStorageSnapshot snapshot(&storage, snapshot_meta);
DB::KeeperSnapshotManager manager(options["output-dir"].as<std::string>(), 1);
auto snp = manager.serializeSnapshotToBuffer(snapshot);
auto path = manager.serializeSnapshotBufferToDisk(*snp, storage.getZXID());
std::cout << "Snapshot serialized to path:" << path << std::endl;
}
catch (...)
{
std::cerr << getCurrentExceptionMessage(true) << '\n';
return getCurrentExceptionCode();
}
return 0;
}

2
programs/keeper-converter/clickhouse-keeper-converter.cpp Normal file
View File

@ -0,0 +1,2 @@
int mainEntryClickHouseKeeperConverter(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseKeeperConverter(argc_, argv_); }

6
programs/main.cpp
View File

@ -59,6 +59,9 @@ int mainEntryClickHouseGitImport(int argc, char ** argv);
#if ENABLE_CLICKHOUSE_KEEPER
int mainEntryClickHouseKeeper(int argc, char ** argv);
#endif
#if ENABLE_CLICKHOUSE_KEEPER
int mainEntryClickHouseKeeperConverter(int argc, char ** argv);
#endif
#if ENABLE_CLICKHOUSE_INSTALL
int mainEntryClickHouseInstall(int argc, char ** argv);
int mainEntryClickHouseStart(int argc, char ** argv);
@ -119,6 +122,9 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
#if ENABLE_CLICKHOUSE_KEEPER
{"keeper", mainEntryClickHouseKeeper},
#endif
#if ENABLE_CLICKHOUSE_KEEPER_CONVERTER
{"keeper-converter", mainEntryClickHouseKeeperConverter},
#endif
#if ENABLE_CLICKHOUSE_INSTALL
{"install", mainEntryClickHouseInstall},
{"start", mainEntryClickHouseStart},

57
src/Coordination/ZooKeeperDataReader.cpp
View File

@ -45,6 +45,8 @@ void deserializeSnapshotMagic(ReadBuffer & in)
int64_t dbid;
Coordination::read(magic_header, in);
Coordination::read(version, in);
if (version != 2)
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot deserialize ZooKeeper data other than version 2, got version {}", version);
Coordination::read(dbid, in);
static constexpr int32_t SNP_HEADER = 1514885966; /// "ZKSN"
if (magic_header != SNP_HEADER)
@ -98,7 +100,7 @@ void deserializeACLMap(KeeperStorage & storage, ReadBuffer & in)
}
}
int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in)
int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in, Poco::Logger * log)
{
int64_t max_zxid = 0;
std::string path;
@ -138,7 +140,7 @@ int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in)
Coordination::read(path, in);
count++;
if (count % 1000 == 0)
std::cerr << "Deserialized nodes from snapshot:" << count << std::endl;
LOG_INFO(log, "Deserialized nodes from snapshot: {}", count);
}
for (const auto & itr : storage.container)
@ -153,23 +155,31 @@ int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in)
return max_zxid;
}
void deserializeKeeperStorageFromSnapshot(KeeperStorage & storage, const std::string & snapshot_path)
void deserializeKeeperStorageFromSnapshot(KeeperStorage & storage, const std::string & snapshot_path, Poco::Logger * log)
{
LOG_INFO(log, "Deserializing storage snapshot {}", snapshot_path);
int64_t zxid = getZxidFromName(snapshot_path);
ReadBufferFromFile reader(snapshot_path);
deserializeSnapshotMagic(reader);
LOG_INFO(log, "Magic deserialized, looks OK");
auto max_session_id = deserializeSessionAndTimeout(storage, reader);
LOG_INFO(log, "Sessions and timeouts deserialized");
storage.session_id_counter = max_session_id;
deserializeACLMap(storage, reader);
LOG_INFO(log, "ACLs deserialized");
int64_t zxid_from_nodes = deserializeStorageData(storage, reader);
LOG_INFO(log, "Deserializing data from snapshot");
int64_t zxid_from_nodes = deserializeStorageData(storage, reader, log);
storage.zxid = std::max(zxid, zxid_from_nodes);
LOG_INFO(log, "Finished, snapshot ZXID {}", storage.zxid);
}
void deserializeKeeperStorageFromSnapshotsDir(KeeperStorage & storage, const std::string & path)
void deserializeKeeperStorageFromSnapshotsDir(KeeperStorage & storage, const std::string & path, Poco::Logger * log)
{
namespace fs = std::filesystem;
std::map<int64_t, std::string> existing_snapshots;
@ -181,9 +191,13 @@ void deserializeKeeperStorageFromSnapshotsDir(KeeperStorage & storage, const std
int64_t zxid = getZxidFromName(log_path);
existing_snapshots[zxid] = p.path();
}
LOG_INFO(log, "Totally have {} snapshots, will use latest", existing_snapshots.size());
/// deserialize only from latest snapshot
if (!existing_snapshots.empty())
deserializeKeeperStorageFromSnapshot(storage, existing_snapshots.rbegin()->second);
deserializeKeeperStorageFromSnapshot(storage, existing_snapshots.rbegin()->second, log);
else
throw Exception(ErrorCodes::CORRUPTED_DATA, "No snapshots found on path {}. At least one snapshot must exist.", path);
}
void deserializeLogMagic(ReadBuffer & in)
@ -197,6 +211,9 @@ void deserializeLogMagic(ReadBuffer & in)
static constexpr int32_t LOG_HEADER = 1514884167; /// "ZKLG"
if (magic_header != LOG_HEADER)
throw Exception(ErrorCodes::CORRUPTED_DATA ,"Incorrect magic header in file, expected {}, got {}", LOG_HEADER, magic_header);
if (version != 2)
throw Exception(ErrorCodes::NOT_IMPLEMENTED,"Cannot deserialize ZooKeeper data other than version 2, got version {}", version);
}
@ -435,15 +452,7 @@ bool deserializeTxn(KeeperStorage & storage, ReadBuffer & in)
Coordination::ZooKeeperRequestPtr request = deserializeTxnImpl(in, false);
/// For Error requests ZooKeeper doesn't store version + tree_digest
if (!isErrorRequest(request))
{
int32_t version;
int64_t tree_digest;
Coordination::read(version, in);
Coordination::read(tree_digest, in);
}
/// Skip all other bytes
int64_t bytes_read = in.count() - count_before;
if (bytes_read < txn_len)
in.ignore(txn_len - bytes_read);
@ -475,25 +484,31 @@ bool deserializeTxn(KeeperStorage & storage, ReadBuffer & in)
return true;
}
void deserializeLogAndApplyToStorage(KeeperStorage & storage, const std::string & log_path)
void deserializeLogAndApplyToStorage(KeeperStorage & storage, const std::string & log_path, Poco::Logger * log)
{
ReadBufferFromFile reader(log_path);
LOG_INFO(log, "Deserializing log {}", log_path);
deserializeLogMagic(reader);
LOG_INFO(log, "Header looks OK");
size_t counter = 0;
while (!reader.eof() && deserializeTxn(storage, reader))
{
counter++;
if (counter % 1000 == 0)
std::cerr << "Deserialized from log: " << counter << std::endl;
LOG_INFO(log, "Deserialized txns log: {}", counter);
int8_t forty_two;
Coordination::read(forty_two, reader);
if (forty_two != 0x42)
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Forty two check byte ({}) is not equal 0x42", forty_two);
}
LOG_INFO(log, "Finished {} deserialization, totally read {} records", log_path, counter);
}
void deserializeLogsAndApplyToStorage(KeeperStorage & storage, const std::string & path)
void deserializeLogsAndApplyToStorage(KeeperStorage & storage, const std::string & path, Poco::Logger * log)
{
namespace fs = std::filesystem;
std::map<int64_t, std::string> existing_logs;
@ -506,10 +521,14 @@ void deserializeLogsAndApplyToStorage(KeeperStorage & storage, const std::string
existing_logs[zxid] = p.path();
}
LOG_INFO(log, "Totally have {} logs", existing_logs.size());
for (auto [zxid, log_path] : existing_logs)
{
if (zxid > storage.zxid)
deserializeLogAndApplyToStorage(storage, log_path);
deserializeLogAndApplyToStorage(storage, log_path, log);
else
LOG_INFO(log, "Skipping log {}, it's ZXID {} is smaller than storages ZXID {}", log_path, zxid, storage.zxid);
}
}

8
src/Coordination/ZooKeeperDataReader.h
View File

@ -6,12 +6,12 @@
namespace DB
{
void deserializeKeeperStorageFromSnapshot(KeeperStorage & storage, const std::string & snapshot_path, Poco::Logger * log = nullptr);
void deserializeKeeperStorageFromSnapshot(KeeperStorage & storage, const std::string & snapshot_path, Poco::Logger * log);
void deserializeKeeperStorageFromSnapshotsDir(KeeperStorage & storage, const std::string & path, Poco::Logger * log = nullptr);
void deserializeKeeperStorageFromSnapshotsDir(KeeperStorage & storage, const std::string & path, Poco::Logger * log);
void deserializeLogAndApplyToStorage(KeeperStorage & storage, const std::string & log_path, Poco::Logger * log = nullptr);
void deserializeLogAndApplyToStorage(KeeperStorage & storage, const std::string & log_path, Poco::Logger * log);
void deserializeLogsAndApplyToStorage(KeeperStorage & storage, const std::string & path, Poco::Logger * log = nullptr);
void deserializeLogsAndApplyToStorage(KeeperStorage & storage, const std::string & path, Poco::Logger * log);
}