thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-27 18:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Test simple serde

Browse Source
This commit is contained in:
alesapin 2021-03-01 17:40:32 +03:00
parent 03960b1eed
commit 73d3c20554
3 changed files with 55 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Coordination/NuKeeperStorageSerializer.cpp
View File

@ -181,6 +181,7 @@ std::string NuKeeperSnapshotManager::serializeSnapshotBufferToDisk(nuraft::buffe
WriteBufferFromFile plain_buf(new_snapshot_path);
copyData(reader, plain_buf);
plain_buf.sync();
existing_snapshots.emplace(up_to_log_idx, new_snapshot_path);
return new_snapshot_path;
}
@ -199,6 +200,7 @@ nuraft::ptr<nuraft::buffer> NuKeeperSnapshotManager::serializeSnapshotToBuffer(c
CompressedWriteBuffer compressed_writer(writer);
NuKeeperStorageSnapshot::serialize(snapshot, compressed_writer);
compressed_writer.finalize();
return writer.getBuffer();
}

6
src/Coordination/NuKeeperStorageSerializer.h
View File

@ -41,11 +41,11 @@ public:
size_t restoreFromLatestSnapshot(NuKeeperStorage * storage) const;
std::string serializeSnapshotBufferToDisk(nuraft::buffer & buffer, size_t up_to_log_idx);
nuraft::ptr<nuraft::buffer> deserializeSnapshotBufferFromDisk(size_t up_to_log_idx) const;
static nuraft::ptr<nuraft::buffer> serializeSnapshotToBuffer(const NuKeeperStorageSnapshot & snapshot);
std::string serializeSnapshotBufferToDisk(nuraft::buffer & buffer, size_t up_to_log_idx);
static void deserializeSnapshotFromBuffer(NuKeeperStorage * storage, nuraft::ptr<nuraft::buffer> buffer);
nuraft::ptr<nuraft::buffer> deserializeSnapshotBufferFromDisk(size_t up_to_log_idx) const;
private:
const std::string snapshots_path;

79
src/Coordination/tests/gtest_for_build.cpp
View File

@ -230,35 +230,6 @@ DB::NuKeeperStorage::ResponsesForSessions getZooKeeperResponses(nuraft::ptr<nura
return results;
}
//TEST(CoordinationTest, TestStorageSerialization)
//{
// DB::NuKeeperStorage storage(500);
// storage.container["/hello"] = DB::NuKeeperStorage::Node{.data="world"};
// storage.container["/hello/somepath"] = DB::NuKeeperStorage::Node{.data="somedata"};
// storage.session_id_counter = 5;
// storage.zxid = 156;
// storage.ephemerals[3] = {"/hello", "/"};
// storage.ephemerals[1] = {"/hello/somepath"};
//
// DB::WriteBufferFromOwnString buffer;
// DB::NuKeeperStorageSerializer serializer;
// serializer.serialize(storage, buffer);
// std::string serialized = buffer.str();
// EXPECT_NE(serialized.size(), 0);
// DB::ReadBufferFromString read(serialized);
// DB::NuKeeperStorage new_storage(500);
// serializer.deserialize(new_storage, read);
//
// EXPECT_EQ(new_storage.container.size(), 3);
// EXPECT_EQ(new_storage.container["/hello"].data, "world");
// EXPECT_EQ(new_storage.container["/hello/somepath"].data, "somedata");
// EXPECT_EQ(new_storage.session_id_counter, 5);
// EXPECT_EQ(new_storage.zxid, 156);
// EXPECT_EQ(new_storage.ephemerals.size(), 2);
// EXPECT_EQ(new_storage.ephemerals[3].size(), 2);
// EXPECT_EQ(new_storage.ephemerals[1].size(), 1);
//}
DB::LogEntryPtr getLogEntry(const std::string & s, size_t term)
{
DB::WriteBufferFromNuraftBuffer bufwriter;
@ -924,6 +895,56 @@ TEST(CoordinationTest, SnapshotableHashMapTrySnapshot)
map_snp.disableSnapshotMode();
}
TEST(CoordinationTest, TestStorageSnapshotSimple)
{
using Node = DB::NuKeeperStorage::Node;
ChangelogDirTest test("./snapshots");
DB::NuKeeperSnapshotManager manager("./snapshots");
DB::NuKeeperStorage storage(500);
storage.container.insert("/hello", Node{.data="world", .ephemeral_owner = 1});
storage.container.insert("/hello/somepath", Node{.data="somedata", .ephemeral_owner = 3});
storage.session_id_counter = 5;
storage.zxid = 2;
storage.ephemerals[3] = {"/hello"};
storage.ephemerals[1] = {"/hello/somepath"};
storage.getSessionID(130);
storage.getSessionID(130);
DB::NuKeeperStorageSnapshot snapshot(&storage, 2);
EXPECT_EQ(snapshot.up_to_log_idx, 2);
EXPECT_EQ(snapshot.zxid, 2);
EXPECT_EQ(snapshot.session_id, 7);
EXPECT_EQ(snapshot.snapshot_container_size, 3);
EXPECT_EQ(snapshot.session_and_timeout.size(), 2);
auto buf = manager.serializeSnapshotToBuffer(snapshot);
manager.serializeSnapshotBufferToDisk(*buf, 2);
EXPECT_TRUE(fs::exists("./snapshots/snapshot_2.bin"));
DB::NuKeeperStorage restored_storage(500);
auto debuf = manager.deserializeSnapshotBufferFromDisk(2);
manager.deserializeSnapshotFromBuffer(&restored_storage, debuf);
EXPECT_EQ(restored_storage.container.size(), 3);
EXPECT_EQ(restored_storage.container.getValue("/").children.size(), 1);
EXPECT_EQ(restored_storage.container.getValue("/hello").children.size(), 1);
EXPECT_EQ(restored_storage.container.getValue("/hello/somepath").children.size(), 0);
EXPECT_EQ(restored_storage.container.getValue("/").data, "");
EXPECT_EQ(restored_storage.container.getValue("/hello").data, "world");
EXPECT_EQ(restored_storage.container.getValue("/hello/somepath").data, "somedata");
EXPECT_EQ(restored_storage.session_id_counter, 7);
EXPECT_EQ(restored_storage.zxid, 2);
EXPECT_EQ(restored_storage.ephemerals.size(), 2);
EXPECT_EQ(restored_storage.ephemerals[3].size(), 1);
EXPECT_EQ(restored_storage.ephemerals[1].size(), 1);
EXPECT_EQ(restored_storage.session_and_timeout.size(), 2);
}
int main(int argc, char ** argv)
{