ClickHouse/src/Storages/MergeTree/MergeTreeWriteAheadLog.h

99 lines
2.8 KiB
C++
Raw Normal View History

2020-04-14 19:47:19 +00:00
#pragma once
#include <DataStreams/NativeBlockInputStream.h>
#include <DataStreams/NativeBlockOutputStream.h>
2020-09-01 22:25:10 +00:00
#include <Core/BackgroundSchedulePool.h>
2020-04-14 19:47:19 +00:00
#include <Disks/IDisk.h>
2020-10-29 16:18:25 +00:00
#include <Storages/MergeTree/IMergeTreeDataPart.h>
#include <Storages/MergeTree/MergeTreeDataPartInMemory.h>
2020-04-14 19:47:19 +00:00
namespace DB
{
class MergeTreeData;
/** WAL stores addditions and removals of data parts in in-memory format.
* Format of data in WAL:
* - version
* - type of action (ADD or DROP)
* - part name
* - part's block in Native format. (for ADD action)
*/
2020-04-14 19:47:19 +00:00
class MergeTreeWriteAheadLog
{
public:
2020-05-29 15:02:12 +00:00
/// Append-only enum. It is serialized to WAL
enum class ActionType : UInt8
{
ADD_PART = 0,
DROP_PART = 1,
};
struct ActionMetadata
{
/// The minimum version of WAL reader that can understand metadata written by current ClickHouse version.
/// This field must be increased when making backwards incompatible changes.
///
/// The same approach can be used recursively inside metadata.
UInt8 min_compatible_version = 0;
2020-10-29 16:18:25 +00:00
/// Actual metadata.
UUID part_uuid = UUIDHelpers::Nil;
void write(WriteBuffer & meta_out) const;
void read(ReadBuffer & meta_in);
2020-11-02 22:36:32 +00:00
private:
static constexpr auto JSON_KEY_PART_UUID = "part_uuid";
String toJSON() const;
void fromJSON(const String & buf);
};
2020-10-29 16:18:25 +00:00
constexpr static UInt8 WAL_VERSION = 1;
2020-04-14 19:47:19 +00:00
constexpr static auto WAL_FILE_NAME = "wal";
constexpr static auto WAL_FILE_EXTENSION = ".bin";
constexpr static auto DEFAULT_WAL_FILE_NAME = "wal.bin";
2020-04-14 19:47:19 +00:00
2020-09-01 22:25:10 +00:00
MergeTreeWriteAheadLog(MergeTreeData & storage_, const DiskPtr & disk_,
const String & name = DEFAULT_WAL_FILE_NAME);
2020-04-14 19:47:19 +00:00
2020-09-11 13:09:26 +00:00
~MergeTreeWriteAheadLog();
2020-10-29 16:18:25 +00:00
void addPart(DataPartInMemoryPtr & part);
2020-05-29 15:02:12 +00:00
void dropPart(const String & part_name);
std::vector<MergeTreeMutableDataPartPtr> restore(const StorageMetadataPtr & metadata_snapshot, ContextPtr context);
2020-04-14 19:47:19 +00:00
using MinMaxBlockNumber = std::pair<Int64, Int64>;
static std::optional<MinMaxBlockNumber> tryParseMinMaxBlockNumber(const String & filename);
2020-04-14 19:47:19 +00:00
private:
void init();
2020-09-10 23:24:16 +00:00
void rotate(const std::unique_lock<std::mutex> & lock);
void sync(std::unique_lock<std::mutex> & lock);
2020-04-14 19:47:19 +00:00
const MergeTreeData & storage;
DiskPtr disk;
2020-05-14 20:08:15 +00:00
String name;
2020-04-14 19:47:19 +00:00
String path;
std::unique_ptr<WriteBuffer> out;
std::unique_ptr<NativeBlockOutputStream> block_out;
Int64 min_block_number = std::numeric_limits<Int64>::max();
2020-05-14 20:08:15 +00:00
Int64 max_block_number = -1;
2020-04-14 19:47:19 +00:00
2020-09-01 22:25:10 +00:00
BackgroundSchedulePool & pool;
BackgroundSchedulePoolTaskHolder sync_task;
2020-09-10 23:24:16 +00:00
std::condition_variable sync_cv;
2020-09-01 22:25:10 +00:00
size_t bytes_at_last_sync = 0;
bool sync_scheduled = false;
2020-04-14 19:47:19 +00:00
mutable std::mutex write_mutex;
2021-08-26 11:01:15 +00:00
Poco::Logger * log;
2020-04-14 19:47:19 +00:00
};
}