ClickHouse/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp

508 lines
16 KiB
C++
Raw Normal View History

#include <Common/ZooKeeper/Types.h>
#include "Access/IAccessEntity.h"
#include <Storages/MergeTree/ReplicatedMergeTreeLogEntry.h>
2020-01-30 12:54:52 +00:00
#include <Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h>
2021-05-13 11:29:59 +00:00
#include <Storages/MergeTree/MergeTreePartInfo.h>
#include <IO/Operators.h>
#include <IO/ReadBufferFromString.h>
2017-07-31 21:39:24 +00:00
#include <IO/WriteBufferFromString.h>
2020-02-13 14:19:43 +00:00
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
namespace DB
{
2020-02-25 18:10:48 +00:00
namespace ErrorCodes
{
extern const int UNKNOWN_FORMAT_VERSION;
extern const int LOGICAL_ERROR;
}
enum FormatVersion : UInt8
{
FORMAT_WITH_CREATE_TIME = 2,
FORMAT_WITH_BLOCK_ID = 3,
FORMAT_WITH_DEDUPLICATE = 4,
FORMAT_WITH_UUID = 5,
FORMAT_WITH_DEDUPLICATE_BY_COLUMNS = 6,
FORMAT_WITH_LOG_ENTRY_ID = 7,
FORMAT_LAST
};
2016-06-08 13:08:20 +00:00
void ReplicatedMergeTreeLogEntryData::writeText(WriteBuffer & out) const
{
UInt8 format_version = FORMAT_WITH_DEDUPLICATE;
if (!deduplicate_by_columns.empty())
format_version = std::max<UInt8>(format_version, FORMAT_WITH_DEDUPLICATE_BY_COLUMNS);
/// Conditionally bump format_version only when uuid has been assigned.
/// If some other feature requires bumping format_version to >= 5 then this code becomes no-op.
if (new_part_uuid != UUIDHelpers::Nil)
format_version = std::max<UInt8>(format_version, FORMAT_WITH_UUID);
if (!log_entry_id.empty())
format_version = std::max<UInt8>(format_version, FORMAT_WITH_LOG_ENTRY_ID);
out << "format version: " << format_version << "\n"
2017-12-18 04:07:26 +00:00
<< "create_time: " << LocalDateTime(create_time ? create_time : time(nullptr)) << "\n"
<< "source replica: " << source_replica << '\n'
<< "block_id: " << escape << block_id << '\n';
if (format_version >= FORMAT_WITH_LOG_ENTRY_ID)
out << "log_entry_id: " << escape << log_entry_id << '\n';
switch (type)
{
case GET_PART:
out << "get\n" << new_part_name;
break;
case CLONE_PART_FROM_SHARD:
out << "clone_part_from_shard\n"
<< new_part_name << "\n"
<< "source_shard: " << source_shard;
break;
case ATTACH_PART:
out << "attach\n" << new_part_name << "\n"
2021-03-03 13:51:41 +00:00
<< "part_checksum: " << part_checksum;
break;
case MERGE_PARTS:
out << "merge\n";
for (const String & s : source_parts)
out << s << '\n';
out << "into\n" << new_part_name;
2017-04-17 15:14:56 +00:00
out << "\ndeduplicate: " << deduplicate;
2020-10-29 16:18:25 +00:00
2020-09-03 13:00:13 +00:00
if (merge_type != MergeType::REGULAR)
out <<"\nmerge_type: " << static_cast<UInt64>(merge_type);
2020-10-29 16:18:25 +00:00
if (new_part_uuid != UUIDHelpers::Nil)
out << "\ninto_uuid: " << new_part_uuid;
if (!deduplicate_by_columns.empty())
{
out << "\ndeduplicate_by_columns: ";
for (size_t i = 0; i < deduplicate_by_columns.size(); ++i)
{
2020-12-19 18:58:23 +00:00
out << quote << deduplicate_by_columns[i];
if (i != deduplicate_by_columns.size() - 1)
out << ",";
}
}
break;
case DROP_RANGE:
if (detach)
out << "detach\n";
else
out << "drop\n";
out << new_part_name;
break;
2020-03-17 13:49:50 +00:00
/// NOTE: Deprecated.
case CLEAR_COLUMN:
out << "clear_column\n"
<< escape << column_name
<< "\nfrom\n"
<< new_part_name;
break;
2020-03-17 13:49:50 +00:00
/// NOTE: Deprecated.
2019-05-09 14:25:18 +00:00
case CLEAR_INDEX:
out << "clear_index\n"
<< escape << index_name
<< "\nfrom\n"
<< new_part_name;
break;
case REPLACE_RANGE:
out << typeToString(REPLACE_RANGE) << "\n";
replace_range_entry->writeText(out);
break;
case MUTATE_PART:
out << "mutate\n"
<< source_parts.at(0) << "\n"
<< "to\n"
<< new_part_name;
if (new_part_uuid != UUIDHelpers::Nil)
out << "\nto_uuid\n"
<< new_part_uuid;
if (isAlterMutation())
out << "\nalter_version\n" << alter_version;
break;
2020-01-28 17:15:22 +00:00
case ALTER_METADATA: /// Just make local /metadata and /columns consistent with global
2020-01-13 16:39:20 +00:00
out << "alter\n";
2020-01-31 19:30:33 +00:00
out << "alter_version\n";
out << alter_version<< "\n";
out << "have_mutation\n";
out << have_mutation << "\n";
2020-01-30 12:54:52 +00:00
out << "columns_str_size:\n";
out << columns_str.size() << "\n";
out << columns_str << "\n";
out << "metadata_str_size:\n";
out << metadata_str.size() << "\n";
out << metadata_str;
2020-01-13 16:39:20 +00:00
break;
2020-01-15 13:00:08 +00:00
case SYNC_PINNED_PART_UUIDS:
out << "sync_pinned_part_uuids\n";
break;
default:
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown log entry type: {}", static_cast<int>(type));
}
out << '\n';
2020-02-13 14:19:43 +00:00
if (new_part_type != MergeTreeDataPartType::WIDE && new_part_type != MergeTreeDataPartType::UNKNOWN)
out << "part_type: " << new_part_type.toString() << "\n";
if (quorum)
out << "quorum: " << quorum << '\n';
}
2016-06-08 13:08:20 +00:00
void ReplicatedMergeTreeLogEntryData::readText(ReadBuffer & in)
{
UInt8 format_version = 0;
String type_str;
in >> "format version: " >> format_version >> "\n";
if (format_version < 1 || format_version >= FORMAT_LAST)
throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unknown ReplicatedMergeTreeLogEntry format version: {}",
DB::toString(format_version));
if (format_version >= FORMAT_WITH_CREATE_TIME)
{
LocalDateTime create_time_dt;
in >> "create_time: " >> create_time_dt >> "\n";
2021-03-15 19:23:27 +00:00
create_time = DateLUT::instance().makeDateTime(
create_time_dt.year(), create_time_dt.month(), create_time_dt.day(),
create_time_dt.hour(), create_time_dt.minute(), create_time_dt.second());
}
in >> "source replica: " >> source_replica >> "\n";
if (format_version >= FORMAT_WITH_BLOCK_ID)
{
in >> "block_id: " >> escape >> block_id >> "\n";
}
if (format_version >= FORMAT_WITH_LOG_ENTRY_ID)
in >> "log_entry_id: " >> escape >> log_entry_id >> "\n";
in >> type_str >> "\n";
bool trailing_newline_found = false;
if (type_str == "get")
{
type = GET_PART;
in >> new_part_name;
}
else if (type_str == "attach")
{
type = ATTACH_PART;
2021-03-03 13:51:41 +00:00
in >> new_part_name >> "\npart_checksum: " >> part_checksum;
}
else if (type_str == "merge")
{
type = MERGE_PARTS;
while (true)
{
String s;
in >> s >> "\n";
if (s == "into")
break;
source_parts.push_back(s);
}
in >> new_part_name;
2020-09-03 13:00:13 +00:00
if (format_version >= FORMAT_WITH_DEDUPLICATE)
2020-09-03 13:00:13 +00:00
{
2017-04-17 15:14:56 +00:00
in >> "\ndeduplicate: " >> deduplicate;
2020-09-04 11:27:27 +00:00
/// Trying to be more backward compatible
2020-10-29 16:18:25 +00:00
while (!trailing_newline_found)
2020-09-03 13:00:13 +00:00
{
2020-10-29 16:18:25 +00:00
in >> "\n";
if (checkString("merge_type: ", in))
{
UInt64 value;
in >> value;
merge_type = checkAndGetMergeType(value);
}
else if (checkString("into_uuid: ", in))
in >> new_part_uuid;
else if (checkString("deduplicate_by_columns: ", in))
{
Strings new_deduplicate_by_columns;
for (;;)
{
String tmp_column_name;
2020-12-19 18:58:23 +00:00
in >> quote >> tmp_column_name;
new_deduplicate_by_columns.emplace_back(std::move(tmp_column_name));
if (!checkString(",", in))
break;
}
deduplicate_by_columns = std::move(new_deduplicate_by_columns);
}
2020-10-29 16:18:25 +00:00
else
trailing_newline_found = true;
2020-09-03 13:00:13 +00:00
}
}
}
else if (type_str == "drop" || type_str == "detach")
{
type = DROP_RANGE;
detach = type_str == "detach";
in >> new_part_name;
}
2020-03-17 13:49:50 +00:00
else if (type_str == "clear_column") /// NOTE: Deprecated.
{
type = CLEAR_COLUMN;
in >> escape >> column_name >> "\nfrom\n" >> new_part_name;
}
2020-03-17 13:49:50 +00:00
else if (type_str == "clear_index") /// NOTE: Deprecated.
2019-05-09 14:25:18 +00:00
{
type = CLEAR_INDEX;
in >> escape >> index_name >> "\nfrom\n" >> new_part_name;
}
else if (type_str == typeToString(REPLACE_RANGE))
{
type = REPLACE_RANGE;
replace_range_entry = std::make_shared<ReplaceRangeEntry>();
replace_range_entry->readText(in);
}
else if (type_str == "mutate")
{
type = MUTATE_PART;
String source_part;
in >> source_part >> "\n"
>> "to\n"
>> new_part_name;
source_parts.push_back(source_part);
2020-03-13 17:23:36 +00:00
2020-10-29 16:18:25 +00:00
while (!trailing_newline_found)
{
in >> "\n";
2020-10-29 16:18:25 +00:00
if (checkString("alter_version\n", in))
in >> alter_version;
else if (checkString("to_uuid\n", in))
in >> new_part_uuid;
else
trailing_newline_found = true;
}
}
2020-01-13 16:39:20 +00:00
else if (type_str == "alter")
{
2020-01-28 17:15:22 +00:00
type = ALTER_METADATA;
2020-01-31 19:30:33 +00:00
in >> "alter_version\n";
in >> alter_version;
in >> "\nhave_mutation\n";
in >> have_mutation;
2020-01-30 12:54:52 +00:00
in >> "\ncolumns_str_size:\n";
size_t columns_size;
in >> columns_size >> "\n";
columns_str.resize(columns_size);
in.readStrict(&columns_str[0], columns_size);
in >> "\nmetadata_str_size:\n";
size_t metadata_size;
in >> metadata_size >> "\n";
metadata_str.resize(metadata_size);
in.readStrict(&metadata_str[0], metadata_size);
2020-01-13 16:39:20 +00:00
}
else if (type_str == "sync_pinned_part_uuids")
{
type = SYNC_PINNED_PART_UUIDS;
}
else if (type_str == "clone_part_from_shard")
{
type = CLONE_PART_FROM_SHARD;
in >> new_part_name;
in >> "\nsource_shard: " >> source_shard;
}
if (!trailing_newline_found)
in >> "\n";
2020-02-13 14:19:43 +00:00
if (checkString("part_type: ", in))
{
String part_type_str;
in >> type_str;
new_part_type.fromString(type_str);
in >> "\n";
}
else
new_part_type = MergeTreeDataPartType::WIDE;
/// Optional field.
if (!in.eof())
in >> "quorum: " >> quorum >> "\n";
}
void ReplicatedMergeTreeLogEntryData::ReplaceRangeEntry::writeText(WriteBuffer & out) const
{
out << "drop_range_name: " << drop_range_part_name << "\n";
out << "from_database: " << escape << from_database << "\n";
out << "from_table: " << escape << from_table << "\n";
out << "source_parts: ";
writeQuoted(src_part_names, out);
out << "\n";
out << "new_parts: ";
writeQuoted(new_part_names, out);
out << "\n";
out << "part_checksums: ";
writeQuoted(part_names_checksums, out);
out << "\n";
out << "columns_version: " << columns_version;
}
void ReplicatedMergeTreeLogEntryData::ReplaceRangeEntry::readText(ReadBuffer & in)
{
in >> "drop_range_name: " >> drop_range_part_name >> "\n";
in >> "from_database: " >> escape >> from_database >> "\n";
in >> "from_table: " >> escape >> from_table >> "\n";
in >> "source_parts: ";
readQuoted(src_part_names, in);
in >> "\n";
in >> "new_parts: ";
readQuoted(new_part_names, in);
in >> "\n";
in >> "part_checksums: ";
readQuoted(part_names_checksums, in);
in >> "\n";
in >> "columns_version: " >> columns_version;
}
2021-05-14 16:11:40 +00:00
bool ReplicatedMergeTreeLogEntryData::ReplaceRangeEntry::isMovePartitionOrAttachFrom(const MergeTreePartInfo & drop_range_info)
{
assert(drop_range_info.getBlocksCount() != 0);
2021-05-14 16:11:40 +00:00
return drop_range_info.getBlocksCount() == 1;
}
2016-06-08 13:08:20 +00:00
String ReplicatedMergeTreeLogEntryData::toString() const
{
2017-07-31 21:39:24 +00:00
WriteBufferFromOwnString out;
writeText(out);
return out.str();
}
ReplicatedMergeTreeLogEntry::Ptr ReplicatedMergeTreeLogEntry::parse(const String & s, const Coordination::Stat & stat)
{
ReadBufferFromString in(s);
Ptr res = std::make_shared<ReplicatedMergeTreeLogEntry>();
res->readText(in);
assertEOF(in);
if (!res->create_time)
res->create_time = stat.ctime / 1000;
return res;
}
2021-06-01 13:25:23 +00:00
std::optional<String> ReplicatedMergeTreeLogEntryData::getDropRange(MergeTreeDataFormatVersion format_version) const
{
if (type == DROP_RANGE)
return new_part_name;
if (type == REPLACE_RANGE)
{
auto drop_range_info = MergeTreePartInfo::fromPartName(replace_range_entry->drop_range_part_name, format_version);
if (!ReplaceRangeEntry::isMovePartitionOrAttachFrom(drop_range_info))
{
/// It's REPLACE, not MOVE or ATTACH, so drop range is real
return replace_range_entry->drop_range_part_name;
}
}
return {};
}
2021-07-06 16:51:23 +00:00
bool ReplicatedMergeTreeLogEntryData::isDropPart(MergeTreeDataFormatVersion format_version) const
{
if (type == DROP_RANGE)
{
auto drop_range_info = MergeTreePartInfo::fromPartName(new_part_name, format_version);
return !drop_range_info.isFakeDropRangePart();
}
return false;
}
2021-05-13 11:29:59 +00:00
Strings ReplicatedMergeTreeLogEntryData::getVirtualPartNames(MergeTreeDataFormatVersion format_version) const
{
/// Doesn't produce any part
if (type == ALTER_METADATA)
return {};
/// DROP_RANGE does not add a real part, but we must disable merges in that range
if (type == DROP_RANGE)
2021-06-30 12:29:09 +00:00
{
auto drop_range_part_info = MergeTreePartInfo::fromPartName(new_part_name, format_version);
/// It's DROP PART and we don't want to add it into virtual parts
/// because it can lead to intersecting parts on stale replicas and this
/// problem is fundamental. So we have very weak guarantees for DROP
/// PART. If any concurrent merge will be assigned then DROP PART will
2021-07-01 12:12:27 +00:00
/// delete nothing and part will be successfully merged into bigger part.
2021-06-30 12:29:09 +00:00
///
/// dropPart used in the following cases:
/// 1) Remove empty parts after TTL.
/// 2) Remove parts after move between shards.
/// 3) User queries: ALTER TABLE DROP PART 'part_name'.
///
/// In the first case merge of empty part is even better than DROP. In
/// the second case part UUIDs used to forbid merges for moding parts so
/// there is no problem with concurrent merges. The third case is quite
/// rare and we give very weak guarantee: there will be no active part
/// with this name, but possibly it was merged to some other part.
if (!drop_range_part_info.isFakeDropRangePart())
return {};
2021-05-13 11:29:59 +00:00
return {new_part_name};
2021-06-30 12:29:09 +00:00
}
2021-05-13 11:29:59 +00:00
if (type == REPLACE_RANGE)
{
Strings res = replace_range_entry->new_part_names;
auto drop_range_info = MergeTreePartInfo::fromPartName(replace_range_entry->drop_range_part_name, format_version);
2021-06-01 13:25:23 +00:00
if (auto drop_range = getDropRange(format_version))
res.emplace_back(*drop_range);
2021-05-13 11:29:59 +00:00
return res;
}
2021-05-17 11:13:18 +00:00
/// Doesn't produce any part.
if (type == SYNC_PINNED_PART_UUIDS)
return {};
/// Doesn't produce any part by itself.
if (type == CLONE_PART_FROM_SHARD)
return {};
2021-05-13 11:29:59 +00:00
return {new_part_name};
}
}