thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-27 10:02:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix local metadata differ zk metadata

Browse Source
This commit is contained in:
liyang830 2022-01-27 16:33:40 +08:00
parent ac1a78f4d9
commit eca0453564
8 changed files with 82 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/Common/IFactoryWithAliases.h
View File

@ -120,12 +120,8 @@ public:
const String & getCanonicalNameIfAny(const String & name) const
{
auto it = case_insensitive_name_mapping.find(Poco::toLower(name));
if (it != case_insensitive_name_mapping.end()) {
if (it->first != name)
{
return it->second;
}
}
if (it != case_insensitive_name_mapping.end())
return it->second;
return name;
}

16
src/Storages/KeyDescription.cpp
View File

@ -8,6 +8,9 @@
#include <Interpreters/TreeRewriter.h>
#include <Storages/extractKeyExpressionList.h>
#include <Common/quoteString.h>
#include <Interpreters/FunctionNameNormalizer.h>
#include <Parsers/ExpressionElementParsers.h>
#include <Parsers/parseQuery.h>
namespace DB
@ -161,4 +164,17 @@ KeyDescription KeyDescription::buildEmptyKey()
return result;
}
KeyDescription KeyDescription::parse(const String & str, const ColumnsDescription & columns, ContextPtr context)
{
KeyDescription result;
if (str.empty())
return result;
ParserExpressionElement parser;
ASTPtr ast = parseQuery(parser, "(" + str + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
FunctionNameNormalizer().visit(ast.get());
return getKeyFromAST(ast, columns, context);
}
}

3
src/Storages/KeyDescription.h
View File

@ -76,6 +76,9 @@ struct KeyDescription
/// Substitute modulo with moduloLegacy. Used in KeyCondition to allow proper comparison with keys.
static bool moduloToModuloLegacyRecursive(ASTPtr node_expr);
/// Parse description from string
static KeyDescription parse(const String & str, const ColumnsDescription & columns, ContextPtr context);
};
}

61
src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp
View File

@ -168,7 +168,7 @@ ReplicatedMergeTreeTableMetadata ReplicatedMergeTreeTableMetadata::parse(const S
}
void ReplicatedMergeTreeTableMetadata::checkImmutableFieldsEquals(const ReplicatedMergeTreeTableMetadata & from_zk) const
void ReplicatedMergeTreeTableMetadata::checkImmutableFieldsEquals(const ReplicatedMergeTreeTableMetadata & from_zk, const ColumnsDescription & columns, ContextPtr context) const
{
if (data_format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
{
@ -203,9 +203,12 @@ void ReplicatedMergeTreeTableMetadata::checkImmutableFieldsEquals(const Replicat
/// NOTE: You can make a less strict check of match expressions so that tables do not break from small changes
/// in formatAST code.
if (primary_key != from_zk.primary_key)
String parsed_zk_primary_key = formattedAST(KeyDescription::parse(from_zk.primary_key, columns, context).expression_list_ast);
if (primary_key != parsed_zk_primary_key)
throw Exception("Existing table metadata in ZooKeeper differs in primary key."
" Stored in ZooKeeper: " + from_zk.primary_key + ", local: " + primary_key,
" Stored in ZooKeeper: " + from_zk.primary_key +
", parsed from ZooKeeper: " + parsed_zk_primary_key +
", local: " + primary_key,
ErrorCodes::METADATA_MISMATCH);
if (data_format_version != from_zk.data_format_version)
@ -214,39 +217,53 @@ void ReplicatedMergeTreeTableMetadata::checkImmutableFieldsEquals(const Replicat
", local: " + DB::toString(data_format_version.toUnderType()),
ErrorCodes::METADATA_MISMATCH);
if (partition_key != from_zk.partition_key)
String parsed_zk_partition_key = formattedAST(KeyDescription::parse(from_zk.partition_key, columns, context).expression_list_ast);
if (partition_key != parsed_zk_partition_key)
throw Exception(
"Existing table metadata in ZooKeeper differs in partition key expression."
" Stored in ZooKeeper: " + from_zk.partition_key + ", local: " + partition_key,
" Stored in ZooKeeper: " + from_zk.partition_key +
", parsed from ZooKeeper: " + parsed_zk_partition_key +
", local: " + partition_key,
ErrorCodes::METADATA_MISMATCH);
}
void ReplicatedMergeTreeTableMetadata::checkEquals(const ReplicatedMergeTreeTableMetadata & from_zk, const ColumnsDescription & columns, ContextPtr context) const
{
checkImmutableFieldsEquals(from_zk);
checkImmutableFieldsEquals(from_zk, columns, context);
if (sampling_expression != from_zk.sampling_expression)
throw Exception("Existing table metadata in ZooKeeper differs in sample expression."
" Stored in ZooKeeper: " + from_zk.sampling_expression + ", local: " + sampling_expression,
ErrorCodes::METADATA_MISMATCH);
if (sorting_key != from_zk.sorting_key)
String parsed_zk_sampling_expression = formattedAST(KeyDescription::parse(from_zk.sampling_expression, columns, context).definition_ast);
if (sampling_expression != parsed_zk_sampling_expression)
{
throw Exception(
"Existing table metadata in ZooKeeper differs in sorting key expression."
" Stored in ZooKeeper: " + from_zk.sorting_key + ", local: " + sorting_key,
"Existing table metadata in ZooKeeper differs in sample expression."
" Stored in ZooKeeper: " + from_zk.sampling_expression +
", parsed from ZooKeeper: " + parsed_zk_sampling_expression +
", local: " + sampling_expression,
ErrorCodes::METADATA_MISMATCH);
}
if (ttl_table != from_zk.ttl_table)
String parsed_zk_sorting_key = formattedAST(extractKeyExpressionList(KeyDescription::parse(from_zk.sorting_key, columns, context).definition_ast));
if (sorting_key != parsed_zk_sorting_key)
{
throw Exception(
"Existing table metadata in ZooKeeper differs in TTL."
" Stored in ZooKeeper: " + from_zk.ttl_table +
", local: " + ttl_table,
ErrorCodes::METADATA_MISMATCH);
"Existing table metadata in ZooKeeper differs in sorting key expression."
" Stored in ZooKeeper: " + from_zk.sorting_key +
", parsed from ZooKeeper: " + parsed_zk_sorting_key +
", local: " + sorting_key,
ErrorCodes::METADATA_MISMATCH);
}
auto parsed_primary_key = KeyDescription::parse(primary_key, columns, context);
String parsed_zk_ttl_table = formattedAST(TTLTableDescription::parse(from_zk.ttl_table, columns, context, parsed_primary_key).definition_ast);
if (ttl_table != parsed_zk_ttl_table)
{
throw Exception(
"Existing table metadata in ZooKeeper differs in TTL."
" Stored in ZooKeeper: " + from_zk.ttl_table +
", parsed from ZooKeeper: " + parsed_zk_ttl_table +
", local: " + ttl_table,
ErrorCodes::METADATA_MISMATCH);
}
String parsed_zk_skip_indices = IndicesDescription::parse(from_zk.skip_indices, columns, context).toString();
@ -290,10 +307,10 @@ void ReplicatedMergeTreeTableMetadata::checkEquals(const ReplicatedMergeTreeTabl
}
ReplicatedMergeTreeTableMetadata::Diff
ReplicatedMergeTreeTableMetadata::checkAndFindDiff(const ReplicatedMergeTreeTableMetadata & from_zk) const
ReplicatedMergeTreeTableMetadata::checkAndFindDiff(const ReplicatedMergeTreeTableMetadata & from_zk, const ColumnsDescription & columns, ContextPtr context) const
{
checkImmutableFieldsEquals(from_zk);
checkImmutableFieldsEquals(from_zk, columns, context);
Diff diff;

4
src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h
View File

@ -70,11 +70,11 @@ struct ReplicatedMergeTreeTableMetadata
void checkEquals(const ReplicatedMergeTreeTableMetadata & from_zk, const ColumnsDescription & columns, ContextPtr context) const;
Diff checkAndFindDiff(const ReplicatedMergeTreeTableMetadata & from_zk) const;
Diff checkAndFindDiff(const ReplicatedMergeTreeTableMetadata & from_zk, const ColumnsDescription & columns, ContextPtr context) const;
private:
void checkImmutableFieldsEquals(const ReplicatedMergeTreeTableMetadata & from_zk) const;
void checkImmutableFieldsEquals(const ReplicatedMergeTreeTableMetadata & from_zk, const ColumnsDescription & columns, ContextPtr context) const;
bool index_granularity_bytes_found_in_zk = false;
};

2
src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -4478,7 +4478,7 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer
auto alter_lock_holder = lockForAlter(getSettings()->lock_acquire_timeout_for_background_operations);
LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally.");
auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this, getInMemoryMetadataPtr()).checkAndFindDiff(metadata_from_entry);
auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this, getInMemoryMetadataPtr()).checkAndFindDiff(metadata_from_entry, getInMemoryMetadataPtr()->getColumns(), getContext());
setTableStructure(std::move(columns_from_entry), metadata_diff);
metadata_version = entry.alter_version;

16
src/Storages/TTLDescription.cpp
View File

@ -15,6 +15,9 @@
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <Interpreters/FunctionNameNormalizer.h>
#include <Parsers/ExpressionElementParsers.h>
#include <Parsers/parseQuery.h>
namespace DB
@ -370,4 +373,17 @@ TTLTableDescription TTLTableDescription::getTTLForTableFromAST(
return result;
}
TTLTableDescription TTLTableDescription::parse(const String & str, const ColumnsDescription & columns, ContextPtr context, const KeyDescription & primary_key)
{
TTLTableDescription result;
if (str.empty())
return result;
ParserTTLElement parser;
ASTPtr ast = parseQuery(parser, str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
FunctionNameNormalizer().visit(ast.get());
return getTTLForTableFromAST(ast, columns, context, primary_key);
}
}

3
src/Storages/TTLDescription.h
View File

@ -118,6 +118,9 @@ struct TTLTableDescription
static TTLTableDescription getTTLForTableFromAST(
const ASTPtr & definition_ast, const ColumnsDescription & columns, ContextPtr context, const KeyDescription & primary_key);
/// Parse description from string
static TTLTableDescription parse(const String & str, const ColumnsDescription & columns, ContextPtr context, const KeyDescription & primary_key);
};
}