diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh index da2d1172f18..e72c28b7167 100644 --- a/docker/test/upgrade/run.sh +++ b/docker/test/upgrade/run.sh @@ -119,6 +119,13 @@ mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/c install_packages package_folder export ZOOKEEPER_FAULT_INJECTION=1 configure + +# Just in case previous version left some garbage in zk +sudo cat /etc/clickhouse-server/config.d/lost_forever_check.xml \ + | sed "s|>1<|>0<|g" \ + > /etc/clickhouse-server/config.d/lost_forever_check.xml.tmp +sudo mv /etc/clickhouse-server/config.d/lost_forever_check.xml.tmp /etc/clickhouse-server/config.d/lost_forever_check.xml + start 500 clickhouse-client --query "SELECT 'Server successfully started', 'OK', NULL, ''" >> /test_output/test_results.tsv \ || (rg --text ".*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt \ diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 7cf37d6765e..b362b5e3067 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1255,6 +1255,10 @@ static void paranoidCheckForCoveredPartsInZooKeeperOnStart(const StorageReplicat if (!paranoid_check_for_covered_parts) return; + /// FIXME https://github.com/ClickHouse/ClickHouse/issues/51182 + if (storage->getSettings()->use_metadata_cache) + return; + ActiveDataPartSet active_set(format_version); for (const auto & part_name : parts_in_zk) active_set.add(part_name); @@ -1992,7 +1996,7 @@ MutableDataPartStoragePtr StorageReplicatedMergeTree::executeFetchShared( } } -static void paranoidCheckForCoveredPartsInZooKeeper(const ZooKeeperPtr & zookeeper, const String & replica_path, +static void paranoidCheckForCoveredPartsInZooKeeper(const StorageReplicatedMergeTree * storage, const ZooKeeperPtr & zookeeper, const String & replica_path, MergeTreeDataFormatVersion format_version, const String & covering_part_name) { #ifdef ABORT_ON_LOGICAL_ERROR @@ -2003,17 +2007,21 @@ static void paranoidCheckForCoveredPartsInZooKeeper(const ZooKeeperPtr & zookeep bool paranoid_check_for_covered_parts = Context::getGlobalContextInstance()->getConfigRef().getBool( "replicated_merge_tree_paranoid_check_on_drop_range", paranoid_check_for_covered_parts_default); - if (paranoid_check_for_covered_parts) + if (!paranoid_check_for_covered_parts) + return; + + /// FIXME https://github.com/ClickHouse/ClickHouse/issues/51182 + if (storage->getSettings()->use_metadata_cache) + return; + + auto drop_range_info = MergeTreePartInfo::fromPartName(covering_part_name, format_version); + Strings parts_remain = zookeeper->getChildren(replica_path + "/parts"); + for (const auto & part_name : parts_remain) { - auto drop_range_info = MergeTreePartInfo::fromPartName(covering_part_name, format_version); - Strings parts_remain = zookeeper->getChildren(replica_path + "/parts"); - for (const auto & part_name : parts_remain) - { - auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); - if (drop_range_info.contains(part_info)) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Part {} remains in ZooKeeper after DROP_RANGE {}", part_name, covering_part_name); - } + auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); + if (drop_range_info.contains(part_info)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Part {} remains in ZooKeeper after DROP_RANGE {}", part_name, covering_part_name); } } @@ -2072,7 +2080,7 @@ void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry) /// Forcibly remove parts from ZooKeeper removePartsFromZooKeeperWithRetries(parts_to_remove); - paranoidCheckForCoveredPartsInZooKeeper(getZooKeeper(), replica_path, format_version, entry.new_part_name); + paranoidCheckForCoveredPartsInZooKeeper(this, getZooKeeper(), replica_path, format_version, entry.new_part_name); if (entry.detach) LOG_DEBUG(log, "Detached {} parts inside {}.", parts_to_remove.size(), entry.new_part_name); @@ -2209,7 +2217,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) LOG_INFO(log, "All parts from REPLACE PARTITION command have been already attached"); removePartsFromZooKeeperWithRetries(parts_to_remove); if (replace) - paranoidCheckForCoveredPartsInZooKeeper(getZooKeeper(), replica_path, format_version, entry_replace.drop_range_part_name); + paranoidCheckForCoveredPartsInZooKeeper(this, getZooKeeper(), replica_path, format_version, entry_replace.drop_range_part_name); return true; } @@ -2510,7 +2518,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) removePartsFromZooKeeperWithRetries(parts_to_remove); if (replace) - paranoidCheckForCoveredPartsInZooKeeper(getZooKeeper(), replica_path, format_version, entry_replace.drop_range_part_name); + paranoidCheckForCoveredPartsInZooKeeper(this, getZooKeeper(), replica_path, format_version, entry_replace.drop_range_part_name); res_parts.clear(); parts_to_remove.clear(); cleanup_thread.wakeup(); diff --git a/tests/config/config.d/lost_forever_check.xml b/tests/config/config.d/lost_forever_check.xml new file mode 100644 index 00000000000..a021d694dff --- /dev/null +++ b/tests/config/config.d/lost_forever_check.xml @@ -0,0 +1,4 @@ + + 1 + 1 + diff --git a/tests/config/config.d/merge_tree.xml b/tests/config/config.d/merge_tree.xml index 5ec577a0c52..2ac468673fa 100644 --- a/tests/config/config.d/merge_tree.xml +++ b/tests/config/config.d/merge_tree.xml @@ -5,6 +5,4 @@ 60 10 - - 1 diff --git a/tests/config/install.sh b/tests/config/install.sh index b2153db1b2c..50f2627d37c 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -33,6 +33,7 @@ ln -sf $SRC_PATH/config.d/test_cluster_with_incorrect_pw.xml $DEST_SERVER_PATH/c ln -sf $SRC_PATH/config.d/keeper_port.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/logging_no_rotate.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/merge_tree.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/lost_forever_check.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/metadata_cache.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/tcp_with_proxy.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/prometheus.xml $DEST_SERVER_PATH/config.d/