Merge pull request #59507 from ClickHouse/alter-table-forget-partition

Add `FORGET PARTITION` query to remove old partition nodes from ZooKeeper

docs/en/sql-reference/statements/alter/partition.md

@@ -9,6 +9,7 @@ The following operations with [partitions](/docs/en/engines/table-engines/merget
 
 - [DETACH PARTITION\|PART](#detach-partitionpart) — Moves a partition or part to the `detached` directory and forget it.
 - [DROP PARTITION\|PART](#drop-partitionpart) — Deletes a partition or part.
+- [FORGET PARTITION](#forget-partition) — Deletes a partition metadata from zookeeper if it's empty.
 - [ATTACH PARTITION\|PART](#attach-partitionpart) — Adds a partition or part from the `detached` directory to the table.
 - [ATTACH PARTITION FROM](#attach-partition-from) — Copies the data partition from one table to another and adds.
 - [REPLACE PARTITION](#replace-partition) — Copies the data partition from one table to another and replaces.
@@ -73,6 +74,22 @@ ALTER TABLE table_name [ON CLUSTER cluster] DROP DETACHED PARTITION|PART partiti
 Removes the specified part or all parts of the specified partition from `detached`.
 Read more about setting the partition expression in a section [How to set the partition expression](#how-to-set-partition-expression).
 
+## FORGET PARTITION
+
+``` sql
+ALTER TABLE table_name FORGET PARTITION partition_expr
+```
+
+Removes all metadata about an empty partition from ZooKeeper. Query fails if partition is not empty or unknown. Make sure to execute only for partitions that will never be used again.
+
+Read about setting the partition expression in a section [How to set the partition expression](#how-to-set-partition-expression).
+
+Example:
+
+``` sql
+ALTER TABLE mt FORGET PARTITION '20201121';
+```
+
 ## ATTACH PARTITION\|PART
 
 ``` sql

src/Common/ErrorCodes.cpp

@@ -595,6 +595,7 @@
     M(713, BROKEN_PROJECTION) \
     M(714, UNEXPECTED_CLUSTER) \
     M(715, CANNOT_DETECT_FORMAT) \
+    M(716, CANNOT_FORGET_PARTITION) \
     \
     M(999, KEEPER_EXCEPTION) \
     M(1000, POCO_EXCEPTION) \

src/Interpreters/InterpreterAlterQuery.cpp

@@ -420,6 +420,7 @@ AccessRightsElements InterpreterAlterQuery::getRequiredAccessForCommand(const AS
         case ASTAlterCommand::APPLY_DELETED_MASK:
         case ASTAlterCommand::DROP_PARTITION:
         case ASTAlterCommand::DROP_DETACHED_PARTITION:
+        case ASTAlterCommand::FORGET_PARTITION:
         {
             required_access.emplace_back(AccessType::ALTER_DELETE, database, table);
             break;

src/Parsers/ASTAlterQuery.cpp

@@ -288,6 +288,12 @@ void ASTAlterCommand::formatImpl(const FormatSettings & settings, FormatState &
                       << (settings.hilite ? hilite_none : "");
         partition->formatImpl(settings, state, frame);
     }
+    else if (type == ASTAlterCommand::FORGET_PARTITION)
+    {
+        settings.ostr << (settings.hilite ? hilite_keyword : "") << "FORGET PARTITION "
+                      << (settings.hilite ? hilite_none : "");
+        partition->formatImpl(settings, state, frame);
+    }
     else if (type == ASTAlterCommand::ATTACH_PARTITION)
     {
         settings.ostr << (settings.hilite ? hilite_keyword : "") << "ATTACH " << (part ? "PART " : "PARTITION ")

src/Parsers/ASTAlterQuery.h

@@ -63,6 +63,7 @@ public:
 
         DROP_PARTITION,
         DROP_DETACHED_PARTITION,
+        FORGET_PARTITION,
         ATTACH_PARTITION,
         MOVE_PARTITION,
         REPLACE_PARTITION,
@@ -140,7 +141,7 @@ public:
 
     IAST * statistic_decl = nullptr;
 
-    /** Used in DROP PARTITION, ATTACH PARTITION FROM, UPDATE, DELETE queries.
+    /** Used in DROP PARTITION, ATTACH PARTITION FROM, FORGET PARTITION, UPDATE, DELETE queries.
      *  The value or ID of the partition is stored here.
      */
     IAST * partition = nullptr;

src/Parsers/ParserAlterQuery.cpp

@@ -75,6 +75,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
     ParserKeyword s_detach_part("DETACH PART");
     ParserKeyword s_drop_partition("DROP PARTITION");
     ParserKeyword s_drop_part("DROP PART");
+    ParserKeyword s_forget_partition("FORGET PARTITION");
     ParserKeyword s_move_partition("MOVE PARTITION");
     ParserKeyword s_move_part("MOVE PART");
     ParserKeyword s_drop_detached_partition("DROP DETACHED PARTITION");
@@ -266,6 +267,13 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
                 command->type = ASTAlterCommand::DROP_PARTITION;
                 command->part = true;
             }
+            else if (s_forget_partition.ignore(pos, expected))
+            {
+                if (!parser_partition.parse(pos, command_partition, expected))
+                    return false;
+
+                command->type = ASTAlterCommand::FORGET_PARTITION;
+            }
             else if (s_drop_detached_partition.ignore(pos, expected))
             {
                 if (!parser_partition.parse(pos, command_partition, expected))

src/Storages/MergeTree/MergeTreeData.cpp

@@ -190,6 +190,7 @@ namespace ErrorCodes
     extern const int TOO_MANY_MUTATIONS;
     extern const int CANNOT_SCHEDULE_TASK;
     extern const int LIMIT_EXCEEDED;
+    extern const int CANNOT_FORGET_PARTITION;
 }
 
 static void checkSuspiciousIndices(const ASTFunction * index_function)
@@ -4888,7 +4889,16 @@ void MergeTreeData::checkAlterPartitionIsPossible(
                         throw DB::Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support DROP/DETACH PARTITION ALL currently");
                 }
                 else
-                    getPartitionIDFromQuery(command.partition, local_context);
+                {
+                    String partition_id = getPartitionIDFromQuery(command.partition, local_context);
+                    if (command.type == PartitionCommand::FORGET_PARTITION)
+                    {
+                        DataPartsLock lock = lockParts();
+                        auto parts_in_partition = getDataPartsPartitionRange(partition_id);
+                        if (!parts_in_partition.empty())
+                            throw Exception(ErrorCodes::CANNOT_FORGET_PARTITION, "Partition {} is not empty", partition_id);
+                    }
+                }
             }
         }
     }
@@ -5109,6 +5119,11 @@ void MergeTreeData::fetchPartition(
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "FETCH PARTITION is not supported by storage {}", getName());
 }
 
+void MergeTreeData::forgetPartition(const ASTPtr & /*partition*/, ContextPtr /*query_context*/)
+{
+    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "FORGET PARTITION is not supported by storage {}", getName());
+}
+
 Pipe MergeTreeData::alterPartition(
     const StorageMetadataPtr & metadata_snapshot,
     const PartitionCommands & commands,
@@ -5145,6 +5160,10 @@ Pipe MergeTreeData::alterPartition(
                 dropDetached(command.partition, command.part, query_context);
                 break;
 
+            case PartitionCommand::FORGET_PARTITION:
+                forgetPartition(command.partition, query_context);
+                break;
+
             case PartitionCommand::ATTACH_PARTITION:
                 current_command_results = attachPartition(command.partition, metadata_snapshot, command.part, query_context);
                 break;

src/Storages/MergeTree/MergeTreeData.h

@@ -831,7 +831,7 @@ public:
         return secondary_index_sizes;
     }
 
-    /// For ATTACH/DETACH/DROP PARTITION.
+    /// For ATTACH/DETACH/DROP/FORGET PARTITION.
     String getPartitionIDFromQuery(const ASTPtr & ast, ContextPtr context, DataPartsLock * acquired_lock = nullptr) const;
     std::unordered_set<String> getPartitionIDsFromQuery(const ASTs & asts, ContextPtr context) const;
     std::set<String> getPartitionIdsAffectedByCommands(const MutationCommands & commands, ContextPtr query_context) const;
@@ -1339,6 +1339,8 @@ protected:
         bool fetch_part,
         ContextPtr query_context);
 
+    virtual void forgetPartition(const ASTPtr & partition, ContextPtr context);
+
     virtual void movePartitionToShard(const ASTPtr & partition, bool move_part, const String & to, ContextPtr query_context);
 
     void writePartLog(

src/Storages/PartitionCommands.cpp

@@ -36,6 +36,13 @@ std::optional<PartitionCommand> PartitionCommand::parse(const ASTAlterCommand *
         res.part = command_ast->part;
         return res;
     }
+    else if (command_ast->type == ASTAlterCommand::FORGET_PARTITION)
+    {
+        PartitionCommand res;
+        res.type = FORGET_PARTITION;
+        res.partition = command_ast->partition->clone();
+        return res;
+    }
     else if (command_ast->type == ASTAlterCommand::ATTACH_PARTITION)
     {
         PartitionCommand res;
@@ -147,6 +154,8 @@ std::string PartitionCommand::typeToString() const
             return "DROP DETACHED PART";
         else
             return "DROP DETACHED PARTITION";
+    case PartitionCommand::Type::FORGET_PARTITION:
+        return "FORGET PARTITION";
     case PartitionCommand::Type::FETCH_PARTITION:
         if (part)
             return "FETCH PART";

src/Storages/PartitionCommands.h

@@ -26,6 +26,7 @@ struct PartitionCommand
         MOVE_PARTITION,
         DROP_PARTITION,
         DROP_DETACHED_PARTITION,
+        FORGET_PARTITION,
         FETCH_PARTITION,
         FREEZE_ALL_PARTITIONS,
         FREEZE_PARTITION,

src/Storages/StorageReplicatedMergeTree.cpp

@@ -188,6 +188,7 @@ namespace ErrorCodes
     extern const int CANNOT_BACKUP_TABLE;
     extern const int SUPPORT_IS_DISABLED;
     extern const int FAULT_INJECTED;
+    extern const int CANNOT_FORGET_PARTITION;
 }
 
 namespace ActionLocks
@@ -7263,6 +7264,24 @@ void StorageReplicatedMergeTree::fetchPartition(
 }
 
 
+void StorageReplicatedMergeTree::forgetPartition(const ASTPtr & partition, ContextPtr query_context)
+{
+    zkutil::ZooKeeperPtr zookeeper = getZooKeeperAndAssertNotReadonly();
+
+    String partition_id = getPartitionIDFromQuery(partition, query_context);
+    String block_numbers_path = fs::path(zookeeper_path) / "block_numbers";
+    String partition_path = fs::path(block_numbers_path) / partition_id;
+
+    auto error_code = zookeeper->tryRemove(partition_path);
+    if (error_code == Coordination::Error::ZOK)
+        LOG_INFO(log, "Forget partition {}", partition_id);
+    else if (error_code == Coordination::Error::ZNONODE)
+        throw Exception(ErrorCodes::CANNOT_FORGET_PARTITION, "Partition {} is unknown", partition_id);
+    else
+        throw zkutil::KeeperException::fromPath(error_code, partition_path);
+}
+
+
 void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, ContextPtr query_context)
 {
     /// Overview of the mutation algorithm.

src/Storages/StorageReplicatedMergeTree.h

@@ -909,6 +909,8 @@ private:
         const String & from,
         bool fetch_part,
         ContextPtr query_context) override;
+    void forgetPartition(const ASTPtr & partition, ContextPtr query_context) override;
+
 
     /// NOTE: there are no guarantees for concurrent merges. Dropping part can
     /// be concurrently merged into some covering part and dropPart will do

tests/queries/0_stateless/02995_forget_partition.reference

@@ -0,0 +1,21 @@
+---before---
+20240101
+20240102
+20240103
+20240104
+20240105
+20240106
+20240107
+20240108
+20240109
+20240110
+---after---
+20240102
+20240103
+20240104
+20240105
+20240106
+20240107
+20240108
+20240109
+20240110

tests/queries/0_stateless/02995_forget_partition.sql

@@ -0,0 +1,33 @@
+-- Tags: zookeeper, no-replicated-database
+
+drop table if exists forget_partition;
+
+create table forget_partition
+(
+    k UInt64,
+    d Date,
+    v String
+)
+engine = ReplicatedMergeTree('/test/02995/{database}/rmt', '1')
+order by (k, d)
+partition by toYYYYMMDD(d);
+
+insert into forget_partition select number, '2024-01-01' + interval number day, randomString(20) from system.numbers limit 10;
+
+alter table forget_partition drop partition '20240101';
+alter table forget_partition drop partition '20240102';
+
+set allow_unrestricted_reads_from_keeper=1;
+
+select '---before---';
+select name from system.zookeeper where path = '/test/02995/' || currentDatabase() || '/rmt/block_numbers' order by name;
+
+alter table forget_partition forget partition '20240103'; -- {serverError CANNOT_FORGET_PARTITION}
+alter table forget_partition forget partition '20240203'; -- {serverError CANNOT_FORGET_PARTITION}
+alter table forget_partition forget partition '20240101';
+
+
+select '---after---';
+select name from system.zookeeper where path = '/test/02995/' || currentDatabase() || '/rmt/block_numbers' order by name;
+
+drop table forget_partition;