ClickHouse/src/Storages/StorageMergeTree.h

201 lines
7.9 KiB
C++
Raw Normal View History

2012-07-16 20:25:19 +00:00
#pragma once
2017-06-06 17:18:32 +00:00
#include <ext/shared_ptr_helper.h>
#include <Core/Names.h>
2019-05-09 14:25:18 +00:00
#include <Storages/AlterCommands.h>
#include <Storages/IStorage.h>
#include <Storages/MergeTree/MergeTreeData.h>
#include <Storages/MergeTree/MergeTreeDataSelectExecutor.h>
#include <Storages/MergeTree/MergeTreeDataWriter.h>
#include <Storages/MergeTree/MergeTreeDataMergerMutator.h>
2019-08-19 14:40:12 +00:00
#include <Storages/MergeTree/MergeTreePartsMover.h>
#include <Storages/MergeTree/MergeTreeMutationEntry.h>
#include <Storages/MergeTree/MergeTreeMutationStatus.h>
#include <Disks/StoragePolicy.h>
#include <Storages/MergeTree/BackgroundProcessingPool.h>
#include <Common/SimpleIncrement.h>
2019-08-21 16:02:13 +00:00
#include <Core/BackgroundSchedulePool.h>
2015-04-16 06:12:35 +00:00
2012-07-16 20:25:19 +00:00
namespace DB
{
2017-04-16 15:00:33 +00:00
/** See the description of the data structure in MergeTreeData.
2012-07-16 20:25:19 +00:00
*/
class StorageMergeTree final : public ext::shared_ptr_helper<StorageMergeTree>, public MergeTreeData
2012-07-16 20:25:19 +00:00
{
2019-08-26 19:07:29 +00:00
friend struct ext::shared_ptr_helper<StorageMergeTree>;
2012-07-16 20:25:19 +00:00
public:
void startup() override;
void shutdown() override;
~StorageMergeTree() override;
2019-05-03 02:00:57 +00:00
std::string getName() const override { return merging_params.getModeName() + "MergeTree"; }
2019-12-12 10:49:15 +00:00
bool supportsParallelInsert() const override { return true; }
bool supportsIndexForIn() const override { return true; }
2020-08-03 13:54:14 +00:00
Pipe read(
const Names & column_names,
const StorageMetadataPtr & /*metadata_snapshot*/,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum processed_stage,
size_t max_block_size,
2017-06-02 15:54:39 +00:00
unsigned num_streams) override;
std::optional<UInt64> totalRows() const override;
std::optional<UInt64> totalBytes() const override;
BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) override;
2017-04-16 15:00:33 +00:00
/** Perform the next step in combining the parts.
*/
2020-06-17 13:39:26 +00:00
bool optimize(
const ASTPtr & query,
const StorageMetadataPtr & /*metadata_snapshot*/,
const ASTPtr & partition,
bool final,
bool deduplicate,
const Context & context) override;
2020-08-03 13:54:14 +00:00
Pipe alterPartition(
const ASTPtr & query,
const StorageMetadataPtr & /* metadata_snapshot */,
const PartitionCommands & commands,
const Context & context) override;
void mutate(const MutationCommands & commands, const Context & context) override;
/// Return introspection information about currently processing or recently processed mutations.
2019-05-03 02:00:57 +00:00
std::vector<MergeTreeMutationStatus> getMutationsStatus() const override;
CancellationCode killMutation(const String & mutation_id) override;
2020-01-22 11:30:11 +00:00
void drop() override;
2020-06-18 16:10:47 +00:00
void truncate(const ASTPtr &, const StorageMetadataPtr &, const Context &, TableExclusiveLockHolder &) override;
2020-06-18 16:10:47 +00:00
void alter(const AlterCommands & commands, const Context & context, TableLockHolder & table_lock_holder) override;
void checkTableCanBeDropped() const override;
ActionLock getActionLock(StorageActionBlockType action_type) override;
2019-07-03 13:17:19 +00:00
CheckResults checkData(const ASTPtr & query, const Context & context) override;
2019-07-03 08:49:52 +00:00
2012-07-16 20:25:19 +00:00
private:
/// Mutex and condvar for synchronous mutations wait
std::mutex mutation_wait_mutex;
std::condition_variable mutation_wait_event;
MergeTreeDataSelectExecutor reader;
MergeTreeDataWriter writer;
MergeTreeDataMergerMutator merger_mutator;
/// For block numbers.
SimpleIncrement increment{0};
/// For clearOldParts, clearOldTemporaryDirectories.
2018-02-25 02:43:27 +00:00
AtomicStopwatch time_after_previous_cleanup;
2019-09-02 11:35:53 +00:00
/// Mutex for parts currently processing in background
/// merging (also with TTL), mutating or moving.
2019-08-15 09:43:31 +00:00
mutable std::mutex currently_processing_in_background_mutex;
2020-06-02 00:41:52 +00:00
mutable std::condition_variable currently_processing_in_background_condition;
2019-09-02 11:35:53 +00:00
/// Parts that currently participate in merge or mutation.
/// This set have to be used with `currently_processing_in_background_mutex`.
DataParts currently_merging_mutating_parts;
std::map<String, MergeTreeMutationEntry> current_mutations_by_id;
std::multimap<Int64, MergeTreeMutationEntry &> current_mutations_by_version;
std::atomic<bool> shutdown_called {false};
2019-09-02 11:35:53 +00:00
/// Task handler for merges, mutations and moves.
2019-09-05 13:12:29 +00:00
BackgroundProcessingPool::TaskHandle merging_mutating_task_handle;
BackgroundProcessingPool::TaskHandle moving_task_handle;
void loadMutations();
/** Determines what parts should be merged and merges it.
* If aggressive - when selects parts don't takes into account their ratio size and novelty (used for OPTIMIZE query).
* Returns true if merge is finished successfully.
*/
2019-05-03 02:00:57 +00:00
bool merge(bool aggressive, const String & partition_id, bool final, bool deduplicate, String * out_disable_reason = nullptr);
ActionLock stopMergesAndWait();
2019-09-05 13:12:29 +00:00
BackgroundProcessingPoolTaskResult movePartsTask();
2019-06-19 17:56:41 +00:00
2020-04-03 11:09:27 +00:00
/// Allocate block number for new mutation, write mutation to disk
/// and into in-memory structures. Wake up merge-mutation task.
Int64 startMutation(const MutationCommands & commands, String & mutation_file_name);
/// Wait until mutation with version will finish mutation for all parts
2020-04-03 16:33:25 +00:00
void waitForMutation(Int64 version, const String & file_name);
2020-03-18 10:02:57 +00:00
/// Try and find a single part to mutate and mutate it. If some part was successfully mutated, return true.
bool tryMutatePart();
2019-09-05 13:12:29 +00:00
BackgroundProcessingPoolTaskResult mergeMutateTask();
Int64 getCurrentMutationVersion(
2019-05-03 02:00:57 +00:00
const DataPartPtr & part,
2020-06-02 00:41:52 +00:00
std::unique_lock<std::mutex> & /* currently_processing_in_background_mutex_lock */) const;
2019-07-29 09:15:46 +00:00
void clearOldMutations(bool truncate = false);
// Partition helpers
void dropPartition(const ASTPtr & partition, bool detach, const Context & context);
2020-07-28 15:10:36 +00:00
PartitionCommandsResultInfo attachPartition(const ASTPtr & partition, bool part, const Context & context);
void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context);
void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, const Context & context);
2019-09-05 13:12:29 +00:00
bool partIsAssignedToBackgroundOperation(const DataPartPtr & part) const override;
2020-07-22 19:29:54 +00:00
/// Update mutation entries after part mutation execution. May reset old
/// errors if mutation was successful. Otherwise update last_failed* fields
/// in mutation entries.
void updateMutationEntriesErrors(FutureMergedMutatedPart result_part, bool is_successful, const String & exception_message);
2019-09-05 13:12:29 +00:00
/// Return empty optional if mutation was killed. Otherwise return partially
/// filled mutation status with information about error (latest_fail*) and
2020-07-31 12:22:32 +00:00
/// is_done. mutation_ids filled with mutations with the same errors,
/// because we can execute several mutations at once. Order is important for
/// better readability of exception message. If mutation was killed doesn't
/// return any ids.
2020-07-31 11:37:16 +00:00
std::optional<MergeTreeMutationStatus> getIncompleteMutationsStatus(Int64 mutation_version, std::set<String> * mutation_ids = nullptr) const;
void startBackgroundMovesIfNeeded() override;
friend class MergeTreeBlockOutputStream;
friend class MergeTreeData;
friend struct CurrentlyMergingPartsTagger;
protected:
2019-07-24 12:56:39 +00:00
2018-11-26 00:56:50 +00:00
/** Attach the table with the appropriate name, along the appropriate path (with / at the end),
* (correctness of names and paths are not checked)
* consisting of the specified columns.
*
* See MergeTreeData constructor for comments on parameters.
*/
StorageMergeTree(
2019-12-04 16:06:55 +00:00
const StorageID & table_id_,
const String & relative_data_path_,
2019-12-27 16:34:50 +00:00
const StorageInMemoryMetadata & metadata,
bool attach,
Context & context_,
const String & date_column_name,
2019-05-03 02:00:57 +00:00
const MergingParams & merging_params_,
2019-08-26 14:24:29 +00:00
std::unique_ptr<MergeTreeSettings> settings_,
bool has_force_restore_data_flag);
2020-03-24 17:05:38 +00:00
2020-04-02 18:24:11 +00:00
MutationCommands getFirtsAlterMutationCommandsForPart(const DataPartPtr & part) const override;
2012-07-16 20:25:19 +00:00
};
}