2012-07-16 20:25:19 +00:00
|
|
|
#pragma once
|
|
|
|
|
2017-06-06 17:18:32 +00:00
|
|
|
#include <ext/shared_ptr_helper.h>
|
2016-08-26 21:25:05 +00:00
|
|
|
|
2018-11-19 14:31:16 +00:00
|
|
|
#include <Core/Names.h>
|
2018-03-06 20:18:34 +00:00
|
|
|
#include <Storages/IStorage.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Storages/MergeTree/MergeTreeData.h>
|
|
|
|
#include <Storages/MergeTree/MergeTreeDataSelectExecutor.h>
|
|
|
|
#include <Storages/MergeTree/MergeTreeDataWriter.h>
|
2018-04-20 16:18:16 +00:00
|
|
|
#include <Storages/MergeTree/MergeTreeDataMergerMutator.h>
|
2018-03-30 19:25:37 +00:00
|
|
|
#include <Storages/MergeTree/MergeTreeMutationEntry.h>
|
2018-07-06 19:04:54 +00:00
|
|
|
#include <Storages/MergeTree/MergeTreeMutationStatus.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Storages/MergeTree/DiskSpaceMonitor.h>
|
|
|
|
#include <Storages/MergeTree/BackgroundProcessingPool.h>
|
2017-05-10 06:49:19 +00:00
|
|
|
#include <Common/SimpleIncrement.h>
|
2015-04-16 06:12:35 +00:00
|
|
|
|
2012-07-16 20:25:19 +00:00
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2017-04-16 15:00:33 +00:00
|
|
|
/** See the description of the data structure in MergeTreeData.
|
2012-07-16 20:25:19 +00:00
|
|
|
*/
|
2019-05-03 02:00:57 +00:00
|
|
|
class StorageMergeTree : public ext::shared_ptr_helper<StorageMergeTree>, public MergeTreeData
|
2012-07-16 20:25:19 +00:00
|
|
|
{
|
|
|
|
public:
|
2017-06-06 17:06:14 +00:00
|
|
|
void startup() override;
|
2017-04-01 07:20:54 +00:00
|
|
|
void shutdown() override;
|
|
|
|
~StorageMergeTree() override;
|
|
|
|
|
2019-05-03 02:00:57 +00:00
|
|
|
std::string getName() const override { return merging_params.getModeName() + "MergeTree"; }
|
2017-04-01 07:20:54 +00:00
|
|
|
std::string getTableName() const override { return table_name; }
|
2019-03-29 20:31:06 +00:00
|
|
|
std::string getDatabaseName() const override { return database_name; }
|
2018-03-19 14:29:40 +00:00
|
|
|
|
|
|
|
bool supportsIndexForIn() const override { return true; }
|
2019-05-17 04:08:03 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
BlockInputStreams read(
|
|
|
|
const Names & column_names,
|
2017-07-15 03:48:36 +00:00
|
|
|
const SelectQueryInfo & query_info,
|
2017-04-01 07:20:54 +00:00
|
|
|
const Context & context,
|
2018-04-19 14:47:09 +00:00
|
|
|
QueryProcessingStage::Enum processed_stage,
|
2019-02-18 23:38:44 +00:00
|
|
|
size_t max_block_size,
|
2017-06-02 15:54:39 +00:00
|
|
|
unsigned num_streams) override;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-02-27 18:26:24 +00:00
|
|
|
BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2017-04-16 15:00:33 +00:00
|
|
|
/** Perform the next step in combining the parts.
|
2017-04-01 07:20:54 +00:00
|
|
|
*/
|
2017-09-06 20:34:26 +00:00
|
|
|
bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-11-26 14:43:40 +00:00
|
|
|
void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) override;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-03-30 19:25:37 +00:00
|
|
|
void mutate(const MutationCommands & commands, const Context & context) override;
|
2019-05-03 02:00:57 +00:00
|
|
|
std::vector<MergeTreeMutationStatus> getMutationsStatus() const override;
|
2019-02-04 13:04:02 +00:00
|
|
|
CancellationCode killMutation(const String & mutation_id) override;
|
2018-07-06 19:04:54 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void drop() override;
|
2018-12-28 13:39:44 +00:00
|
|
|
void truncate(const ASTPtr &, const Context &) override;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
|
|
|
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override;
|
|
|
|
|
2019-03-05 10:12:20 +00:00
|
|
|
void alter(
|
|
|
|
const AlterCommands & params, const String & database_name, const String & table_name,
|
2019-03-07 20:52:25 +00:00
|
|
|
const Context & context, TableStructureWriteLockHolder & table_lock_holder) override;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-08-03 09:39:01 +00:00
|
|
|
void checkTableCanBeDropped() const override;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-08-03 09:39:01 +00:00
|
|
|
void checkPartitionCanBeDropped(const ASTPtr & partition) override;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-05-28 15:37:30 +00:00
|
|
|
ActionLock getActionLock(StorageActionBlockType action_type) override;
|
2018-05-21 13:49:54 +00:00
|
|
|
|
2018-02-21 19:26:59 +00:00
|
|
|
String getDataPath() const override { return full_path; }
|
|
|
|
|
2012-07-16 20:25:19 +00:00
|
|
|
private:
|
2017-04-01 07:20:54 +00:00
|
|
|
String path;
|
|
|
|
|
|
|
|
BackgroundProcessingPool & background_pool;
|
|
|
|
|
|
|
|
MergeTreeDataSelectExecutor reader;
|
|
|
|
MergeTreeDataWriter writer;
|
2018-03-30 19:25:37 +00:00
|
|
|
MergeTreeDataMergerMutator merger_mutator;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
|
|
|
/// For block numbers.
|
|
|
|
SimpleIncrement increment{0};
|
|
|
|
|
|
|
|
/// For clearOldParts, clearOldTemporaryDirectories.
|
2018-02-25 02:43:27 +00:00
|
|
|
AtomicStopwatch time_after_previous_cleanup;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-07-06 19:04:54 +00:00
|
|
|
mutable std::mutex currently_merging_mutex;
|
2019-05-03 02:00:57 +00:00
|
|
|
DataParts currently_merging;
|
2019-01-14 12:25:25 +00:00
|
|
|
std::map<String, MergeTreeMutationEntry> current_mutations_by_id;
|
|
|
|
std::multimap<Int64, MergeTreeMutationEntry &> current_mutations_by_version;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
|
|
|
std::atomic<bool> shutdown_called {false};
|
|
|
|
|
2018-07-09 15:34:11 +00:00
|
|
|
BackgroundProcessingPool::TaskHandle background_task_handle;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-03-12 13:26:48 +00:00
|
|
|
std::vector<MergeTreeData::AlterDataPartTransactionPtr> prepareAlterTransactions(
|
2019-04-25 21:37:57 +00:00
|
|
|
const ColumnsDescription & new_columns, const IndicesDescription & new_indices, const Context & context);
|
2019-03-10 19:47:26 +00:00
|
|
|
|
2018-07-11 12:43:55 +00:00
|
|
|
void loadMutations();
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2017-11-04 03:20:18 +00:00
|
|
|
/** Determines what parts should be merged and merges it.
|
|
|
|
* If aggressive - when selects parts don't takes into account their ratio size and novelty (used for OPTIMIZE query).
|
|
|
|
* Returns true if merge is finished successfully.
|
|
|
|
*/
|
2019-05-03 02:00:57 +00:00
|
|
|
bool merge(bool aggressive, const String & partition_id, bool final, bool deduplicate, String * out_disable_reason = nullptr);
|
2017-11-04 03:20:18 +00:00
|
|
|
|
2018-07-09 15:34:11 +00:00
|
|
|
/// Try and find a single part to mutate and mutate it. If some part was successfully mutated, return true.
|
|
|
|
bool tryMutatePart();
|
|
|
|
|
2018-12-26 17:03:29 +00:00
|
|
|
BackgroundProcessingPoolTaskResult backgroundTask();
|
2017-11-04 03:20:18 +00:00
|
|
|
|
2018-03-30 19:25:37 +00:00
|
|
|
Int64 getCurrentMutationVersion(
|
2019-05-03 02:00:57 +00:00
|
|
|
const DataPartPtr & part,
|
2018-03-30 19:25:37 +00:00
|
|
|
std::lock_guard<std::mutex> & /* currently_merging_mutex_lock */) const;
|
|
|
|
|
2018-07-31 12:34:34 +00:00
|
|
|
void clearOldMutations();
|
2017-11-04 03:20:18 +00:00
|
|
|
|
2018-11-13 13:48:53 +00:00
|
|
|
// Partition helpers
|
|
|
|
void dropPartition(const ASTPtr & partition, bool detach, const Context & context);
|
|
|
|
void clearColumnInPartition(const ASTPtr & partition, const Field & column_name, const Context & context);
|
|
|
|
void attachPartition(const ASTPtr & partition, bool part, const Context & context);
|
|
|
|
void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, const Context & context);
|
|
|
|
|
2018-05-21 13:49:54 +00:00
|
|
|
friend class MergeTreeBlockOutputStream;
|
|
|
|
friend class MergeTreeData;
|
|
|
|
friend struct CurrentlyMergingPartsTagger;
|
|
|
|
|
2017-11-04 03:20:18 +00:00
|
|
|
protected:
|
2018-11-26 00:56:50 +00:00
|
|
|
/** Attach the table with the appropriate name, along the appropriate path (with / at the end),
|
2017-06-06 18:36:13 +00:00
|
|
|
* (correctness of names and paths are not checked)
|
|
|
|
* consisting of the specified columns.
|
|
|
|
*
|
2018-11-06 18:25:36 +00:00
|
|
|
* See MergeTreeData constructor for comments on parameters.
|
2017-06-06 18:36:13 +00:00
|
|
|
*/
|
2017-04-01 07:20:54 +00:00
|
|
|
StorageMergeTree(
|
|
|
|
const String & path_,
|
|
|
|
const String & database_name_,
|
|
|
|
const String & table_name_,
|
2018-03-06 20:18:34 +00:00
|
|
|
const ColumnsDescription & columns_,
|
2019-02-05 14:50:25 +00:00
|
|
|
const IndicesDescription & indices_,
|
2019-05-18 08:05:52 +00:00
|
|
|
const ConstraintsDescription & constraints_,
|
2017-04-01 07:20:54 +00:00
|
|
|
bool attach,
|
|
|
|
Context & context_,
|
2017-09-08 18:11:09 +00:00
|
|
|
const String & date_column_name,
|
2018-11-06 18:25:36 +00:00
|
|
|
const ASTPtr & partition_by_ast_,
|
|
|
|
const ASTPtr & order_by_ast_,
|
|
|
|
const ASTPtr & primary_key_ast_,
|
2018-11-09 19:01:39 +00:00
|
|
|
const ASTPtr & sample_by_ast_, /// nullptr, if sampling is not supported.
|
2019-04-15 09:30:45 +00:00
|
|
|
const ASTPtr & ttl_table_ast_,
|
2019-05-03 02:00:57 +00:00
|
|
|
const MergingParams & merging_params_,
|
2017-09-19 20:42:42 +00:00
|
|
|
const MergeTreeSettings & settings_,
|
|
|
|
bool has_force_restore_data_flag);
|
2012-07-16 20:25:19 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
}
|