2015-08-16 07:01:41 +00:00
|
|
|
#pragma once
|
|
|
|
|
|
|
|
#include <map>
|
2017-07-28 17:34:02 +00:00
|
|
|
#include <shared_mutex>
|
2016-08-26 21:25:05 +00:00
|
|
|
|
2019-12-12 08:57:25 +00:00
|
|
|
#include <Core/Defines.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Storages/IStorage.h>
|
2021-08-26 22:15:24 +00:00
|
|
|
#include <Formats/IndexForNativeFormat.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Common/FileChecker.h>
|
|
|
|
#include <Common/escapeForFileName.h>
|
2022-01-24 17:41:13 +00:00
|
|
|
#include <Disks/IDisk.h>
|
2015-08-16 07:01:41 +00:00
|
|
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
2021-08-26 22:15:24 +00:00
|
|
|
struct IndexForNativeFormat;
|
2022-05-31 09:33:23 +00:00
|
|
|
class IBackup;
|
|
|
|
using BackupPtr = std::shared_ptr<const IBackup>;
|
2021-08-26 22:15:24 +00:00
|
|
|
|
2017-06-25 00:17:08 +00:00
|
|
|
/** Implements a table engine that is suitable for small chunks of the log.
|
2017-04-16 15:00:33 +00:00
|
|
|
* In doing so, stores all the columns in a single Native file, with a nearby index.
|
2015-08-16 07:01:41 +00:00
|
|
|
*/
|
2022-07-13 20:35:24 +00:00
|
|
|
class StorageStripeLog final : public IStorage, public WithMutableContext
|
2015-08-16 07:01:41 +00:00
|
|
|
{
|
2022-04-19 20:47:29 +00:00
|
|
|
friend class StripeLogSource;
|
|
|
|
friend class StripeLogSink;
|
2015-08-16 07:01:41 +00:00
|
|
|
|
|
|
|
public:
|
2022-04-19 20:47:29 +00:00
|
|
|
StorageStripeLog(
|
|
|
|
DiskPtr disk_,
|
|
|
|
const String & relative_path_,
|
|
|
|
const StorageID & table_id_,
|
|
|
|
const ColumnsDescription & columns_,
|
|
|
|
const ConstraintsDescription & constraints_,
|
|
|
|
const String & comment,
|
|
|
|
bool attach,
|
2022-07-13 20:35:24 +00:00
|
|
|
ContextMutablePtr context_);
|
2022-04-19 20:47:29 +00:00
|
|
|
|
2021-08-26 22:15:24 +00:00
|
|
|
~StorageStripeLog() override;
|
|
|
|
|
2019-12-12 08:57:25 +00:00
|
|
|
String getName() const override { return "StripeLog"; }
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2020-08-03 13:54:14 +00:00
|
|
|
Pipe read(
|
2015-08-16 07:01:41 +00:00
|
|
|
const Names & column_names,
|
2021-07-09 03:15:41 +00:00
|
|
|
const StorageSnapshotPtr & storage_snapshot,
|
2020-09-20 17:52:17 +00:00
|
|
|
SelectQueryInfo & query_info,
|
2022-07-13 20:35:24 +00:00
|
|
|
ContextPtr local_context,
|
2018-04-19 14:47:09 +00:00
|
|
|
QueryProcessingStage::Enum processed_stage,
|
2019-02-18 23:38:44 +00:00
|
|
|
size_t max_block_size,
|
2022-10-07 10:46:45 +00:00
|
|
|
size_t num_streams) override;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-06-07 18:33:08 +00:00
|
|
|
SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr local_context, bool async_insert) override;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2020-04-07 14:05:51 +00:00
|
|
|
void rename(const String & new_path_to_table_data, const StorageID & new_table_id) override;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-10-24 12:50:24 +00:00
|
|
|
DataValidationTasksPtr getCheckTaskList(const CheckTaskFilter & check_task_filter, ContextPtr context) override;
|
2023-10-23 10:12:30 +00:00
|
|
|
std::optional<CheckResult> checkDataNext(DataValidationTasksPtr & check_task_list) override;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2020-11-01 17:38:43 +00:00
|
|
|
bool storesDataOnDisk() const override { return true; }
|
2019-12-12 08:57:25 +00:00
|
|
|
Strings getDataPaths() const override { return {DB::fullPath(disk, table_path)}; }
|
2015-08-16 07:01:41 +00:00
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
void truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder&) override;
|
2018-04-21 00:35:20 +00:00
|
|
|
|
2022-05-03 17:55:45 +00:00
|
|
|
std::optional<UInt64> totalRows(const Settings & settings) const override;
|
|
|
|
std::optional<UInt64> totalBytes(const Settings & settings) const override;
|
|
|
|
|
2022-05-31 09:33:23 +00:00
|
|
|
void backupData(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, const std::optional<ASTs> & partitions) override;
|
|
|
|
void restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional<ASTs> & partitions) override;
|
2021-10-26 09:48:31 +00:00
|
|
|
|
2015-08-16 07:01:41 +00:00
|
|
|
private:
|
2021-08-26 22:15:24 +00:00
|
|
|
using ReadLock = std::shared_lock<std::shared_timed_mutex>;
|
|
|
|
using WriteLock = std::unique_lock<std::shared_timed_mutex>;
|
2019-12-12 08:57:25 +00:00
|
|
|
|
2021-08-26 22:15:24 +00:00
|
|
|
/// Reads the index file if it hasn't read yet.
|
|
|
|
/// It is done lazily, so that with a large number of tables, the server starts quickly.
|
|
|
|
void loadIndices(std::chrono::seconds lock_timeout);
|
|
|
|
void loadIndices(const WriteLock &);
|
|
|
|
|
|
|
|
/// Saves the index file.
|
|
|
|
void saveIndices(const WriteLock &);
|
|
|
|
|
|
|
|
/// Removes all unsaved indices.
|
|
|
|
void removeUnsavedIndices(const WriteLock &);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2021-08-26 22:15:24 +00:00
|
|
|
/// Saves the sizes of the data and index files.
|
|
|
|
void saveFileSizes(const WriteLock &);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2022-05-03 17:55:45 +00:00
|
|
|
/// Recalculates the number of rows stored in this table.
|
|
|
|
void updateTotalRows(const WriteLock &);
|
|
|
|
|
2022-05-31 09:33:23 +00:00
|
|
|
/// Restores the data of this table from backup.
|
|
|
|
void restoreDataImpl(const BackupPtr & backup, const String & data_path_in_backup, std::chrono::seconds lock_timeout);
|
2022-05-29 19:53:56 +00:00
|
|
|
|
2021-08-26 22:15:24 +00:00
|
|
|
const DiskPtr disk;
|
|
|
|
String table_path;
|
|
|
|
|
2023-08-14 09:58:08 +00:00
|
|
|
struct DataValidationTasks : public IStorage::DataValidationTasksBase
|
|
|
|
{
|
|
|
|
DataValidationTasks(FileChecker::DataValidationTasksPtr file_checker_tasks_, ReadLock && lock_)
|
|
|
|
: file_checker_tasks(std::move(file_checker_tasks_)), lock(std::move(lock_))
|
|
|
|
{}
|
|
|
|
|
|
|
|
size_t size() const override { return file_checker_tasks->size(); }
|
|
|
|
|
|
|
|
FileChecker::DataValidationTasksPtr file_checker_tasks;
|
|
|
|
|
|
|
|
/// Lock to prevent table modification while checking
|
|
|
|
ReadLock lock;
|
|
|
|
};
|
|
|
|
|
2021-08-26 22:15:24 +00:00
|
|
|
String data_file_path;
|
|
|
|
String index_file_path;
|
2015-08-16 07:01:41 +00:00
|
|
|
FileChecker file_checker;
|
2021-08-26 22:15:24 +00:00
|
|
|
|
|
|
|
IndexForNativeFormat indices;
|
|
|
|
std::atomic<bool> indices_loaded = false;
|
|
|
|
size_t num_indices_saved = 0;
|
|
|
|
|
2022-05-03 17:55:45 +00:00
|
|
|
std::atomic<UInt64> total_rows = 0;
|
|
|
|
std::atomic<UInt64> total_bytes = 0;
|
2021-08-26 22:15:24 +00:00
|
|
|
|
|
|
|
const size_t max_compress_block_size;
|
|
|
|
|
2021-10-26 09:48:31 +00:00
|
|
|
mutable std::shared_timed_mutex rwlock;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2024-01-23 17:04:50 +00:00
|
|
|
LoggerPtr log;
|
2015-08-16 07:01:41 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
}
|