mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-30 05:30:51 +00:00
Merge pull request #2 from ObjatieGroba/questions
Add space reservation for each new MargeTreeDataPart.
This commit is contained in:
commit
fab2c64110
@ -16,6 +16,7 @@ class ActiveDataPartSet
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
struct PartPathName {
|
struct PartPathName {
|
||||||
|
/// path + name is absolute path to DataPart
|
||||||
String path;
|
String path;
|
||||||
String name;
|
String name;
|
||||||
};
|
};
|
||||||
|
@ -30,6 +30,8 @@ namespace ErrorCodes
|
|||||||
* Could "reserve" space, for different operations to plan disk space usage.
|
* Could "reserve" space, for different operations to plan disk space usage.
|
||||||
* Reservations are not separated for different filesystems,
|
* Reservations are not separated for different filesystems,
|
||||||
* instead it is assumed, that all reservations are done within same filesystem.
|
* instead it is assumed, that all reservations are done within same filesystem.
|
||||||
|
*
|
||||||
|
* It is necessary to set all paths in map before MergeTreeData starts
|
||||||
*/
|
*/
|
||||||
class DiskSpaceMonitor
|
class DiskSpaceMonitor
|
||||||
{
|
{
|
||||||
@ -86,8 +88,17 @@ public:
|
|||||||
return size;
|
return size;
|
||||||
}
|
}
|
||||||
|
|
||||||
Reservation(UInt64 size_, DiskReserve * reserves_)
|
const String & getPath() const {
|
||||||
: size(size_), metric_increment(CurrentMetrics::DiskSpaceReservedForMerge, size), reserves(reserves_)
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
void addEnclosedDirToPath(const String & dir) {
|
||||||
|
path += dir + '/';
|
||||||
|
}
|
||||||
|
|
||||||
|
Reservation(UInt64 size_, DiskReserve * reserves_, const String & path_)
|
||||||
|
: size(size_), metric_increment(CurrentMetrics::DiskSpaceReservedForMerge, size), reserves(reserves_),
|
||||||
|
path(path_)
|
||||||
{
|
{
|
||||||
std::lock_guard lock(DiskSpaceMonitor::mutex);
|
std::lock_guard lock(DiskSpaceMonitor::mutex);
|
||||||
reserves->reserved_bytes += size;
|
reserves->reserved_bytes += size;
|
||||||
@ -98,15 +109,16 @@ public:
|
|||||||
UInt64 size;
|
UInt64 size;
|
||||||
CurrentMetrics::Increment metric_increment;
|
CurrentMetrics::Increment metric_increment;
|
||||||
DiskReserve * reserves;
|
DiskReserve * reserves;
|
||||||
|
String path;
|
||||||
};
|
};
|
||||||
|
|
||||||
using ReservationPtr = std::unique_ptr<Reservation>;
|
using ReservationPtr = std::unique_ptr<Reservation>;
|
||||||
|
|
||||||
static UInt64 getUnreservedFreeSpace(const std::string & path)
|
static UInt64 getUnreservedFreeSpace(const String & disk_path)
|
||||||
{
|
{
|
||||||
struct statvfs fs;
|
struct statvfs fs;
|
||||||
|
|
||||||
if (statvfs(path.c_str(), &fs) != 0)
|
if (statvfs(disk_path.c_str(), &fs) != 0)
|
||||||
throwFromErrno("Could not calculate available disk space (statvfs)", ErrorCodes::CANNOT_STATVFS);
|
throwFromErrno("Could not calculate available disk space (statvfs)", ErrorCodes::CANNOT_STATVFS);
|
||||||
|
|
||||||
UInt64 res = fs.f_bfree * fs.f_bsize;
|
UInt64 res = fs.f_bfree * fs.f_bsize;
|
||||||
@ -116,7 +128,7 @@ public:
|
|||||||
|
|
||||||
std::lock_guard lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
|
|
||||||
auto & reserved_bytes = reserved[path].reserved_bytes;
|
auto & reserved_bytes = reserved[disk_path].reserved_bytes;
|
||||||
|
|
||||||
if (reserved_bytes > res)
|
if (reserved_bytes > res)
|
||||||
res = 0;
|
res = 0;
|
||||||
@ -126,26 +138,60 @@ public:
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
static UInt64 getReservedSpace(const std::string & path)
|
/** Returns max of unreserved free space on all disks
|
||||||
|
* It is necessary to have guarantee that all paths are set
|
||||||
|
*/
|
||||||
|
static UInt64 getMaxUnreservedFreeSpace()
|
||||||
{
|
{
|
||||||
std::lock_guard lock(mutex);
|
UInt64 max_unreserved = 0;
|
||||||
return reserved[path].reserved_bytes;
|
for (auto& [disk_path, reserve] : reserved) {
|
||||||
|
struct statvfs fs;
|
||||||
|
|
||||||
|
if (statvfs(disk_path.c_str(), &fs) != 0)
|
||||||
|
throwFromErrno("Could not calculate available disk space (statvfs)", ErrorCodes::CANNOT_STATVFS);
|
||||||
|
|
||||||
|
UInt64 res = fs.f_bfree * fs.f_bsize;
|
||||||
|
|
||||||
|
/// Heuristic by Michael Kolupaev: reserve 30 MB more, because statvfs shows few megabytes more space than df.
|
||||||
|
res -= std::min(res, static_cast<UInt64>(30 * (1ul << 20)));
|
||||||
|
|
||||||
|
///@TODO_IGR ASK Maybe mutex out of for
|
||||||
|
std::lock_guard lock(mutex);
|
||||||
|
|
||||||
|
auto &reserved_bytes = reserved[disk_path].reserved_bytes;
|
||||||
|
|
||||||
|
if (reserved_bytes > res)
|
||||||
|
res = 0;
|
||||||
|
else
|
||||||
|
res -= reserved_bytes;
|
||||||
|
|
||||||
|
max_unreserved = std::max(max_unreserved, res);
|
||||||
|
}
|
||||||
|
return max_unreserved;
|
||||||
}
|
}
|
||||||
|
|
||||||
static UInt64 getReservationCount(const std::string & path)
|
static UInt64 getReservedSpace(const String & disk_path)
|
||||||
{
|
{
|
||||||
std::lock_guard lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
return reserved[path].reservation_count;
|
return reserved[disk_path].reserved_bytes;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// If not enough (approximately) space, throw an exception.
|
static UInt64 getReservationCount(const String & disk_path)
|
||||||
static ReservationPtr reserve(const std::string & path, UInt64 size)
|
|
||||||
{
|
{
|
||||||
UInt64 free_bytes = getUnreservedFreeSpace(path);
|
std::lock_guard lock(mutex);
|
||||||
|
return reserved[disk_path].reservation_count;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// If not enough (approximately) space, do not reserve.
|
||||||
|
static ReservationPtr tryToReserve(const String & disk_path, UInt64 size)
|
||||||
|
{
|
||||||
|
UInt64 free_bytes = getUnreservedFreeSpace(disk_path);
|
||||||
|
///@TODO_IGR ASK twice reservation?
|
||||||
if (free_bytes < size)
|
if (free_bytes < size)
|
||||||
throw Exception("Not enough free disk space to reserve: " + formatReadableSizeWithBinarySuffix(free_bytes) + " available, "
|
{
|
||||||
+ formatReadableSizeWithBinarySuffix(size) + " requested", ErrorCodes::NOT_ENOUGH_SPACE);
|
return {};
|
||||||
return std::make_unique<Reservation>(size, &reserved[path]);
|
}
|
||||||
|
return std::make_unique<Reservation>(size, &reserved[disk_path], disk_path);
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
@ -153,4 +199,61 @@ private:
|
|||||||
static std::mutex mutex;
|
static std::mutex mutex;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
class Schema
|
||||||
|
{
|
||||||
|
class Volume {
|
||||||
|
friend class Schema;
|
||||||
|
|
||||||
|
public:
|
||||||
|
Volume(std::vector<String> paths_) : paths(std::move(paths_))
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
DiskSpaceMonitor::ReservationPtr reserve(UInt64 expected_size) const {
|
||||||
|
for (size_t i = 0; i != paths.size(); ++i) {
|
||||||
|
last_used = (last_used + 1) % paths.size();
|
||||||
|
auto reservation = DiskSpaceMonitor::tryToReserve(paths[last_used], expected_size);
|
||||||
|
if (reservation) {
|
||||||
|
return reservation;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
const Strings paths;
|
||||||
|
mutable size_t last_used = 0; ///@TODO_IGR ASK It is thread safe, but it is not consistent. :(
|
||||||
|
/// P.S. I do not want to use mutex here
|
||||||
|
};
|
||||||
|
|
||||||
|
public:
|
||||||
|
Schema(const std::vector<Strings> & disks) {
|
||||||
|
for (const Strings & volume : disks) {
|
||||||
|
volumes.emplace_back(volume);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
///@TODO_IGR ASK maybe iterator without copy?
|
||||||
|
Strings getFullPaths() const {
|
||||||
|
Strings res;
|
||||||
|
for (const auto & volume : volumes) {
|
||||||
|
std::copy(volume.paths.begin(), volume.paths.end(), std::back_inserter(res));
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
DiskSpaceMonitor::ReservationPtr reserve(UInt64 expected_size) const {
|
||||||
|
for (auto & volume : volumes) {
|
||||||
|
auto reservation = volume.reserve(expected_size);
|
||||||
|
if (reservation) {
|
||||||
|
return reservation;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
std::vector<Volume> volumes;
|
||||||
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -89,7 +89,7 @@ namespace ErrorCodes
|
|||||||
|
|
||||||
MergeTreeData::MergeTreeData(
|
MergeTreeData::MergeTreeData(
|
||||||
const String & database_, const String & table_,
|
const String & database_, const String & table_,
|
||||||
const Strings & full_paths_, const ColumnsDescription & columns_,
|
const Schema & schema_, const ColumnsDescription & columns_,
|
||||||
const IndicesDescription & indices_,
|
const IndicesDescription & indices_,
|
||||||
Context & context_,
|
Context & context_,
|
||||||
const String & date_column_name,
|
const String & date_column_name,
|
||||||
@ -110,7 +110,7 @@ MergeTreeData::MergeTreeData(
|
|||||||
sample_by_ast(sample_by_ast_),
|
sample_by_ast(sample_by_ast_),
|
||||||
require_part_metadata(require_part_metadata_),
|
require_part_metadata(require_part_metadata_),
|
||||||
database_name(database_), table_name(table_),
|
database_name(database_), table_name(table_),
|
||||||
full_paths(full_paths_),
|
schema(schema_),
|
||||||
broken_part_callback(broken_part_callback_),
|
broken_part_callback(broken_part_callback_),
|
||||||
log_name(database_name + "." + table_name), log(&Logger::get(log_name + " (Data)")),
|
log_name(database_name + "." + table_name), log(&Logger::get(log_name + " (Data)")),
|
||||||
data_parts_by_info(data_parts_indexes.get<TagByInfo>()),
|
data_parts_by_info(data_parts_indexes.get<TagByInfo>()),
|
||||||
@ -159,7 +159,10 @@ MergeTreeData::MergeTreeData(
|
|||||||
min_format_version = MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING;
|
min_format_version = MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
auto full_paths = getFullPaths();
|
||||||
|
|
||||||
auto format_path = full_paths[0]; ///@TODO_IGR ASK What path should we use for format file?
|
auto format_path = full_paths[0]; ///@TODO_IGR ASK What path should we use for format file?
|
||||||
|
/// Use first disk. If format file not there move it.
|
||||||
auto path_exists = Poco::File(format_path).exists();
|
auto path_exists = Poco::File(format_path).exists();
|
||||||
|
|
||||||
for (const String & path : full_paths) {
|
for (const String & path : full_paths) {
|
||||||
@ -629,6 +632,8 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
|
|||||||
{
|
{
|
||||||
LOG_DEBUG(log, "Loading data parts");
|
LOG_DEBUG(log, "Loading data parts");
|
||||||
|
|
||||||
|
const auto full_paths = getFullPaths();
|
||||||
|
|
||||||
std::vector<std::pair<String, size_t>> part_file_names;
|
std::vector<std::pair<String, size_t>> part_file_names;
|
||||||
Poco::DirectoryIterator end;
|
Poco::DirectoryIterator end;
|
||||||
for (size_t i = 0; i != full_paths.size(); ++i)
|
for (size_t i = 0; i != full_paths.size(); ++i)
|
||||||
@ -840,6 +845,8 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life
|
|||||||
? current_time - custom_directories_lifetime_seconds
|
? current_time - custom_directories_lifetime_seconds
|
||||||
: current_time - settings.temporary_directories_lifetime.totalSeconds();
|
: current_time - settings.temporary_directories_lifetime.totalSeconds();
|
||||||
|
|
||||||
|
const auto full_paths = getFullPaths();
|
||||||
|
|
||||||
/// Delete temporary directories older than a day.
|
/// Delete temporary directories older than a day.
|
||||||
Poco::DirectoryIterator end;
|
Poco::DirectoryIterator end;
|
||||||
for (auto && full_path : full_paths)
|
for (auto && full_path : full_paths)
|
||||||
@ -1006,7 +1013,7 @@ void MergeTreeData::dropAllData()
|
|||||||
|
|
||||||
LOG_TRACE(log, "dropAllData: removing data from filesystem.");
|
LOG_TRACE(log, "dropAllData: removing data from filesystem.");
|
||||||
|
|
||||||
for (auto && full_path : full_paths) {
|
for (auto && full_path : getFullPaths()) {
|
||||||
Poco::File(full_path).remove(true);
|
Poco::File(full_path).remove(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2273,6 +2280,8 @@ size_t MergeTreeData::getPartitionSize(const std::string & partition_id) const
|
|||||||
|
|
||||||
Poco::DirectoryIterator end;
|
Poco::DirectoryIterator end;
|
||||||
|
|
||||||
|
const auto full_paths = getFullPaths();
|
||||||
|
|
||||||
for (const String & full_path : full_paths)
|
for (const String & full_path : full_paths)
|
||||||
{
|
{
|
||||||
for (Poco::DirectoryIterator it(full_path); it != end; ++it)
|
for (Poco::DirectoryIterator it(full_path); it != end; ++it)
|
||||||
@ -2413,7 +2422,7 @@ MergeTreeData::DataPartsVector MergeTreeData::getAllDataPartsVector(MergeTreeDat
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
String MergeTreeData::getFullPathForPart(UInt64 expected_size) const
|
DiskSpaceMonitor::ReservationPtr MergeTreeData::reserveSpaceForPart(UInt64 expected_size) const
|
||||||
{
|
{
|
||||||
std::cerr << "Exp size " << expected_size << std::endl;
|
std::cerr << "Exp size " << expected_size << std::endl;
|
||||||
constexpr UInt64 SIZE_100MB = 100ull << 20;
|
constexpr UInt64 SIZE_100MB = 100ull << 20;
|
||||||
@ -2422,16 +2431,14 @@ String MergeTreeData::getFullPathForPart(UInt64 expected_size) const
|
|||||||
if (expected_size < SIZE_100MB) {
|
if (expected_size < SIZE_100MB) {
|
||||||
expected_size = SIZE_100MB;
|
expected_size = SIZE_100MB;
|
||||||
}
|
}
|
||||||
for (const String & path : full_paths) {
|
auto reservation = reserveSpaceAtDisk(expected_size * MAGIC_CONST);
|
||||||
UInt64 free_space = DiskSpaceMonitor::getUnreservedFreeSpace(path); ///@TODO_IGR ASK reserve? YES, we are
|
if (reservation) {
|
||||||
|
return reservation;
|
||||||
if (free_space > expected_size * MAGIC_CONST) {
|
|
||||||
std::cerr << "Choosed " << free_space << " " << path << std::endl;
|
|
||||||
return path;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
std::cerr << "Choosed last " << full_paths[full_paths.size() - 1] << std::endl;
|
|
||||||
return full_paths[full_paths.size() - 1];
|
throw Exception("Not enough free disk space to reserve: " + formatReadableSizeWithBinarySuffix(expected_size) + " requested, "
|
||||||
|
+ formatReadableSizeWithBinarySuffix(DiskSpaceMonitor::getMaxUnreservedFreeSpace()) + " available",
|
||||||
|
ErrorCodes::NOT_ENOUGH_SPACE);
|
||||||
}
|
}
|
||||||
|
|
||||||
MergeTreeData::DataParts MergeTreeData::getDataParts(const DataPartStates & affordable_states) const
|
MergeTreeData::DataParts MergeTreeData::getDataParts(const DataPartStates & affordable_states) const
|
||||||
@ -2619,8 +2626,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPart(const Merg
|
|||||||
String dst_part_name = src_part->getNewName(dst_part_info);
|
String dst_part_name = src_part->getNewName(dst_part_info);
|
||||||
String tmp_dst_part_name = tmp_part_prefix + dst_part_name;
|
String tmp_dst_part_name = tmp_part_prefix + dst_part_name;
|
||||||
|
|
||||||
///@TODO_IGR ASK Maybe flag that it is not recent part? Or choose same dir if it is possible
|
auto reservation = reserveSpaceForPart(src_part->bytes_on_disk);
|
||||||
Poco::Path dst_part_absolute_path = Poco::Path(getFullPathForPart(src_part->bytes_on_disk) + tmp_dst_part_name).absolute();
|
String dst_part_path = reservation->getPath();
|
||||||
|
Poco::Path dst_part_absolute_path = Poco::Path(dst_part_path + tmp_dst_part_name).absolute();
|
||||||
Poco::Path src_part_absolute_path = Poco::Path(src_part->getFullPath()).absolute();
|
Poco::Path src_part_absolute_path = Poco::Path(src_part->getFullPath()).absolute();
|
||||||
|
|
||||||
if (Poco::File(dst_part_absolute_path).exists())
|
if (Poco::File(dst_part_absolute_path).exists())
|
||||||
@ -2629,7 +2637,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPart(const Merg
|
|||||||
LOG_DEBUG(log, "Cloning part " << src_part_absolute_path.toString() << " to " << dst_part_absolute_path.toString());
|
LOG_DEBUG(log, "Cloning part " << src_part_absolute_path.toString() << " to " << dst_part_absolute_path.toString());
|
||||||
localBackup(src_part_absolute_path, dst_part_absolute_path);
|
localBackup(src_part_absolute_path, dst_part_absolute_path);
|
||||||
|
|
||||||
MergeTreeData::MutableDataPartPtr dst_data_part = std::make_shared<MergeTreeData::DataPart>(*this, dst_part_storage_path, dst_part_name, dst_part_info);
|
MergeTreeData::MutableDataPartPtr dst_data_part = std::make_shared<MergeTreeData::DataPart>(*this, dst_part_path, dst_part_name, dst_part_info);
|
||||||
dst_data_part->relative_path = tmp_dst_part_name;
|
dst_data_part->relative_path = tmp_dst_part_name;
|
||||||
dst_data_part->is_temp = true;
|
dst_data_part->is_temp = true;
|
||||||
|
|
||||||
@ -2638,6 +2646,15 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPart(const Merg
|
|||||||
return dst_data_part;
|
return dst_data_part;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
DiskSpaceMonitor::ReservationPtr MergeTreeData::reserveSpaceAtDisk(UInt64 expected_size) const {
|
||||||
|
auto reservation = schema.reserve(expected_size);
|
||||||
|
if (reservation) {
|
||||||
|
/// Add path to table at disk
|
||||||
|
reservation->addEnclosedDirToPath(table_name); ///@TODO_IGR ASK can we use table_name here? Could path be different?
|
||||||
|
}
|
||||||
|
return reservation;
|
||||||
|
}
|
||||||
|
|
||||||
void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String & with_name, const Context & context)
|
void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String & with_name, const Context & context)
|
||||||
{
|
{
|
||||||
String clickhouse_path = Poco::Path(context.getPath()).makeAbsolute().toString();
|
String clickhouse_path = Poco::Path(context.getPath()).makeAbsolute().toString();
|
||||||
|
@ -15,6 +15,7 @@
|
|||||||
#include <DataStreams/GraphiteRollupSortedBlockInputStream.h>
|
#include <DataStreams/GraphiteRollupSortedBlockInputStream.h>
|
||||||
#include <Storages/MergeTree/MergeTreeDataPart.h>
|
#include <Storages/MergeTree/MergeTreeDataPart.h>
|
||||||
#include <Storages/IndicesDescription.h>
|
#include <Storages/IndicesDescription.h>
|
||||||
|
#include <Storages/MergeTree/DiskSpaceMonitor.h>
|
||||||
|
|
||||||
#include <boost/multi_index_container.hpp>
|
#include <boost/multi_index_container.hpp>
|
||||||
#include <boost/multi_index/ordered_index.hpp>
|
#include <boost/multi_index/ordered_index.hpp>
|
||||||
@ -286,7 +287,7 @@ public:
|
|||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
/// Attach the table corresponding to the directory in full_path (must end with /), with the given columns.
|
/// Attach the table corresponding to the directory in full_path inside schema (must end with /), with the given columns.
|
||||||
/// Correctness of names and paths is not checked.
|
/// Correctness of names and paths is not checked.
|
||||||
///
|
///
|
||||||
/// date_column_name - if not empty, the name of the Date column used for partitioning by month.
|
/// date_column_name - if not empty, the name of the Date column used for partitioning by month.
|
||||||
@ -303,7 +304,7 @@ public:
|
|||||||
/// require_part_metadata - should checksums.txt and columns.txt exist in the part directory.
|
/// require_part_metadata - should checksums.txt and columns.txt exist in the part directory.
|
||||||
/// attach - whether the existing table is attached or the new table is created.
|
/// attach - whether the existing table is attached or the new table is created.
|
||||||
MergeTreeData(const String & database_, const String & table_,
|
MergeTreeData(const String & database_, const String & table_,
|
||||||
const Strings & full_paths_,
|
const Schema & schema_,
|
||||||
const ColumnsDescription & columns_,
|
const ColumnsDescription & columns_,
|
||||||
const IndicesDescription & indices_,
|
const IndicesDescription & indices_,
|
||||||
Context & context_,
|
Context & context_,
|
||||||
@ -363,7 +364,7 @@ public:
|
|||||||
|
|
||||||
String getTableName() const { return table_name; }
|
String getTableName() const { return table_name; }
|
||||||
|
|
||||||
String getFullPathForPart(UInt64 expected_size) const;
|
DiskSpaceMonitor::ReservationPtr reserveSpaceForPart(UInt64 expected_size) const; ///@TODO_IGR ASK Is it realy const?
|
||||||
|
|
||||||
String getLogName() const { return log_name; }
|
String getLogName() const { return log_name; }
|
||||||
|
|
||||||
@ -569,6 +570,17 @@ public:
|
|||||||
MergeTreeData::MutableDataPartPtr cloneAndLoadDataPart(const MergeTreeData::DataPartPtr & src_part, const String & tmp_part_prefix,
|
MergeTreeData::MutableDataPartPtr cloneAndLoadDataPart(const MergeTreeData::DataPartPtr & src_part, const String & tmp_part_prefix,
|
||||||
const MergeTreePartInfo & dst_part_info);
|
const MergeTreePartInfo & dst_part_info);
|
||||||
|
|
||||||
|
DiskSpaceMonitor::ReservationPtr reserveSpaceAtDisk(UInt64 expected_size) const; ///@TODO_IGR ASK Maybe set this method as private?
|
||||||
|
|
||||||
|
Strings getFullPaths() const {
|
||||||
|
auto paths = schema.getFullPaths();
|
||||||
|
for (auto && path : paths) {
|
||||||
|
path += table_name + '/'; ///@TODO_IGR ASK It is too slow(
|
||||||
|
/// Maybe store in full_paths variable?
|
||||||
|
}
|
||||||
|
return paths;
|
||||||
|
}
|
||||||
|
|
||||||
MergeTreeDataFormatVersion format_version;
|
MergeTreeDataFormatVersion format_version;
|
||||||
|
|
||||||
Context global_context;
|
Context global_context;
|
||||||
@ -633,7 +645,8 @@ private:
|
|||||||
|
|
||||||
String database_name;
|
String database_name;
|
||||||
String table_name;
|
String table_name;
|
||||||
Strings full_paths;
|
|
||||||
|
Schema schema;
|
||||||
|
|
||||||
/// Current column sizes in compressed and uncompressed form.
|
/// Current column sizes in compressed and uncompressed form.
|
||||||
ColumnSizeByName column_sizes;
|
ColumnSizeByName column_sizes;
|
||||||
|
@ -119,14 +119,6 @@ void FutureMergedMutatedPart::assign(MergeTreeData::DataPartsVector parts_)
|
|||||||
name = part_info.getPartName();
|
name = part_info.getPartName();
|
||||||
}
|
}
|
||||||
|
|
||||||
UInt64 FutureMergedMutatedPart::expectedSize() const { ///TODO_IGR ASK sum size?
|
|
||||||
size_t size = 0;
|
|
||||||
for (auto &part : parts) {
|
|
||||||
size += part->getFileSizeOrZero(""); ///@TODO_IGR ASK file name?
|
|
||||||
}
|
|
||||||
return size;
|
|
||||||
}
|
|
||||||
|
|
||||||
MergeTreeDataMergerMutator::MergeTreeDataMergerMutator(MergeTreeData & data_, const BackgroundProcessingPool & pool_)
|
MergeTreeDataMergerMutator::MergeTreeDataMergerMutator(MergeTreeData & data_, const BackgroundProcessingPool & pool_)
|
||||||
: data(data_), pool(pool_), log(&Logger::get(data.getLogName() + " (MergerMutator)"))
|
: data(data_), pool(pool_), log(&Logger::get(data.getLogName() + " (MergerMutator)"))
|
||||||
{
|
{
|
||||||
@ -522,8 +514,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor
|
|||||||
<< parts.front()->name << " to " << parts.back()->name
|
<< parts.front()->name << " to " << parts.back()->name
|
||||||
<< " into " << TMP_PREFIX + future_part.name);
|
<< " into " << TMP_PREFIX + future_part.name);
|
||||||
|
|
||||||
size_t expected_size = future_part.expectedSize();
|
String part_path = disk_reservation->getPath();
|
||||||
String part_path = data.getFullPathForPart(expected_size); ///@TODO_IGR ASK EXPECTED SIZE
|
|
||||||
|
|
||||||
String new_part_tmp_path = part_path + TMP_PREFIX + future_part.name + "/";
|
String new_part_tmp_path = part_path + TMP_PREFIX + future_part.name + "/";
|
||||||
if (Poco::File(new_part_tmp_path).exists())
|
if (Poco::File(new_part_tmp_path).exists())
|
||||||
@ -830,7 +821,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor
|
|||||||
const FutureMergedMutatedPart & future_part,
|
const FutureMergedMutatedPart & future_part,
|
||||||
const std::vector<MutationCommand> & commands,
|
const std::vector<MutationCommand> & commands,
|
||||||
MergeListEntry & merge_entry,
|
MergeListEntry & merge_entry,
|
||||||
const Context & context)
|
const Context & context,
|
||||||
|
DiskSpaceMonitor::Reservation * disk_reservation)
|
||||||
{
|
{
|
||||||
auto check_not_cancelled = [&]()
|
auto check_not_cancelled = [&]()
|
||||||
{
|
{
|
||||||
@ -865,8 +857,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor
|
|||||||
else
|
else
|
||||||
LOG_TRACE(log, "Mutating part " << source_part->name << " to mutation version " << future_part.part_info.mutation);
|
LOG_TRACE(log, "Mutating part " << source_part->name << " to mutation version " << future_part.part_info.mutation);
|
||||||
|
|
||||||
size_t expected_size = future_part.expectedSize();
|
String part_path = disk_reservation->getPath();
|
||||||
String part_path = data.getFullPathForPart(expected_size); ///@TODO_IGR ASK EXPECTED_SIZE
|
|
||||||
|
|
||||||
MergeTreeData::MutableDataPartPtr new_data_part = std::make_shared<MergeTreeData::DataPart>(
|
MergeTreeData::MutableDataPartPtr new_data_part = std::make_shared<MergeTreeData::DataPart>(
|
||||||
data, part_path, future_part.name, future_part.part_info);
|
data, part_path, future_part.name, future_part.part_info);
|
||||||
|
@ -30,8 +30,6 @@ struct FutureMergedMutatedPart
|
|||||||
}
|
}
|
||||||
|
|
||||||
void assign(MergeTreeData::DataPartsVector parts_);
|
void assign(MergeTreeData::DataPartsVector parts_);
|
||||||
|
|
||||||
UInt64 expectedSize() const;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/** Can select the parts to merge and merge them.
|
/** Can select the parts to merge and merge them.
|
||||||
@ -98,7 +96,7 @@ public:
|
|||||||
MergeTreeData::MutableDataPartPtr mutatePartToTemporaryPart(
|
MergeTreeData::MutableDataPartPtr mutatePartToTemporaryPart(
|
||||||
const FutureMergedMutatedPart & future_part,
|
const FutureMergedMutatedPart & future_part,
|
||||||
const std::vector<MutationCommand> & commands,
|
const std::vector<MutationCommand> & commands,
|
||||||
MergeListEntry & merge_entry, const Context & context);
|
MergeListEntry & merge_entry, const Context & context, DiskSpaceMonitor::Reservation * disk_reservation);
|
||||||
|
|
||||||
MergeTreeData::DataPartPtr renameMergedTemporaryPart(
|
MergeTreeData::DataPartPtr renameMergedTemporaryPart(
|
||||||
MergeTreeData::MutableDataPartPtr & new_data_part,
|
MergeTreeData::MutableDataPartPtr & new_data_part,
|
||||||
|
@ -65,7 +65,7 @@ StorageMergeTree::StorageMergeTree(
|
|||||||
: path(path_), database_name(database_name_), table_name(table_name_), full_paths{path + escapeForFileName(table_name) + '/', "/mnt/data/Data2/" + escapeForFileName(table_name) + '/'},
|
: path(path_), database_name(database_name_), table_name(table_name_), full_paths{path + escapeForFileName(table_name) + '/', "/mnt/data/Data2/" + escapeForFileName(table_name) + '/'},
|
||||||
global_context(context_), background_pool(context_.getBackgroundPool()),
|
global_context(context_), background_pool(context_.getBackgroundPool()),
|
||||||
data(database_name, table_name,
|
data(database_name, table_name,
|
||||||
full_paths, columns_, indices_,
|
Schema(std::vector<Strings>{full_paths}), columns_, indices_, ///@TODO_IGR generate Schema from config
|
||||||
context_, date_column_name, partition_by_ast_, order_by_ast_, primary_key_ast_,
|
context_, date_column_name, partition_by_ast_, order_by_ast_, primary_key_ast_,
|
||||||
sample_by_ast_, merging_params_, settings_, false, attach),
|
sample_by_ast_, merging_params_, settings_, false, attach),
|
||||||
reader(data), writer(data), merger_mutator(data, global_context.getBackgroundPool()),
|
reader(data), writer(data), merger_mutator(data, global_context.getBackgroundPool()),
|
||||||
@ -273,12 +273,15 @@ public:
|
|||||||
: future_part(future_part_), storage(storage_)
|
: future_part(future_part_), storage(storage_)
|
||||||
{
|
{
|
||||||
/// Assume mutex is already locked, because this method is called from mergeTask.
|
/// Assume mutex is already locked, because this method is called from mergeTask.
|
||||||
reserved_space = DiskSpaceMonitor::reserve(storage.full_paths[0], total_size); /// May throw. @TODO_IGR ASK WHERE TO RESERVE
|
reserved_space = storage.data.reserveSpaceForPart(total_size);
|
||||||
|
if (!reserved_space) {
|
||||||
|
throw Exception("Not enought space", ErrorCodes::NOT_ENOUGH_SPACE); ///@TODO_IGR Edit exception msg
|
||||||
|
}
|
||||||
|
|
||||||
for (const auto & part : future_part.parts)
|
for (const auto & part : future_part.parts)
|
||||||
{
|
{
|
||||||
if (storage.currently_merging.count(part))
|
if (storage.currently_merging.count(part))
|
||||||
throw Exception("Tagging alreagy tagged part " + part->name + ". This is a bug.", ErrorCodes::LOGICAL_ERROR);
|
throw Exception("Tagging already tagged part " + part->name + ". This is a bug.", ErrorCodes::LOGICAL_ERROR);
|
||||||
}
|
}
|
||||||
storage.currently_merging.insert(future_part.parts.begin(), future_part.parts.end());
|
storage.currently_merging.insert(future_part.parts.begin(), future_part.parts.end());
|
||||||
}
|
}
|
||||||
@ -481,7 +484,8 @@ bool StorageMergeTree::merge(
|
|||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
UInt64 disk_space = DiskSpaceMonitor::getUnreservedFreeSpace(full_paths[0]); ///@TODO_IGR ASK DISK OR DISKS
|
/// DataPArt can be store only at one disk. Get Max of free space at all disks
|
||||||
|
UInt64 disk_space = DiskSpaceMonitor::getMaxUnreservedFreeSpace(); ///@TODO_IGR ASK Maybe reserve max space at this disk there and then change to exactly space
|
||||||
selected = merger_mutator.selectAllPartsToMergeWithinPartition(future_part, disk_space, can_merge, partition_id, final, out_disable_reason);
|
selected = merger_mutator.selectAllPartsToMergeWithinPartition(future_part, disk_space, can_merge, partition_id, final, out_disable_reason);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -570,7 +574,8 @@ bool StorageMergeTree::tryMutatePart()
|
|||||||
/// You must call destructor with unlocked `currently_merging_mutex`.
|
/// You must call destructor with unlocked `currently_merging_mutex`.
|
||||||
std::optional<CurrentlyMergingPartsTagger> tagger;
|
std::optional<CurrentlyMergingPartsTagger> tagger;
|
||||||
{
|
{
|
||||||
auto disk_space = DiskSpaceMonitor::getUnreservedFreeSpace(full_paths[0]); ///@TODO_IGR ASK DISK OR DISKS
|
/// DataPArt can be store only at one disk. Get Max of free space at all disks
|
||||||
|
UInt64 disk_space = DiskSpaceMonitor::getMaxUnreservedFreeSpace(); ///@TODO_IGR ASK Maybe reserve max space at this disk there and then change to exactly space
|
||||||
|
|
||||||
std::lock_guard lock(currently_merging_mutex);
|
std::lock_guard lock(currently_merging_mutex);
|
||||||
|
|
||||||
@ -660,7 +665,8 @@ bool StorageMergeTree::tryMutatePart()
|
|||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
new_part = merger_mutator.mutatePartToTemporaryPart(future_part, commands, *merge_entry, global_context);
|
new_part = merger_mutator.mutatePartToTemporaryPart(future_part, commands, *merge_entry, global_context,
|
||||||
|
tagger->reserved_space.get());
|
||||||
data.renameTempPartAndReplace(new_part);
|
data.renameTempPartAndReplace(new_part);
|
||||||
tagger->is_successful = true;
|
tagger->is_successful = true;
|
||||||
write_part_log({});
|
write_part_log({});
|
||||||
|
Loading…
Reference in New Issue
Block a user