thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-11 01:54:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

allow to randomize part type in MergeTree

Browse Source
This commit is contained in:
Anton Popov 2020-09-10 04:27:36 +03:00
parent d1c35dc794
commit b9bf67b6ac
2 changed files with 38 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/Storages/MergeTree/MergeTreeSettings.h
View File

@ -20,7 +20,7 @@ struct Settings;
M(UInt64, index_granularity, 8192, "How many rows correspond to one primary key value.", 0) \
\
/** Data storing format settings. */ \
M(UInt64, min_bytes_for_wide_part, 0, "Minimal uncompressed size in bytes to create part in wide format instead of compact", 0) \
M(UInt64, min_bytes_for_wide_part, 10485760, "Minimal uncompressed size in bytes to create part in wide format instead of compact", 0) \
M(UInt64, min_rows_for_wide_part, 0, "Minimal number of rows to create part in wide format instead of compact", 0) \
M(UInt64, min_bytes_for_compact_part, 0, "Experimental. Minimal uncompressed size in bytes to create part in compact format instead of saving it in RAM", 0) \
M(UInt64, min_rows_for_compact_part, 0, "Experimental. Minimal number of rows to create part in compact format instead of saving it in RAM", 0) \
@ -97,6 +97,9 @@ struct Settings;
M(String, storage_policy, "default", "Name of storage disk policy", 0) \
M(Bool, allow_nullable_key, false, "Allow Nullable types as primary keys.", 0) \
\
/** Settings for testing purposes */ \
M(Bool, randomize_part_type, false, "For testing purposes only. Randomizes part type between wide and compact", 0) \
\
/** Obsolete settings. Kept for backward compatibility only. */ \
M(UInt64, min_relative_delay_to_yield_leadership, 120, "Obsolete setting, does nothing.", 0) \
M(UInt64, check_delay_period, 60, "Obsolete setting, does nothing.", 0) \

34
src/Storages/MergeTree/registerStorageMergeTree.cpp
View File

@ -8,6 +8,7 @@
#include <Common/Macros.h>
#include <Common/OptimizedRegularExpression.h>
#include <Common/typeid_cast.h>
#include <Common/thread_local_rng.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTExpressionList.h>
@ -233,6 +234,25 @@ If you use the Replicated version of engines, see https://clickhouse.tech/docs/e
}
static void randomizePartTypeSettings(const std::unique_ptr<MergeTreeSettings> & storage_settings)
{
static constexpr auto MAX_THRESHOLD_FOR_ROWS = 100000;
static constexpr auto MAX_THRESHOLD_FOR_BYTES = 1024 * 1024 * 10;
/// Create all parts in wide format with probability 1/3.
if (thread_local_rng() % 3 == 0)
{
storage_settings->min_rows_for_wide_part = 0;
storage_settings->min_bytes_for_wide_part = 0;
}
else
{
storage_settings->min_rows_for_wide_part = std::uniform_int_distribution{0, MAX_THRESHOLD_FOR_ROWS}(thread_local_rng);
storage_settings->min_bytes_for_wide_part = std::uniform_int_distribution{0, MAX_THRESHOLD_FOR_BYTES}(thread_local_rng);
}
}
static StoragePtr create(const StorageFactory::Arguments & args)
{
/** [Replicated][|Summing|Collapsing|Aggregating|Replacing|Graphite]MergeTree (2 * 7 combinations) engines
@ -652,6 +672,20 @@ static StoragePtr create(const StorageFactory::Arguments & args)
++arg_num;
}
/// Allow to randomize part type for tests to cover more cases.
/// But if settings were set explicitly restrict it.
if (storage_settings->randomize_part_type
&& !storage_settings->min_rows_for_wide_part.changed
&& !storage_settings->min_bytes_for_wide_part.changed)
{
randomizePartTypeSettings(storage_settings);
LOG_INFO(&Poco::Logger::get(args.table_id.getNameForLogs() + " (registerStorageMergeTree)"),
"Applied setting 'randomize_part_type'. "
"Setting 'min_rows_for_wide_part' changed to {}. "
"Setting 'min_bytes_for_wide_part' changed to {}.",
storage_settings->min_rows_for_wide_part, storage_settings->min_bytes_for_wide_part);
}
if (arg_num != arg_cnt)
throw Exception("Wrong number of engine arguments.", ErrorCodes::BAD_ARGUMENTS);