thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-21 01:00:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into roundUpToPowerOfTwoOrZero-safety

Browse Source 
* upstream/master:
  fix left asof join with join_use_nulls (#13291)
  Revert "Add QueryTimeMicroseconds, SelectQueryTimeMicroseconds and InsertQuer…"
  Add test
  Added another check just in case
  Fixup
  Fix range checks in h3 functions
  Fix configs
  Update tests
  Check ALTERs; update test
  Sanity checks for MergeTreeSettings
This commit is contained in:
Azat Khuzhin 2020-08-04 09:29:46 +03:00
commit d28c4c4a6c
26 changed files with 340 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
programs/server/Server.cpp
View File

@ -644,6 +644,9 @@ int Server::main(const std::vector<std::string> & /*args*/)
global_context->setFormatSchemaPath(format_schema_path.path());
format_schema_path.createDirectories();
/// Check sanity of MergeTreeSettings on server startup
global_context->getMergeTreeSettings().sanityCheck(settings);
/// Limit on total memory usage
size_t max_server_memory_usage = config().getUInt64("max_server_memory_usage", 0);

3
src/Common/ProfileEvents.cpp
View File

@ -11,9 +11,6 @@
M(FailedQuery, "Number of failed queries.") \
M(FailedSelectQuery, "Same as FailedQuery, but only for SELECT queries.") \
M(FailedInsertQuery, "Same as FailedQuery, but only for INSERT queries.") \
M(QueryTimeMicroseconds, "Total time of all queries.") \
M(SelectQueryTimeMicroseconds, "Total time of SELECT queries.") \
M(InsertQueryTimeMicroseconds, "Total time of INSERT queries.") \
M(FileOpen, "Number of files opened.") \
M(Seek, "Number of times the 'lseek' function was called.") \
M(ReadBufferFromFileDescriptorRead, "Number of reads (read/pread) from a file descriptor. Does not include sockets.") \

20
src/Functions/h3ToChildren.cpp
View File

@ -5,17 +5,26 @@
#include <Functions/FunctionFactory.h>
#include <Functions/IFunction.h>
#include <Common/typeid_cast.h>
#include <IO/WriteHelpers.h>
#include <ext/range.h>
#include <constants.h>
#include <h3api.h>
static constexpr size_t MAX_ARRAY_SIZE = 1 << 30;
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int ARGUMENT_OUT_OF_BOUND;
extern const int TOO_LARGE_ARRAY_SIZE;
}
class FunctionH3ToChildren : public IFunction
{
public:
@ -63,7 +72,16 @@ public:
const UInt64 parent_hindex = col_hindex->getUInt(row);
const UInt8 child_resolution = col_resolution->getUInt(row);
const auto vec_size = maxH3ToChildrenSize(parent_hindex, child_resolution);
if (child_resolution > MAX_H3_RES)
throw Exception("The argument 'resolution' (" + toString(child_resolution) + ") of function " + getName()
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
const size_t vec_size = maxH3ToChildrenSize(parent_hindex, child_resolution);
if (vec_size > MAX_ARRAY_SIZE)
throw Exception("The result of function" + getName()
+ " (array of " + toString(vec_size) + " elements) will be too large with resolution argument = "
+ toString(child_resolution), ErrorCodes::TOO_LARGE_ARRAY_SIZE);
hindex_vec.resize(vec_size);
h3ToChildren(parent_hindex, child_resolution, hindex_vec.data());

9
src/Functions/h3ToParent.cpp
View File

@ -3,17 +3,22 @@
#include <Functions/FunctionFactory.h>
#include <Functions/IFunction.h>
#include <Common/typeid_cast.h>
#include <IO/WriteHelpers.h>
#include <ext/range.h>
#include <constants.h>
#include <h3api.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int ARGUMENT_OUT_OF_BOUND;
}
class FunctionH3ToParent : public IFunction
{
public:
@ -57,6 +62,10 @@ public:
const UInt64 hindex = col_hindex->getUInt(row);
const UInt8 resolution = col_resolution->getUInt(row);
if (resolution > MAX_H3_RES)
throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
UInt64 res = h3ToParent(hindex, resolution);
dst_data[row] = res;

102
src/Interpreters/HashJoin.cpp
View File

@ -33,7 +33,6 @@ namespace DB
namespace ErrorCodes
{
extern const int BAD_TYPE_OF_FIELD;
extern const int NOT_IMPLEMENTED;
extern const int NO_SUCH_COLUMN_IN_TABLE;
extern const int INCOMPATIBLE_TYPE_OF_JOIN;
@ -158,25 +157,21 @@ HashJoin::HashJoin(std::shared_ptr<TableJoin> table_join_, const Block & right_s
}
else if (strictness == ASTTableJoin::Strictness::Asof)
{
if (kind != ASTTableJoin::Kind::Left and kind != ASTTableJoin::Kind::Inner)
throw Exception("ASOF only supports LEFT and INNER as base joins", ErrorCodes::NOT_IMPLEMENTED);
/// @note ASOF JOIN is not INNER. It's better avoid use of 'INNER ASOF' combination in messages.
/// In fact INNER means 'LEFT SEMI ASOF' while LEFT means 'LEFT OUTER ASOF'.
if (!isLeft(kind) && !isInner(kind))
throw Exception("Wrong ASOF JOIN type. Only ASOF and LEFT ASOF joins are supported", ErrorCodes::NOT_IMPLEMENTED);
if (key_columns.size() <= 1)
throw Exception("ASOF join needs at least one equi-join column", ErrorCodes::SYNTAX_ERROR);
if (right_table_keys.getByName(key_names_right.back()).type->isNullable())
throw Exception("ASOF join over right table Nullable column is not implemented", ErrorCodes::NOT_IMPLEMENTED);
const IColumn * asof_column = key_columns.back();
size_t asof_size;
asof_type = AsofRowRefs::getTypeSize(asof_column, asof_size);
if (!asof_type)
{
std::string msg = "ASOF join not supported for type: ";
msg += asof_column->getFamilyName();
throw Exception(msg, ErrorCodes::BAD_TYPE_OF_FIELD);
}
asof_type = AsofRowRefs::getTypeSize(*key_columns.back(), asof_size);
key_columns.pop_back();
if (key_columns.empty())
throw Exception("ASOF join cannot be done without a joining column", ErrorCodes::SYNTAX_ERROR);
/// this is going to set up the appropriate hash table for the direct lookup part of the join
/// However, this does not depend on the size of the asof join key (as that goes into the BST)
/// Therefore, add it back in such that it can be extracted appropriately from the full stored
@ -248,11 +243,6 @@ HashJoin::Type HashJoin::chooseMethod(const ColumnRawPtrs & key_columns, Sizes &
return Type::hashed;
}
static const IColumn * extractAsofColumn(const ColumnRawPtrs & key_columns)
{
return key_columns.back();
}
template<typename KeyGetter, bool is_asof_join>
static KeyGetter createKeyGetter(const ColumnRawPtrs & key_columns, const Sizes & key_sizes)
{
@ -428,14 +418,15 @@ namespace
}
static ALWAYS_INLINE void insertAsof(HashJoin & join, Map & map, KeyGetter & key_getter, Block * stored_block, size_t i, Arena & pool,
const IColumn * asof_column)
const IColumn & asof_column)
{
auto emplace_result = key_getter.emplaceKey(map, i, pool);
typename Map::mapped_type * time_series_map = &emplace_result.getMapped();
TypeIndex asof_type = *join.getAsofType();
if (emplace_result.isInserted())
time_series_map = new (time_series_map) typename Map::mapped_type(join.getAsofType());
time_series_map->insert(join.getAsofType(), asof_column, stored_block, i);
time_series_map = new (time_series_map) typename Map::mapped_type(asof_type);
time_series_map->insert(asof_type, asof_column, stored_block, i);
}
};
@ -451,7 +442,7 @@ namespace
const IColumn * asof_column [[maybe_unused]] = nullptr;
if constexpr (is_asof_join)
asof_column = extractAsofColumn(key_columns);
asof_column = key_columns.back();
auto key_getter = createKeyGetter<KeyGetter, is_asof_join>(key_columns, key_sizes);
@ -461,7 +452,7 @@ namespace
continue;
if constexpr (is_asof_join)
Inserter<Map, KeyGetter>::insertAsof(join, map, key_getter, stored_block, i, pool, asof_column);
Inserter<Map, KeyGetter>::insertAsof(join, map, key_getter, stored_block, i, pool, *asof_column);
else if constexpr (mapped_one)
Inserter<Map, KeyGetter>::insertOne(join, map, key_getter, stored_block, i, pool);
else
@ -614,21 +605,22 @@ class AddedColumns
public:
using TypeAndNames = std::vector<std::pair<decltype(ColumnWithTypeAndName::type), decltype(ColumnWithTypeAndName::name)>>;
AddedColumns(const Block & sample_block_with_columns_to_add,
const Block & block_with_columns_to_add,
AddedColumns(const Block & block_with_columns_to_add,
const Block & block,
const Block & saved_block_sample,
const ColumnsWithTypeAndName & extras,
const HashJoin & join_,
const HashJoin & join,
const ColumnRawPtrs & key_columns_,
const Sizes & key_sizes_)
: join(join_)
, key_columns(key_columns_)
const Sizes & key_sizes_,
bool is_asof_join)
: key_columns(key_columns_)
, key_sizes(key_sizes_)
, rows_to_add(block.rows())
, need_filter(false)
, asof_type(join.getAsofType())
, asof_inequality(join.getAsofInequality())
{
size_t num_columns_to_add = sample_block_with_columns_to_add.columns();
size_t num_columns_to_add = block_with_columns_to_add.columns();
if (is_asof_join)
++num_columns_to_add;
columns.reserve(num_columns_to_add);
type_name.reserve(num_columns_to_add);
@ -641,8 +633,12 @@ public:
addColumn(src_column);
}
for (const auto & extra : extras)
addColumn(extra);
if (is_asof_join)
{
const ColumnWithTypeAndName & right_asof_column = join.rightAsofKeyColumn();
addColumn(right_asof_column);
left_asof_key = key_columns.back();
}
for (auto & tn : type_name)
right_indexes.push_back(saved_block_sample.getPositionByName(tn.second));
@ -680,18 +676,25 @@ public:
}
}
const HashJoin & join;
TypeIndex asofType() const { return *asof_type; }
ASOF::Inequality asofInequality() const { return asof_inequality; }
const IColumn & leftAsofKey() const { return *left_asof_key; }
const ColumnRawPtrs & key_columns;
const Sizes & key_sizes;
size_t rows_to_add;
std::unique_ptr<IColumn::Offsets> offsets_to_replicate;
bool need_filter;
bool need_filter = false;
private:
TypeAndNames type_name;
MutableColumns columns;
std::vector<size_t> right_indexes;
size_t lazy_defaults_count = 0;
/// for ASOF
std::optional<TypeIndex> asof_type;
ASOF::Inequality asof_inequality;
const IColumn * left_asof_key = nullptr;
void addColumn(const ColumnWithTypeAndName & src_column)
{
@ -760,10 +763,6 @@ NO_INLINE IColumn::Filter joinRightColumns(const Map & map, AddedColumns & added
if constexpr (need_replication)
added_columns.offsets_to_replicate = std::make_unique<IColumn::Offsets>(rows);
const IColumn * asof_column [[maybe_unused]] = nullptr;
if constexpr (is_asof_join)
asof_column = extractAsofColumn(added_columns.key_columns);
auto key_getter = createKeyGetter<KeyGetter, is_asof_join>(added_columns.key_columns, added_columns.key_sizes);
IColumn::Offset current_offset = 0;
@ -790,8 +789,11 @@ NO_INLINE IColumn::Filter joinRightColumns(const Map & map, AddedColumns & added
if constexpr (is_asof_join)
{
const HashJoin & join = added_columns.join;
if (const RowRef * found = mapped.findAsof(join.getAsofType(), join.getAsofInequality(), asof_column, i))
TypeIndex asof_type = added_columns.asofType();
ASOF::Inequality asof_inequality = added_columns.asofInequality();
const IColumn & left_asof_key = added_columns.leftAsofKey();
if (const RowRef * found = mapped.findAsof(asof_type, asof_inequality, left_asof_key, i))
{
setUsed<need_filter>(filter, i);
mapped.setUsed();
@ -932,11 +934,11 @@ void HashJoin::joinBlockImpl(
/// Rare case, when keys are constant or low cardinality. To avoid code bloat, simply materialize them.
Columns materialized_keys = JoinCommon::materializeColumns(block, key_names_left);
ColumnRawPtrs key_columns = JoinCommon::getRawPointers(materialized_keys);
ColumnRawPtrs left_key_columns = JoinCommon::getRawPointers(materialized_keys);
/// Keys with NULL value in any column won't join to anything.
ConstNullMapPtr null_map{};
ColumnPtr null_map_holder = extractNestedColumnsAndNullMap(key_columns, null_map);
ColumnPtr null_map_holder = extractNestedColumnsAndNullMap(left_key_columns, null_map);
size_t existing_columns = block.columns();
@ -957,12 +959,8 @@ void HashJoin::joinBlockImpl(
* but they will not be used at this stage of joining (and will be in `AdderNonJoined`), and they need to be skipped.
* For ASOF, the last column is used as the ASOF column
*/
ColumnsWithTypeAndName extras;
if constexpr (is_asof_join)
extras.push_back(right_table_keys.getByName(key_names_right.back()));
AddedColumns added_columns(sample_block_with_columns_to_add, block_with_columns_to_add, block, savedBlockSample(),
extras, *this, key_columns, key_sizes);
AddedColumns added_columns(block_with_columns_to_add, block, savedBlockSample(), *this, left_key_columns, key_sizes, is_asof_join);
bool has_required_right_keys = (required_right_keys.columns() != 0);
added_columns.need_filter = need_filter || has_required_right_keys;

8
src/Interpreters/HashJoin.h
View File

@ -191,10 +191,16 @@ public:
ASTTableJoin::Kind getKind() const { return kind; }
ASTTableJoin::Strictness getStrictness() const { return strictness; }
TypeIndex getAsofType() const { return *asof_type; }
const std::optional<TypeIndex> & getAsofType() const { return asof_type; }
ASOF::Inequality getAsofInequality() const { return asof_inequality; }
bool anyTakeLastRow() const { return any_take_last_row; }
const ColumnWithTypeAndName & rightAsofKeyColumn() const
{
/// It should be nullable if nullable_right_side is true
return savedBlockSample().getByName(key_names_right.back());
}
/// Different types of keys for maps.
#define APPLY_FOR_JOIN_VARIANTS(M) \
M(key8) \

24
src/Interpreters/RowRefs.cpp
View File

@ -12,6 +12,11 @@
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_TYPE_OF_FIELD;
}
namespace
{
@ -56,7 +61,7 @@ AsofRowRefs::AsofRowRefs(TypeIndex type)
callWithType(type, call);
}
void AsofRowRefs::insert(TypeIndex type, const IColumn * asof_column, const Block * block, size_t row_num)
void AsofRowRefs::insert(TypeIndex type, const IColumn & asof_column, const Block * block, size_t row_num)
{
auto call = [&](const auto & t)
{
@ -64,9 +69,9 @@ void AsofRowRefs::insert(TypeIndex type, const IColumn * asof_column, const Bloc
using LookupPtr = typename Entry<T>::LookupPtr;
using ColumnType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;
auto * column = typeid_cast<const ColumnType *>(asof_column);
const auto & column = typeid_cast<const ColumnType &>(asof_column);
T key = column->getElement(row_num);
T key = column.getElement(row_num);
auto entry = Entry<T>(key, RowRef(block, row_num));
std::get<LookupPtr>(lookups)->insert(entry);
};
@ -74,7 +79,7 @@ void AsofRowRefs::insert(TypeIndex type, const IColumn * asof_column, const Bloc
callWithType(type, call);
}
const RowRef * AsofRowRefs::findAsof(TypeIndex type, ASOF::Inequality inequality, const IColumn * asof_column, size_t row_num) const
const RowRef * AsofRowRefs::findAsof(TypeIndex type, ASOF::Inequality inequality, const IColumn & asof_column, size_t row_num) const
{
const RowRef * out = nullptr;
@ -88,8 +93,8 @@ const RowRef * AsofRowRefs::findAsof(TypeIndex type, ASOF::Inequality inequality
using LookupPtr = typename EntryType::LookupPtr;
using ColumnType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;
auto * column = typeid_cast<const ColumnType *>(asof_column);
T key = column->getElement(row_num);
const auto & column = typeid_cast<const ColumnType &>(asof_column);
T key = column.getElement(row_num);
auto & typed_lookup = std::get<LookupPtr>(lookups);
if (is_strict)
@ -102,9 +107,9 @@ const RowRef * AsofRowRefs::findAsof(TypeIndex type, ASOF::Inequality inequality
return out;
}
std::optional<TypeIndex> AsofRowRefs::getTypeSize(const IColumn * asof_column, size_t & size)
std::optional<TypeIndex> AsofRowRefs::getTypeSize(const IColumn & asof_column, size_t & size)
{
TypeIndex idx = asof_column->getDataType();
TypeIndex idx = asof_column.getDataType();
switch (idx)
{
@ -152,8 +157,7 @@ std::optional<TypeIndex> AsofRowRefs::getTypeSize(const IColumn * asof_column, s
break;
}
size = 0;
return {};
throw Exception("ASOF join not supported for type: " + std::string(asof_column.getFamilyName()), ErrorCodes::BAD_TYPE_OF_FIELD);
}
}

6
src/Interpreters/RowRefs.h
View File

@ -233,13 +233,13 @@ public:
AsofRowRefs() {}
AsofRowRefs(TypeIndex t);
static std::optional<TypeIndex> getTypeSize(const IColumn * asof_column, size_t & type_size);
static std::optional<TypeIndex> getTypeSize(const IColumn & asof_column, size_t & type_size);
// This will be synchronized by the rwlock mutex in Join.h
void insert(TypeIndex type, const IColumn * asof_column, const Block * block, size_t row_num);
void insert(TypeIndex type, const IColumn & asof_column, const Block * block, size_t row_num);
// This will internally synchronize
const RowRef * findAsof(TypeIndex type, ASOF::Inequality inequality, const IColumn * asof_column, size_t row_num) const;
const RowRef * findAsof(TypeIndex type, ASOF::Inequality inequality, const IColumn & asof_column, size_t row_num) const;
private:
// Lookups can be stored in a HashTable because it is memmovable

54
src/Interpreters/executeQuery.cpp
View File

@ -51,9 +51,6 @@ namespace ProfileEvents
extern const Event FailedQuery;
extern const Event FailedInsertQuery;
extern const Event FailedSelectQuery;
extern const Event QueryTimeMicroseconds;
extern const Event SelectQueryTimeMicroseconds;
extern const Event InsertQueryTimeMicroseconds;
}
namespace DB
@ -483,34 +480,8 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
query_log->add(elem);
}
/// Coomon code for finish and exception callbacks
auto status_info_to_query_log = [ast](QueryLogElement &element, const QueryStatusInfo &info) mutable
{
DB::UInt64 query_time = info.elapsed_seconds * 1000000;
ProfileEvents::increment(ProfileEvents::QueryTimeMicroseconds, query_time);
if (ast->as<ASTSelectQuery>() || ast->as<ASTSelectWithUnionQuery>())
{
ProfileEvents::increment(ProfileEvents::SelectQueryTimeMicroseconds, query_time);
}
else if (ast->as<ASTInsertQuery>())
{
ProfileEvents::increment(ProfileEvents::InsertQueryTimeMicroseconds, query_time);
}
element.query_duration_ms = info.elapsed_seconds * 1000;
element.read_rows = info.read_rows;
element.read_bytes = info.read_bytes;
element.memory_usage = info.peak_memory_usage > 0 ? info.peak_memory_usage : 0;
element.thread_ids = std::move(info.thread_ids);
element.profile_counters = std::move(info.profile_counters);
};
/// Also make possible for caller to log successful query finish and exception during execution.
auto finish_callback = [elem, &context, ast, log_queries, log_queries_min_type = settings.log_queries_min_type,
status_info_to_query_log]
auto finish_callback = [elem, &context, log_queries, log_queries_min_type = settings.log_queries_min_type]
(IBlockInputStream * stream_in, IBlockOutputStream * stream_out, QueryPipeline * query_pipeline) mutable
{
QueryStatus * process_list_elem = context.getProcessListElement();
@ -528,14 +499,21 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
elem.type = QueryLogElementType::QUERY_FINISH;
elem.event_time = time(nullptr);
elem.query_duration_ms = elapsed_seconds * 1000;
status_info_to_query_log(elem, info);
elem.read_rows = info.read_rows;
elem.read_bytes = info.read_bytes;
elem.written_rows = info.written_rows;
elem.written_bytes = info.written_bytes;
auto progress_callback = context.getProgressCallback();
if (progress_callback)
progress_callback(Progress(WriteProgress(info.written_rows, info.written_bytes)));
elem.memory_usage = info.peak_memory_usage > 0 ? info.peak_memory_usage : 0;
if (stream_in)
{
const BlockStreamProfileInfo & stream_in_info = stream_in->getProfileInfo();
@ -580,8 +558,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
}
};
auto exception_callback = [elem, &context, ast, log_queries, log_queries_min_type = settings.log_queries_min_type, quota(quota),
status_info_to_query_log] () mutable
auto exception_callback = [elem, &context, ast, log_queries, log_queries_min_type = settings.log_queries_min_type, quota(quota)] () mutable
{
if (quota)
quota->used(Quota::ERRORS, 1, /* check_exceeded = */ false);
@ -602,7 +579,16 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
if (process_list_elem)
{
QueryStatusInfo info = process_list_elem->getInfo(true, current_settings.log_profile_events, false);
status_info_to_query_log(elem, info);
elem.query_duration_ms = info.elapsed_seconds * 1000;
elem.read_rows = info.read_rows;
elem.read_bytes = info.read_bytes;
elem.memory_usage = info.peak_memory_usage > 0 ? info.peak_memory_usage : 0;
elem.thread_ids = std::move(info.thread_ids);
elem.profile_counters = std::move(info.profile_counters);
}
if (current_settings.calculate_text_stack_trace)

9
src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -146,6 +146,10 @@ MergeTreeData::MergeTreeData(
if (relative_data_path.empty())
throw Exception("MergeTree storages require data path", ErrorCodes::INCORRECT_FILE_NAME);
/// Check sanity of MergeTreeSettings. Only when table is created.
if (!attach)
settings->sanityCheck(global_context.getSettingsRef());
MergeTreeDataFormatVersion min_format_version(0);
if (!date_column_name.empty())
{
@ -1608,6 +1612,7 @@ void MergeTreeData::changeSettings(
const auto & new_changes = new_settings->as<const ASTSetQuery &>().changes;
for (const auto & change : new_changes)
{
if (change.name == "storage_policy")
{
StoragePolicyPtr new_storage_policy = global_context.getStoragePolicy(change.value.safeGet<String>());
@ -1642,9 +1647,13 @@ void MergeTreeData::changeSettings(
has_storage_policy_changed = true;
}
}
}
MergeTreeSettings copy = *getSettings();
copy.applyChanges(new_changes);
copy.sanityCheck(global_context.getSettingsRef());
storage_settings.set(std::make_unique<const MergeTreeSettings>(copy));
StorageInMemoryMetadata new_metadata = getInMemoryMetadata();
new_metadata.setSettingsChanges(new_settings);

27
src/Storages/MergeTree/MergeTreeSettings.cpp
View File

@ -75,4 +75,31 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def)
#undef ADD_IF_ABSENT
}
void MergeTreeSettings::sanityCheck(const Settings & query_settings) const
{
if (number_of_free_entries_in_pool_to_execute_mutation >= query_settings.background_pool_size)
{
throw Exception(ErrorCodes::BAD_ARGUMENTS, "The value of 'number_of_free_entries_in_pool_to_execute_mutation' setting"
" ({}) (default values are defined in <merge_tree> section of config.xml"
" or the value can be specified per table in SETTINGS section of CREATE TABLE query)"
" is greater or equals to the value of 'background_pool_size'"
" ({}) (the value is defined in users.xml for default profile)."
" This indicates incorrect configuration because mutations cannot work with these settings.",
number_of_free_entries_in_pool_to_execute_mutation,
query_settings.background_pool_size);
}
if (number_of_free_entries_in_pool_to_lower_max_size_of_merge >= query_settings.background_pool_size)
{
throw Exception(ErrorCodes::BAD_ARGUMENTS, "The value of 'number_of_free_entries_in_pool_to_lower_max_size_of_merge' setting"
" ({}) (default values are defined in <merge_tree> section of config.xml"
" or the value can be specified per table in SETTINGS section of CREATE TABLE query)"
" is greater or equals to the value of 'background_pool_size'"
" ({}) (the value is defined in users.xml for default profile)."
" This indicates incorrect configuration because the maximum size of merge will be always lowered.",
number_of_free_entries_in_pool_to_execute_mutation,
query_settings.background_pool_size);
}
}
}

4
src/Storages/MergeTree/MergeTreeSettings.h
View File

@ -13,6 +13,7 @@ namespace Poco::Util
namespace DB
{
class ASTStorage;
struct Settings;
#define LIST_OF_MERGE_TREE_SETTINGS(M) \
@ -123,6 +124,9 @@ struct MergeTreeSettings : public BaseSettings<MergeTreeSettingsTraits>
return name == "min_bytes_for_wide_part" || name == "min_rows_for_wide_part"
|| name == "min_bytes_for_compact_part" || name == "min_rows_for_compact_part";
}
/// Check that the values are sane taking also query-level settings into account.
void sanityCheck(const Settings & query_settings) const;
};
using MergeTreeSettingsPtr = std::shared_ptr<const MergeTreeSettings>;

9
tests/integration/test_insert_into_distributed/configs/forbid_background_merges.xml
View File

@ -1,7 +1,6 @@
<yandex>
<profiles>
<default>
<background_pool_size>0</background_pool_size>
</default>
</profiles>
<merge_tree>
<max_bytes_to_merge_at_min_space_in_pool>1</max_bytes_to_merge_at_min_space_in_pool>
<max_bytes_to_merge_at_max_space_in_pool>2</max_bytes_to_merge_at_max_space_in_pool>
</merge_tree>
</yandex>

2
tests/integration/test_insert_into_distributed/test.py
View File

@ -12,7 +12,7 @@ instance_test_reconnect = cluster.add_instance('instance_test_reconnect', main_c
instance_test_inserts_batching = cluster.add_instance(
'instance_test_inserts_batching',
main_configs=['configs/remote_servers.xml'], user_configs=['configs/enable_distributed_inserts_batching.xml'])
remote = cluster.add_instance('remote', user_configs=['configs/forbid_background_merges.xml'])
remote = cluster.add_instance('remote', main_configs=['configs/forbid_background_merges.xml'])
instance_test_inserts_local_cluster = cluster.add_instance(
'instance_test_inserts_local_cluster',

9
tests/integration/test_insert_into_distributed_through_materialized_view/configs/forbid_background_merges.xml
View File

@ -1,7 +1,6 @@
<yandex>
<profiles>
<default>
<background_pool_size>0</background_pool_size>
</default>
</profiles>
<merge_tree>
<max_bytes_to_merge_at_min_space_in_pool>1</max_bytes_to_merge_at_min_space_in_pool>
<max_bytes_to_merge_at_max_space_in_pool>2</max_bytes_to_merge_at_max_space_in_pool>
</merge_tree>
</yandex>

2
tests/integration/test_insert_into_distributed_through_materialized_view/test.py
View File

@ -12,7 +12,7 @@ instance_test_reconnect = cluster.add_instance('instance_test_reconnect', main_c
instance_test_inserts_batching = cluster.add_instance(
'instance_test_inserts_batching',
main_configs=['configs/remote_servers.xml'], user_configs=['configs/enable_distributed_inserts_batching.xml'])
remote = cluster.add_instance('remote', user_configs=['configs/forbid_background_merges.xml'])
remote = cluster.add_instance('remote', main_configs=['configs/forbid_background_merges.xml'])
instance_test_inserts_local_cluster = cluster.add_instance(
'instance_test_inserts_local_cluster',

1
tests/integration/test_polymorphic_parts/configs/do_not_merge.xml
View File

@ -2,7 +2,6 @@
<merge_tree>
<max_bytes_to_merge_at_min_space_in_pool>1</max_bytes_to_merge_at_min_space_in_pool>
<max_bytes_to_merge_at_max_space_in_pool>2</max_bytes_to_merge_at_max_space_in_pool>
<number_of_free_entries_in_pool_to_lower_max_size_of_merge>100</number_of_free_entries_in_pool_to_lower_max_size_of_merge>
<max_replicated_merges_in_queue>0</max_replicated_merges_in_queue>
</merge_tree>
</yandex>

2
tests/queries/0_stateless/01415_inconsistent_merge_tree_settings.sql
View File

@ -1,7 +1,7 @@
DROP TABLE IF EXISTS t;
SET mutations_sync = 1;
CREATE TABLE t (x UInt8, s String) ENGINE = MergeTree ORDER BY x SETTINGS number_of_free_entries_in_pool_to_execute_mutation = 1000;
CREATE TABLE t (x UInt8, s String) ENGINE = MergeTree ORDER BY x SETTINGS number_of_free_entries_in_pool_to_execute_mutation = 15;
INSERT INTO t VALUES (1, 'hello');
SELECT * FROM t;

3
tests/queries/0_stateless/01417_query_time_in_system_events.reference
View File

@ -1,3 +0,0 @@
QueryTimeMicroseconds: Ok
SelectQueryTimeMicroseconds: Ok
InsertQueryTimeMicroseconds: Ok

51
tests/queries/0_stateless/01417_query_time_in_system_events.sh
View File

@ -1,51 +0,0 @@
#!/usr/bin/env bash
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. $CURDIR/../shell_config.sh
DATA_BEFORE=`${CLICKHOUSE_CLIENT} --query="SELECT event,value FROM system.events WHERE event IN ('QueryTimeMicroseconds','SelectQueryTimeMicroseconds','InsertQueryTimeMicroseconds') FORMAT CSV"`
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test"
${CLICKHOUSE_CLIENT} --query="CREATE TABLE test (k UInt32) ENGINE=MergeTree ORDER BY k"
${CLICKHOUSE_CLIENT} --query="INSERT INTO test (k) SELECT sleep(1)"
${CLICKHOUSE_CLIENT} --query="SELECT sleep(1)" > /dev/null
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test"
DATA_AFTER=`${CLICKHOUSE_CLIENT} --query="SELECT event,value FROM system.events WHERE event IN ('QueryTimeMicroseconds','SelectQueryTimeMicroseconds','InsertQueryTimeMicroseconds') FORMAT CSV"`
declare -A VALUES_BEFORE
VALUES_BEFORE=(["\"QueryTimeMicroseconds\""]="0" ["\"SelectQueryTimeMicroseconds\""]="0" ["\"InsertQueryTimeMicroseconds\""]="0")
declare -A VALUES_AFTER
VALUES_AFTER=(["\"QueryTimeMicroseconds\""]="0" ["\"SelectQueryTimeMicroseconds\""]="0" ["\"InsertQueryTimeMicroseconds\""]="0")
for RES in ${DATA_BEFORE}
do
IFS=',' read -ra FIELDS <<< ${RES}
VALUES_BEFORE[${FIELDS[0]}]=${FIELDS[1]}
done
for RES in ${DATA_AFTER}
do
IFS=',' read -ra FIELDS <<< ${RES}
VALUES_AFTER[${FIELDS[0]}]=${FIELDS[1]}
done
let QUERY_TIME=${VALUES_AFTER[\"QueryTimeMicroseconds\"]}-${VALUES_BEFORE[\"QueryTimeMicroseconds\"]}
let SELECT_QUERY_TIME=${VALUES_AFTER[\"SelectQueryTimeMicroseconds\"]}-${VALUES_BEFORE[\"SelectQueryTimeMicroseconds\"]}
let INSERT_QUERY_TIME=${VALUES_AFTER[\"InsertQueryTimeMicroseconds\"]}-${VALUES_BEFORE[\"InsertQueryTimeMicroseconds\"]}
if [[ "${QUERY_TIME}" -lt "2000000" ]]; then
echo "QueryTimeMicroseconds: Fail (${QUERY_TIME})"
else
echo "QueryTimeMicroseconds: Ok"
fi
if [[ "${SELECT_QUERY_TIME}" -lt "1000000" ]]; then
echo "SelectQueryTimeMicroseconds: Fail (${SELECT_QUERY_TIME})"
else
echo "SelectQueryTimeMicroseconds: Ok"
fi
if [[ "${INSERT_QUERY_TIME}" -lt "1000000" ]]; then
echo "InsertQueryTimeMicroseconds: Fail (${INSERT_QUERY_TIME})"
else
echo "InsertQueryTimeMicroseconds: Ok"
fi

0
tests/queries/0_stateless/01419_merge_tree_settings_sanity_check.reference Normal file
View File

37
tests/queries/0_stateless/01419_merge_tree_settings_sanity_check.sql Normal file
View File

@ -0,0 +1,37 @@
DROP TABLE IF EXISTS mytable_local;
CREATE TABLE mytable_local
(
created DateTime,
eventday Date,
user_id UInt32
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(eventday)
ORDER BY (eventday, user_id)
SETTINGS number_of_free_entries_in_pool_to_execute_mutation = 100; -- { serverError 36 }
CREATE TABLE mytable_local
(
created DateTime,
eventday Date,
user_id UInt32
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(eventday)
ORDER BY (eventday, user_id)
SETTINGS number_of_free_entries_in_pool_to_lower_max_size_of_merge = 100; -- { serverError 36 }
CREATE TABLE mytable_local
(
created DateTime,
eventday Date,
user_id UInt32
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(eventday)
ORDER BY (eventday, user_id);
ALTER TABLE mytable_local MODIFY SETTING number_of_free_entries_in_pool_to_execute_mutation = 100; -- { serverError 36 }
DROP TABLE mytable_local;

0
tests/queries/0_stateless/01428_h3_range_check.reference Normal file
View File

2
tests/queries/0_stateless/01428_h3_range_check.sql Normal file
View File

@ -0,0 +1,2 @@
SELECT h3ToChildren(599405990164561919, 100); -- { serverError 69 }
SELECT h3ToParent(599405990164561919, 100); -- { serverError 69 }

20
tests/queries/0_stateless/01428_nullable_asof_join.reference Normal file
View File

@ -0,0 +1,20 @@
left asof using
0 \N 0 \N UInt8 Nullable(UInt8) UInt8 Nullable(UInt8)
1 \N 1 \N UInt8 Nullable(UInt8) UInt8 Nullable(UInt8)
1 1 2 2 UInt8 Nullable(UInt8) UInt8 Nullable(UInt8)
0 \N 0 \N UInt8 Nullable(UInt8) Nullable(UInt8) Nullable(UInt8)
1 \N 1 \N UInt8 Nullable(UInt8) Nullable(UInt8) Nullable(UInt8)
1 1 2 2 UInt8 Nullable(UInt8) Nullable(UInt8) Nullable(UInt8)
left asof on
0 \N 0 \N UInt8 Nullable(UInt8) UInt8 Nullable(UInt8)
1 \N 1 \N UInt8 Nullable(UInt8) UInt8 Nullable(UInt8)
1 1 2 2 UInt8 Nullable(UInt8) UInt8 Nullable(UInt8)
0 \N 0 \N UInt8 Nullable(UInt8) Nullable(UInt8) Nullable(UInt8)
1 \N 1 \N UInt8 Nullable(UInt8) Nullable(UInt8) Nullable(UInt8)
1 1 2 2 UInt8 Nullable(UInt8) Nullable(UInt8) Nullable(UInt8)
asof using
1 1 2 2 UInt8 UInt8 UInt8 UInt8
1 1 2 2 UInt8 UInt8 Nullable(UInt8) UInt8
asof on
1 1 2 2 UInt8 UInt8 UInt8 UInt8
1 1 2 2 UInt8 UInt8 Nullable(UInt8) UInt8

105
tests/queries/0_stateless/01428_nullable_asof_join.sql Normal file
View File

@ -0,0 +1,105 @@
SET join_use_nulls = 1;
select 'left asof using';
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
ASOF LEFT JOIN (SELECT 1 as pk, 2 as dt) b
USING(pk, dt)
ORDER BY a.dt;
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
ASOF LEFT JOIN (SELECT 1 as pk, 2 as dt) b
USING(pk, dt)
ORDER BY a.dt;
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b
USING(pk, dt)
ORDER BY a.dt;-- { serverError 48 }
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b
USING(pk, dt)
ORDER BY a.dt;-- { serverError 48 }
select 'left asof on';
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
ASOF LEFT JOIN (SELECT 1 as pk, 2 as dt) b
ON a.pk = b.pk AND a.dt >= b.dt
ORDER BY a.dt;
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
ASOF LEFT JOIN (SELECT 1 as pk, 2 as dt) b
ON a.pk = b.pk AND a.dt >= b.dt
ORDER BY a.dt;
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b
ON a.pk = b.pk AND a.dt >= b.dt
ORDER BY a.dt;-- { serverError 48 }
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b
ON a.pk = b.pk AND a.dt >= b.dt
ORDER BY a.dt;-- { serverError 48 }
select 'asof using';
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
ASOF JOIN (SELECT 1 as pk, 2 as dt) b
USING(pk, dt)
ORDER BY a.dt;
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
ASOF JOIN (SELECT 1 as pk, 2 as dt) b
USING(pk, dt)
ORDER BY a.dt;
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b
USING(pk, dt)
ORDER BY a.dt;-- { serverError 48 }
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b
USING(pk, dt)
ORDER BY a.dt;-- { serverError 48 }
select 'asof on';
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
ASOF JOIN (SELECT 1 as pk, 2 as dt) b
ON a.pk = b.pk AND a.dt >= b.dt
ORDER BY a.dt;
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
ASOF JOIN (SELECT 1 as pk, 2 as dt) b
ON a.pk = b.pk AND a.dt >= b.dt
ORDER BY a.dt;
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b
ON a.pk = b.pk AND a.dt >= b.dt
ORDER BY a.dt;-- { serverError 48 }
SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b
ON a.pk = b.pk AND a.dt >= b.dt
ORDER BY a.dt;-- { serverError 48 }