From 31105670a8a62050f87780f595ac9c2fb2e8492d Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Wed, 6 Jan 2021 18:53:14 +0800 Subject: [PATCH 01/46] Introduce mapContains, mapKeys, mapValues functions for Map data type --- src/Functions/map.cpp | 161 ++++++++++++++++++ .../0_stateless/01651_map_functions.reference | 16 ++ .../0_stateless/01651_map_functions.sql | 21 +++ 3 files changed, 198 insertions(+) create mode 100644 tests/queries/0_stateless/01651_map_functions.reference create mode 100644 tests/queries/0_stateless/01651_map_functions.sql diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 5993ab3706e..dd74d3efa47 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -1,20 +1,28 @@ #include #include +#include #include #include #include #include #include +#include +#include #include #include #include +#include +#include +#include "array/arrayIndex.h" + namespace DB { namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; } namespace @@ -130,11 +138,164 @@ public: } }; +struct NameMapContains { static constexpr auto name = "mapContains"; }; + +class FunctionMapContains : public IFunction +{ +public: + static constexpr auto name = NameMapContains::name; + static FunctionPtr create(const Context &) { return std::make_shared(); } + + String getName() const override + { + return NameMapContains::name ; + } + + size_t getNumberOfArguments() const override { return 2; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() != 2) + throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + + toString(arguments.size()) + ", should be 2", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const DataTypeMap * map_type = checkAndGetDataType(arguments[0].type.get()); + + if (!map_type) + throw Exception{"First argument for function " + getName() + " must be a map.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + + auto key_type = map_type->getKeyType(); + + if (!(isNumber(arguments[1].type) && isNumber(key_type)) + && key_type->getName() != arguments[1].type->getName()) + throw Exception{"Second argument for function " + getName() + " must be a " + key_type->getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + { + const ColumnMap * col_map = typeid_cast(arguments[0].column.get()); + if (!col_map) + return nullptr; + + const auto & nested_column = col_map->getNestedColumn(); + const auto & keys_data = col_map->getNestedData().getColumn(0); + + /// Prepare arguments to call arrayIndex for check has the array element. + ColumnsWithTypeAndName new_arguments = + { + { + ColumnArray::create(keys_data.getPtr(), nested_column.getOffsetsPtr()), + std::make_shared(result_type), + "" + }, + arguments[1] + }; + + return FunctionArrayIndex().executeImpl(new_arguments, result_type, input_rows_count); + } +}; + +class FunctionMapKeys : public IFunction +{ +public: + static constexpr auto name = "mapKeys"; + static FunctionPtr create(const Context &) { return std::make_shared(); } + + String getName() const override + { + return name ; + } + + size_t getNumberOfArguments() const override { return 1; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() != 1) + throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + + toString(arguments.size()) + ", should be 1", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const DataTypeMap * map_type = checkAndGetDataType(arguments[0].type.get()); + + if (!map_type) + throw Exception{"First argument for function " + getName() + " must be a map.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + + auto key_type = map_type->getKeyType(); + + return std::make_shared(key_type); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const override + { + const ColumnMap * col_map = typeid_cast(arguments[0].column.get()); + if (!col_map) + return nullptr; + + const auto & nested_column = col_map->getNestedColumn(); + const auto & keys_data = col_map->getNestedData().getColumn(0); + + return ColumnArray::create(keys_data.getPtr(), nested_column.getOffsetsPtr()); + } +}; + +class FunctionMapValues : public IFunction +{ +public: + static constexpr auto name = "mapValues"; + static FunctionPtr create(const Context &) { return std::make_shared(); } + + String getName() const override + { + return name ; + } + + size_t getNumberOfArguments() const override { return 1; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() != 1) + throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + + toString(arguments.size()) + ", should be 1", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const DataTypeMap * map_type = checkAndGetDataType(arguments[0].type.get()); + + if (!map_type) + throw Exception{"First argument for function " + getName() + " must be a map.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + + auto value_type = map_type->getValueType(); + + return std::make_shared(value_type); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const override + { + const ColumnMap * col_map = typeid_cast(arguments[0].column.get()); + if (!col_map) + return nullptr; + + const auto & nested_column = col_map->getNestedColumn(); + const auto & values_data = col_map->getNestedData().getColumn(1); + + return ColumnArray::create(values_data.getPtr(), nested_column.getOffsetsPtr()); + } +}; + } void registerFunctionsMap(FunctionFactory & factory) { factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); } } diff --git a/tests/queries/0_stateless/01651_map_functions.reference b/tests/queries/0_stateless/01651_map_functions.reference new file mode 100644 index 00000000000..efcd9ce8bcd --- /dev/null +++ b/tests/queries/0_stateless/01651_map_functions.reference @@ -0,0 +1,16 @@ +1 +1 +0 +1 +0 +0 +1 +0 +['name','age'] +['name','gender'] +1 0 0 +1 0 1 +1 0 0 +[232] +[233] +[234] diff --git a/tests/queries/0_stateless/01651_map_functions.sql b/tests/queries/0_stateless/01651_map_functions.sql new file mode 100644 index 00000000000..30ca3a4aeea --- /dev/null +++ b/tests/queries/0_stateless/01651_map_functions.sql @@ -0,0 +1,21 @@ +set allow_experimental_map_type = 1; + +-- String type +drop table if exists table_map; +create table table_map (a Map(String, String), b String) engine = Memory; +insert into table_map values ({'name':'zhangsan', 'age':'10'}, 'name'), ({'name':'lisi', 'gender':'female'},'age'); +select mapContains(a, 'name') from table_map; +select mapContains(a, 'gender') from table_map; +select mapContains(a, 'abc') from table_map; +select mapContains(a, b) from table_map; +select mapContains(a, 10) from table_map; -- { serverError 43 } +select mapKeys(a) from table_map; +drop table if exists table_map; + +CREATE TABLE table_map (a Map(UInt8, Int), b UInt8, c UInt32) engine = MergeTree order by tuple(); +insert into table_map select map(number, number), number, number from numbers(1000, 3); +select mapContains(a, b), mapContains(a, c), mapContains(a, 233) from table_map; +select mapContains(a, 'aaa') from table_map; -- { serverError 43 } +select mapContains(b, 'aaa') from table_map; -- { serverError 43 } +select mapKeys(a) from table_map; +drop table if exists table_map; From 41fe290b2bd9131e3895eb4ad9b0c1ddc6facbdf Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 9 Jan 2021 07:15:28 +0300 Subject: [PATCH 02/46] Update map.cpp --- src/Functions/map.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index dd74d3efa47..f1c5a26ce7d 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -138,17 +138,16 @@ public: } }; -struct NameMapContains { static constexpr auto name = "mapContains"; }; class FunctionMapContains : public IFunction { public: - static constexpr auto name = NameMapContains::name; + static constexpr auto name = "mapContains"; static FunctionPtr create(const Context &) { return std::make_shared(); } String getName() const override { - return NameMapContains::name ; + return name; } size_t getNumberOfArguments() const override { return 2; } @@ -200,6 +199,7 @@ public: } }; + class FunctionMapKeys : public IFunction { public: @@ -208,7 +208,7 @@ public: String getName() const override { - return name ; + return name; } size_t getNumberOfArguments() const override { return 1; } @@ -244,6 +244,7 @@ public: } }; + class FunctionMapValues : public IFunction { public: @@ -252,7 +253,7 @@ public: String getName() const override { - return name ; + return name; } size_t getNumberOfArguments() const override { return 1; } From 9e3b4a67deb9bfcbb0d3d280ad96aa140b0380ed Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Mon, 11 Jan 2021 10:56:13 +0800 Subject: [PATCH 03/46] Add mapValues test --- tests/queries/0_stateless/01651_map_functions.reference | 3 +++ tests/queries/0_stateless/01651_map_functions.sql | 1 + 2 files changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/01651_map_functions.reference b/tests/queries/0_stateless/01651_map_functions.reference index efcd9ce8bcd..ede7a6f5e68 100644 --- a/tests/queries/0_stateless/01651_map_functions.reference +++ b/tests/queries/0_stateless/01651_map_functions.reference @@ -14,3 +14,6 @@ [232] [233] [234] +[1000] +[1001] +[1002] diff --git a/tests/queries/0_stateless/01651_map_functions.sql b/tests/queries/0_stateless/01651_map_functions.sql index 30ca3a4aeea..c3b5bd7edd1 100644 --- a/tests/queries/0_stateless/01651_map_functions.sql +++ b/tests/queries/0_stateless/01651_map_functions.sql @@ -18,4 +18,5 @@ select mapContains(a, b), mapContains(a, c), mapContains(a, 233) from table_map; select mapContains(a, 'aaa') from table_map; -- { serverError 43 } select mapContains(b, 'aaa') from table_map; -- { serverError 43 } select mapKeys(a) from table_map; +select mapValues(a) from table_map; drop table if exists table_map; From bd05d9db2f01382b76b202177db27a5932810932 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 6 Jan 2021 00:57:05 +0300 Subject: [PATCH 04/46] Fix memory tracking for OPTIMIZE TABLE queries Because of BlockerInThread in MergeTreeDataPartWriterOnDisk::calculateAndSerializePrimaryIndex memory was accounted incorrectly and grows constantly. And IIUC there is no need in that blocker, since INSERT SELECT shares the same thread group. --- .../MergeTree/MergeTreeDataPartWriterOnDisk.cpp | 8 -------- .../01641_memory_tracking_insert_optimize.reference | 0 .../01641_memory_tracking_insert_optimize.sql | 13 +++++++++++++ tests/queries/skip_list.json | 6 ++++-- 4 files changed, 17 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/01641_memory_tracking_insert_optimize.reference create mode 100644 tests/queries/0_stateless/01641_memory_tracking_insert_optimize.sql diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp index fd3338c8a70..9390180d51c 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp @@ -180,14 +180,6 @@ void MergeTreeDataPartWriterOnDisk::calculateAndSerializePrimaryIndex(const Bloc index_columns[i] = primary_index_block.getByPosition(i).column->cloneEmpty(); } - /** While filling index (index_columns), disable memory tracker. - * Because memory is allocated here (maybe in context of INSERT query), - * but then freed in completely different place (while merging parts), where query memory_tracker is not available. - * And otherwise it will look like excessively growing memory consumption in context of query. - * (observed in long INSERT SELECTs) - */ - MemoryTracker::BlockerInThread temporarily_disable_memory_tracker; - /// Write index. The index contains Primary Key value for each `index_granularity` row. for (const auto & granule : granules_to_write) { diff --git a/tests/queries/0_stateless/01641_memory_tracking_insert_optimize.reference b/tests/queries/0_stateless/01641_memory_tracking_insert_optimize.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01641_memory_tracking_insert_optimize.sql b/tests/queries/0_stateless/01641_memory_tracking_insert_optimize.sql new file mode 100644 index 00000000000..59b5098bbd1 --- /dev/null +++ b/tests/queries/0_stateless/01641_memory_tracking_insert_optimize.sql @@ -0,0 +1,13 @@ +drop table if exists data_01641; + +create table data_01641 (key Int, value String) engine=MergeTree order by (key, repeat(value, 10)) settings old_parts_lifetime=0, min_bytes_for_wide_part=0; + +-- peak memory usage is 170MiB +set max_memory_usage='200Mi'; +system stop merges data_01641; +insert into data_01641 select number, toString(number) from numbers(toUInt64(120e6)); + +-- FIXME: this limit does not work +set max_memory_usage='10Mi'; +system start merges data_01641; +optimize table data_01641 final; diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index cfbac463932..d380ab2ecf0 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -16,7 +16,8 @@ "01474_executable_dictionary", /// informational stderr from sanitizer at start "functions_bad_arguments", /// Too long for TSan "01603_read_with_backoff_bug", /// Too long for TSan - "01646_system_restart_replicas_smoke" /// RESTART REPLICAS can acquire too much locks, while only 64 is possible from one thread under TSan + "01646_system_restart_replicas_smoke", /// RESTART REPLICAS can acquire too much locks, while only 64 is possible from one thread under TSan + "01641_memory_tracking_insert_optimize" /// INSERT lots of rows is too heavy for TSan ], "address-sanitizer": [ "00877", @@ -62,7 +63,8 @@ "hyperscan", "01193_metadata_loading", "01473_event_time_microseconds", - "01396_inactive_replica_cleanup_nodes" + "01396_inactive_replica_cleanup_nodes", + "01641_memory_tracking_insert_optimize" /// INSERT lots of rows is too heavy in debug build ], "unbundled-build": [ "00429", From 82edbfb5816fc60df3dcc1d064834e7915531f67 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 6 Jan 2021 02:20:26 +0300 Subject: [PATCH 05/46] Account query memory limits and sampling for OPTIMIZE TABLE/merges --- src/Storages/MergeTree/MergeList.cpp | 12 ++++++++++++ .../01641_memory_tracking_insert_optimize.sql | 12 ++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeList.cpp b/src/Storages/MergeTree/MergeList.cpp index ba6c2a3d462..dbdfe650713 100644 --- a/src/Storages/MergeTree/MergeList.cpp +++ b/src/Storages/MergeTree/MergeList.cpp @@ -40,6 +40,18 @@ MergeListElement::MergeListElement(const std::string & database_, const std::str background_thread_memory_tracker = CurrentThread::getMemoryTracker(); if (background_thread_memory_tracker) { + /// From the query context it will be ("for thread") memory tracker with VariableContext::Thread level, + /// which does not have any limits and sampling settings configured. + /// And parent for this memory tracker should be ("(for query)") with VariableContext::Process level, + /// that has limits and sampling configured. + MemoryTracker * parent; + if (background_thread_memory_tracker->level == VariableContext::Thread && + (parent = background_thread_memory_tracker->getParent()) && + parent != &total_memory_tracker) + { + background_thread_memory_tracker = parent; + } + background_thread_memory_tracker_prev_parent = background_thread_memory_tracker->getParent(); background_thread_memory_tracker->setParent(&memory_tracker); } diff --git a/tests/queries/0_stateless/01641_memory_tracking_insert_optimize.sql b/tests/queries/0_stateless/01641_memory_tracking_insert_optimize.sql index 59b5098bbd1..f059da20755 100644 --- a/tests/queries/0_stateless/01641_memory_tracking_insert_optimize.sql +++ b/tests/queries/0_stateless/01641_memory_tracking_insert_optimize.sql @@ -7,7 +7,15 @@ set max_memory_usage='200Mi'; system stop merges data_01641; insert into data_01641 select number, toString(number) from numbers(toUInt64(120e6)); --- FIXME: this limit does not work -set max_memory_usage='10Mi'; +-- peak: +-- - is 21MiB if background merges already scheduled +-- - is ~60MiB otherwise +set max_memory_usage='80Mi'; system start merges data_01641; optimize table data_01641 final; + +-- definitely should fail +set max_memory_usage='1Mi'; +optimize table data_01641 final; -- { serverError 241 } + +drop table data_01641; From 05608687d6e3f188bb26f6e576b414f8f64fee99 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 12 Jan 2021 00:50:37 +0300 Subject: [PATCH 06/46] Get back memory tracking blocker in calculateAndSerializePrimaryIndex() But reduce scope, to avoid leaking too much memory, since there are old values in last_block_index_columns. The scope of the MemoryTracker::BlockerInThread has been increased in #8290 --- .../MergeTreeDataPartWriterOnDisk.cpp | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp index 9390180d51c..1339127e660 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp @@ -180,19 +180,30 @@ void MergeTreeDataPartWriterOnDisk::calculateAndSerializePrimaryIndex(const Bloc index_columns[i] = primary_index_block.getByPosition(i).column->cloneEmpty(); } - /// Write index. The index contains Primary Key value for each `index_granularity` row. - for (const auto & granule : granules_to_write) { - if (metadata_snapshot->hasPrimaryKey() && granule.mark_on_start) + /** While filling index (index_columns), disable memory tracker. + * Because memory is allocated here (maybe in context of INSERT query), + * but then freed in completely different place (while merging parts), where query memory_tracker is not available. + * And otherwise it will look like excessively growing memory consumption in context of query. + * (observed in long INSERT SELECTs) + */ + MemoryTracker::BlockerInThread temporarily_disable_memory_tracker; + + /// Write index. The index contains Primary Key value for each `index_granularity` row. + for (const auto & granule : granules_to_write) { - for (size_t j = 0; j < primary_columns_num; ++j) + if (metadata_snapshot->hasPrimaryKey() && granule.mark_on_start) { - const auto & primary_column = primary_index_block.getByPosition(j); - index_columns[j]->insertFrom(*primary_column.column, granule.start_row); - primary_column.type->serializeBinary(*primary_column.column, granule.start_row, *index_stream); + for (size_t j = 0; j < primary_columns_num; ++j) + { + const auto & primary_column = primary_index_block.getByPosition(j); + index_columns[j]->insertFrom(*primary_column.column, granule.start_row); + primary_column.type->serializeBinary(*primary_column.column, granule.start_row, *index_stream); + } } } } + /// store last index row to write final mark at the end of column for (size_t j = 0; j < primary_columns_num; ++j) last_block_index_columns[j] = primary_index_block.getByPosition(j).column; From eda9ca82030a2e74cd808899316090ddf3fdf1e6 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 13 Jan 2021 22:52:09 +0300 Subject: [PATCH 07/46] Creating multiword-types.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Создал страницу multiword-types.md и обновил таблицу соответствия типов данных. --- docs/en/sql-reference/ansi.md | 4 ++-- .../sql-reference/data-types/multiword-types.md | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 docs/en/sql-reference/data-types/multiword-types.md diff --git a/docs/en/sql-reference/ansi.md b/docs/en/sql-reference/ansi.md index fc759f9f79a..eb6e0152fb0 100644 --- a/docs/en/sql-reference/ansi.md +++ b/docs/en/sql-reference/ansi.md @@ -25,14 +25,14 @@ The following table lists cases when query feature works in ClickHouse, but beha |------------|--------------------------------------------------------------------------------------------------------------------------|----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | **E011** | **Numeric data types** | **Partial**{.text-warning} | | | E011-01 | INTEGER and SMALLINT data types | Yes {.text-success} | | -| E011-02 | REAL, DOUBLE PRECISION and FLOAT data types data types | Partial {.text-warning} | `FLOAT()`, `REAL` and `DOUBLE PRECISION` are not supported | +| E011-02 | REAL, DOUBLE PRECISION and FLOAT data types data types | Partial {.text-warning} | `FLOAT()` and `REAL` are not supported | | E011-03 | DECIMAL and NUMERIC data types | Partial {.text-warning} | Only `DECIMAL(p,s)` is supported, not `NUMERIC` | | E011-04 | Arithmetic operators | Yes {.text-success} | | | E011-05 | Numeric comparison | Yes {.text-success} | | | E011-06 | Implicit casting among the numeric data types | No {.text-danger} | ANSI SQL allows arbitrary implicit cast between numeric types, while ClickHouse relies on functions having multiple overloads instead of implicit cast | | **E021** | **Character string types** | **Partial**{.text-warning} | | | E021-01 | CHARACTER data type | No {.text-danger} | | -| E021-02 | CHARACTER VARYING data type | No {.text-danger} | `String` behaves similarly, but without length limit in parentheses | +| E021-02 | CHARACTER VARYING data type | Yes {.text-danger} | | | E021-03 | Character literals | Partial {.text-warning} | No automatic concatenation of consecutive literals and character set support | | E021-04 | CHARACTER_LENGTH function | Partial {.text-warning} | No `USING` clause | | E021-05 | OCTET_LENGTH function | No {.text-danger} | `LENGTH` behaves similarly | diff --git a/docs/en/sql-reference/data-types/multiword-types.md b/docs/en/sql-reference/data-types/multiword-types.md new file mode 100644 index 00000000000..ea6a12ac82e --- /dev/null +++ b/docs/en/sql-reference/data-types/multiword-types.md @@ -0,0 +1,17 @@ +--- +toc_priority: 61 +toc_title: Multiword Type Names +--- + +# Multiword Types {#multiword-types} + +When creating tables, you can also use data types with a name consisting of several words. This is necessary for better SQL compatibility. + +## Multiword Types Support {#multiword-types-support} + +| Multiword types | Simple types | +|----------------------------------|--------------------------------------------------------------| +| DOUBLE PRECISION | [Float64](../../sql-reference/data-types/float.md) | +| CHAR VARYING | [String](../../sql-reference/data-types/string.md) | + +[Original article](https://clickhouse.tech/docs/en/sql-reference/data-types/multiword-types/) From ce086197b7a711c4c84e9fc6d4bd28ba2ce32a3e Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Thu, 14 Jan 2021 20:29:10 +0300 Subject: [PATCH 08/46] DOCSUP-5272: Fix query syntax for DOCSUP-4261 --- docs/en/operations/settings/settings.md | 6 +----- docs/ru/operations/settings/settings.md | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index d3a4d50d21c..4433c27e181 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2491,11 +2491,7 @@ Default value: 0. Consider the following query with aggregate functions: ```sql -SELECT - SUM(-1), - MAX(0) -FROM system.one -WHERE 0 +SELECT SUM(-1), MAX(0) FROM system.one WHERE 0; ``` With `aggregate_functions_null_for_empty = 0` it would produce: diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index aa549fc5776..21bb58ca01c 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2360,11 +2360,7 @@ SELECT number FROM numbers(3) FORMAT JSONEachRow; Рассмотрим запрос с агрегирующими функциями: ```sql -SELECT - SUM(-1), - MAX(0) -FROM system.one -WHERE 0 +SELECT SUM(-1), MAX(0) FROM system.one WHERE 0; ``` Результат запроса с настройкой `aggregate_functions_null_for_empty = 0`: From c448542b5c83d02861e49e6fc4b49002814a1c27 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Thu, 14 Jan 2021 20:36:10 +0300 Subject: [PATCH 09/46] DOCSUP-5272: Fix PR 17121 mispelling in RU --- docs/ru/engines/table-engines/mergetree-family/replication.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/mergetree-family/replication.md b/docs/ru/engines/table-engines/mergetree-family/replication.md index f17e1b035d4..a8a308b104f 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replication.md +++ b/docs/ru/engines/table-engines/mergetree-family/replication.md @@ -153,7 +153,7 @@ CREATE TABLE table_name ```xml /clickhouse/tables/{shard}/{database}/{table} -{replica} +{replica} ``` В этом случае можно опустить аргументы при создании таблиц: From 5c97e473931f20017d496694ab3451aaf8408b15 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Thu, 14 Jan 2021 20:50:32 +0300 Subject: [PATCH 10/46] DOCSUP-5272: Add PR#17213 translation to RU --- .../sql-reference/statements/create/table.md | 14 ++------------ .../sql-reference/statements/create/table.md | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index b1a5fdb19b5..1dd9238a9f2 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -114,23 +114,13 @@ You can define a [primary key](../../../engines/table-engines/mergetree-family/m - inside the column list ``` sql -CREATE TABLE db.table_name -( - name1 type1, name2 type2, ..., - PRIMARY KEY(expr1[, expr2,...])] -) -ENGINE = engine; +CREATE TABLE db.table_name (name1 type1, name2 type2, ..., PRIMARY KEY (expr1[, expr2,...])]) ENGINE = engine; ``` - outside the column list ``` sql -CREATE TABLE db.table_name -( - name1 type1, name2 type2, ... -) -ENGINE = engine -PRIMARY KEY(expr1[, expr2,...]); +CREATE TABLE db.table_name (name1 type1, name2 type2, ...) ENGINE = engine PRIMARY KEY(expr1[, expr2,...]); ``` You can't combine both ways in one query. diff --git a/docs/ru/sql-reference/statements/create/table.md b/docs/ru/sql-reference/statements/create/table.md index d54ec189a1a..0a3e187cc3b 100644 --- a/docs/ru/sql-reference/statements/create/table.md +++ b/docs/ru/sql-reference/statements/create/table.md @@ -22,6 +22,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] Описание столбца, это `name type`, в простейшем случае. Пример: `RegionID UInt32`. Также могут быть указаны выражения для значений по умолчанию - смотрите ниже. +При необходимости можно указать первичный ключ с одним или несколькими ключевыми выражениями. ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name AS [db2.]name2 [ENGINE = engine] ``` @@ -80,6 +81,24 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ... Отсутствует возможность задать значения по умолчанию для элементов вложенных структур данных. +## Первичный ключ {#primary-key} + +Вы можете определить [первичный ключ](../../../engines/table-engines/mergetree-family/mergetree.md#primary-keys-and-indexes-in-queries) при создании таблицы. Первичный ключ может быть указан двумя способами: + +- В списке столбцов: + +``` sql +CREATE TABLE db.table_name (name1 type1, name2 type2, ..., PRIMARY KEY (expr1[, expr2,...])]) ENGINE = engine; +``` + +- Вне списка столбцов: + +``` sql +CREATE TABLE db.table_name (name1 type1, name2 type2, ...) ENGINE = engine PRIMARY KEY(expr1[, expr2,...]); +``` + +Вы не можете сочетать оба способа в одном запросе. + ### Ограничения (constraints) {#constraints} Наряду с объявлением столбцов можно объявить ограничения на значения в столбцах таблицы: From 812f8ee19701e14c1d4de0d0d88b02f8cfdd2f74 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Thu, 14 Jan 2021 21:10:32 +0300 Subject: [PATCH 11/46] DOCSUP-5272: primary key crosslink in doc --- docs/en/sql-reference/statements/create/table.md | 2 +- docs/ru/sql-reference/statements/create/table.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 1dd9238a9f2..95ac0252eaa 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -23,7 +23,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ``` Creates a table named `name` in the `db` database or the current database if `db` is not set, with the structure specified in brackets and the `engine` engine. -The structure of the table is a list of column descriptions, secondary indexes and constraints . If primary key is supported by the engine, it will be indicated as parameter for the table engine. +The structure of the table is a list of column descriptions, secondary indexes and constraints . If [primary key](#primary-key) is supported by the engine, it will be indicated as parameter for the table engine. A column description is `name type` in the simplest case. Example: `RegionID UInt32`. diff --git a/docs/ru/sql-reference/statements/create/table.md b/docs/ru/sql-reference/statements/create/table.md index 0a3e187cc3b..e91a7f15903 100644 --- a/docs/ru/sql-reference/statements/create/table.md +++ b/docs/ru/sql-reference/statements/create/table.md @@ -22,7 +22,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] Описание столбца, это `name type`, в простейшем случае. Пример: `RegionID UInt32`. Также могут быть указаны выражения для значений по умолчанию - смотрите ниже. -При необходимости можно указать первичный ключ с одним или несколькими ключевыми выражениями. +При необходимости можно указать [первичный ключ](#primary-key) с одним или несколькими ключевыми выражениями. ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name AS [db2.]name2 [ENGINE = engine] ``` From 322a5e515300cef09ba52bef4f730e84dc8c1295 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sat, 16 Jan 2021 23:00:00 +0300 Subject: [PATCH 12/46] First draft --- docs/en/operations/caches.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 docs/en/operations/caches.md diff --git a/docs/en/operations/caches.md b/docs/en/operations/caches.md new file mode 100644 index 00000000000..0107c340019 --- /dev/null +++ b/docs/en/operations/caches.md @@ -0,0 +1,24 @@ +--- +toc_priority: 65 +toc_title: Caches +--- + +# Cache Types {#cache-types} + +When performing queries, ClichHouse uses different caches. + +Main cache types: +- `mark_cache` — Cache of marks used by table engines of the [MergeTree](../engines/table-engines/mergetree-family/mergetree.md) family. +- `uncompressed_cache` — Cache of uncompressed data used by table engines of the [MergeTree](../engines/table-engines/mergetree-family/mergetree.md) family. + +Additional cache types: +- DNS cache +- [regexp](../interfaces/formats.md#data-format-regexp) cache +- compiled expressions cache +- [Avro format](../interfaces/formats.md#data-format-avro) schemas cache +- [dictionaries data cache](../sql-reference/dictionaries/index.md) + +Not directly used: +- page cache OS + +[Original article](https://clickhouse.tech/docs/en/operations/caches/) From f6d1f76b42bcc1a1161802fe961239a6ffff2f91 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sun, 17 Jan 2021 04:30:02 +0300 Subject: [PATCH 13/46] Add some multiword types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Обновил таблицу ANSI и добавил новые типы данных. --- docs/en/sql-reference/ansi.md | 2 +- docs/en/sql-reference/data-types/multiword-types.md | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/ansi.md b/docs/en/sql-reference/ansi.md index eb6e0152fb0..5ca216d11fa 100644 --- a/docs/en/sql-reference/ansi.md +++ b/docs/en/sql-reference/ansi.md @@ -25,7 +25,7 @@ The following table lists cases when query feature works in ClickHouse, but beha |------------|--------------------------------------------------------------------------------------------------------------------------|----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | **E011** | **Numeric data types** | **Partial**{.text-warning} | | | E011-01 | INTEGER and SMALLINT data types | Yes {.text-success} | | -| E011-02 | REAL, DOUBLE PRECISION and FLOAT data types data types | Partial {.text-warning} | `FLOAT()` and `REAL` are not supported | +| E011-02 | REAL, DOUBLE PRECISION and FLOAT data types data types | Yes {.text-warning} | | | E011-03 | DECIMAL and NUMERIC data types | Partial {.text-warning} | Only `DECIMAL(p,s)` is supported, not `NUMERIC` | | E011-04 | Arithmetic operators | Yes {.text-success} | | | E011-05 | Numeric comparison | Yes {.text-success} | | diff --git a/docs/en/sql-reference/data-types/multiword-types.md b/docs/en/sql-reference/data-types/multiword-types.md index ea6a12ac82e..f55efcd7a51 100644 --- a/docs/en/sql-reference/data-types/multiword-types.md +++ b/docs/en/sql-reference/data-types/multiword-types.md @@ -12,6 +12,18 @@ When creating tables, you can also use data types with a name consisting of seve | Multiword types | Simple types | |----------------------------------|--------------------------------------------------------------| | DOUBLE PRECISION | [Float64](../../sql-reference/data-types/float.md) | +| CHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | | CHAR VARYING | [String](../../sql-reference/data-types/string.md) | +| CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | +| CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) | +| NCHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | +| NCHAR VARYING | [String](../../sql-reference/data-types/string.md) | +| NATIONAL CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | +| NATIONAL CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) | +| NATIONAL CHAR VARYING | [String](../../sql-reference/data-types/string.md) | +| NATIONAL CHARACTER | [String](../../sql-reference/data-types/string.md) | +| NATIONAL CHAR | [String](../../sql-reference/data-types/string.md) | +| BINARY LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | +| BINARY VARYING | [String](../../sql-reference/data-types/string.md) | [Original article](https://clickhouse.tech/docs/en/sql-reference/data-types/multiword-types/) From 6a78b10e0b1ef3e341dfc7959ef24b7dede0dc1d Mon Sep 17 00:00:00 2001 From: hexiaoting Date: Mon, 18 Jan 2021 10:58:07 +0800 Subject: [PATCH 14/46] fix build error --- src/Functions/map.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index f1c5a26ce7d..2561f06d9b8 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -139,15 +139,17 @@ public: }; +struct NameMapContains { static constexpr auto name = "mapContains"; }; + class FunctionMapContains : public IFunction { public: - static constexpr auto name = "mapContains"; + static constexpr auto name = NameMapContains::name; static FunctionPtr create(const Context &) { return std::make_shared(); } String getName() const override { - return name; + return NameMapContains::name; } size_t getNumberOfArguments() const override { return 2; } From e313209a103f47f362b41653b252337e5c361774 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Tue, 19 Jan 2021 13:26:56 +0300 Subject: [PATCH 15/46] Support CHARACTER data type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Добавил поддержку типа данных CHARACTER. --- docs/en/sql-reference/ansi.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/ansi.md b/docs/en/sql-reference/ansi.md index 5ca216d11fa..84e47902f3b 100644 --- a/docs/en/sql-reference/ansi.md +++ b/docs/en/sql-reference/ansi.md @@ -31,7 +31,7 @@ The following table lists cases when query feature works in ClickHouse, but beha | E011-05 | Numeric comparison | Yes {.text-success} | | | E011-06 | Implicit casting among the numeric data types | No {.text-danger} | ANSI SQL allows arbitrary implicit cast between numeric types, while ClickHouse relies on functions having multiple overloads instead of implicit cast | | **E021** | **Character string types** | **Partial**{.text-warning} | | -| E021-01 | CHARACTER data type | No {.text-danger} | | +| E021-01 | CHARACTER data type | Yes {.text-danger} | | | E021-02 | CHARACTER VARYING data type | Yes {.text-danger} | | | E021-03 | Character literals | Partial {.text-warning} | No automatic concatenation of consecutive literals and character set support | | E021-04 | CHARACTER_LENGTH function | Partial {.text-warning} | No `USING` clause | From 3668885f6fd2b3e5d01771edf7656df03227d992 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Tue, 19 Jan 2021 17:42:34 +0300 Subject: [PATCH 16/46] Fix translation of multiword types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Заменил выражение "типы с названием из нескольких слов" на "составные типы". --- .../data-types/multiword-types.md | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/ru/sql-reference/data-types/multiword-types.md diff --git a/docs/ru/sql-reference/data-types/multiword-types.md b/docs/ru/sql-reference/data-types/multiword-types.md new file mode 100644 index 00000000000..4c08ea8ee92 --- /dev/null +++ b/docs/ru/sql-reference/data-types/multiword-types.md @@ -0,0 +1,29 @@ +--- +toc_priority: 61 +toc_title: Составные типы +--- + +# Составные типы {#multiword-types} + +При создании таблиц вы также можете использовать типы данных с названием, состоящим из нескольких слов. Это необходимо для лучшей совместимости с SQL. + +## Поддержка составных типов {#multiword-types-support} + +| Составные типы | Обычные типы | +|-------------------------------------|-----------------------------------------------------------| +| DOUBLE PRECISION | [Float64](../../sql-reference/data-types/float.md) | +| CHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | +| CHAR VARYING | [String](../../sql-reference/data-types/string.md) | +| CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | +| CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) | +| NCHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | +| NCHAR VARYING | [String](../../sql-reference/data-types/string.md) | +| NATIONAL CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | +| NATIONAL CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) | +| NATIONAL CHAR VARYING | [String](../../sql-reference/data-types/string.md) | +| NATIONAL CHARACTER | [String](../../sql-reference/data-types/string.md) | +| NATIONAL CHAR | [String](../../sql-reference/data-types/string.md) | +| BINARY LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | +| BINARY VARYING | [String](../../sql-reference/data-types/string.md) | + +[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/data-types/multiword-types/) From 5b7af74f84f14f3a45c1b09851a703767acd48e8 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Tue, 19 Jan 2021 20:10:23 +0300 Subject: [PATCH 17/46] Fix multiword types.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Внес небольшие поправки. --- docs/en/sql-reference/data-types/multiword-types.md | 2 +- docs/ru/sql-reference/data-types/multiword-types.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/data-types/multiword-types.md b/docs/en/sql-reference/data-types/multiword-types.md index f55efcd7a51..5012fbb404e 100644 --- a/docs/en/sql-reference/data-types/multiword-types.md +++ b/docs/en/sql-reference/data-types/multiword-types.md @@ -5,7 +5,7 @@ toc_title: Multiword Type Names # Multiword Types {#multiword-types} -When creating tables, you can also use data types with a name consisting of several words. This is necessary for better SQL compatibility. +When creating tables, you can use data types with a name consisting of several words. This is implemented for better SQL compatibility. ## Multiword Types Support {#multiword-types-support} diff --git a/docs/ru/sql-reference/data-types/multiword-types.md b/docs/ru/sql-reference/data-types/multiword-types.md index 4c08ea8ee92..559755ef989 100644 --- a/docs/ru/sql-reference/data-types/multiword-types.md +++ b/docs/ru/sql-reference/data-types/multiword-types.md @@ -5,7 +5,7 @@ toc_title: Составные типы # Составные типы {#multiword-types} -При создании таблиц вы также можете использовать типы данных с названием, состоящим из нескольких слов. Это необходимо для лучшей совместимости с SQL. +При создании таблиц вы можете использовать типы данных с названием, состоящим из нескольких слов. Такие названия поддерживаются для лучшей совместимости с SQL. ## Поддержка составных типов {#multiword-types-support} From e4350e078ce1754597b5ace38d05ed5cf4e74b70 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 20 Jan 2021 00:42:31 +0300 Subject: [PATCH 18/46] Add ability to distinguish remote exceptions from local --- src/Client/Connection.cpp | 5 ++++- src/Common/Exception.cpp | 3 ++- src/Common/Exception.h | 7 ++++++- src/IO/ReadHelpers.cpp | 4 ++-- src/IO/ReadHelpers.h | 2 +- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index ef114490c51..084f169bb9f 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -803,6 +803,9 @@ Packet Connection::receivePacket(std::function async_ } catch (Exception & e) { + /// This is to consider ATTEMPT_TO_READ_AFTER_EOF as a remote exception. + e.setRemoteException(); + /// Add server address to exception message, if need. if (e.code() != ErrorCodes::UNKNOWN_PACKET_FROM_SERVER) e.addMessage("while receiving packet from " + getDescription()); @@ -892,7 +895,7 @@ void Connection::setDescription() std::unique_ptr Connection::receiveException() { - return std::make_unique(readException(*in, "Received from " + getDescription())); + return std::make_unique(readException(*in, "Received from " + getDescription(), true /* remote */)); } diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index b782471a4e8..231b45a49c6 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -50,8 +50,9 @@ void handle_error_code([[maybe_unused]] const std::string & msg, int code) ErrorCodes::increment(code); } -Exception::Exception(const std::string & msg, int code) +Exception::Exception(const std::string & msg, int code, bool remote_) : Poco::Exception(msg, code) + , remote(remote_) { handle_error_code(msg, code); } diff --git a/src/Common/Exception.h b/src/Common/Exception.h index a6190c7ca24..661d31469fe 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -25,7 +25,7 @@ class Exception : public Poco::Exception { public: Exception() = default; - Exception(const std::string & msg, int code); + Exception(const std::string & msg, int code, bool remote_ = false); Exception(const std::string & msg, const Exception & nested, int code); Exception(int code, const std::string & message) @@ -61,12 +61,17 @@ public: extendedMessage(message); } + /// Used to distinguish local exceptions from the one that was received from remote node. + void setRemoteException(bool remote_ = true) { remote = remote_; } + bool isRemoteException() const { return remote; } + std::string getStackTraceString() const; private: #ifndef STD_EXCEPTION_HAS_STACK_TRACE StackTrace trace; #endif + bool remote = false; const char * className() const throw() override { return "DB::Exception"; } }; diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index 9cd8747da64..76a722a8ad1 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -1014,7 +1014,7 @@ void skipJSONField(ReadBuffer & buf, const StringRef & name_of_field) } -Exception readException(ReadBuffer & buf, const String & additional_message) +Exception readException(ReadBuffer & buf, const String & additional_message, bool remote_exception) { int code = 0; String name; @@ -1041,7 +1041,7 @@ Exception readException(ReadBuffer & buf, const String & additional_message) if (!stack_trace.empty()) out << " Stack trace:\n\n" << stack_trace; - return Exception(out.str(), code); + return Exception(out.str(), code, remote_exception); } void readAndThrowException(ReadBuffer & buf, const String & additional_message) diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 56c795324e3..de4e87440a2 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -1073,7 +1073,7 @@ void skipJSONField(ReadBuffer & buf, const StringRef & name_of_field); * (type is cut to base class, 'message' replaced by 'displayText', and stack trace is appended to 'message') * Some additional message could be appended to exception (example: you could add information about from where it was received). */ -Exception readException(ReadBuffer & buf, const String & additional_message = ""); +Exception readException(ReadBuffer & buf, const String & additional_message = "", bool remote_exception = false); void readAndThrowException(ReadBuffer & buf, const String & additional_message = ""); From 8a0081639612b2d2221991976615b8cbf882f551 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 19 Jan 2021 22:22:58 +0300 Subject: [PATCH 19/46] Do not mark file for distributed send as broken on EOF - the sender will got ATTEMPT_TO_READ_AFTER_EOF (added in 946c275dfbe901cfec87deecc845f72215350b9d) when the client just go away, i.e. server had been restarted, and this is incorrect to mark the file as broken in this case. - since #18853 the file will be checked on the sender locally, and in case the file was truncated CANNOT_READ_ALL_DATA will be thrown. But before #18853 the sender will not receive ATTEMPT_TO_READ_AFTER_EOF from the client in case of file was truncated on the sender, since the client will just wait for more data, IOW just hang. - and I don't see how ATTEMPT_TO_READ_AFTER_EOF can be received while reading local file. --- src/Storages/Distributed/DirectoryMonitor.cpp | 35 ++++++++++++------- src/Storages/Distributed/DirectoryMonitor.h | 1 - 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 2a29f2559b6..1b0b78ba0d9 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -155,6 +155,27 @@ namespace return header; } + + /// remote_error argument is used to decide whether some errors should be + /// ignored or not, in particular: + /// + /// - ATTEMPT_TO_READ_AFTER_EOF should not be ignored + /// if we receive it from remote (receiver), since: + /// - the sender will got ATTEMPT_TO_READ_AFTER_EOF when the client just go away, + /// i.e. server had been restarted + /// - since #18853 the file will be checked on the sender locally, and + /// if there is something wrong with the file itself, we will receive + /// ATTEMPT_TO_READ_AFTER_EOF not from the remote at first + /// and mark batch as broken. + bool isFileBrokenErrorCode(int code, bool remote_error) + { + return code == ErrorCodes::CHECKSUM_DOESNT_MATCH + || code == ErrorCodes::TOO_LARGE_SIZE_COMPRESSED + || code == ErrorCodes::CANNOT_READ_ALL_DATA + || code == ErrorCodes::UNKNOWN_CODEC + || code == ErrorCodes::CANNOT_DECOMPRESS + || (!remote_error && code == ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF); + } } @@ -571,7 +592,7 @@ struct StorageDistributedDirectoryMonitor::Batch } catch (const Exception & e) { - if (isFileBrokenErrorCode(e.code())) + if (isFileBrokenErrorCode(e.code(), e.isRemoteException())) { tryLogCurrentException(parent.log, "Failed to send batch due to"); batch_broken = true; @@ -801,16 +822,6 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map } } -bool StorageDistributedDirectoryMonitor::isFileBrokenErrorCode(int code) -{ - return code == ErrorCodes::CHECKSUM_DOESNT_MATCH - || code == ErrorCodes::TOO_LARGE_SIZE_COMPRESSED - || code == ErrorCodes::CANNOT_READ_ALL_DATA - || code == ErrorCodes::UNKNOWN_CODEC - || code == ErrorCodes::CANNOT_DECOMPRESS - || code == ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF; -} - void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_path) const { const auto last_path_separator_pos = file_path.rfind('/'); @@ -837,7 +848,7 @@ void StorageDistributedDirectoryMonitor::markAsBroken(const std::string & file_p bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & file_path, const Exception & e) const { /// mark file as broken if necessary - if (isFileBrokenErrorCode(e.code())) + if (isFileBrokenErrorCode(e.code(), e.isRemoteException())) { markAsBroken(file_path); return true; diff --git a/src/Storages/Distributed/DirectoryMonitor.h b/src/Storages/Distributed/DirectoryMonitor.h index bc897136786..c73b79761ca 100644 --- a/src/Storages/Distributed/DirectoryMonitor.h +++ b/src/Storages/Distributed/DirectoryMonitor.h @@ -70,7 +70,6 @@ private: void processFile(const std::string & file_path); void processFilesWithBatching(const std::map & files); - static bool isFileBrokenErrorCode(int code); void markAsBroken(const std::string & file_path) const; bool maybeMarkAsBroken(const std::string & file_path, const Exception & e) const; From cf888740887917bb5de3e84d0f01dcbbe9890ee8 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 20 Jan 2021 15:30:41 +0300 Subject: [PATCH 20/46] done --- src/Interpreters/Context.cpp | 2 +- src/Interpreters/Context.h | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 2a8fdce869b..abd39cc3d4b 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -946,7 +946,7 @@ bool Context::hasScalar(const String & name) const void Context::addQueryAccessInfo(const String & quoted_database_name, const String & full_quoted_table_name, const Names & column_names) { assert(global_context != this || getApplicationType() == ApplicationType::LOCAL); - auto lock = getLock(); + std::lock_guard lock(query_access_info.mutex); query_access_info.databases.emplace(quoted_database_name); query_access_info.tables.emplace(full_quoted_table_name); for (const auto & column_name : column_names) diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 79140f0d209..1bd901d40c6 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -194,9 +194,36 @@ private: /// Record entities accessed by current query, and store this information in system.query_log. struct QueryAccessInfo { - std::set databases; - std::set tables; - std::set columns; + QueryAccessInfo() = default; + + QueryAccessInfo(const QueryAccessInfo & rhs) + { + std::lock_guard lock(rhs.mutex); + databases = rhs.databases; + tables = rhs.tables; + columns = rhs.columns; + } + + QueryAccessInfo(QueryAccessInfo && rhs) = delete; + + QueryAccessInfo & operator=(QueryAccessInfo rhs) + { + swap(rhs); + return *this; + } + + void swap(QueryAccessInfo & rhs) + { + std::swap(databases, rhs.databases); + std::swap(tables, rhs.tables); + std::swap(columns, rhs.columns); + } + + /// To prevent a race between copy-constructor and other uses of this structure. + mutable std::mutex mutex{}; + std::set databases{}; + std::set tables{}; + std::set columns{}; }; QueryAccessInfo query_access_info; From 1d1e53f5d3c68f82a5491e15a972983c6f7f8add Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 20 Jan 2021 15:37:53 +0300 Subject: [PATCH 21/46] style --- src/Interpreters/Context.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 1bd901d40c6..a74875376ec 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -212,7 +212,7 @@ private: return *this; } - void swap(QueryAccessInfo & rhs) + void swap(QueryAccessInfo & rhs) { std::swap(databases, rhs.databases); std::swap(tables, rhs.tables); From ac438f8ee7cedb85fef4a9888fe506e3d25e87c2 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 20 Jan 2021 17:31:56 +0300 Subject: [PATCH 22/46] add Sanitizer report issue template --- .github/ISSUE_TEMPLATE/95_sanitizer-report.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/95_sanitizer-report.md diff --git a/.github/ISSUE_TEMPLATE/95_sanitizer-report.md b/.github/ISSUE_TEMPLATE/95_sanitizer-report.md new file mode 100644 index 00000000000..4dfcf17c60f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/95_sanitizer-report.md @@ -0,0 +1,19 @@ +--- +name: Sanitizer alert +about: Potential issue has been found by special code instrumentation +title: '' +labels: testing +assignees: '' + +--- + +(you don't have to strictly follow this form) + +**Describe the bug** +A link to the report + +**How to reproduce** +Try to reproduce the report and copy the tables and queries involved. + +**Error message and/or stacktrace** +You can find additional information in server logs. From 8da4aa5bf08a2898b8e40634e1f45f69422d805d Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Wed, 20 Jan 2021 20:10:35 +0300 Subject: [PATCH 23/46] DOCSUP-5272: fix PR and ticket comments --- .../sql-reference/statements/create/table.md | 21 ++++++++++++++----- .../sql-reference/statements/create/table.md | 21 ++++++++++++++----- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 95ac0252eaa..3b7506ae89a 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -111,19 +111,30 @@ It is not possible to set default values for elements in nested data structures. You can define a [primary key](../../../engines/table-engines/mergetree-family/mergetree.md#primary-keys-and-indexes-in-queries) when creating a table. Primary key can be specified in two ways: -- inside the column list +- Inside the column list ``` sql -CREATE TABLE db.table_name (name1 type1, name2 type2, ..., PRIMARY KEY (expr1[, expr2,...])]) ENGINE = engine; +CREATE TABLE db.table_name +( + name1 type1, name2 type2, ..., + PRIMARY KEY(expr1[, expr2,...])] +) +ENGINE = engine; ``` -- outside the column list +- Outside the column list ``` sql -CREATE TABLE db.table_name (name1 type1, name2 type2, ...) ENGINE = engine PRIMARY KEY(expr1[, expr2,...]); +CREATE TABLE db.table_name +( + name1 type1, name2 type2, ... +) +ENGINE = engine +PRIMARY KEY(expr1[, expr2,...]); ``` -You can't combine both ways in one query. +!!! warning "Warning" + You can't combine both ways in one query. ## Constraints {#constraints} diff --git a/docs/ru/sql-reference/statements/create/table.md b/docs/ru/sql-reference/statements/create/table.md index e91a7f15903..5244774c58c 100644 --- a/docs/ru/sql-reference/statements/create/table.md +++ b/docs/ru/sql-reference/statements/create/table.md @@ -85,19 +85,30 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ... Вы можете определить [первичный ключ](../../../engines/table-engines/mergetree-family/mergetree.md#primary-keys-and-indexes-in-queries) при создании таблицы. Первичный ключ может быть указан двумя способами: -- В списке столбцов: +- в списке столбцов: ``` sql -CREATE TABLE db.table_name (name1 type1, name2 type2, ..., PRIMARY KEY (expr1[, expr2,...])]) ENGINE = engine; +CREATE TABLE db.table_name +( + name1 type1, name2 type2, ..., + PRIMARY KEY(expr1[, expr2,...])] +) +ENGINE = engine; ``` -- Вне списка столбцов: +- вне списка столбцов: ``` sql -CREATE TABLE db.table_name (name1 type1, name2 type2, ...) ENGINE = engine PRIMARY KEY(expr1[, expr2,...]); +CREATE TABLE db.table_name +( + name1 type1, name2 type2, ... +) +ENGINE = engine +PRIMARY KEY(expr1[, expr2,...]); ``` -Вы не можете сочетать оба способа в одном запросе. +!!! warning "Предупреждение" + Вы не можете сочетать оба способа в одном запросе. ### Ограничения (constraints) {#constraints} From 7d61f27abb2cd8ecd8a0d09657035c70441f58e8 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 20 Jan 2021 20:51:14 +0300 Subject: [PATCH 24/46] Fix ansi.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Разрешаю конфликт с мастером. --- docs/en/sql-reference/ansi.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/sql-reference/ansi.md b/docs/en/sql-reference/ansi.md index 84e47902f3b..18243a5f9f5 100644 --- a/docs/en/sql-reference/ansi.md +++ b/docs/en/sql-reference/ansi.md @@ -25,15 +25,15 @@ The following table lists cases when query feature works in ClickHouse, but beha |------------|--------------------------------------------------------------------------------------------------------------------------|----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | **E011** | **Numeric data types** | **Partial**{.text-warning} | | | E011-01 | INTEGER and SMALLINT data types | Yes {.text-success} | | -| E011-02 | REAL, DOUBLE PRECISION and FLOAT data types data types | Yes {.text-warning} | | -| E011-03 | DECIMAL and NUMERIC data types | Partial {.text-warning} | Only `DECIMAL(p,s)` is supported, not `NUMERIC` | +| E011-02 | REAL, DOUBLE PRECISION and FLOAT data types data types | Yes {.text-success} | | +| E011-03 | DECIMAL and NUMERIC data types | Yes {.text-success} | | | E011-04 | Arithmetic operators | Yes {.text-success} | | | E011-05 | Numeric comparison | Yes {.text-success} | | | E011-06 | Implicit casting among the numeric data types | No {.text-danger} | ANSI SQL allows arbitrary implicit cast between numeric types, while ClickHouse relies on functions having multiple overloads instead of implicit cast | | **E021** | **Character string types** | **Partial**{.text-warning} | | -| E021-01 | CHARACTER data type | Yes {.text-danger} | | -| E021-02 | CHARACTER VARYING data type | Yes {.text-danger} | | -| E021-03 | Character literals | Partial {.text-warning} | No automatic concatenation of consecutive literals and character set support | +| E021-01 | CHARACTER data type | Yes {.text-success} | | +| E021-02 | CHARACTER VARYING data type | Yes {.text-success} | | +| E021-03 | Character literals | Yes {.text-success} | | | E021-04 | CHARACTER_LENGTH function | Partial {.text-warning} | No `USING` clause | | E021-05 | OCTET_LENGTH function | No {.text-danger} | `LENGTH` behaves similarly | | E021-06 | SUBSTRING | Partial {.text-warning} | No support for `SIMILAR` and `ESCAPE` clauses, no `SUBSTRING_REGEX` variant | From 2d87e52b3a2a9db9db6a045b8c24a4dc88a29e5f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Jan 2021 03:24:44 +0300 Subject: [PATCH 25/46] Fix bad formatting of IPv4 addresses --- src/DataTypes/DataTypeNumberBase.h | 8 +++++++- src/DataTypes/IDataType.h | 2 +- .../0_stateless/01656_ipv4_bad_formatting.reference | 4 ++++ tests/queries/0_stateless/01656_ipv4_bad_formatting.sql | 1 + 4 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/01656_ipv4_bad_formatting.reference create mode 100644 tests/queries/0_stateless/01656_ipv4_bad_formatting.sql diff --git a/src/DataTypes/DataTypeNumberBase.h b/src/DataTypes/DataTypeNumberBase.h index 0390da2cb6f..cbbc203bf4f 100644 --- a/src/DataTypes/DataTypeNumberBase.h +++ b/src/DataTypes/DataTypeNumberBase.h @@ -51,7 +51,13 @@ public: bool isParametric() const override { return false; } bool haveSubtypes() const override { return false; } - bool shouldAlignRightInPrettyFormats() const override { return true; } + + bool shouldAlignRightInPrettyFormats() const override + { + /// Just a number, without customizations. Counterexample: IPv4. + return !custom_text_serialization; + } + bool textCanContainOnlyValidUTF8() const override { return true; } bool isComparable() const override { return true; } bool isValueRepresentedByNumber() const override { return true; } diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index b67c5ee1846..cb9fc7f122c 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -497,7 +497,7 @@ public: /// For all other substreams (like ArraySizes, NullMasks, etc.) we use only /// generic compression codecs like LZ4. static bool isSpecialCompressionAllowed(const SubstreamPath & path); -private: +protected: friend class DataTypeFactory; friend class AggregateFunctionSimpleState; /// Customize this DataType diff --git a/tests/queries/0_stateless/01656_ipv4_bad_formatting.reference b/tests/queries/0_stateless/01656_ipv4_bad_formatting.reference new file mode 100644 index 00000000000..a7b5c448f13 --- /dev/null +++ b/tests/queries/0_stateless/01656_ipv4_bad_formatting.reference @@ -0,0 +1,4 @@ +┌─x───────────────┬─y───────────────┬──────────z─┐ +│ 1.1.1.1 │ 1.1.1.1 │ 16843009 │ +│ 255.255.255.255 │ 255.255.255.255 │ 4294967295 │ +└─────────────────┴─────────────────┴────────────┘ diff --git a/tests/queries/0_stateless/01656_ipv4_bad_formatting.sql b/tests/queries/0_stateless/01656_ipv4_bad_formatting.sql new file mode 100644 index 00000000000..a0b253ea31a --- /dev/null +++ b/tests/queries/0_stateless/01656_ipv4_bad_formatting.sql @@ -0,0 +1 @@ +SELECT arrayJoin(['1.1.1.1', '255.255.255.255']) AS x, toIPv4(x) AS y, toUInt32(y) AS z FORMAT PrettyCompactNoEscapes; From 2094eae23df8ae19002ff65af842b5ed6913798b Mon Sep 17 00:00:00 2001 From: feng lv Date: Thu, 21 Jan 2021 04:49:35 +0000 Subject: [PATCH 26/46] fix sleep with infinite input --- src/Functions/sleep.h | 4 ++-- .../queries/0_stateless/01655_sleep_infinite_float.reference | 0 tests/queries/0_stateless/01655_sleep_infinite_float.sql | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/01655_sleep_infinite_float.reference create mode 100644 tests/queries/0_stateless/01655_sleep_infinite_float.sql diff --git a/src/Functions/sleep.h b/src/Functions/sleep.h index 6dca6b5f84c..65566e36d1f 100644 --- a/src/Functions/sleep.h +++ b/src/Functions/sleep.h @@ -78,8 +78,8 @@ public: Float64 seconds = applyVisitor(FieldVisitorConvertToNumber(), assert_cast(*col).getField()); - if (seconds < 0) - throw Exception("Cannot sleep negative amount of time (not implemented)", ErrorCodes::BAD_ARGUMENTS); + if (seconds < 0 || !std::isfinite(seconds)) + throw Exception("Cannot sleep infinite or negative amount of time (not implemented)", ErrorCodes::BAD_ARGUMENTS); size_t size = col->size(); diff --git a/tests/queries/0_stateless/01655_sleep_infinite_float.reference b/tests/queries/0_stateless/01655_sleep_infinite_float.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01655_sleep_infinite_float.sql b/tests/queries/0_stateless/01655_sleep_infinite_float.sql new file mode 100644 index 00000000000..a469ba9674a --- /dev/null +++ b/tests/queries/0_stateless/01655_sleep_infinite_float.sql @@ -0,0 +1,2 @@ +SELECT sleep(nan); -- { serverError 36 } +SELECT sleep(inf); -- { serverError 36 } From 2edf69fe6459657a0b1783b3236d1ce7445db308 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Thu, 21 Jan 2021 09:17:12 +0300 Subject: [PATCH 27/46] Fix --- docs/en/operations/caches.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/caches.md b/docs/en/operations/caches.md index 0107c340019..0323fc84ef4 100644 --- a/docs/en/operations/caches.md +++ b/docs/en/operations/caches.md @@ -19,6 +19,6 @@ Additional cache types: - [dictionaries data cache](../sql-reference/dictionaries/index.md) Not directly used: -- page cache OS +- OS page cache [Original article](https://clickhouse.tech/docs/en/operations/caches/) From 1bb8cc5c9ab8659b97082abc2a678cbe857c92ce Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Jan 2021 11:10:31 +0300 Subject: [PATCH 28/46] Avoid UBSan report in arrayElement --- src/Functions/array/arrayElement.cpp | 27 ++++++++++++++----- .../01657_array_element_ubsan.reference | 26 ++++++++++++++++++ .../0_stateless/01657_array_element_ubsan.sql | 19 +++++++++++++ 3 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 tests/queries/0_stateless/01657_array_element_ubsan.reference create mode 100644 tests/queries/0_stateless/01657_array_element_ubsan.sql diff --git a/src/Functions/array/arrayElement.cpp b/src/Functions/array/arrayElement.cpp index 88166f04e0e..7d053988cae 100644 --- a/src/Functions/array/arrayElement.cpp +++ b/src/Functions/array/arrayElement.cpp @@ -231,7 +231,7 @@ struct ArrayElementNumImpl if (builder) builder.update(j); } - else if (index < 0 && static_cast(-index) <= array_size) + else if (index < 0 && -static_cast(index) <= array_size) { size_t j = offsets[i] + index; result[i] = data[j]; @@ -329,7 +329,7 @@ struct ArrayElementStringImpl TIndex index = indices[i]; if (index > 0 && static_cast(index) <= array_size) adjusted_index = index - 1; - else if (index < 0 && static_cast(-index) <= array_size) + else if (index < 0 && -static_cast(index) <= array_size) adjusted_index = array_size + index; else adjusted_index = array_size; /// means no element should be taken @@ -427,7 +427,7 @@ struct ArrayElementGenericImpl if (builder) builder.update(j); } - else if (index < 0 && static_cast(-index) <= array_size) + else if (index < 0 && -static_cast(index) <= array_size) { size_t j = offsets[i] + index; result.insertFrom(data, j); @@ -472,11 +472,24 @@ ColumnPtr FunctionArrayElement::executeNumberConst( auto col_res = ColumnVector::create(); if (index.getType() == Field::Types::UInt64) + { ArrayElementNumImpl::template vectorConst( col_nested->getData(), col_array->getOffsets(), safeGet(index) - 1, col_res->getData(), builder); + } else if (index.getType() == Field::Types::Int64) + { + /// Cast to UInt64 before negation allows to avoid undefined behaviour for negation of the most negative number. + /// NOTE: this would be undefined behaviour in C++ sense, but nevertheless, compiler cannot see it on user provided data, + /// and generates the code that we want on supported CPU architectures (overflow in sense of two's complement arithmetic). + /// This is only needed to avoid UBSan report. + + /// Negative array indices work this way: + /// arr[-1] is the element at offset 0 from the last + /// arr[-2] is the element at offset 1 from the last and so on. + ArrayElementNumImpl::template vectorConst( - col_nested->getData(), col_array->getOffsets(), -safeGet(index) - 1, col_res->getData(), builder); + col_nested->getData(), col_array->getOffsets(), -(UInt64(safeGet(index)) + 1), col_res->getData(), builder); + } else throw Exception("Illegal type of array index", ErrorCodes::LOGICAL_ERROR); @@ -534,7 +547,7 @@ FunctionArrayElement::executeStringConst(const ColumnsWithTypeAndName & argument col_nested->getChars(), col_array->getOffsets(), col_nested->getOffsets(), - -safeGet(index) - 1, + -(UInt64(safeGet(index)) + 1), col_res->getChars(), col_res->getOffsets(), builder); @@ -588,7 +601,7 @@ ColumnPtr FunctionArrayElement::executeGenericConst( col_nested, col_array->getOffsets(), safeGet(index) - 1, *col_res, builder); else if (index.getType() == Field::Types::Int64) ArrayElementGenericImpl::vectorConst( - col_nested, col_array->getOffsets(), -safeGet(index) - 1, *col_res, builder); + col_nested, col_array->getOffsets(), -(UInt64(safeGet(index) + 1)), *col_res, builder); else throw Exception("Illegal type of array index", ErrorCodes::LOGICAL_ERROR); @@ -639,7 +652,7 @@ ColumnPtr FunctionArrayElement::executeConst(const ColumnsWithTypeAndName & argu if (builder) builder.update(j); } - else if (index < 0 && static_cast(-index) <= array_size) + else if (index < 0 && -static_cast(index) <= array_size) { size_t j = array_size + index; res->insertFrom(array_elements, j); diff --git a/tests/queries/0_stateless/01657_array_element_ubsan.reference b/tests/queries/0_stateless/01657_array_element_ubsan.reference new file mode 100644 index 00000000000..14e3161f529 --- /dev/null +++ b/tests/queries/0_stateless/01657_array_element_ubsan.reference @@ -0,0 +1,26 @@ +0 +0 +0 +0 +--- +0 +0 +0 +--- +0 +0 +0 +0 +0 +0 +0 +1 +--- +0 +0 +0 +0 +0 +0 +0 +1 diff --git a/tests/queries/0_stateless/01657_array_element_ubsan.sql b/tests/queries/0_stateless/01657_array_element_ubsan.sql new file mode 100644 index 00000000000..82ddf643389 --- /dev/null +++ b/tests/queries/0_stateless/01657_array_element_ubsan.sql @@ -0,0 +1,19 @@ +SELECT [number][10000000000] FROM numbers(1); +SELECT [number][-10000000000] FROM numbers(1); + +SELECT [number][-0x8000000000000000] FROM numbers(1); +SELECT [number][0xFFFFFFFFFFFFFFFF] FROM numbers(1); + +SELECT '---'; + +SELECT [materialize(1)][0xFFFFFFFFFFFFFFFF]; +SELECT [materialize(1)][materialize(18446744073709551615)]; +SELECT [materialize(1)][-0x8000000000000000]; + +SELECT '---'; + +SELECT [number][arrayJoin([-0x8000000000000000, -10000000000, 0, -1])] FROM numbers(2); + +SELECT '---'; + +SELECT [number][arrayJoin([0xFFFFFFFFFFFFFFFF, 10000000000, 0, 1])] FROM numbers(2); From 110089086bbf0148743a92ef14fe5f51e287b5a5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Jan 2021 11:41:19 +0300 Subject: [PATCH 29/46] Fix UBSan report in GatherUtils #19287 --- src/Functions/GatherUtils/Algorithms.h | 6 +++--- .../0_stateless/01658_substring_ubsan.reference | 0 tests/queries/0_stateless/01658_substring_ubsan.sql | 10 ++++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/01658_substring_ubsan.reference create mode 100644 tests/queries/0_stateless/01658_substring_ubsan.sql diff --git a/src/Functions/GatherUtils/Algorithms.h b/src/Functions/GatherUtils/Algorithms.h index 620d6439af2..101e1354bc6 100644 --- a/src/Functions/GatherUtils/Algorithms.h +++ b/src/Functions/GatherUtils/Algorithms.h @@ -342,7 +342,7 @@ void NO_INLINE sliceDynamicOffsetUnbounded(Source && src, Sink && sink, const IC if (offset > 0) slice = src.getSliceFromLeft(offset - 1); else - slice = src.getSliceFromRight(-offset); + slice = src.getSliceFromRight(-UInt64(offset)); writeSlice(slice, sink); } @@ -374,7 +374,7 @@ void NO_INLINE sliceDynamicOffsetBounded(Source && src, Sink && sink, const ICol Int64 size = has_length ? length_nested_column->getInt(row_num) : static_cast(src.getElementSize()); if (size < 0) - size += offset > 0 ? static_cast(src.getElementSize()) - (offset - 1) : -offset; + size += offset > 0 ? static_cast(src.getElementSize()) - (offset - 1) : -UInt64(offset); if (offset != 0 && size > 0) { @@ -383,7 +383,7 @@ void NO_INLINE sliceDynamicOffsetBounded(Source && src, Sink && sink, const ICol if (offset > 0) slice = src.getSliceFromLeft(offset - 1, size); else - slice = src.getSliceFromRight(-offset, size); + slice = src.getSliceFromRight(-UInt64(offset), size); writeSlice(slice, sink); } diff --git a/tests/queries/0_stateless/01658_substring_ubsan.reference b/tests/queries/0_stateless/01658_substring_ubsan.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01658_substring_ubsan.sql b/tests/queries/0_stateless/01658_substring_ubsan.sql new file mode 100644 index 00000000000..3d7968b8d6b --- /dev/null +++ b/tests/queries/0_stateless/01658_substring_ubsan.sql @@ -0,0 +1,10 @@ +/** NOTE: The behaviour of substring and substringUTF8 is inconsistent when negative offset is greater than string size: + * substring: + * hello + * ^-----^ - offset -10, length 7, result: "he" + * substringUTF8: + * hello + * ^-----^ - offset -10, length 7, result: "hello" + * This may be subject for change. + */ +SELECT substringUTF8('hello, пÑ�ивеÑ�', -9223372036854775808, number) FROM numbers(16) FORMAT Null; From 2cd04e8923051323b906c33786ccac58cff87eae Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Jan 2021 14:42:34 +0300 Subject: [PATCH 30/46] Fix UBSan report in arraySum --- src/Functions/array/arrayAggregation.cpp | 3 ++- .../0_stateless/01659_array_aggregation_ubsan.reference | 1 + tests/queries/0_stateless/01659_array_aggregation_ubsan.sql | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01659_array_aggregation_ubsan.reference create mode 100644 tests/queries/0_stateless/01659_array_aggregation_ubsan.sql diff --git a/src/Functions/array/arrayAggregation.cpp b/src/Functions/array/arrayAggregation.cpp index 992a331d05b..40afd657abb 100644 --- a/src/Functions/array/arrayAggregation.cpp +++ b/src/Functions/array/arrayAggregation.cpp @@ -5,6 +5,7 @@ #include #include "FunctionArrayMapped.h" #include +#include namespace DB @@ -121,7 +122,7 @@ struct ArrayAggregateImpl } template - static bool executeType(const ColumnPtr & mapped, const ColumnArray::Offsets & offsets, ColumnPtr & res_ptr) + static NO_SANITIZE_UNDEFINED bool executeType(const ColumnPtr & mapped, const ColumnArray::Offsets & offsets, ColumnPtr & res_ptr) { using Result = ArrayAggregateResult; using ColVecType = std::conditional_t, ColumnDecimal, ColumnVector>; diff --git a/tests/queries/0_stateless/01659_array_aggregation_ubsan.reference b/tests/queries/0_stateless/01659_array_aggregation_ubsan.reference new file mode 100644 index 00000000000..62c80bed251 --- /dev/null +++ b/tests/queries/0_stateless/01659_array_aggregation_ubsan.reference @@ -0,0 +1 @@ +446744073709551616 diff --git a/tests/queries/0_stateless/01659_array_aggregation_ubsan.sql b/tests/queries/0_stateless/01659_array_aggregation_ubsan.sql new file mode 100644 index 00000000000..1b8b506b26e --- /dev/null +++ b/tests/queries/0_stateless/01659_array_aggregation_ubsan.sql @@ -0,0 +1 @@ +SELECT arraySum([-9000000000000000000, -9000000000000000000]); From 4968f1733e2a284215889f947d03246e4a4fd56d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Jan 2021 15:01:24 +0300 Subject: [PATCH 31/46] Annotations for DateLUT --- base/common/DateLUTImpl.h | 8 ++++---- src/Functions/FunctionDateOrDateTimeAddInterval.h | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 3698276d0e1..ef74182b333 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -780,7 +780,7 @@ public: return lut[index].date + time_offset; } - inline time_t addWeeks(time_t t, Int64 delta) const + inline NO_SANITIZE_UNDEFINED time_t addWeeks(time_t t, Int64 delta) const { return addDays(t, delta * 7); } @@ -812,7 +812,7 @@ public: return lut[result_day].date + time_offset; } - inline DayNum addMonths(DayNum d, Int64 delta) const + inline NO_SANITIZE_UNDEFINED DayNum addMonths(DayNum d, Int64 delta) const { const Values & values = lut[d]; @@ -836,12 +836,12 @@ public: } } - inline time_t addQuarters(time_t t, Int64 delta) const + inline NO_SANITIZE_UNDEFINED time_t addQuarters(time_t t, Int64 delta) const { return addMonths(t, delta * 3); } - inline DayNum addQuarters(DayNum d, Int64 delta) const + inline NO_SANITIZE_UNDEFINED DayNum addQuarters(DayNum d, Int64 delta) const { return addMonths(d, delta * 3); } diff --git a/src/Functions/FunctionDateOrDateTimeAddInterval.h b/src/Functions/FunctionDateOrDateTimeAddInterval.h index 70e2616eeac..aa753916987 100644 --- a/src/Functions/FunctionDateOrDateTimeAddInterval.h +++ b/src/Functions/FunctionDateOrDateTimeAddInterval.h @@ -224,8 +224,9 @@ struct SubtractIntervalImpl : public Transform using Transform::Transform; template - inline auto execute(T t, Int64 delta, const DateLUTImpl & time_zone) const + inline NO_SANITIZE_UNDEFINED auto execute(T t, Int64 delta, const DateLUTImpl & time_zone) const { + /// Signed integer overflow is Ok. return Transform::execute(t, -delta, time_zone); } }; From 56012402e2b9d7e8b657f6a140e478378dc747c8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Jan 2021 15:19:00 +0300 Subject: [PATCH 32/46] Fix potential nullptr dereference in table function VALUES --- src/TableFunctions/TableFunctionValues.cpp | 10 +++++++--- tests/queries/0_stateless/01658_values_ubsan.reference | 1 + tests/queries/0_stateless/01658_values_ubsan.sql | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/01658_values_ubsan.reference create mode 100644 tests/queries/0_stateless/01658_values_ubsan.sql diff --git a/src/TableFunctions/TableFunctionValues.cpp b/src/TableFunctions/TableFunctionValues.cpp index 2d01d581fda..4127a30892f 100644 --- a/src/TableFunctions/TableFunctionValues.cpp +++ b/src/TableFunctions/TableFunctionValues.cpp @@ -47,12 +47,18 @@ static void parseAndInsertValues(MutableColumns & res_columns, const ASTs & args for (size_t i = 1; i < args.size(); ++i) { const auto & [value_field, value_type_ptr] = evaluateConstantExpression(args[i], context); - const DataTypes & value_types_tuple = typeid_cast(value_type_ptr.get())->getElements(); + + const DataTypeTuple * type_tuple = typeid_cast(value_type_ptr.get()); + if (!type_tuple) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Table function VALUES requires all but first argument (rows specification) to be either tuples or single values"); + const Tuple & value_tuple = value_field.safeGet(); if (value_tuple.size() != sample_block.columns()) throw Exception("Values size should match with number of columns", ErrorCodes::BAD_ARGUMENTS); + const DataTypes & value_types_tuple = type_tuple->getElements(); for (size_t j = 0; j < value_tuple.size(); ++j) { Field value = convertFieldToTypeOrThrow(value_tuple[j], *sample_block.getByPosition(j).type, value_types_tuple[j].get()); @@ -64,8 +70,6 @@ static void parseAndInsertValues(MutableColumns & res_columns, const ASTs & args void TableFunctionValues::parseArguments(const ASTPtr & ast_function, const Context & /*context*/) { - - ASTs & args_func = ast_function->children; if (args_func.size() != 1) diff --git a/tests/queries/0_stateless/01658_values_ubsan.reference b/tests/queries/0_stateless/01658_values_ubsan.reference new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/queries/0_stateless/01658_values_ubsan.reference @@ -0,0 +1 @@ + diff --git a/tests/queries/0_stateless/01658_values_ubsan.sql b/tests/queries/0_stateless/01658_values_ubsan.sql new file mode 100644 index 00000000000..10d17c2f00a --- /dev/null +++ b/tests/queries/0_stateless/01658_values_ubsan.sql @@ -0,0 +1 @@ +SELECT * FROM VALUES('x UInt8, y UInt16', 1 + 2, 'Hello'); -- { serverError 36 } From 375db8ce7e2a042229fb0d0d26ff7c7a74c8d73c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Jan 2021 16:42:47 +0300 Subject: [PATCH 33/46] Update reference --- tests/queries/0_stateless/01658_values_ubsan.reference | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/01658_values_ubsan.reference b/tests/queries/0_stateless/01658_values_ubsan.reference index 8b137891791..e69de29bb2d 100644 --- a/tests/queries/0_stateless/01658_values_ubsan.reference +++ b/tests/queries/0_stateless/01658_values_ubsan.reference @@ -1 +0,0 @@ - From 3b2a87b57e11596a96db28ff45e0e4ae88f46e80 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 21 Jan 2021 16:48:07 +0300 Subject: [PATCH 34/46] Update SECURITY.md (outdated long time ago) --- SECURITY.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 6d64dd2d24b..906b2966348 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -15,9 +15,13 @@ currently being supported with security updates: | 20.4 | :x: | | 20.5 | :x: | | 20.6 | :x: | -| 20.7 | :white_check_mark: | +| 20.7 | :x: | | 20.8 | :white_check_mark: | -| 20.9 | :white_check_mark: | +| 20.9 | :x: | +| 20.10 | :x: | +| 20.11 | :white_check_mark: | +| 20.12 | :white_check_mark: | +| 21.1 | :white_check_mark: | ## Reporting a Vulnerability From 0a155ee2f150d1aaed2c223ea7ec7bb816132ce5 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Thu, 21 Jan 2021 19:38:13 +0300 Subject: [PATCH 35/46] Fix IDisk::open parameters to match posix open --- src/Disks/DiskDecorator.cpp | 4 ++-- src/Disks/DiskDecorator.h | 2 +- src/Disks/DiskLocal.cpp | 4 ++-- src/Disks/DiskLocal.h | 2 +- src/Disks/DiskMemory.cpp | 2 +- src/Disks/DiskMemory.h | 2 +- src/Disks/IDisk.h | 2 +- src/Disks/S3/DiskS3.cpp | 2 +- src/Disks/S3/DiskS3.h | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Disks/DiskDecorator.cpp b/src/Disks/DiskDecorator.cpp index a475f85f9d2..6e8a2798212 100644 --- a/src/Disks/DiskDecorator.cpp +++ b/src/Disks/DiskDecorator.cpp @@ -175,9 +175,9 @@ void DiskDecorator::truncateFile(const String & path, size_t size) delegate->truncateFile(path, size); } -int DiskDecorator::open(const String & path, mode_t mode) const +int DiskDecorator::open(const String & path, int flags) const { - return delegate->open(path, mode); + return delegate->open(path, flags); } void DiskDecorator::close(int fd) const diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index 87668e82a3d..4414959cca3 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -48,7 +48,7 @@ public: void setReadOnly(const String & path) override; void createHardLink(const String & src_path, const String & dst_path) override; void truncateFile(const String & path, size_t size) override; - int open(const String & path, mode_t mode) const override; + int open(const String & path, int flags) const override; void close(int fd) const override; void sync(int fd) const override; const String getType() const override { return delegate->getType(); } diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index 2193d2a2a73..ed6b82fa73d 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -315,10 +315,10 @@ void DiskLocal::copy(const String & from_path, const std::shared_ptr & to IDisk::copy(from_path, to_disk, to_path); /// Copy files through buffers. } -int DiskLocal::open(const String & path, mode_t mode) const +int DiskLocal::open(const String & path, int flags) const { String full_path = disk_path + path; - int fd = ::open(full_path.c_str(), mode); + int fd = ::open(full_path.c_str(), flags); if (-1 == fd) throwFromErrnoWithPath("Cannot open file " + full_path, full_path, errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE); diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h index e82bac57825..09f08a01d27 100644 --- a/src/Disks/DiskLocal.h +++ b/src/Disks/DiskLocal.h @@ -98,7 +98,7 @@ public: void createHardLink(const String & src_path, const String & dst_path) override; - int open(const String & path, mode_t mode) const override; + int open(const String & path, int flags) const override; void close(int fd) const override; void sync(int fd) const override; diff --git a/src/Disks/DiskMemory.cpp b/src/Disks/DiskMemory.cpp index a3e57f32dcc..7317d8a75be 100644 --- a/src/Disks/DiskMemory.cpp +++ b/src/Disks/DiskMemory.cpp @@ -436,7 +436,7 @@ void DiskMemory::setReadOnly(const String &) throw Exception("Method setReadOnly is not implemented for memory disks", ErrorCodes::NOT_IMPLEMENTED); } -int DiskMemory::open(const String & /*path*/, mode_t /*mode*/) const +int DiskMemory::open(const String & /*path*/, int /*flags*/) const { throw Exception("Method open is not implemented for memory disks", ErrorCodes::NOT_IMPLEMENTED); } diff --git a/src/Disks/DiskMemory.h b/src/Disks/DiskMemory.h index f8c2815b269..10d1dc670af 100644 --- a/src/Disks/DiskMemory.h +++ b/src/Disks/DiskMemory.h @@ -89,7 +89,7 @@ public: void createHardLink(const String & src_path, const String & dst_path) override; - int open(const String & path, mode_t mode) const override; + int open(const String & path, int flags) const override; void close(int fd) const override; void sync(int fd) const override; diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 36f07c3b110..b68bca07de1 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -175,7 +175,7 @@ public: virtual void createHardLink(const String & src_path, const String & dst_path) = 0; /// Wrapper for POSIX open - virtual int open(const String & path, mode_t mode) const = 0; + virtual int open(const String & path, int flags) const = 0; /// Wrapper for POSIX close virtual void close(int fd) const = 0; diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 9a60879085e..081c62bd047 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -878,7 +878,7 @@ void DiskS3::setReadOnly(const String & path) metadata.save(); } -int DiskS3::open(const String & /*path*/, mode_t /*mode*/) const +int DiskS3::open(const String & /*path*/, int /*flags*/) const { throw Exception("Method open is not implemented for S3 disks", ErrorCodes::NOT_IMPLEMENTED); } diff --git a/src/Disks/S3/DiskS3.h b/src/Disks/S3/DiskS3.h index abdc049a58e..7f742368d19 100644 --- a/src/Disks/S3/DiskS3.h +++ b/src/Disks/S3/DiskS3.h @@ -105,7 +105,7 @@ public: void setReadOnly(const String & path) override; - int open(const String & path, mode_t mode) const override; + int open(const String & path, int flags) const override; void close(int fd) const override; void sync(int fd) const override; From e8a9768bd0daf9c4bf0362b1fedb8d8625c4488b Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 21 Jan 2021 20:47:57 +0300 Subject: [PATCH 36/46] Update map.cpp --- src/Functions/map.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 2561f06d9b8..25b3a96a9df 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -65,7 +65,7 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (arguments.size() % 2 != 0) - throw Exception("Function " + getName() + " even number of arguments.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception("Function " + getName() + " even number of arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); DataTypes keys, values; for (size_t i = 0; i < arguments.size(); i += 2) @@ -164,7 +164,7 @@ public: const DataTypeMap * map_type = checkAndGetDataType(arguments[0].type.get()); if (!map_type) - throw Exception{"First argument for function " + getName() + " must be a map.", + throw Exception{"First argument for function " + getName() + " must be a map", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; auto key_type = map_type->getKeyType(); @@ -225,7 +225,7 @@ public: const DataTypeMap * map_type = checkAndGetDataType(arguments[0].type.get()); if (!map_type) - throw Exception{"First argument for function " + getName() + " must be a map.", + throw Exception{"First argument for function " + getName() + " must be a map", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; auto key_type = map_type->getKeyType(); @@ -270,7 +270,7 @@ public: const DataTypeMap * map_type = checkAndGetDataType(arguments[0].type.get()); if (!map_type) - throw Exception{"First argument for function " + getName() + " must be a map.", + throw Exception{"First argument for function " + getName() + " must be a map", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; auto value_type = map_type->getValueType(); From e9e331402634cdce8265a3a19612c9931d533396 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 21 Jan 2021 23:05:35 +0300 Subject: [PATCH 37/46] Update run-fuzzer.sh --- docker/test/fuzzer/run-fuzzer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 2c44f44bd65..200c7b6f4ce 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -75,7 +75,7 @@ function fuzz { # Obtain the list of newly added tests. They will be fuzzed in more extreme way than other tests. cd ch - NEW_TESTS=$(git diff --name-only master "$SHA_TO_TEST" | grep -P 'tests/queries/0_stateless/.*\.sql' | sed -r -e 's!^!ch/!' | sort -R) + NEW_TESTS=$(git diff --name-only origin/master "$SHA_TO_TEST" | grep -P 'tests/queries/0_stateless/.*\.sql' | sed -r -e 's!^!ch/!' | sort -R) cd .. if [[ -n "$NEW_TESTS" ]] then From 56687b2d84c23afbc43bd7a8e67d9efaa7d72fd4 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 21 Jan 2021 23:29:01 +0300 Subject: [PATCH 38/46] Update run.sh --- docker/test/fasttest/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 437740506e5..fb3b3e3b459 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -332,7 +332,7 @@ function run_tests 01622_defaults_for_url_engine ) - time clickhouse-test -j 8 --order=random --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" -- "$FASTTEST_FOCUS" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt" + time clickhouse-test -j 8 --order=random --use-skip-list --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" -- "$FASTTEST_FOCUS" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt" # substr is to remove semicolon after test name readarray -t FAILED_TESTS < <(awk '/\[ FAIL|TIMEOUT|ERROR \]/ { print substr($3, 1, length($3)-1) }' "$FASTTEST_OUTPUT/test_log.txt" | tee "$FASTTEST_OUTPUT/failed-parallel-tests.txt") From cf5c0dad94eff60d0ddec72abcf63c3c4b983250 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Thu, 21 Jan 2021 23:56:29 +0300 Subject: [PATCH 39/46] Minor fix --- docs/en/operations/caches.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/caches.md b/docs/en/operations/caches.md index 0323fc84ef4..66f2eaeadc9 100644 --- a/docs/en/operations/caches.md +++ b/docs/en/operations/caches.md @@ -18,7 +18,7 @@ Additional cache types: - [Avro format](../interfaces/formats.md#data-format-avro) schemas cache - [dictionaries data cache](../sql-reference/dictionaries/index.md) -Not directly used: +Indirectly used: - OS page cache [Original article](https://clickhouse.tech/docs/en/operations/caches/) From dcd03eb15508db5e0ed37d5bc5a00cb8247ede26 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 22 Jan 2021 00:56:49 +0300 Subject: [PATCH 40/46] Remove useless file --- src/Common/bitonicSortKernels.cl | 1250 ------------------------------ 1 file changed, 1250 deletions(-) delete mode 100644 src/Common/bitonicSortKernels.cl diff --git a/src/Common/bitonicSortKernels.cl b/src/Common/bitonicSortKernels.cl deleted file mode 100644 index c124dfe85bc..00000000000 --- a/src/Common/bitonicSortKernels.cl +++ /dev/null @@ -1,1250 +0,0 @@ -// Copyright (c) 2009-2011 Intel Corporation -// All rights reserved. -// -// WARRANTY DISCLAIMER -// -// THESE MATERIALS ARE PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL OR ITS -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THESE -// MATERIALS, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Intel Corporation is the author of the Materials, and requests that all -// problem reports or change requests be submitted to it directly - -static const char * bitonic_sort_kernels = R"( - -uchar4 makeMask_uchar(uchar4 srcLeft, uchar4 srcRight, uint4 indexLeft, uint4 indexRight) -{ - uchar4 mask = convert_uchar4(srcLeft < srcRight); - - if (srcLeft.x == srcRight.x) - mask.x = (indexLeft.x < indexRight.x) ? UCHAR_MAX : 0; - if (srcLeft.y == srcRight.y) - mask.y = (indexLeft.y < indexRight.y) ? UCHAR_MAX : 0; - if (srcLeft.z == srcRight.z) - mask.z = (indexLeft.z < indexRight.z) ? UCHAR_MAX : 0; - if (srcLeft.w == srcRight.w) - mask.w = (indexLeft.w < indexRight.w) ? UCHAR_MAX : 0; - - return mask; -} - - -char4 makeMask_char(char4 srcLeft, char4 srcRight, uint4 indexLeft, uint4 indexRight) -{ - char4 mask = srcLeft < srcRight; - - if (srcLeft.x == srcRight.x) - mask.x = -(indexLeft.x < indexRight.x); - if (srcLeft.y == srcRight.y) - mask.y = -(indexLeft.y < indexRight.y); - if (srcLeft.z == srcRight.z) - mask.z = -(indexLeft.z < indexRight.z); - if (srcLeft.w == srcRight.w) - mask.w = -(indexLeft.w < indexRight.w); - - return mask; -} - - -ushort4 makeMask_ushort(ushort4 srcLeft, ushort4 srcRight, uint4 indexLeft, uint4 indexRight) -{ - ushort4 mask = convert_ushort4(srcLeft < srcRight); - - if (srcLeft.x == srcRight.x) - mask.x = (indexLeft.x < indexRight.x) ? USHRT_MAX : 0; - if (srcLeft.y == srcRight.y) - mask.y = (indexLeft.y < indexRight.y) ? USHRT_MAX : 0; - if (srcLeft.z == srcRight.z) - mask.z = (indexLeft.z < indexRight.z) ? USHRT_MAX : 0; - if (srcLeft.w == srcRight.w) - mask.w = (indexLeft.w < indexRight.w) ? USHRT_MAX : 0; - - return mask; -} - - -short4 makeMask_short(short4 srcLeft, short4 srcRight, uint4 indexLeft, uint4 indexRight) -{ - short4 mask = srcLeft < srcRight; - - if (srcLeft.x == srcRight.x) - mask.x = -(indexLeft.x < indexRight.x); - if (srcLeft.y == srcRight.y) - mask.y = -(indexLeft.y < indexRight.y); - if (srcLeft.z == srcRight.z) - mask.z = -(indexLeft.z < indexRight.z); - if (srcLeft.w == srcRight.w) - mask.w = -(indexLeft.w < indexRight.w); - - return mask; -} - - -uint4 makeMask_uint(uint4 srcLeft, uint4 srcRight, uint4 indexLeft, uint4 indexRight) -{ - uint4 mask = convert_uint4(srcLeft < srcRight); - - if (srcLeft.x == srcRight.x) - mask.x = (indexLeft.x < indexRight.x) ? UINT_MAX : 0; - if (srcLeft.y == srcRight.y) - mask.y = (indexLeft.y < indexRight.y) ? UINT_MAX : 0; - if (srcLeft.z == srcRight.z) - mask.z = (indexLeft.z < indexRight.z) ? UINT_MAX : 0; - if (srcLeft.w == srcRight.w) - mask.w = (indexLeft.w < indexRight.w) ? UINT_MAX : 0; - - return mask; -} - - -int4 makeMask_int(int4 srcLeft, int4 srcRight, uint4 indexLeft, uint4 indexRight) -{ - int4 mask = srcLeft < srcRight; - - if (srcLeft.x == srcRight.x) - mask.x = -(indexLeft.x < indexRight.x); - if (srcLeft.y == srcRight.y) - mask.y = -(indexLeft.y < indexRight.y); - if (srcLeft.z == srcRight.z) - mask.z = -(indexLeft.z < indexRight.z); - if (srcLeft.w == srcRight.w) - mask.w = -(indexLeft.w < indexRight.w); - - return mask; -} - - -ulong4 makeMask_ulong(ulong4 srcLeft, ulong4 srcRight, uint4 indexLeft, uint4 indexRight) -{ - ulong4 mask = convert_ulong4(srcLeft < srcRight); - - if (srcLeft.x == srcRight.x) - mask.x = (indexLeft.x < indexRight.x) ? ULONG_MAX : 0; - if (srcLeft.y == srcRight.y) - mask.y = (indexLeft.y < indexRight.y) ? ULONG_MAX : 0; - if (srcLeft.z == srcRight.z) - mask.z = (indexLeft.z < indexRight.z) ? ULONG_MAX : 0; - if (srcLeft.w == srcRight.w) - mask.w = (indexLeft.w < indexRight.w) ? ULONG_MAX : 0; - - return mask; -} - - -long4 makeMask_long(long4 srcLeft, long4 srcRight, uint4 indexLeft, uint4 indexRight) -{ - long4 mask = srcLeft < srcRight; - - if (srcLeft.x == srcRight.x) - mask.x = -(indexLeft.x < indexRight.x); - if (srcLeft.y == srcRight.y) - mask.y = -(indexLeft.y < indexRight.y); - if (srcLeft.z == srcRight.z) - mask.z = -(indexLeft.z < indexRight.z); - if (srcLeft.w == srcRight.w) - mask.w = -(indexLeft.w < indexRight.w); - - return mask; -} - - -__kernel void __attribute__((vec_type_hint(char4))) __attribute__((vec_type_hint(uint4))) bitonicSort_char(__global char4 * theArray, - __global uint4 * indices, - const uint stage, - const uint passOfStage, - const uint dir) -{ - size_t i = get_global_id(0); - char4 srcLeft, srcRight, mask; - char4 imask10 = (char4)(0, 0, -1, -1); - char4 imask11 = (char4)(0, -1, 0, -1); - uint4 indexLeft, indexRight; - - if (stage > 0) - { - if (passOfStage > 0) // upper level pass, exchange between two fours - { - size_t r = 1 << (passOfStage - 1); // length of arrow - size_t lmask = r - 1; - size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); - size_t right = left + r; - - srcLeft = theArray[left]; - srcRight = theArray[right]; - indexLeft = indices[left]; - indexRight = indices[right]; - - mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight); - - char4 imin = (srcLeft & mask) | (srcRight & ~mask); - char4 imax = (srcLeft & ~mask) | (srcRight & mask); - - if (((i >> (stage - 1)) & 1) ^ dir) - { - theArray[left] = imin; - theArray[right] = imax; - - indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - else - { - theArray[right] = imin; - theArray[left] = imax; - - indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - else // last pass, sort inside one four - { - srcLeft = theArray[i]; - srcRight = srcLeft.zwxy; - indexLeft = indices[i]; - indexRight = indexLeft.zwxy; - - mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if (((i >> stage) & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - } - else // first stage, sort inside one four - { - char4 imask0 = (char4)(0, -1, -1, 0); - - srcLeft = theArray[i]; - srcRight = srcLeft.yxwz; - indexLeft = indices[i]; - indexRight = indexLeft.yxwz; - - mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; - - if (dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - - srcRight = srcLeft.zwxy; - indexRight = indexLeft.zwxy; - mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if ((i & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_char(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } -} - - -__kernel void /*__attribute__((vec_type_hint(uchar4)))*/ bitonicSort_uchar(__global uchar4 * theArray, - __global uint4 * indices, - const uint stage, - const uint passOfStage, - const uint dir) -{ - size_t i = get_global_id(0); - uchar4 srcLeft, srcRight, mask; - uchar4 imask10 = (uchar4)(0, 0, -1, -1); - uchar4 imask11 = (uchar4)(0, -1, 0, -1); - uint4 indexLeft, indexRight; - - if (stage > 0) - { - if (passOfStage > 0) // upper level pass, exchange between two fours - { - size_t r = 1 << (passOfStage - 1); // length of arrow - size_t lmask = r - 1; - size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); - size_t right = left + r; - - srcLeft = theArray[left]; - srcRight = theArray[right]; - indexLeft = indices[left]; - indexRight = indices[right]; - - mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight); - - uchar4 imin = (srcLeft & mask) | (srcRight & ~mask); - uchar4 imax = (srcLeft & ~mask) | (srcRight & mask); - - if (((i >> (stage - 1)) & 1) ^ dir) - { - theArray[left] = imin; - theArray[right] = imax; - - indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - else - { - theArray[right] = imin; - theArray[left] = imax; - - indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - else // last pass, sort inside one four - { - srcLeft = theArray[i]; - srcRight = srcLeft.zwxy; - indexLeft = indices[i]; - indexRight = indexLeft.zwxy; - - mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if (((i >> stage) & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - } - else // first stage, sort inside one four - { - uchar4 imask0 = (uchar4)(0, -1, -1, 0); - - srcLeft = theArray[i]; - srcRight = srcLeft.yxwz; - indexLeft = indices[i]; - indexRight = indexLeft.yxwz; - - mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; - - if (dir) { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } else { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - - srcRight = srcLeft.zwxy; - indexRight = indexLeft.zwxy; - mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if ((i & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_uchar(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } -} - - -__kernel void /*__attribute__((vec_type_hint(short4)))*/ bitonicSort_short(__global short4 * theArray, - __global uint4 * indices, - const uint stage, - const uint passOfStage, - const uint dir) -{ - size_t i = get_global_id(0); - short4 srcLeft, srcRight, mask; - short4 imask10 = (short4)(0, 0, -1, -1); - short4 imask11 = (short4)(0, -1, 0, -1); - uint4 indexLeft, indexRight; - - if (stage > 0) - { - if (passOfStage > 0) // upper level pass, exchange between two fours - { - size_t r = 1 << (passOfStage - 1); // length of arrow - size_t lmask = r - 1; - size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); - size_t right = left + r; - - srcLeft = theArray[left]; - srcRight = theArray[right]; - indexLeft = indices[left]; - indexRight = indices[right]; - - mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight); - - short4 imin = (srcLeft & mask) | (srcRight & ~mask); - short4 imax = (srcLeft & ~mask) | (srcRight & mask); - - if (((i >> (stage - 1)) & 1) ^ dir) - { - theArray[left] = imin; - theArray[right] = imax; - - indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - else - { - theArray[right] = imin; - theArray[left] = imax; - - indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - else // last pass, sort inside one four - { - srcLeft = theArray[i]; - srcRight = srcLeft.zwxy; - indexLeft = indices[i]; - indexRight = indexLeft.zwxy; - - mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if (((i >> stage) & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - } - else // first stage, sort inside one four - { - short4 imask0 = (short4)(0, -1, -1, 0); - - srcLeft = theArray[i]; - srcRight = srcLeft.yxwz; - indexLeft = indices[i]; - indexRight = indexLeft.yxwz; - - mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; - - if (dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - - srcRight = srcLeft.zwxy; - indexRight = indexLeft.zwxy; - mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if ((i & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_short(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } -} - - -__kernel void /*__attribute__((vec_type_hint(ushort4)))*/ bitonicSort_ushort(__global ushort4 * theArray, - __global uint4 * indices, - const uint stage, - const uint passOfStage, - const uint dir) -{ - size_t i = get_global_id(0); - ushort4 srcLeft, srcRight, mask; - ushort4 imask10 = (ushort4)(0, 0, -1, -1); - ushort4 imask11 = (ushort4)(0, -1, 0, -1); - uint4 indexLeft, indexRight; - - if (stage > 0) - { - if (passOfStage > 0) // upper level pass, exchange between two fours - { - size_t r = 1 << (passOfStage - 1); // length of arrow - size_t lmask = r - 1; - size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); - size_t right = left + r; - - srcLeft = theArray[left]; - srcRight = theArray[right]; - indexLeft = indices[left]; - indexRight = indices[right]; - - mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight); - - ushort4 imin = (srcLeft & mask) | (srcRight & ~mask); - ushort4 imax = (srcLeft & ~mask) | (srcRight & mask); - - if (((i >> (stage - 1)) & 1) ^ dir) - { - theArray[left] = imin; - theArray[right] = imax; - - indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - else - { - theArray[right] = imin; - theArray[left] = imax; - - indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - else // last pass, sort inside one four - { - srcLeft = theArray[i]; - srcRight = srcLeft.zwxy; - indexLeft = indices[i]; - indexRight = indexLeft.zwxy; - - mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if (((i >> stage) & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - } - else // first stage, sort inside one four - { - ushort4 imask0 = (ushort4)(0, -1, -1, 0); - - srcLeft = theArray[i]; - srcRight = srcLeft.yxwz; - indexLeft = indices[i]; - indexRight = indexLeft.yxwz; - - mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; - - if (dir) { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } else { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - - srcRight = srcLeft.zwxy; - indexRight = indexLeft.zwxy; - mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if ((i & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_ushort(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } -} - - -__kernel void /*__attribute__((vec_type_hint(int4)))*/ bitonicSort_int(__global int4 * theArray, - __global uint4 * indices, - const uint stage, - const uint passOfStage, - const uint dir) -{ - - size_t i = get_global_id(0); - int4 srcLeft, srcRight, mask; - int4 imask10 = (int4)(0, 0, -1, -1); - int4 imask11 = (int4)(0, -1, 0, -1); - uint4 indexLeft, indexRight; - - int random_id = (i * 5 + stage * 42 + passOfStage * 47 + dir * 88) % 100; - - if (stage > 0) - { - if (passOfStage > 0) // upper level pass, exchange between two fours - { - size_t r = 1 << (passOfStage - 1); // length of arrow - size_t lmask = r - 1; - size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); - size_t right = left + r; - - srcLeft = theArray[left]; - srcRight = theArray[right]; - indexLeft = indices[left]; - indexRight = indices[right]; - - mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight); - - int4 imin = (srcLeft & mask) | (srcRight & ~mask); - int4 imax = (srcLeft & ~mask) | (srcRight & mask); - - if (((i >> (stage - 1)) & 1) ^ dir) - { - theArray[left] = imin; - theArray[right] = imax; - - indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - else - { - theArray[right] = imin; - theArray[left] = imax; - - indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - else // last pass, sort inside one four - { - srcLeft = theArray[i]; - srcRight = srcLeft.zwxy; - indexLeft = indices[i]; - indexRight = indexLeft.zwxy; - - mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if (((i >> stage) & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - } - else // first stage, sort inside one four - { - int4 imask0 = (int4)(0, -1, -1, 0); - - srcLeft = theArray[i]; - srcRight = srcLeft.yxwz; - indexLeft = indices[i]; - indexRight = indexLeft.yxwz; - - mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; - - if (dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - - srcRight = srcLeft.zwxy; - indexRight = indexLeft.zwxy; - mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if ((i & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_int(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } -} - - -__kernel void /*__attribute__((vec_type_hint(uint4)))*/ bitonicSort_uint(__global uint4 * theArray, - __global uint4 * indices, - const uint stage, - const uint passOfStage, - const uint dir) -{ - size_t i = get_global_id(0); - uint4 srcLeft, srcRight, mask; - uint4 imask10 = (uint4)(0, 0, -1, -1); - uint4 imask11 = (uint4)(0, -1, 0, -1); - uint4 indexLeft, indexRight; - - if (stage > 0) - { - if (passOfStage > 0) // upper level pass, exchange between two fours - { - size_t r = 1 << (passOfStage - 1); // length of arrow - size_t lmask = r - 1; - size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); - size_t right = left + r; - - srcLeft = theArray[left]; - srcRight = theArray[right]; - indexLeft = indices[left]; - indexRight = indices[right]; - - mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight); - - uint4 imin = (srcLeft & mask) | (srcRight & ~mask); - uint4 imax = (srcLeft & ~mask) | (srcRight & mask); - - if (((i >> (stage - 1)) & 1) ^ dir) - { - theArray[left] = imin; - theArray[right] = imax; - - indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - else - { - theArray[right] = imin; - theArray[left] = imax; - - indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - else // last pass, sort inside one four - { - srcLeft = theArray[i]; - srcRight = srcLeft.zwxy; - indexLeft = indices[i]; - indexRight = indexLeft.zwxy; - - mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if (((i >> stage) & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - } - else // first stage, sort inside one four - { - uint4 imask0 = (uint4)(0, -1, -1, 0); - - srcLeft = theArray[i]; - srcRight = srcLeft.yxwz; - indexLeft = indices[i]; - indexRight = indexLeft.yxwz; - - mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; - - if (dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - - srcRight = srcLeft.zwxy; - indexRight = indexLeft.zwxy; - mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if ((i & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_uint(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } -} - - -__kernel void /*__attribute__((vec_type_hint(long4)))*/ bitonicSort_long(__global long4 * theArray, - __global uint4 * indices, - const uint stage, - const uint passOfStage, - const uint dir) -{ - size_t i = get_global_id(0); - long4 srcLeft, srcRight, mask; - long4 imask10 = (long4)(0, 0, -1, -1); - long4 imask11 = (long4)(0, -1, 0, -1); - uint4 indexLeft, indexRight; - - if (stage > 0) - { - if (passOfStage > 0) // upper level pass, exchange between two fours - { - size_t r = 1 << (passOfStage - 1); // length of arrow - size_t lmask = r - 1; - size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); - size_t right = left + r; - - srcLeft = theArray[left]; - srcRight = theArray[right]; - indexLeft = indices[left]; - indexRight = indices[right]; - - mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight); - - long4 imin = (srcLeft & mask) | (srcRight & ~mask); - long4 imax = (srcLeft & ~mask) | (srcRight & mask); - - if (((i >> (stage - 1)) & 1) ^ dir) - { - theArray[left] = imin; - theArray[right] = imax; - - indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - else - { - theArray[right] = imin; - theArray[left] = imax; - - indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - else // last pass, sort inside one four - { - srcLeft = theArray[i]; - srcRight = srcLeft.zwxy; - indexLeft = indices[i]; - indexRight = indexLeft.zwxy; - - mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if (((i >> stage) & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - } - else // first stage, sort inside one four - { - long4 imask0 = (long4)(0, -1, -1, 0); - - srcLeft = theArray[i]; - srcRight = srcLeft.yxwz; - indexLeft = indices[i]; - indexRight = indexLeft.yxwz; - - mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; - - if (dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - - srcRight = srcLeft.zwxy; - indexRight = indexLeft.zwxy; - mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if ((i & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_long(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } -} - - -__kernel void /*__attribute__((vec_type_hint(ulong4)))*/ bitonicSort_ulong(__global ulong4 * theArray, - __global uint4 * indices, - const uint stage, - const uint passOfStage, - const uint dir) -{ - size_t i = get_global_id(0); - ulong4 srcLeft, srcRight, mask; - ulong4 imask10 = (ulong4)(0, 0, -1, -1); - ulong4 imask11 = (ulong4)(0, -1, 0, -1); - uint4 indexLeft, indexRight; - - if (stage > 0) - { - if (passOfStage > 0) // upper level pass, exchange between two fours - { - size_t r = 1 << (passOfStage - 1); // length of arrow - size_t lmask = r - 1; - size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask); - size_t right = left + r; - - srcLeft = theArray[left]; - srcRight = theArray[right]; - indexLeft = indices[left]; - indexRight = indices[right]; - - mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight); - - ulong4 imin = (srcLeft & mask) | (srcRight & ~mask); - ulong4 imax = (srcLeft & ~mask) | (srcRight & mask); - - if (((i >> (stage - 1)) & 1) ^ dir) - { - theArray[left] = imin; - theArray[right] = imax; - - indices[left] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[right] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - else - { - theArray[right] = imin; - theArray[left] = imax; - - indices[right] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indices[left] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - else // last pass, sort inside one four - { - srcLeft = theArray[i]; - srcRight = srcLeft.zwxy; - indexLeft = indices[i]; - indexRight = indexLeft.zwxy; - - mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if (((i >> stage) & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } - } - else // first stage, sort inside one four - { - ulong4 imask0 = (ulong4)(0, -1, -1, 0); - - srcLeft = theArray[i]; - srcRight = srcLeft.yxwz; - indexLeft = indices[i]; - indexRight = indexLeft.yxwz; - - mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight) ^ imask0; - - if (dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - - srcRight = srcLeft.zwxy; - indexRight = indexLeft.zwxy; - mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight) ^ imask10; - - if ((i & 1) ^ dir) - { - srcLeft = (srcLeft & mask) | (srcRight & ~mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & mask) | (srcRight & ~mask); - indices[i] = (indexLeft & convert_uint4(mask)) | (indexRight & convert_uint4(~mask)); - } - else - { - srcLeft = (srcLeft & ~mask) | (srcRight & mask); - srcRight = srcLeft.yxwz; - indexLeft = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - indexRight = indexLeft.yxwz; - - mask = makeMask_ulong(srcLeft, srcRight, indexLeft, indexRight) ^ imask11; - - theArray[i] = (srcLeft & ~mask) | (srcRight & mask); - indices[i] = (indexLeft & convert_uint4(~mask)) | (indexRight & convert_uint4(mask)); - } - } -} - -)"; From b0fca03d79fed83f4d3bee2344641b06531baef6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 22 Jan 2021 01:13:48 +0300 Subject: [PATCH 41/46] Update run-fuzzer.sh --- docker/test/fuzzer/run-fuzzer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 200c7b6f4ce..0a282133690 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -75,7 +75,7 @@ function fuzz { # Obtain the list of newly added tests. They will be fuzzed in more extreme way than other tests. cd ch - NEW_TESTS=$(git diff --name-only origin/master "$SHA_TO_TEST" | grep -P 'tests/queries/0_stateless/.*\.sql' | sed -r -e 's!^!ch/!' | sort -R) + NEW_TESTS=$(git diff --name-only $(git merge-base origin/master "$SHA_TO_TEST"~) "$SHA_TO_TEST" | grep -P 'tests/queries/0_stateless/.*\.sql' | sed -r -e 's!^!ch/!' | sort -R) cd .. if [[ -n "$NEW_TESTS" ]] then From 963699d9c89c68b977951669cfd51093fd553357 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 22 Jan 2021 02:26:49 +0300 Subject: [PATCH 42/46] Update run-fuzzer.sh --- docker/test/fuzzer/run-fuzzer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 0a282133690..0ce52c1eb4e 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -75,7 +75,7 @@ function fuzz { # Obtain the list of newly added tests. They will be fuzzed in more extreme way than other tests. cd ch - NEW_TESTS=$(git diff --name-only $(git merge-base origin/master "$SHA_TO_TEST"~) "$SHA_TO_TEST" | grep -P 'tests/queries/0_stateless/.*\.sql' | sed -r -e 's!^!ch/!' | sort -R) + NEW_TESTS=$(git diff --name-only "$(git merge-base origin/master "$SHA_TO_TEST"~)" "$SHA_TO_TEST" | grep -P 'tests/queries/0_stateless/.*\.sql' | sed -r -e 's!^!ch/!' | sort -R) cd .. if [[ -n "$NEW_TESTS" ]] then From fe5d02a06f1f403296476ae27f5d4599525d757d Mon Sep 17 00:00:00 2001 From: achimbab <07c00h@gmail.com> Date: Fri, 22 Jan 2021 12:59:21 +0900 Subject: [PATCH 43/46] Add kakao to adopters.md --- docs/en/introduction/adopters.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index 9b08de99fc0..2684e6fdd3a 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -120,5 +120,6 @@ toc_title: Adopters | ЦВТ | Software Development | Metrics, Logging | — | — | [Blog Post, March 2019, in Russian](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) | | МКБ | Bank | Web-system monitoring | — | — | [Slides in Russian, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) | | ЦФТ | Banking, Financial products, Payments | — | — | — | [Meetup in Russian, April 2020](https://team.cft.ru/events/162) | +| kakaocorp | Internet company | — | — | — | [if(kakao)2020 conference](https://if.kakao.com/session/117) | [Original article](https://clickhouse.tech/docs/en/introduction/adopters/) From 83862799cdcf2f4ebc4046812dd564af41e34df9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 22 Jan 2021 13:02:48 +0300 Subject: [PATCH 44/46] Fix one more race in TestKeeper --- src/Server/TestKeeperTCPHandler.cpp | 51 ++++++++++++++++++----------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/src/Server/TestKeeperTCPHandler.cpp b/src/Server/TestKeeperTCPHandler.cpp index bf407ba96b7..354540c797d 100644 --- a/src/Server/TestKeeperTCPHandler.cpp +++ b/src/Server/TestKeeperTCPHandler.cpp @@ -33,9 +33,9 @@ namespace ErrorCodes struct PollResult { - bool has_responses; - bool has_requests; - bool error; + size_t ready_responses_count{0}; + bool has_requests{false}; + bool error{false}; }; /// Queue with mutex. As simple as possible. @@ -191,10 +191,17 @@ struct SocketInterruptablePollWrapper result.has_requests = true; else { - /// Skip all of them, we are not interested in exact - /// amount because responses ordered in responses queue. - response_in.ignore(); - result.has_responses = true; + UInt8 dummy; + do + { + /// All ready responses stored in responses queue, + /// but we have to count amount of ready responses in pipe + /// and process them only. Otherwise states of response_in + /// and response queue will be inconsistent and race condition is possible. + readIntBinary(dummy, response_in); + result.ready_responses_count++; + } + while (response_in.available()); } } } @@ -349,23 +356,27 @@ void TestKeeperTCPHandler::runImpl() while (in->available()); } - if (result.has_responses) + /// Process exact amout of responses from pipe + /// otherwise state of responses queue and signaling pipe + /// became inconsistent and race condition is possible. + while (result.ready_responses_count != 0) { Coordination::ZooKeeperResponsePtr response; - while (responses->tryPop(response)) - { - if (response->xid == close_xid) - { - LOG_DEBUG(log, "Session #{} successfully closed", session_id); - return; - } + if (!responses->tryPop(response)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "We must have at least {} ready responses, but queue is empty. It's a bug.", result.ready_responses_count); - if (response->error == Coordination::Error::ZOK) - response->write(*out); - else if (response->xid != Coordination::WATCH_XID) - response->write(*out); - /// skipping bad response for watch + if (response->xid == close_xid) + { + LOG_DEBUG(log, "Session #{} successfully closed", session_id); + return; } + + if (response->error == Coordination::Error::ZOK) + response->write(*out); + else if (response->xid != Coordination::WATCH_XID) + response->write(*out); + /// skipping bad response for watch + result.ready_responses_count--; } if (result.error) From b2a492155868c922b7484bd7e73a4c0d5fb33c6e Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 22 Jan 2021 13:14:53 +0300 Subject: [PATCH 45/46] Fix style --- src/Server/TestKeeperTCPHandler.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Server/TestKeeperTCPHandler.cpp b/src/Server/TestKeeperTCPHandler.cpp index 354540c797d..6bd8eaa65f7 100644 --- a/src/Server/TestKeeperTCPHandler.cpp +++ b/src/Server/TestKeeperTCPHandler.cpp @@ -28,6 +28,7 @@ namespace DB namespace ErrorCodes { extern const int SYSTEM_ERROR; + extern const int LOGICAL_ERROR; extern const int UNEXPECTED_PACKET_FROM_CLIENT; } From cc2788af881c96701076b366259bc99525d926b1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 22 Jan 2021 13:48:40 +0300 Subject: [PATCH 46/46] Fix typo --- src/Server/TestKeeperTCPHandler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Server/TestKeeperTCPHandler.cpp b/src/Server/TestKeeperTCPHandler.cpp index 6bd8eaa65f7..97999c2b1c1 100644 --- a/src/Server/TestKeeperTCPHandler.cpp +++ b/src/Server/TestKeeperTCPHandler.cpp @@ -357,7 +357,7 @@ void TestKeeperTCPHandler::runImpl() while (in->available()); } - /// Process exact amout of responses from pipe + /// Process exact amount of responses from pipe /// otherwise state of responses queue and signaling pipe /// became inconsistent and race condition is possible. while (result.ready_responses_count != 0)