thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 23:52:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #68132 from Avogar/better-dynamic

Browse Source 
Reimplement Dynamic type
This commit is contained in:
Kruglov Pavel 2024-08-13 19:21:52 +00:00 committed by GitHub
commit 6c8ad63104
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
75 changed files with 6796 additions and 1961 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
docs/en/sql-reference/data-types/dynamic.md
View File

@ -14,7 +14,7 @@ To declare a column of `Dynamic` type, use the following syntax:
<column_name> Dynamic(max_types=N) <column_name> Dynamic(max_types=N)
``` ```
Where `N` is an optional parameter between `1` and `255` indicating how many different data types can be stored inside a column with type `Dynamic` across single block of data that is stored separately (for example across single data part for MergeTree table). If this limit is exceeded, all new types will be converted to type `String`. Default value of `max_types` is `32`. Where `N` is an optional parameter between `0` and `254` indicating how many different data types can be stored as separate subcolumns inside a column with type `Dynamic` across single block of data that is stored separately (for example across single data part for MergeTree table). If this limit is exceeded, all values with new types will be stored together in a special shared data structure in binary form. Default value of `max_types` is `32`.
:::note :::note
The Dynamic data type is an experimental feature. To use it, set `allow_experimental_dynamic_type = 1`. The Dynamic data type is an experimental feature. To use it, set `allow_experimental_dynamic_type = 1`.
@ -224,41 +224,43 @@ SELECT d::Dynamic(max_types=5) as d2, dynamicType(d2) FROM test;
└───────┴────────────────┘ └───────┴────────────────┘
``` ```
If `K < N`, then the values with the rarest types are converted to `String`: If `K < N`, then the values with the rarest types will be inserted into a single special subcolumn, but still will be accessible:
```text ```text
CREATE TABLE test (d Dynamic(max_types=4)) ENGINE = Memory; CREATE TABLE test (d Dynamic(max_types=4)) ENGINE = Memory;
INSERT INTO test VALUES (NULL), (42), (43), ('42.42'), (true), ([1, 2, 3]); INSERT INTO test VALUES (NULL), (42), (43), ('42.42'), (true), ([1, 2, 3]);
SELECT d, dynamicType(d), d::Dynamic(max_types=2) as d2, dynamicType(d2) FROM test; SELECT d, dynamicType(d), d::Dynamic(max_types=2) as d2, dynamicType(d2), isDynamicElementInSharedData(d2) FROM test;
``` ```
```text ```text
┌─d───────┬─dynamicType(d)─┬─d2──────┬─dynamicType(d2)─┐ ┌─d───────┬─dynamicType(d)─┬─d2──────┬─dynamicType(d2)─┬─isDynamicElementInSharedData(d2)─┐
│ ᴺᵁᴸᴸ │ None │ ᴺᵁᴸᴸ │ None │ │ ᴺᵁᴸᴸ │ None │ ᴺᵁᴸᴸ │ None │ false │
│ 42 │ Int64 │ 42 │ Int64 │ │ 42 │ Int64 │ 42 │ Int64 │ false │
│ 43 │ Int64 │ 43 │ Int64 │ │ 43 │ Int64 │ 43 │ Int64 │ false │
│ 42.42 │ String │ 42.42 │ String │ │ 42.42 │ String │ 42.42 │ String │ false │
│ true │ Bool │ true │ String │ │ true │ Bool │ true │ Bool │ true │
│ [1,2,3] │ Array(Int64) │ [1,2,3] │ String │ │ [1,2,3] │ Array(Int64) │ [1,2,3] │ Array(Int64) │ true │
└─────────┴────────────────┴─────────┴─────────────────┘ └─────────┴────────────────┴─────────┴─────────────────┴──────────────────────────────────┘
``` ```
If `K=1`, all types are converted to `String`: Functions `isDynamicElementInSharedData` returns `true` for rows that are stored in a special shared data structure inside `Dynamic` and as we can see, resulting column contains only 2 types that are not stored in shared data structure.
If `K=0`, all types will be inserted into single special subcolumn:
```text ```text
CREATE TABLE test (d Dynamic(max_types=4)) ENGINE = Memory; CREATE TABLE test (d Dynamic(max_types=4)) ENGINE = Memory;
INSERT INTO test VALUES (NULL), (42), (43), ('42.42'), (true), ([1, 2, 3]); INSERT INTO test VALUES (NULL), (42), (43), ('42.42'), (true), ([1, 2, 3]);
SELECT d, dynamicType(d), d::Dynamic(max_types=1) as d2, dynamicType(d2) FROM test; SELECT d, dynamicType(d), d::Dynamic(max_types=0) as d2, dynamicType(d2), isDynamicElementInSharedData(d2) FROM test;
``` ```
```text ```text
┌─d───────┬─dynamicType(d)─┬─d2──────┬─dynamicType(d2)─┐ ┌─d───────┬─dynamicType(d)─┬─d2──────┬─dynamicType(d2)─┬─isDynamicElementInSharedData(d2)─┐
│ ᴺᵁᴸᴸ │ None │ ᴺᵁᴸᴸ │ None │ │ ᴺᵁᴸᴸ │ None │ ᴺᵁᴸᴸ │ None │ false │
│ 42 │ Int64 │ 42 │ String │ │ 42 │ Int64 │ 42 │ Int64 │ true │
│ 43 │ Int64 │ 43 │ String │ │ 43 │ Int64 │ 43 │ Int64 │ true │
│ 42.42 │ String │ 42.42 │ String │ │ 42.42 │ String │ 42.42 │ String │ true │
│ true │ Bool │ true │ String │ │ true │ Bool │ true │ Bool │ true │
│ [1,2,3] │ Array(Int64) │ [1,2,3] │ String │ │ [1,2,3] │ Array(Int64) │ [1,2,3] │ Array(Int64) │ true │
└─────────┴────────────────┴─────────┴─────────────────┘ └─────────┴────────────────┴─────────┴─────────────────┴──────────────────────────────────┘
``` ```
## Reading Dynamic type from the data ## Reading Dynamic type from the data
@ -411,17 +413,17 @@ SELECT d, dynamicType(d) FROM test ORDER by d;
## Reaching the limit in number of different data types stored inside Dynamic ## Reaching the limit in number of different data types stored inside Dynamic
`Dynamic` data type can store only limited number of different data types inside. By default, this limit is 32, but you can change it in type declaration using syntax `Dynamic(max_types=N)` where N is between 1 and 255 (due to implementation details, it's impossible to have more than 255 different data types inside Dynamic). `Dynamic` data type can store only limited number of different data types as separate subcolumns. By default, this limit is 32, but you can change it in type declaration using syntax `Dynamic(max_types=N)` where N is between 0 and 254 (due to implementation details, it's impossible to have more than 254 different data types that can be stored as separate subcolumns inside Dynamic).
When the limit is reached, all new data types inserted to `Dynamic` column will be casted to `String` and stored as `String` values. When the limit is reached, all new data types inserted to `Dynamic` column will be inserted into a single shared data structure that stores values with different data types in binary form.
Let's see what happens when the limit is reached in different scenarios. Let's see what happens when the limit is reached in different scenarios.
### Reaching the limit during data parsing ### Reaching the limit during data parsing
During parsing of `Dynamic` values from the data, when the limit is reached for current block of data, all new values will be inserted as `String` values: During parsing of `Dynamic` values from the data, when the limit is reached for current block of data, all new values will be inserted into shared data structure:
```sql ```sql
SELECT d, dynamicType(d) FROM format(JSONEachRow, 'd Dynamic(max_types=3)', ' SELECT d, dynamicType(d), isDynamicElementInSharedData(d) FROM format(JSONEachRow, 'd Dynamic(max_types=3)', '
{"d" : 42} {"d" : 42}
{"d" : [1, 2, 3]} {"d" : [1, 2, 3]}
{"d" : "Hello, World!"} {"d" : "Hello, World!"}
@ -432,22 +434,22 @@ SELECT d, dynamicType(d) FROM format(JSONEachRow, 'd Dynamic(max_types=3)', '
``` ```
```text ```text
┌─d──────────────────────────┬─dynamicType(d)─┐ ┌─d──────────────────────┬─dynamicType(d)─────────────────┬─isDynamicElementInSharedData(d)─┐
│ 42 │ Int64 │ │ 42 │ Int64 │ false │
│ [1,2,3] │ Array(Int64) │ │ [1,2,3] │ Array(Int64) │ false │
│ Hello, World! │ String │ │ Hello, World! │ String │ false │
│ 2020-01-01 │ String │ │ 2020-01-01 │ Date │ true │
│ ["str1", "str2", "str3"] │ String │ │ ['str1','str2','str3'] │ Array(String) │ true │
│ {"a" : 1, "b" : [1, 2, 3]} │ String │ │ (1,[1,2,3]) │ Tuple(a Int64, b Array(Int64)) │ true │
└────────────────────────────┴────────────────┘ └────────────────────────┴────────────────────────────────┴─────────────────────────────────┘
``` ```
As we can see, after inserting 3 different data types `Int64`, `Array(Int64)` and `String` all new types were converted to `String`. As we can see, after inserting 3 different data types `Int64`, `Array(Int64)` and `String` all new types were inserted into special shared data structure.
### During merges of data parts in MergeTree table engines ### During merges of data parts in MergeTree table engines
During merge of several data parts in MergeTree table the `Dynamic` column in the resulting data part can reach the limit of different data types inside and won't be able to store all types from source parts. During merge of several data parts in MergeTree table the `Dynamic` column in the resulting data part can reach the limit of different data types that can be stored in separate subcolumns inside and won't be able to store all types as subcolumns from source parts.
In this case ClickHouse chooses what types will remain after merge and what types will be casted to `String`. In most cases ClickHouse tries to keep the most frequent types and cast the rarest types to `String`, but it depends on the implementation. In this case ClickHouse chooses what types will remain as separate subcolumns after merge and what types will be inserted into shared data structure. In most cases ClickHouse tries to keep the most frequent types and store the rarest types in shared data structure, but it depends on the implementation.
Let's see an example of such merge. First, let's create a table with `Dynamic` column, set the limit of different data types to `3` and insert values with `5` different types: Let's see an example of such merge. First, let's create a table with `Dynamic` column, set the limit of different data types to `3` and insert values with `5` different types:
@ -463,17 +465,17 @@ INSERT INTO test SELECT number, 'str_' || toString(number) FROM numbers(1);
Each insert will create a separate data pert with `Dynamic` column containing single type: Each insert will create a separate data pert with `Dynamic` column containing single type:
```sql ```sql
SELECT count(), dynamicType(d), _part FROM test GROUP BY _part, dynamicType(d) ORDER BY _part; SELECT count(), dynamicType(d), isDynamicElementInSharedData(d), _part FROM test GROUP BY _part, dynamicType(d), isDynamicElementInSharedData(d) ORDER BY _part, count();
``` ```
```text ```text
┌─count()─┬─dynamicType(d)──────┬─_part─────┐ ┌─count()─┬─dynamicType(d)──────┬─isDynamicElementInSharedData(d)─┬─_part─────┐
│ 5 │ UInt64 │ all_1_1_0 │ │ 5 │ UInt64 │ false │ all_1_1_0 │
│ 4 │ Array(UInt64) │ all_2_2_0 │ │ 4 │ Array(UInt64) │ false │ all_2_2_0 │
│ 3 │ Date │ all_3_3_0 │ │ 3 │ Date │ false │ all_3_3_0 │
│ 2 │ Map(UInt64, UInt64) │ all_4_4_0 │ │ 2 │ Map(UInt64, UInt64) │ false │ all_4_4_0 │
│ 1 │ String │ all_5_5_0 │ │ 1 │ String │ false │ all_5_5_0 │
└─────────┴─────────────────────┴───────────┘ └─────────┴─────────────────────┴─────────────────────────────────┴───────────┘
``` ```
Now, let's merge all parts into one and see what will happen: Now, let's merge all parts into one and see what will happen:
@ -481,18 +483,20 @@ Now, let's merge all parts into one and see what will happen:
```sql ```sql
SYSTEM START MERGES test; SYSTEM START MERGES test;
OPTIMIZE TABLE test FINAL; OPTIMIZE TABLE test FINAL;
SELECT count(), dynamicType(d), _part FROM test GROUP BY _part, dynamicType(d) ORDER BY _part; SELECT count(), dynamicType(d), isDynamicElementInSharedData(d), _part FROM test GROUP BY _part, dynamicType(d), isDynamicElementInSharedData(d) ORDER BY _part, count() desc;
``` ```
```text ```text
┌─count()─┬─dynamicType(d)─┬─_part─────┐ ┌─count()─┬─dynamicType(d)──────┬─isDynamicElementInSharedData(d)─┬─_part─────┐
│ 5 │ UInt64 │ all_1_5_2 │ │ 5 │ UInt64 │ false │ all_1_5_2 │
│ 6 │ String │ all_1_5_2 │ │ 4 │ Array(UInt64) │ false │ all_1_5_2 │
│ 4 │ Array(UInt64) │ all_1_5_2 │ │ 3 │ Date │ false │ all_1_5_2 │
└─────────┴────────────────┴───────────┘ │ 2 │ Map(UInt64, UInt64) │ true │ all_1_5_2 │
│ 1 │ String │ true │ all_1_5_2 │
└─────────┴─────────────────────┴─────────────────────────────────┴───────────┘
``` ```
As we can see, ClickHouse kept the most frequent types `UInt64` and `Array(UInt64)` and casted all other types to `String`. As we can see, ClickHouse kept the most frequent types `UInt64` and `Array(UInt64)` as subcolumns and inserted all other types into shared data.
## JSONExtract functions with Dynamic ## JSONExtract functions with Dynamic
@ -509,22 +513,23 @@ SELECT JSONExtract('{"a" : [1, 2, 3]}', 'a', 'Dynamic') AS dynamic, dynamicType(
``` ```
```sql ```sql
SELECT JSONExtract('{"obj" : {"a" : 42, "b" : "Hello", "c" : [1,2,3]}}', 'obj', 'Map(String, Variant(UInt32, String, Array(UInt32)))') AS map_of_dynamics, mapApply((k, v) -> (k, variantType(v)), map_of_dynamics) AS map_of_dynamic_types``` SELECT JSONExtract('{"obj" : {"a" : 42, "b" : "Hello", "c" : [1,2,3]}}', 'obj', 'Map(String, Dynamic)') AS map_of_dynamics, mapApply((k, v) -> (k, dynamicType(v)), map_of_dynamics) AS map_of_dynamic_types
```
```text ```text
┌─map_of_dynamics──────────────────┬─map_of_dynamic_types────────────────────────────┐ ┌─map_of_dynamics──────────────────┬─map_of_dynamic_types────────────────────────────────────┐
│ {'a':42,'b':'Hello','c':[1,2,3]} │ {'a':'UInt32','b':'String','c':'Array(UInt32)'} │ │ {'a':42,'b':'Hello','c':[1,2,3]} │ {'a':'Int64','b':'String','c':'Array(Nullable(Int64))'} │
└──────────────────────────────────┴─────────────────────────────────────────────────┘ └──────────────────────────────────┴─────────────────────────────────────────────────────────┘
``` ```
```sql ```sql
SELECT JSONExtractKeysAndValues('{"a" : 42, "b" : "Hello", "c" : [1,2,3]}', 'Variant(UInt32, String, Array(UInt32))') AS dynamics, arrayMap(x -> (x.1, variantType(x.2)), dynamics) AS dynamic_types``` SELECT JSONExtractKeysAndValues('{"a" : 42, "b" : "Hello", "c" : [1,2,3]}', 'Dynamic') AS dynamics, arrayMap(x -> (x.1, dynamicType(x.2)), dynamics) AS dynamic_types```
``` ```
```text ```text
┌─dynamics───────────────────────────────┬─dynamic_types─────────────────────────────────────────┐ ┌─dynamics───────────────────────────────┬─dynamic_types─────────────────────────────────────────────────┐
│ [('a',42),('b','Hello'),('c',[1,2,3])] │ [('a','UInt32'),('b','String'),('c','Array(UInt32)')] │ │ [('a',42),('b','Hello'),('c',[1,2,3])] │ [('a','Int64'),('b','String'),('c','Array(Nullable(Int64))')] │
└────────────────────────────────────────┴───────────────────────────────────────────────────────┘ └────────────────────────────────────────┴───────────────────────────────────────────────────────────────┘
``` ```
### Binary output format ### Binary output format

953
src/Columns/ColumnDynamic.cpp
View File

File diff suppressed because it is too large Load Diff

176
src/Columns/ColumnDynamic.h
View File

@ -3,6 +3,7 @@
#include <Columns/IColumn.h> #include <Columns/IColumn.h>
#include <Columns/ColumnVector.h> #include <Columns/ColumnVector.h>
#include <Columns/ColumnVariant.h> #include <Columns/ColumnVariant.h>
#include <Columns/ColumnString.h>
#include <DataTypes/IDataType.h> #include <DataTypes/IDataType.h>
#include <Common/WeakHash.h> #include <Common/WeakHash.h>
@ -19,11 +20,19 @@ namespace DB
* *
* When new values are inserted into Dynamic column, the internal Variant * When new values are inserted into Dynamic column, the internal Variant
* type and column are extended if the inserted value has new type. * type and column are extended if the inserted value has new type.
* When the limit on number of dynamic types is exceeded, all values
* with new types are inserted into special shared variant with type String
* that contains values and their types in binary format.
*/ */
class ColumnDynamic final : public COWHelper<IColumnHelper<ColumnDynamic>, ColumnDynamic> class ColumnDynamic final : public COWHelper<IColumnHelper<ColumnDynamic>, ColumnDynamic>
{ {
public: public:
/// /// Maximum limit on dynamic types. We use ColumnVariant to store all the types,
/// so the limit cannot be greater then ColumnVariant::MAX_NESTED_COLUMNS.
/// We also always have reserved variant for shared variant.
static constexpr size_t MAX_DYNAMIC_TYPES_LIMIT = ColumnVariant::MAX_NESTED_COLUMNS - 1;
static constexpr const char * SHARED_VARIANT_TYPE_NAME = "SharedVariant";
struct Statistics struct Statistics
{ {
enum class Source enum class Source
@ -32,12 +41,27 @@ public:
MERGE, /// Statistics were calculated during merge of several MergeTree parts. MERGE, /// Statistics were calculated during merge of several MergeTree parts.
}; };
explicit Statistics(Source source_) : source(source_) {}
/// Source of the statistics. /// Source of the statistics.
Source source; Source source;
/// Statistics data: (variant name) -> (total variant size in data part). /// Statistics data for usual variants: (variant name) -> (total variant size in data part).
std::unordered_map<String, size_t> data; std::unordered_map<String, size_t> variants_statistics;
/// Statistics data for variants from shared variant: (variant name) -> (total variant size in data part).
/// For shared variant we store statistics only for first 256 variants (should cover almost all cases and it's not expensive).
static constexpr const size_t MAX_SHARED_VARIANT_STATISTICS_SIZE = 256;
std::unordered_map<String, size_t> shared_variants_statistics;
}; };
using StatisticsPtr = std::shared_ptr<const Statistics>;
struct ComparatorBase;
using ComparatorAscendingUnstable = ComparatorAscendingUnstableImpl<ComparatorBase>;
using ComparatorAscendingStable = ComparatorAscendingStableImpl<ComparatorBase>;
using ComparatorDescendingUnstable = ComparatorDescendingUnstableImpl<ComparatorBase>;
using ComparatorDescendingStable = ComparatorDescendingStableImpl<ComparatorBase>;
using ComparatorEqual = ComparatorEqualImpl<ComparatorBase>;
private: private:
friend class COWHelper<IColumnHelper<ColumnDynamic>, ColumnDynamic>; friend class COWHelper<IColumnHelper<ColumnDynamic>, ColumnDynamic>;
@ -54,28 +78,32 @@ private:
}; };
explicit ColumnDynamic(size_t max_dynamic_types_); explicit ColumnDynamic(size_t max_dynamic_types_);
ColumnDynamic(MutableColumnPtr variant_column_, const VariantInfo & variant_info_, size_t max_dynamic_types_, const Statistics & statistics_ = {}); ColumnDynamic(MutableColumnPtr variant_column_, const DataTypePtr & variant_type_, size_t max_dynamic_types_, size_t global_max_dynamic_types_, const StatisticsPtr & statistics_ = {});
ColumnDynamic(MutableColumnPtr variant_column_, const VariantInfo & variant_info_, size_t max_dynamic_types_, size_t global_max_dynamic_types_, const StatisticsPtr & statistics_ = {});
public: public:
/** Create immutable column using immutable arguments. This arguments may be shared with other columns. /** Create immutable column using immutable arguments. This arguments may be shared with other columns.
* Use IColumn::mutate in order to make mutable column and mutate shared nested columns. * Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
*/ */
using Base = COWHelper<IColumnHelper<ColumnDynamic>, ColumnDynamic>; using Base = COWHelper<IColumnHelper<ColumnDynamic>, ColumnDynamic>;
static Ptr create(const ColumnPtr & variant_column_, const VariantInfo & variant_info_, size_t max_dynamic_types_, const Statistics & statistics_ = {}) static Ptr create(const ColumnPtr & variant_column_, const VariantInfo & variant_info_, size_t max_dynamic_types_, size_t global_max_dynamic_types_, const StatisticsPtr & statistics_ = {})
{ {
return ColumnDynamic::create(variant_column_->assumeMutable(), variant_info_, max_dynamic_types_, statistics_); return ColumnDynamic::create(variant_column_->assumeMutable(), variant_info_, max_dynamic_types_, global_max_dynamic_types_, statistics_);
} }
static MutablePtr create(MutableColumnPtr variant_column_, const VariantInfo & variant_info_, size_t max_dynamic_types_, const Statistics & statistics_ = {}) static MutablePtr create(MutableColumnPtr variant_column_, const VariantInfo & variant_info_, size_t max_dynamic_types_, size_t global_max_dynamic_types_, const StatisticsPtr & statistics_ = {})
{ {
return Base::create(std::move(variant_column_), variant_info_, max_dynamic_types_, statistics_); return Base::create(std::move(variant_column_), variant_info_, max_dynamic_types_, global_max_dynamic_types_, statistics_);
} }
static MutablePtr create(MutableColumnPtr variant_column_, const DataTypePtr & variant_type, size_t max_dynamic_types_, const Statistics & statistics_ = {}); static MutablePtr create(MutableColumnPtr variant_column_, const DataTypePtr & variant_type_, size_t max_dynamic_types_, size_t global_max_dynamic_types_, const StatisticsPtr & statistics_ = {})
static ColumnPtr create(ColumnPtr variant_column_, const DataTypePtr & variant_type, size_t max_dynamic_types_, const Statistics & statistics_ = {})
{ {
return create(variant_column_->assumeMutable(), variant_type, max_dynamic_types_, statistics_); return Base::create(std::move(variant_column_), variant_type_, max_dynamic_types_, global_max_dynamic_types_, statistics_);
}
static ColumnPtr create(ColumnPtr variant_column_, const DataTypePtr & variant_type, size_t max_dynamic_types_, size_t global_max_dynamic_types_, const StatisticsPtr & statistics_ = {})
{
return create(variant_column_->assumeMutable(), variant_type, max_dynamic_types_, global_max_dynamic_types_, statistics_);
} }
static MutablePtr create(size_t max_dynamic_types_) static MutablePtr create(size_t max_dynamic_types_)
@ -83,7 +111,7 @@ public:
return Base::create(max_dynamic_types_); return Base::create(max_dynamic_types_);
} }
std::string getName() const override { return "Dynamic(max_types=" + std::to_string(max_dynamic_types) + ")"; } std::string getName() const override { return "Dynamic(max_types=" + std::to_string(global_max_dynamic_types) + ")"; }
const char * getFamilyName() const override const char * getFamilyName() const override
{ {
@ -98,12 +126,12 @@ public:
MutableColumnPtr cloneEmpty() const override MutableColumnPtr cloneEmpty() const override
{ {
/// Keep current dynamic structure /// Keep current dynamic structure
return Base::create(variant_column->cloneEmpty(), variant_info, max_dynamic_types, statistics); return Base::create(variant_column->cloneEmpty(), variant_info, max_dynamic_types, global_max_dynamic_types, statistics);
} }
MutableColumnPtr cloneResized(size_t size) const override MutableColumnPtr cloneResized(size_t size) const override
{ {
return Base::create(variant_column->cloneResized(size), variant_info, max_dynamic_types, statistics); return Base::create(variant_column->cloneResized(size), variant_info, max_dynamic_types, global_max_dynamic_types, statistics);
} }
size_t size() const override size_t size() const override
@ -111,15 +139,9 @@ public:
return variant_column->size(); return variant_column->size();
} }
Field operator[](size_t n) const override Field operator[](size_t n) const override;
{
return (*variant_column)[n];
}
void get(size_t n, Field & res) const override void get(size_t n, Field & res) const override;
{
variant_column->get(n, res);
}
bool isDefaultAt(size_t n) const override bool isDefaultAt(size_t n) const override
{ {
@ -187,7 +209,7 @@ public:
ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override
{ {
return create(variant_column->filter(filt, result_size_hint), variant_info, max_dynamic_types); return create(variant_column->filter(filt, result_size_hint), variant_info, max_dynamic_types, global_max_dynamic_types);
} }
void expand(const Filter & mask, bool inverted) override void expand(const Filter & mask, bool inverted) override
@ -197,17 +219,17 @@ public:
ColumnPtr permute(const Permutation & perm, size_t limit) const override ColumnPtr permute(const Permutation & perm, size_t limit) const override
{ {
return create(variant_column->permute(perm, limit), variant_info, max_dynamic_types); return create(variant_column->permute(perm, limit), variant_info, max_dynamic_types, global_max_dynamic_types);
} }
ColumnPtr index(const IColumn & indexes, size_t limit) const override ColumnPtr index(const IColumn & indexes, size_t limit) const override
{ {
return create(variant_column->index(indexes, limit), variant_info, max_dynamic_types); return create(variant_column->index(indexes, limit), variant_info, max_dynamic_types, global_max_dynamic_types);
} }
ColumnPtr replicate(const Offsets & replicate_offsets) const override ColumnPtr replicate(const Offsets & replicate_offsets) const override
{ {
return create(variant_column->replicate(replicate_offsets), variant_info, max_dynamic_types); return create(variant_column->replicate(replicate_offsets), variant_info, max_dynamic_types, global_max_dynamic_types);
} }
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override
@ -216,7 +238,7 @@ public:
MutableColumns scattered_columns; MutableColumns scattered_columns;
scattered_columns.reserve(num_columns); scattered_columns.reserve(num_columns);
for (auto & scattered_variant_column : scattered_variant_columns) for (auto & scattered_variant_column : scattered_variant_columns)
scattered_columns.emplace_back(create(std::move(scattered_variant_column), variant_info, max_dynamic_types)); scattered_columns.emplace_back(create(std::move(scattered_variant_column), variant_info, max_dynamic_types, global_max_dynamic_types));
return scattered_columns; return scattered_columns;
} }
@ -238,16 +260,10 @@ public:
} }
void getPermutation(IColumn::PermutationSortDirection direction, IColumn::PermutationSortStability stability, void getPermutation(IColumn::PermutationSortDirection direction, IColumn::PermutationSortStability stability,
size_t limit, int nan_direction_hint, IColumn::Permutation & res) const override size_t limit, int nan_direction_hint, IColumn::Permutation & res) const override;
{
variant_column->getPermutation(direction, stability, limit, nan_direction_hint, res);
}
void updatePermutation(IColumn::PermutationSortDirection direction, IColumn::PermutationSortStability stability, void updatePermutation(IColumn::PermutationSortDirection direction, IColumn::PermutationSortStability stability,
size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_ranges) const override size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_ranges) const override;
{
variant_column->updatePermutation(direction, stability, limit, nan_direction_hint, res, equal_ranges);
}
void reserve(size_t n) override void reserve(size_t n) override
{ {
@ -295,7 +311,7 @@ public:
bool structureEquals(const IColumn & rhs) const override bool structureEquals(const IColumn & rhs) const override
{ {
if (const auto * rhs_concrete = typeid_cast<const ColumnDynamic *>(&rhs)) if (const auto * rhs_concrete = typeid_cast<const ColumnDynamic *>(&rhs))
return max_dynamic_types == rhs_concrete->max_dynamic_types; return global_max_dynamic_types == rhs_concrete->global_max_dynamic_types;
return false; return false;
} }
@ -338,17 +354,75 @@ public:
const ColumnVariant & getVariantColumn() const { return assert_cast<const ColumnVariant &>(*variant_column); } const ColumnVariant & getVariantColumn() const { return assert_cast<const ColumnVariant &>(*variant_column); }
ColumnVariant & getVariantColumn() { return assert_cast<ColumnVariant &>(*variant_column); } ColumnVariant & getVariantColumn() { return assert_cast<ColumnVariant &>(*variant_column); }
bool addNewVariant(const DataTypePtr & new_variant); bool addNewVariant(const DataTypePtr & new_variant, const String & new_variant_name);
void addStringVariant(); bool addNewVariant(const DataTypePtr & new_variant) { return addNewVariant(new_variant, new_variant->getName()); }
bool hasDynamicStructure() const override { return true; } bool hasDynamicStructure() const override { return true; }
void takeDynamicStructureFromSourceColumns(const Columns & source_columns) override; void takeDynamicStructureFromSourceColumns(const Columns & source_columns) override;
const Statistics & getStatistics() const { return statistics; } const StatisticsPtr & getStatistics() const { return statistics; }
void setStatistics(const StatisticsPtr & statistics_) { statistics = statistics_; }
size_t getMaxDynamicTypes() const { return max_dynamic_types; } size_t getMaxDynamicTypes() const { return max_dynamic_types; }
/// Check if we can add new variant types.
/// Shared variant doesn't count in the limit but always presents,
/// so we should subtract 1 from the total types count.
bool canAddNewVariants(size_t current_variants_count, size_t new_variants_count) const { return current_variants_count + new_variants_count - 1 <= max_dynamic_types; }
bool canAddNewVariant(size_t current_variants_count) const { return canAddNewVariants(current_variants_count, 1); }
bool canAddNewVariants(size_t new_variants_count) const { return canAddNewVariants(variant_info.variant_names.size(), new_variants_count); }
bool canAddNewVariant() const { return canAddNewVariants(variant_info.variant_names.size(), 1); }
void setVariantType(const DataTypePtr & variant_type);
void setMaxDynamicPaths(size_t max_dynamic_type_);
static const String & getSharedVariantTypeName()
{
static const String name = SHARED_VARIANT_TYPE_NAME;
return name;
}
static DataTypePtr getSharedVariantDataType();
ColumnVariant::Discriminator getSharedVariantDiscriminator() const
{
return variant_info.variant_name_to_discriminator.at(getSharedVariantTypeName());
}
ColumnString & getSharedVariant()
{
return assert_cast<ColumnString &>(getVariantColumn().getVariantByGlobalDiscriminator(getSharedVariantDiscriminator()));
}
const ColumnString & getSharedVariant() const
{
return assert_cast<const ColumnString &>(getVariantColumn().getVariantByGlobalDiscriminator(getSharedVariantDiscriminator()));
}
/// Serializes type and value in binary format into provided shared variant. Doesn't update Variant discriminators and offsets.
static void serializeValueIntoSharedVariant(ColumnString & shared_variant, const IColumn & src, const DataTypePtr & type, const SerializationPtr & serialization, size_t n);
/// Insert value into shared variant. Also updates Variant discriminators and offsets.
void insertValueIntoSharedVariant(const IColumn & src, const DataTypePtr & type, const String & type_name, size_t n);
const SerializationPtr & getVariantSerialization(const DataTypePtr & variant_type, const String & variant_name) const
{
/// Get serialization for provided data type.
/// To avoid calling type->getDefaultSerialization() every time we use simple cache with max size.
/// When max size is reached, just clear the cache.
if (serialization_cache.size() == SERIALIZATION_CACHE_MAX_SIZE)
serialization_cache.clear();
if (auto it = serialization_cache.find(variant_name); it != serialization_cache.end())
return it->second;
return serialization_cache.emplace(variant_name, variant_type->getDefaultSerialization()).first->second;
}
const SerializationPtr & getVariantSerialization(const DataTypePtr & variant_type) const { return getVariantSerialization(variant_type, variant_type->getName()); }
private: private:
void createVariantInfo(const DataTypePtr & variant_type);
/// Combine current variant with the other variant and return global discriminators mapping /// Combine current variant with the other variant and return global discriminators mapping
/// from other variant to the combined one. It's used for inserting from /// from other variant to the combined one. It's used for inserting from
/// different variants. /// different variants.
@ -361,12 +435,19 @@ private:
/// Store the type of current variant with some additional information. /// Store the type of current variant with some additional information.
VariantInfo variant_info; VariantInfo variant_info;
/// The maximum number of different types that can be stored in this Dynamic column. /// The maximum number of different types that can be stored in this Dynamic column.
/// If exceeded, all new variants will be converted to String. /// If exceeded, all new variants will be added to a special shared variant with type String
/// in binary format. This limit can be different for different instances of Dynamic column.
/// When max_dynamic_types = 0, we will have only shared variant and insert all values into it.
size_t max_dynamic_types; size_t max_dynamic_types;
/// The types limit specified in the data type by the user Dynamic(max_types=N).
/// max_dynamic_types in all column instances of this Dynamic type can be only smaller
/// (for example, max_dynamic_types can be reduced in takeDynamicStructureFromSourceColumns
/// before merge of different Dynamic columns).
size_t global_max_dynamic_types;
/// Size statistics of each variants from MergeTree data part. /// Size statistics of each variants from MergeTree data part.
/// Used in takeDynamicStructureFromSourceColumns and set during deserialization. /// Used in takeDynamicStructureFromSourceColumns and set during deserialization.
Statistics statistics; StatisticsPtr statistics;
/// Cache (Variant name) -> (global discriminators mapping from this variant to current variant in Dynamic column). /// Cache (Variant name) -> (global discriminators mapping from this variant to current variant in Dynamic column).
/// Used to avoid mappings recalculation in combineVariants for the same Variant types. /// Used to avoid mappings recalculation in combineVariants for the same Variant types.
@ -374,6 +455,17 @@ private:
/// Cache of Variant types that couldn't be combined with current variant in Dynamic column. /// Cache of Variant types that couldn't be combined with current variant in Dynamic column.
/// Used to avoid checking if combination is possible for the same Variant types. /// Used to avoid checking if combination is possible for the same Variant types.
std::unordered_set<String> variants_with_failed_combination; std::unordered_set<String> variants_with_failed_combination;
/// We can use serializations of different data types to serialize values into shared variant.
/// To avoid creating the same serialization multiple times, use simple cache.
static const size_t SERIALIZATION_CACHE_MAX_SIZE = 256;
mutable std::unordered_map<String, SerializationPtr> serialization_cache;
}; };
void extendVariantColumn(
IColumn & variant_column,
const DataTypePtr & old_variant_type,
const DataTypePtr & new_variant_type,
std::unordered_map<String, UInt8> old_variant_name_to_discriminator);
} }

29
src/Columns/ColumnVariant.cpp
View File

@ -476,7 +476,7 @@ void ColumnVariant::insertFromImpl(const DB::IColumn & src_, size_t n, const std
} }
} }
void ColumnVariant::insertRangeFromImpl(const DB::IColumn & src_, size_t start, size_t length, const std::vector<ColumnVariant::Discriminator> * global_discriminators_mapping) void ColumnVariant::insertRangeFromImpl(const DB::IColumn & src_, size_t start, size_t length, const std::vector<ColumnVariant::Discriminator> * global_discriminators_mapping, const Discriminator * skip_discriminator)
{ {
const size_t num_variants = variants.size(); const size_t num_variants = variants.size();
const auto & src = assert_cast<const ColumnVariant &>(src_); const auto & src = assert_cast<const ColumnVariant &>(src_);
@ -557,9 +557,12 @@ void ColumnVariant::insertRangeFromImpl(const DB::IColumn & src_, size_t start,
Discriminator global_discr = src_global_discr; Discriminator global_discr = src_global_discr;
if (global_discriminators_mapping && src_global_discr != NULL_DISCRIMINATOR) if (global_discriminators_mapping && src_global_discr != NULL_DISCRIMINATOR)
global_discr = (*global_discriminators_mapping)[src_global_discr]; global_discr = (*global_discriminators_mapping)[src_global_discr];
Discriminator local_discr = localDiscriminatorByGlobal(global_discr); if (!skip_discriminator || global_discr != *skip_discriminator)
if (nested_length) {
variants[local_discr]->insertRangeFrom(*src.variants[src_local_discr], nested_start, nested_length); Discriminator local_discr = localDiscriminatorByGlobal(global_discr);
if (nested_length)
variants[local_discr]->insertRangeFrom(*src.variants[src_local_discr], nested_start, nested_length);
}
} }
} }
@ -610,7 +613,7 @@ void ColumnVariant::insertRangeFrom(const IColumn & src_, size_t start, size_t l
void ColumnVariant::doInsertRangeFrom(const IColumn & src_, size_t start, size_t length) void ColumnVariant::doInsertRangeFrom(const IColumn & src_, size_t start, size_t length)
#endif #endif
{ {
insertRangeFromImpl(src_, start, length, nullptr); insertRangeFromImpl(src_, start, length, nullptr, nullptr);
} }
#if !defined(DEBUG_OR_SANITIZER_BUILD) #if !defined(DEBUG_OR_SANITIZER_BUILD)
@ -627,9 +630,9 @@ void ColumnVariant::insertFrom(const DB::IColumn & src_, size_t n, const std::ve
insertFromImpl(src_, n, &global_discriminators_mapping); insertFromImpl(src_, n, &global_discriminators_mapping);
} }
void ColumnVariant::insertRangeFrom(const IColumn & src_, size_t start, size_t length, const std::vector<ColumnVariant::Discriminator> & global_discriminators_mapping) void ColumnVariant::insertRangeFrom(const IColumn & src_, size_t start, size_t length, const std::vector<ColumnVariant::Discriminator> & global_discriminators_mapping, Discriminator skip_discriminator)
{ {
insertRangeFromImpl(src_, start, length, &global_discriminators_mapping); insertRangeFromImpl(src_, start, length, &global_discriminators_mapping, &skip_discriminator);
} }
void ColumnVariant::insertManyFrom(const DB::IColumn & src_, size_t position, size_t length, const std::vector<ColumnVariant::Discriminator> & global_discriminators_mapping) void ColumnVariant::insertManyFrom(const DB::IColumn & src_, size_t position, size_t length, const std::vector<ColumnVariant::Discriminator> & global_discriminators_mapping)
@ -673,6 +676,14 @@ void ColumnVariant::insertManyIntoVariantFrom(DB::ColumnVariant::Discriminator g
variants[local_discr]->insertManyFrom(src_, position, length); variants[local_discr]->insertManyFrom(src_, position, length);
} }
void ColumnVariant::deserializeBinaryIntoVariant(ColumnVariant::Discriminator global_discr, const SerializationPtr & serialization, ReadBuffer & buf, const FormatSettings & format_settings)
{
auto local_discr = localDiscriminatorByGlobal(global_discr);
serialization->deserializeBinary(*variants[local_discr], buf, format_settings);
getLocalDiscriminators().push_back(local_discr);
getOffsets().push_back(variants[local_discr]->size() - 1);
}
void ColumnVariant::insertDefault() void ColumnVariant::insertDefault()
{ {
getLocalDiscriminators().push_back(NULL_DISCRIMINATOR); getLocalDiscriminators().push_back(NULL_DISCRIMINATOR);
@ -1213,9 +1224,7 @@ struct ColumnVariant::ComparatorBase
ALWAYS_INLINE int compare(size_t lhs, size_t rhs) const ALWAYS_INLINE int compare(size_t lhs, size_t rhs) const
{ {
int res = parent.compareAt(lhs, rhs, parent, nan_direction_hint); return parent.compareAt(lhs, rhs, parent, nan_direction_hint);
return res;
} }
}; };

11
src/Columns/ColumnVariant.h
View File

@ -2,6 +2,8 @@
#include <Columns/IColumn.h> #include <Columns/IColumn.h>
#include <Columns/ColumnVector.h> #include <Columns/ColumnVector.h>
#include <Formats/FormatSettings.h>
#include <DataTypes/Serializations/ISerialization.h>
namespace DB namespace DB
@ -196,13 +198,15 @@ public:
/// Methods for insertion from another Variant but with known mapping between global discriminators. /// Methods for insertion from another Variant but with known mapping between global discriminators.
void insertFrom(const IColumn & src_, size_t n, const std::vector<ColumnVariant::Discriminator> & global_discriminators_mapping); void insertFrom(const IColumn & src_, size_t n, const std::vector<ColumnVariant::Discriminator> & global_discriminators_mapping);
void insertRangeFrom(const IColumn & src_, size_t start, size_t length, const std::vector<ColumnVariant::Discriminator> & global_discriminators_mapping); /// Don't insert data into variant with skip_discriminator global discriminator, it will be processed separately.
void insertRangeFrom(const IColumn & src_, size_t start, size_t length, const std::vector<ColumnVariant::Discriminator> & global_discriminators_mapping, Discriminator skip_discriminator);
void insertManyFrom(const IColumn & src_, size_t position, size_t length, const std::vector<ColumnVariant::Discriminator> & global_discriminators_mapping); void insertManyFrom(const IColumn & src_, size_t position, size_t length, const std::vector<ColumnVariant::Discriminator> & global_discriminators_mapping);
/// Methods for insertion into a specific variant. /// Methods for insertion into a specific variant.
void insertIntoVariantFrom(Discriminator global_discr, const IColumn & src_, size_t n); void insertIntoVariantFrom(Discriminator global_discr, const IColumn & src_, size_t n);
void insertRangeIntoVariantFrom(Discriminator global_discr, const IColumn & src_, size_t start, size_t length); void insertRangeIntoVariantFrom(Discriminator global_discr, const IColumn & src_, size_t start, size_t length);
void insertManyIntoVariantFrom(Discriminator global_discr, const IColumn & src_, size_t position, size_t length); void insertManyIntoVariantFrom(Discriminator global_discr, const IColumn & src_, size_t position, size_t length);
void deserializeBinaryIntoVariant(Discriminator global_discr, const SerializationPtr & serialization, ReadBuffer & buf, const FormatSettings & format_settings);
void insertDefault() override; void insertDefault() override;
void insertManyDefaults(size_t length) override; void insertManyDefaults(size_t length) override;
@ -264,6 +268,7 @@ public:
ColumnPtr & getVariantPtrByGlobalDiscriminator(size_t discr) { return variants[global_to_local_discriminators.at(discr)]; } ColumnPtr & getVariantPtrByGlobalDiscriminator(size_t discr) { return variants[global_to_local_discriminators.at(discr)]; }
const NestedColumns & getVariants() const { return variants; } const NestedColumns & getVariants() const { return variants; }
NestedColumns & getVariants() { return variants; }
const IColumn & getLocalDiscriminatorsColumn() const { return *local_discriminators; } const IColumn & getLocalDiscriminatorsColumn() const { return *local_discriminators; }
IColumn & getLocalDiscriminatorsColumn() { return *local_discriminators; } IColumn & getLocalDiscriminatorsColumn() { return *local_discriminators; }
@ -303,6 +308,8 @@ public:
return true; return true;
} }
std::vector<Discriminator> getLocalToGlobalDiscriminatorsMapping() const { return local_to_global_discriminators; }
/// Check if we have only 1 non-empty variant and no NULL values, /// Check if we have only 1 non-empty variant and no NULL values,
/// and if so, return the discriminator of this non-empty column. /// and if so, return the discriminator of this non-empty column.
std::optional<Discriminator> getLocalDiscriminatorOfOneNoneEmptyVariantNoNulls() const; std::optional<Discriminator> getLocalDiscriminatorOfOneNoneEmptyVariantNoNulls() const;
@ -323,7 +330,7 @@ public:
private: private:
void insertFromImpl(const IColumn & src_, size_t n, const std::vector<ColumnVariant::Discriminator> * global_discriminators_mapping); void insertFromImpl(const IColumn & src_, size_t n, const std::vector<ColumnVariant::Discriminator> * global_discriminators_mapping);
void insertRangeFromImpl(const IColumn & src_, size_t start, size_t length, const std::vector<ColumnVariant::Discriminator> * global_discriminators_mapping); void insertRangeFromImpl(const IColumn & src_, size_t start, size_t length, const std::vector<ColumnVariant::Discriminator> * global_discriminators_mapping, const Discriminator * skip_discriminator);
void insertManyFromImpl(const IColumn & src_, size_t position, size_t length, const std::vector<ColumnVariant::Discriminator> * global_discriminators_mapping); void insertManyFromImpl(const IColumn & src_, size_t position, size_t length, const std::vector<ColumnVariant::Discriminator> * global_discriminators_mapping);
void initIdentityGlobalToLocalDiscriminatorsMapping(); void initIdentityGlobalToLocalDiscriminatorsMapping();

486
src/Columns/tests/gtest_column_dynamic.cpp
View File

@ -7,28 +7,34 @@ using namespace DB;
TEST(ColumnDynamic, CreateEmpty) TEST(ColumnDynamic, CreateEmpty)
{ {
auto column = ColumnDynamic::create(255); auto column = ColumnDynamic::create(254);
ASSERT_TRUE(column->empty()); ASSERT_TRUE(column->empty());
ASSERT_EQ(column->getVariantInfo().variant_type->getName(), "Variant()"); ASSERT_EQ(column->getVariantInfo().variant_type->getName(), "Variant(SharedVariant)");
ASSERT_TRUE(column->getVariantInfo().variant_names.empty()); ASSERT_EQ(column->getVariantInfo().variant_names.size(), 1);
ASSERT_TRUE(column->getVariantInfo().variant_name_to_discriminator.empty()); ASSERT_EQ(column->getVariantInfo().variant_names[0], "SharedVariant");
ASSERT_EQ(column->getVariantInfo().variant_name_to_discriminator.size(), 1);
ASSERT_EQ(column->getVariantInfo().variant_name_to_discriminator.at("SharedVariant"), 0);
ASSERT_TRUE(column->getVariantColumn().getVariantByGlobalDiscriminator(0).empty());
} }
TEST(ColumnDynamic, InsertDefault) TEST(ColumnDynamic, InsertDefault)
{ {
auto column = ColumnDynamic::create(255); auto column = ColumnDynamic::create(254);
column->insertDefault(); column->insertDefault();
ASSERT_TRUE(column->size() == 1); ASSERT_TRUE(column->size() == 1);
ASSERT_EQ(column->getVariantInfo().variant_type->getName(), "Variant()"); ASSERT_EQ(column->getVariantInfo().variant_type->getName(), "Variant(SharedVariant)");
ASSERT_TRUE(column->getVariantInfo().variant_names.empty()); ASSERT_EQ(column->getVariantInfo().variant_names.size(), 1);
ASSERT_TRUE(column->getVariantInfo().variant_name_to_discriminator.empty()); ASSERT_EQ(column->getVariantInfo().variant_names[0], "SharedVariant");
ASSERT_EQ(column->getVariantInfo().variant_name_to_discriminator.size(), 1);
ASSERT_EQ(column->getVariantInfo().variant_name_to_discriminator.at("SharedVariant"), 0);
ASSERT_TRUE(column->getVariantColumn().getVariantByGlobalDiscriminator(0).empty());
ASSERT_TRUE(column->isNullAt(0)); ASSERT_TRUE(column->isNullAt(0));
ASSERT_EQ((*column)[0], Field(Null())); ASSERT_EQ((*column)[0], Field(Null()));
} }
TEST(ColumnDynamic, InsertFields) TEST(ColumnDynamic, InsertFields)
{ {
auto column = ColumnDynamic::create(255); auto column = ColumnDynamic::create(254);
column->insert(Field(42)); column->insert(Field(42));
column->insert(Field(-42)); column->insert(Field(-42));
column->insert(Field("str1")); column->insert(Field("str1"));
@ -41,16 +47,16 @@ TEST(ColumnDynamic, InsertFields)
column->insert(Field(43.43)); column->insert(Field(43.43));
ASSERT_TRUE(column->size() == 10); ASSERT_TRUE(column->size() == 10);
ASSERT_EQ(column->getVariantInfo().variant_type->getName(), "Variant(Float64, Int8, String)"); ASSERT_EQ(column->getVariantInfo().variant_type->getName(), "Variant(Float64, Int8, SharedVariant, String)");
std::vector<String> expected_names = {"Float64", "Int8", "String"}; std::vector<String> expected_names = {"Float64", "Int8", "SharedVariant", "String"};
ASSERT_EQ(column->getVariantInfo().variant_names, expected_names); ASSERT_EQ(column->getVariantInfo().variant_names, expected_names);
std::unordered_map<String, UInt8> expected_variant_name_to_discriminator = {{"Float64", 0}, {"Int8", 1}, {"String", 2}}; std::unordered_map<String, UInt8> expected_variant_name_to_discriminator = {{"Float64", 0}, {"Int8", 1}, {"SharedVariant", 2}, {"String", 3}};
ASSERT_TRUE(column->getVariantInfo().variant_name_to_discriminator == expected_variant_name_to_discriminator); ASSERT_TRUE(column->getVariantInfo().variant_name_to_discriminator == expected_variant_name_to_discriminator);
} }
ColumnDynamic::MutablePtr getDynamicWithManyVariants(size_t num_variants, Field tuple_element = Field(42)) ColumnDynamic::MutablePtr getDynamicWithManyVariants(size_t num_variants, Field tuple_element = Field(42))
{ {
auto column = ColumnDynamic::create(255); auto column = ColumnDynamic::create(254);
for (size_t i = 0; i != num_variants; ++i) for (size_t i = 0; i != num_variants; ++i)
{ {
Tuple tuple; Tuple tuple;
@ -66,61 +72,71 @@ TEST(ColumnDynamic, InsertFieldsOverflow1)
{ {
auto column = getDynamicWithManyVariants(253); auto column = getDynamicWithManyVariants(253);
ASSERT_EQ(column->getVariantInfo().variant_names.size(), 253); ASSERT_EQ(column->getVariantInfo().variant_names.size(), 254);
column->insert(Field(42.42)); column->insert(Field(42.42));
ASSERT_EQ(column->getVariantInfo().variant_names.size(), 254); ASSERT_EQ(column->size(), 254);
ASSERT_EQ(column->getVariantInfo().variant_names.size(), 255);
ASSERT_TRUE(column->getVariantInfo().variant_name_to_discriminator.contains("Float64")); ASSERT_TRUE(column->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
column->insert(Field(42)); column->insert(Field(42));
ASSERT_EQ(column->size(), 255);
ASSERT_EQ(column->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column->getVariantInfo().variant_names.size(), 255);
ASSERT_FALSE(column->getVariantInfo().variant_name_to_discriminator.contains("Int8")); ASSERT_FALSE(column->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_TRUE(column->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_EQ(column->getSharedVariant().size(), 1);
Field field = (*column)[column->size() - 1]; Field field = (*column)[column->size() - 1];
ASSERT_EQ(field, "42"); ASSERT_EQ(field, 42);
column->insert(Field(43)); column->insert(Field(43));
ASSERT_EQ(column->size(), 256);
ASSERT_EQ(column->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column->getVariantInfo().variant_names.size(), 255);
ASSERT_FALSE(column->getVariantInfo().variant_name_to_discriminator.contains("Int8")); ASSERT_FALSE(column->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_TRUE(column->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_EQ(column->getSharedVariant().size(), 2);
field = (*column)[column->size() - 1]; field = (*column)[column->size() - 1];
ASSERT_EQ(field, "43"); ASSERT_EQ(field, 43);
column->insert(Field("str1")); column->insert(Field("str1"));
ASSERT_EQ(column->size(), 257);
ASSERT_EQ(column->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column->getVariantInfo().variant_names.size(), 255);
ASSERT_FALSE(column->getVariantInfo().variant_name_to_discriminator.contains("Int8")); ASSERT_FALSE(column->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_TRUE(column->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_FALSE(column->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_EQ(column->getSharedVariant().size(), 3);
field = (*column)[column->size() - 1]; field = (*column)[column->size() - 1];
ASSERT_EQ(field, "str1"); ASSERT_EQ(field, "str1");
column->insert(Field(Array({Field(42), Field(43)}))); column->insert(Field(Array({Field(42), Field(43)})));
ASSERT_EQ(column->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column->getVariantInfo().variant_names.size(), 255);
ASSERT_FALSE(column->getVariantInfo().variant_name_to_discriminator.contains("Array(Int8)")); ASSERT_FALSE(column->getVariantInfo().variant_name_to_discriminator.contains("Array(Int8)"));
ASSERT_TRUE(column->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_FALSE(column->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_EQ(column->getSharedVariant().size(), 4);
field = (*column)[column->size() - 1]; field = (*column)[column->size() - 1];
ASSERT_EQ(field, "[42, 43]"); ASSERT_EQ(field, Field(Array({Field(42), Field(43)})));
} }
TEST(ColumnDynamic, InsertFieldsOverflow2) TEST(ColumnDynamic, InsertFieldsOverflow2)
{ {
auto column = getDynamicWithManyVariants(254); auto column = getDynamicWithManyVariants(254);
ASSERT_EQ(column->getVariantInfo().variant_names.size(), 254); ASSERT_EQ(column->getVariantInfo().variant_names.size(), 255);
column->insert(Field("str1")); column->insert(Field("str1"));
ASSERT_EQ(column->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column->getVariantInfo().variant_names.size(), 255);
ASSERT_TRUE(column->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_FALSE(column->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_EQ(column->getSharedVariant().size(), 1);
Field field = (*column)[column->size() - 1];
ASSERT_EQ(field, "str1");
column->insert(Field(42)); column->insert(Field(42));
ASSERT_EQ(column->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column->getVariantInfo().variant_names.size(), 255);
ASSERT_FALSE(column->getVariantInfo().variant_name_to_discriminator.contains("Int8")); ASSERT_FALSE(column->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_TRUE(column->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_FALSE(column->getVariantInfo().variant_name_to_discriminator.contains("String"));
Field field = (*column)[column->size() - 1]; ASSERT_EQ(column->getSharedVariant().size(), 2);
ASSERT_EQ(field, "42"); field = (*column)[column->size() - 1];
ASSERT_EQ(field, 42);
} }
ColumnDynamic::MutablePtr getInsertFromColumn(size_t num = 1) ColumnDynamic::MutablePtr getInsertFromColumn(size_t num = 1)
{ {
auto column_from = ColumnDynamic::create(255); auto column_from = ColumnDynamic::create(254);
for (size_t i = 0; i != num; ++i) for (size_t i = 0; i != num; ++i)
{ {
column_from->insert(Field(42)); column_from->insert(Field(42));
@ -154,41 +170,41 @@ void checkInsertFrom(const ColumnDynamic::MutablePtr & column_from, ColumnDynami
TEST(ColumnDynamic, InsertFrom1) TEST(ColumnDynamic, InsertFrom1)
{ {
auto column_to = ColumnDynamic::create(255); auto column_to = ColumnDynamic::create(254);
checkInsertFrom(getInsertFromColumn(), column_to, "Variant(Float64, Int8, String)", {"Float64", "Int8", "String"}, {{"Float64", 0}, {"Int8", 1}, {"String", 2}}); checkInsertFrom(getInsertFromColumn(), column_to, "Variant(Float64, Int8, SharedVariant, String)", {"Float64", "Int8", "SharedVariant", "String"}, {{"Float64", 0}, {"Int8", 1}, {"SharedVariant", 2}, {"String", 3}});
} }
TEST(ColumnDynamic, InsertFrom2) TEST(ColumnDynamic, InsertFrom2)
{ {
auto column_to = ColumnDynamic::create(255); auto column_to = ColumnDynamic::create(254);
column_to->insert(Field(42)); column_to->insert(Field(42));
column_to->insert(Field(42.42)); column_to->insert(Field(42.42));
column_to->insert(Field("str")); column_to->insert(Field("str"));
checkInsertFrom(getInsertFromColumn(), column_to, "Variant(Float64, Int8, String)", {"Float64", "Int8", "String"}, {{"Float64", 0}, {"Int8", 1}, {"String", 2}}); checkInsertFrom(getInsertFromColumn(), column_to, "Variant(Float64, Int8, SharedVariant, String)", {"Float64", "Int8", "SharedVariant", "String"}, {{"Float64", 0}, {"Int8", 1}, {"SharedVariant", 2}, {"String", 3}});
} }
TEST(ColumnDynamic, InsertFrom3) TEST(ColumnDynamic, InsertFrom3)
{ {
auto column_to = ColumnDynamic::create(255); auto column_to = ColumnDynamic::create(254);
column_to->insert(Field(42)); column_to->insert(Field(42));
column_to->insert(Field(42.42)); column_to->insert(Field(42.42));
column_to->insert(Field("str")); column_to->insert(Field("str"));
column_to->insert(Array({Field(42)})); column_to->insert(Array({Field(42)}));
checkInsertFrom(getInsertFromColumn(), column_to, "Variant(Array(Int8), Float64, Int8, String)", {"Array(Int8)", "Float64", "Int8", "String"}, {{"Array(Int8)", 0}, {"Float64", 1}, {"Int8", 2}, {"String", 3}}); checkInsertFrom(getInsertFromColumn(), column_to, "Variant(Array(Int8), Float64, Int8, SharedVariant, String)", {"Array(Int8)", "Float64", "Int8", "SharedVariant", "String"}, {{"Array(Int8)", 0}, {"Float64", 1}, {"Int8", 2}, {"SharedVariant", 3}, {"String", 4}});
} }
TEST(ColumnDynamic, InsertFromOverflow1) TEST(ColumnDynamic, InsertFromOverflow1)
{ {
auto column_from = ColumnDynamic::create(255); auto column_from = ColumnDynamic::create(254);
column_from->insert(Field(42)); column_from->insert(Field(42));
column_from->insert(Field(42.42)); column_from->insert(Field(42.42));
column_from->insert(Field("str")); column_from->insert(Field("str"));
auto column_to = getDynamicWithManyVariants(253); auto column_to = getDynamicWithManyVariants(253);
column_to->insertFrom(*column_from, 0); column_to->insertFrom(*column_from, 0);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 254); ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8")); ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
auto field = (*column_to)[column_to->size() - 1]; auto field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, 42); ASSERT_EQ(field, 42);
@ -196,20 +212,22 @@ TEST(ColumnDynamic, InsertFromOverflow1)
column_to->insertFrom(*column_from, 1); column_to->insertFrom(*column_from, 1);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255);
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_EQ(column_to->getSharedVariant().size(), 1);
field = (*column_to)[column_to->size() - 1]; field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, "42.42"); ASSERT_EQ(field, 42.42);
column_to->insertFrom(*column_from, 2); column_to->insertFrom(*column_from, 2);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_EQ(column_to->getSharedVariant().size(), 2);
field = (*column_to)[column_to->size() - 1]; field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, "str"); ASSERT_EQ(field, "str");
} }
TEST(ColumnDynamic, InsertFromOverflow2) TEST(ColumnDynamic, InsertFromOverflow2)
{ {
auto column_from = ColumnDynamic::create(255); auto column_from = ColumnDynamic::create(254);
column_from->insert(Field(42)); column_from->insert(Field(42));
column_from->insert(Field(42.42)); column_from->insert(Field(42.42));
@ -221,9 +239,32 @@ TEST(ColumnDynamic, InsertFromOverflow2)
column_to->insertFrom(*column_from, 1); column_to->insertFrom(*column_from, 1);
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_EQ(column_to->getSharedVariant().size(), 1);
field = (*column_to)[column_to->size() - 1]; field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, "42.42"); ASSERT_EQ(field, 42.42);
}
TEST(ColumnDynamic, InsertFromOverflow3)
{
auto column_from = ColumnDynamic::create(1);
column_from->insert(Field(42));
column_from->insert(Field(42.42));
auto column_to = ColumnDynamic::create(254);
column_to->insert(Field(41));
column_to->insertFrom(*column_from, 0);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_EQ(column_to->getSharedVariant().size(), 0);
auto field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, 42);
column_to->insertFrom(*column_from, 1);
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_EQ(column_to->getSharedVariant().size(), 1);
field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, 42.42);
} }
void checkInsertManyFrom(const ColumnDynamic::MutablePtr & column_from, ColumnDynamic::MutablePtr & column_to, const std::string & expected_variant, const std::vector<String> & expected_names, const std::unordered_map<String, UInt8> & expected_variant_name_to_discriminator) void checkInsertManyFrom(const ColumnDynamic::MutablePtr & column_from, ColumnDynamic::MutablePtr & column_to, const std::string & expected_variant, const std::vector<String> & expected_names, const std::unordered_map<String, UInt8> & expected_variant_name_to_discriminator)
@ -256,42 +297,43 @@ void checkInsertManyFrom(const ColumnDynamic::MutablePtr & column_from, ColumnDy
TEST(ColumnDynamic, InsertManyFrom1) TEST(ColumnDynamic, InsertManyFrom1)
{ {
auto column_to = ColumnDynamic::create(255); auto column_to = ColumnDynamic::create(254);
checkInsertManyFrom(getInsertFromColumn(), column_to, "Variant(Float64, Int8, String)", {"Float64", "Int8", "String"}, {{"Float64", 0}, {"Int8", 1}, {"String", 2}}); checkInsertManyFrom(getInsertFromColumn(), column_to, "Variant(Float64, Int8, SharedVariant, String)", {"Float64", "Int8", "SharedVariant", "String"}, {{"Float64", 0}, {"Int8", 1}, {"SharedVariant", 2}, {"String", 3}});
} }
TEST(ColumnDynamic, InsertManyFrom2) TEST(ColumnDynamic, InsertManyFrom2)
{ {
auto column_to = ColumnDynamic::create(255); auto column_to = ColumnDynamic::create(254);
column_to->insert(Field(42)); column_to->insert(Field(42));
column_to->insert(Field(42.42)); column_to->insert(Field(42.42));
column_to->insert(Field("str")); column_to->insert(Field("str"));
checkInsertManyFrom(getInsertFromColumn(), column_to, "Variant(Float64, Int8, String)", {"Float64", "Int8", "String"}, {{"Float64", 0}, {"Int8", 1}, {"String", 2}}); checkInsertManyFrom(getInsertFromColumn(), column_to, "Variant(Float64, Int8, SharedVariant, String)", {"Float64", "Int8", "SharedVariant", "String"}, {{"Float64", 0}, {"Int8", 1}, {"SharedVariant", 2}, {"String", 3}});
} }
TEST(ColumnDynamic, InsertManyFrom3) TEST(ColumnDynamic, InsertManyFrom3)
{ {
auto column_to = ColumnDynamic::create(255); auto column_to = ColumnDynamic::create(254);
column_to->insert(Field(42)); column_to->insert(Field(42));
column_to->insert(Field(42.42)); column_to->insert(Field(42.42));
column_to->insert(Field("str")); column_to->insert(Field("str"));
column_to->insert(Array({Field(42)})); column_to->insert(Array({Field(42)}));
checkInsertManyFrom(getInsertFromColumn(), column_to, "Variant(Array(Int8), Float64, Int8, String)", {"Array(Int8)", "Float64", "Int8", "String"}, {{"Array(Int8)", 0}, {"Float64", 1}, {"Int8", 2}, {"String", 3}}); checkInsertManyFrom(getInsertFromColumn(), column_to, "Variant(Array(Int8), Float64, Int8, SharedVariant, String)", {"Array(Int8)", "Float64", "Int8", "SharedVariant", "String"}, {{"Array(Int8)", 0}, {"Float64", 1}, {"Int8", 2}, {"SharedVariant", 3}, {"String", 4}});
} }
TEST(ColumnDynamic, InsertManyFromOverflow1) TEST(ColumnDynamic, InsertManyFromOverflow1)
{ {
auto column_from = ColumnDynamic::create(255); auto column_from = ColumnDynamic::create(254);
column_from->insert(Field(42)); column_from->insert(Field(42));
column_from->insert(Field(42.42)); column_from->insert(Field(42.42));
column_from->insert(Field("str")); column_from->insert(Field("str"));
auto column_to = getDynamicWithManyVariants(253); auto column_to = getDynamicWithManyVariants(253);
column_to->insertManyFrom(*column_from, 0, 2); column_to->insertManyFrom(*column_from, 0, 2);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 254); ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8")); ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_EQ(column_to->getSharedVariant().size(), 0);
auto field = (*column_to)[column_to->size() - 2]; auto field = (*column_to)[column_to->size() - 2];
ASSERT_EQ(field, 42); ASSERT_EQ(field, 42);
field = (*column_to)[column_to->size() - 1]; field = (*column_to)[column_to->size() - 1];
@ -300,15 +342,17 @@ TEST(ColumnDynamic, InsertManyFromOverflow1)
column_to->insertManyFrom(*column_from, 1, 2); column_to->insertManyFrom(*column_from, 1, 2);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255);
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_EQ(column_to->getSharedVariant().size(), 2);
field = (*column_to)[column_to->size() - 2]; field = (*column_to)[column_to->size() - 2];
ASSERT_EQ(field, "42.42"); ASSERT_EQ(field, 42.42);
field = (*column_to)[column_to->size() - 1]; field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, "42.42"); ASSERT_EQ(field, 42.42);
column_to->insertManyFrom(*column_from, 2, 2); column_to->insertManyFrom(*column_from, 2, 2);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_EQ(column_to->getSharedVariant().size(), 4);
field = (*column_to)[column_to->size() - 1]; field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, "str"); ASSERT_EQ(field, "str");
field = (*column_to)[column_to->size() - 2]; field = (*column_to)[column_to->size() - 2];
@ -317,14 +361,15 @@ TEST(ColumnDynamic, InsertManyFromOverflow1)
TEST(ColumnDynamic, InsertManyFromOverflow2) TEST(ColumnDynamic, InsertManyFromOverflow2)
{ {
auto column_from = ColumnDynamic::create(255); auto column_from = ColumnDynamic::create(254);
column_from->insert(Field(42)); column_from->insert(Field(42));
column_from->insert(Field(42.42)); column_from->insert(Field(42.42));
auto column_to = getDynamicWithManyVariants(253); auto column_to = getDynamicWithManyVariants(253);
column_to->insertManyFrom(*column_from, 0, 2); column_to->insertManyFrom(*column_from, 0, 2);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 254); ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8")); ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_EQ(column_to->getSharedVariant().size(), 0);
auto field = (*column_to)[column_to->size() - 2]; auto field = (*column_to)[column_to->size() - 2];
ASSERT_EQ(field, 42); ASSERT_EQ(field, 42);
field = (*column_to)[column_to->size() - 1]; field = (*column_to)[column_to->size() - 1];
@ -333,11 +378,39 @@ TEST(ColumnDynamic, InsertManyFromOverflow2)
column_to->insertManyFrom(*column_from, 1, 2); column_to->insertManyFrom(*column_from, 1, 2);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255);
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_EQ(column_to->getSharedVariant().size(), 2);
field = (*column_to)[column_to->size() - 2]; field = (*column_to)[column_to->size() - 2];
ASSERT_EQ(field, "42.42"); ASSERT_EQ(field, 42.42);
field = (*column_to)[column_to->size() - 1]; field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, "42.42"); ASSERT_EQ(field, 42.42);
}
TEST(ColumnDynamic, InsertManyFromOverflow3)
{
auto column_from = ColumnDynamic::create(1);
column_from->insert(Field(42));
column_from->insert(Field(42.42));
auto column_to = ColumnDynamic::create(254);
column_to->insert(Field(41));
column_to->insertManyFrom(*column_from, 0, 2);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_EQ(column_to->getSharedVariant().size(), 0);
auto field = (*column_to)[column_to->size() - 2];
ASSERT_EQ(field, 42);
field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, 42);
column_to->insertManyFrom(*column_from, 1, 2);
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_EQ(column_to->getSharedVariant().size(), 2);
field = (*column_to)[column_to->size() - 2];
ASSERT_EQ(field, 42.42);
field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, 42.42);
} }
void checkInsertRangeFrom(const ColumnDynamic::MutablePtr & column_from, ColumnDynamic::MutablePtr & column_to, const std::string & expected_variant, const std::vector<String> & expected_names, const std::unordered_map<String, UInt8> & expected_variant_name_to_discriminator) void checkInsertRangeFrom(const ColumnDynamic::MutablePtr & column_from, ColumnDynamic::MutablePtr & column_to, const std::string & expected_variant, const std::vector<String> & expected_names, const std::unordered_map<String, UInt8> & expected_variant_name_to_discriminator)
@ -368,34 +441,34 @@ void checkInsertRangeFrom(const ColumnDynamic::MutablePtr & column_from, ColumnD
TEST(ColumnDynamic, InsertRangeFrom1) TEST(ColumnDynamic, InsertRangeFrom1)
{ {
auto column_to = ColumnDynamic::create(255); auto column_to = ColumnDynamic::create(254);
checkInsertRangeFrom(getInsertFromColumn(2), column_to, "Variant(Float64, Int8, String)", {"Float64", "Int8", "String"}, {{"Float64", 0}, {"Int8", 1}, {"String", 2}}); checkInsertRangeFrom(getInsertFromColumn(2), column_to, "Variant(Float64, Int8, SharedVariant, String)", {"Float64", "Int8", "SharedVariant", "String"}, {{"Float64", 0}, {"Int8", 1}, {"SharedVariant", 2}, {"String", 3}});
} }
TEST(ColumnDynamic, InsertRangeFrom2) TEST(ColumnDynamic, InsertRangeFrom2)
{ {
auto column_to = ColumnDynamic::create(255); auto column_to = ColumnDynamic::create(254);
column_to->insert(Field(42)); column_to->insert(Field(42));
column_to->insert(Field(42.42)); column_to->insert(Field(42.42));
column_to->insert(Field("str1")); column_to->insert(Field("str1"));
checkInsertRangeFrom(getInsertFromColumn(2), column_to, "Variant(Float64, Int8, String)", {"Float64", "Int8", "String"}, {{"Float64", 0}, {"Int8", 1}, {"String", 2}}); checkInsertRangeFrom(getInsertFromColumn(2), column_to, "Variant(Float64, Int8, SharedVariant, String)", {"Float64", "Int8", "SharedVariant", "String"}, {{"Float64", 0}, {"Int8", 1}, {"SharedVariant", 2}, {"String", 3}});
} }
TEST(ColumnDynamic, InsertRangeFrom3) TEST(ColumnDynamic, InsertRangeFrom3)
{ {
auto column_to = ColumnDynamic::create(255); auto column_to = ColumnDynamic::create(254);
column_to->insert(Field(42)); column_to->insert(Field(42));
column_to->insert(Field(42.42)); column_to->insert(Field(42.42));
column_to->insert(Field("str1")); column_to->insert(Field("str1"));
column_to->insert(Array({Field(42)})); column_to->insert(Array({Field(42)}));
checkInsertRangeFrom(getInsertFromColumn(2), column_to, "Variant(Array(Int8), Float64, Int8, String)", {"Array(Int8)", "Float64", "Int8", "String"}, {{"Array(Int8)", 0}, {"Float64", 1}, {"Int8", 2}, {"String", 3}}); checkInsertRangeFrom(getInsertFromColumn(2), column_to, "Variant(Array(Int8), Float64, Int8, SharedVariant, String)", {"Array(Int8)", "Float64", "Int8", "SharedVariant", "String"}, {{"Array(Int8)", 0}, {"Float64", 1}, {"Int8", 2}, {"SharedVariant", 3}, {"String", 4}});
} }
TEST(ColumnDynamic, InsertRangeFromOverflow1) TEST(ColumnDynamic, InsertRangeFromOverflow1)
{ {
auto column_from = ColumnDynamic::create(255); auto column_from = ColumnDynamic::create(254);
column_from->insert(Field(42)); column_from->insert(Field(42));
column_from->insert(Field(43)); column_from->insert(Field(43));
column_from->insert(Field(42.42)); column_from->insert(Field(42.42));
@ -403,23 +476,25 @@ TEST(ColumnDynamic, InsertRangeFromOverflow1)
auto column_to = getDynamicWithManyVariants(253); auto column_to = getDynamicWithManyVariants(253);
column_to->insertRangeFrom(*column_from, 0, 4); column_to->insertRangeFrom(*column_from, 0, 4);
ASSERT_EQ(column_to->size(), 257);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8")); ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_EQ(column_to->getSharedVariant().size(), 2);
auto field = (*column_to)[column_to->size() - 4]; auto field = (*column_to)[column_to->size() - 4];
ASSERT_EQ(field, Field(42)); ASSERT_EQ(field, Field(42));
field = (*column_to)[column_to->size() - 3]; field = (*column_to)[column_to->size() - 3];
ASSERT_EQ(field, Field(43)); ASSERT_EQ(field, Field(43));
field = (*column_to)[column_to->size() - 2]; field = (*column_to)[column_to->size() - 2];
ASSERT_EQ(field, Field("42.42")); ASSERT_EQ(field, Field(42.42));
field = (*column_to)[column_to->size() - 1]; field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, Field("str")); ASSERT_EQ(field, Field("str"));
} }
TEST(ColumnDynamic, InsertRangeFromOverflow2) TEST(ColumnDynamic, InsertRangeFromOverflow2)
{ {
auto column_from = ColumnDynamic::create(255); auto column_from = ColumnDynamic::create(254);
column_from->insert(Field(42)); column_from->insert(Field(42));
column_from->insert(Field(43)); column_from->insert(Field(43));
column_from->insert(Field(42.42)); column_from->insert(Field(42.42));
@ -428,19 +503,20 @@ TEST(ColumnDynamic, InsertRangeFromOverflow2)
column_to->insertRangeFrom(*column_from, 0, 3); column_to->insertRangeFrom(*column_from, 0, 3);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8")); ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_EQ(column_to->getSharedVariant().size(), 1);
auto field = (*column_to)[column_to->size() - 3]; auto field = (*column_to)[column_to->size() - 3];
ASSERT_EQ(field, Field(42)); ASSERT_EQ(field, Field(42));
field = (*column_to)[column_to->size() - 2]; field = (*column_to)[column_to->size() - 2];
ASSERT_EQ(field, Field(43)); ASSERT_EQ(field, Field(43));
field = (*column_to)[column_to->size() - 1]; field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, Field("42.42")); ASSERT_EQ(field, Field(42.42));
} }
TEST(ColumnDynamic, InsertRangeFromOverflow3) TEST(ColumnDynamic, InsertRangeFromOverflow3)
{ {
auto column_from = ColumnDynamic::create(255); auto column_from = ColumnDynamic::create(254);
column_from->insert(Field(42)); column_from->insert(Field(42));
column_from->insert(Field(43)); column_from->insert(Field(43));
column_from->insert(Field(42.42)); column_from->insert(Field(42.42));
@ -449,20 +525,21 @@ TEST(ColumnDynamic, InsertRangeFromOverflow3)
column_to->insert(Field("Str")); column_to->insert(Field("Str"));
column_to->insertRangeFrom(*column_from, 0, 3); column_to->insertRangeFrom(*column_from, 0, 3);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_EQ(column_to->getSharedVariant().size(), 3);
auto field = (*column_to)[column_to->size() - 3]; auto field = (*column_to)[column_to->size() - 3];
ASSERT_EQ(field, Field(42)); ASSERT_EQ(field, Field(42));
field = (*column_to)[column_to->size() - 2]; field = (*column_to)[column_to->size() - 2];
ASSERT_EQ(field, Field(43)); ASSERT_EQ(field, Field(43));
field = (*column_to)[column_to->size() - 1]; field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, Field("42.42")); ASSERT_EQ(field, Field(42.42));
} }
TEST(ColumnDynamic, InsertRangeFromOverflow4) TEST(ColumnDynamic, InsertRangeFromOverflow4)
{ {
auto column_from = ColumnDynamic::create(255); auto column_from = ColumnDynamic::create(254);
column_from->insert(Field(42)); column_from->insert(Field(42));
column_from->insert(Field(42.42)); column_from->insert(Field(42.42));
column_from->insert(Field("str")); column_from->insert(Field("str"));
@ -471,19 +548,20 @@ TEST(ColumnDynamic, InsertRangeFromOverflow4)
column_to->insertRangeFrom(*column_from, 0, 3); column_to->insertRangeFrom(*column_from, 0, 3);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255);
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_EQ(column_to->getSharedVariant().size(), 3);
auto field = (*column_to)[column_to->size() - 3]; auto field = (*column_to)[column_to->size() - 3];
ASSERT_EQ(field, Field("42")); ASSERT_EQ(field, Field(42));
field = (*column_to)[column_to->size() - 2]; field = (*column_to)[column_to->size() - 2];
ASSERT_EQ(field, Field("42.42")); ASSERT_EQ(field, Field(42.42));
field = (*column_to)[column_to->size() - 1]; field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, Field("str")); ASSERT_EQ(field, Field("str"));
} }
TEST(ColumnDynamic, InsertRangeFromOverflow5) TEST(ColumnDynamic, InsertRangeFromOverflow5)
{ {
auto column_from = ColumnDynamic::create(255); auto column_from = ColumnDynamic::create(254);
column_from->insert(Field(42)); column_from->insert(Field(42));
column_from->insert(Field(43)); column_from->insert(Field(43));
column_from->insert(Field(42.42)); column_from->insert(Field(42.42));
@ -493,22 +571,23 @@ TEST(ColumnDynamic, InsertRangeFromOverflow5)
column_to->insert(Field("str")); column_to->insert(Field("str"));
column_to->insertRangeFrom(*column_from, 0, 4); column_to->insertRangeFrom(*column_from, 0, 4);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_EQ(column_to->getSharedVariant().size(), 3);
auto field = (*column_to)[column_to->size() - 4]; auto field = (*column_to)[column_to->size() - 4];
ASSERT_EQ(field, Field(42)); ASSERT_EQ(field, Field(42));
field = (*column_to)[column_to->size() - 3]; field = (*column_to)[column_to->size() - 3];
ASSERT_EQ(field, Field(43)); ASSERT_EQ(field, Field(43));
field = (*column_to)[column_to->size() - 2]; field = (*column_to)[column_to->size() - 2];
ASSERT_EQ(field, Field("42.42")); ASSERT_EQ(field, Field(42.42));
field = (*column_to)[column_to->size() - 1]; field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, Field("str")); ASSERT_EQ(field, Field("str"));
} }
TEST(ColumnDynamic, InsertRangeFromOverflow6) TEST(ColumnDynamic, InsertRangeFromOverflow6)
{ {
auto column_from = ColumnDynamic::create(255); auto column_from = ColumnDynamic::create(254);
column_from->insert(Field(42)); column_from->insert(Field(42));
column_from->insert(Field(43)); column_from->insert(Field(43));
column_from->insert(Field(44)); column_from->insert(Field(44));
@ -520,13 +599,14 @@ TEST(ColumnDynamic, InsertRangeFromOverflow6)
auto column_to = getDynamicWithManyVariants(253); auto column_to = getDynamicWithManyVariants(253);
column_to->insertRangeFrom(*column_from, 2, 5); column_to->insertRangeFrom(*column_from, 2, 5);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255); ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 255);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8")); ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Array(Int8)")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Array(Int8)"));
ASSERT_EQ(column_to->getSharedVariant().size(), 4);
auto field = (*column_to)[column_to->size() - 5]; auto field = (*column_to)[column_to->size() - 5];
ASSERT_EQ(field, Field("44")); ASSERT_EQ(field, Field(44));
field = (*column_to)[column_to->size() - 4]; field = (*column_to)[column_to->size() - 4];
ASSERT_EQ(field, Field(42.42)); ASSERT_EQ(field, Field(42.42));
field = (*column_to)[column_to->size() - 3]; field = (*column_to)[column_to->size() - 3];
@ -534,12 +614,136 @@ TEST(ColumnDynamic, InsertRangeFromOverflow6)
field = (*column_to)[column_to->size() - 2]; field = (*column_to)[column_to->size() - 2];
ASSERT_EQ(field, Field("str")); ASSERT_EQ(field, Field("str"));
field = (*column_to)[column_to->size() - 1]; field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, Field("[42]")); ASSERT_EQ(field, Field(Array({Field(42)})));
}
TEST(ColumnDynamic, InsertRangeFromOverflow7)
{
auto column_from = ColumnDynamic::create(2);
column_from->insert(Field(42.42));
column_from->insert(Field("str1"));
column_from->insert(Field(42));
column_from->insert(Field(43.43));
column_from->insert(Field(Array({Field(41)})));
column_from->insert(Field(43));
column_from->insert(Field("str2"));
column_from->insert(Field(Array({Field(42)})));
auto column_to = ColumnDynamic::create(254);
column_to->insert(Field(42));
column_to->insertRangeFrom(*column_from, 0, 8);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 4);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Array(Int8)"));
ASSERT_EQ(column_to->getSharedVariant().size(), 2);
auto field = (*column_to)[column_to->size() - 8];
ASSERT_EQ(field, Field(42.42));
field = (*column_to)[column_to->size() - 7];
ASSERT_EQ(field, Field("str1"));
field = (*column_to)[column_to->size() - 6];
ASSERT_EQ(field, Field(42));
field = (*column_to)[column_to->size() - 5];
ASSERT_EQ(field, Field(43.43));
field = (*column_to)[column_to->size() - 4];
ASSERT_EQ(field, Field(Array({Field(41)})));
field = (*column_to)[column_to->size() - 3];
ASSERT_EQ(field, Field(43));
field = (*column_to)[column_to->size() - 2];
ASSERT_EQ(field, Field("str2"));
field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, Field(Array({Field(42)})));
}
TEST(ColumnDynamic, InsertRangeFromOverflow8)
{
auto column_from = ColumnDynamic::create(2);
column_from->insert(Field(42.42));
column_from->insert(Field("str1"));
column_from->insert(Field(42));
column_from->insert(Field(43.43));
column_from->insert(Field(Array({Field(41)})));
column_from->insert(Field(43));
column_from->insert(Field("str2"));
column_from->insert(Field(Array({Field(42)})));
auto column_to = ColumnDynamic::create(2);
column_to->insert(Field(42));
column_from->insert(Field("str1"));
column_to->insertRangeFrom(*column_from, 0, 8);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 3);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Array(Int8)"));
ASSERT_EQ(column_to->getSharedVariant().size(), 4);
auto field = (*column_to)[column_to->size() - 8];
ASSERT_EQ(field, Field(42.42));
field = (*column_to)[column_to->size() - 7];
ASSERT_EQ(field, Field("str1"));
field = (*column_to)[column_to->size() - 6];
ASSERT_EQ(field, Field(42));
field = (*column_to)[column_to->size() - 5];
ASSERT_EQ(field, Field(43.43));
field = (*column_to)[column_to->size() - 4];
ASSERT_EQ(field, Field(Array({Field(41)})));
field = (*column_to)[column_to->size() - 3];
ASSERT_EQ(field, Field(43));
field = (*column_to)[column_to->size() - 2];
ASSERT_EQ(field, Field("str2"));
field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, Field(Array({Field(42)})));
}
TEST(ColumnDynamic, InsertRangeFromOverflow9)
{
auto column_from = ColumnDynamic::create(3);
column_from->insert(Field("str1"));
column_from->insert(Field(42.42));
column_from->insert(Field("str2"));
column_from->insert(Field(42));
column_from->insert(Field(43.43));
column_from->insert(Field(Array({Field(41)})));
column_from->insert(Field(43));
column_from->insert(Field("str2"));
column_from->insert(Field(Array({Field(42)})));
auto column_to = ColumnDynamic::create(2);
column_to->insert(Field(42));
column_to->insertRangeFrom(*column_from, 0, 9);
ASSERT_EQ(column_to->getVariantInfo().variant_names.size(), 3);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Array(Int8)"));
ASSERT_EQ(column_to->getSharedVariant().size(), 4);
auto field = (*column_to)[column_to->size() - 9];
ASSERT_EQ(field, Field("str1"));
field = (*column_to)[column_to->size() - 8];
ASSERT_EQ(field, Field(42.42));
field = (*column_to)[column_to->size() - 7];
ASSERT_EQ(field, Field("str2"));
field = (*column_to)[column_to->size() - 6];
ASSERT_EQ(field, Field(42));
field = (*column_to)[column_to->size() - 5];
ASSERT_EQ(field, Field(43.43));
field = (*column_to)[column_to->size() - 4];
ASSERT_EQ(field, Field(Array({Field(41)})));
field = (*column_to)[column_to->size() - 3];
ASSERT_EQ(field, Field(43));
field = (*column_to)[column_to->size() - 2];
ASSERT_EQ(field, Field("str2"));
field = (*column_to)[column_to->size() - 1];
ASSERT_EQ(field, Field(Array({Field(42)})));
} }
TEST(ColumnDynamic, SerializeDeserializeFromArena1) TEST(ColumnDynamic, SerializeDeserializeFromArena1)
{ {
auto column = ColumnDynamic::create(255); auto column = ColumnDynamic::create(254);
column->insert(Field(42)); column->insert(Field(42));
column->insert(Field(42.42)); column->insert(Field(42.42));
column->insert(Field("str")); column->insert(Field("str"));
@ -564,7 +768,7 @@ TEST(ColumnDynamic, SerializeDeserializeFromArena1)
TEST(ColumnDynamic, SerializeDeserializeFromArena2) TEST(ColumnDynamic, SerializeDeserializeFromArena2)
{ {
auto column_from = ColumnDynamic::create(255); auto column_from = ColumnDynamic::create(254);
column_from->insert(Field(42)); column_from->insert(Field(42));
column_from->insert(Field(42.42)); column_from->insert(Field(42.42));
column_from->insert(Field("str")); column_from->insert(Field("str"));
@ -577,26 +781,26 @@ TEST(ColumnDynamic, SerializeDeserializeFromArena2)
column_from->serializeValueIntoArena(2, arena, pos); column_from->serializeValueIntoArena(2, arena, pos);
column_from->serializeValueIntoArena(3, arena, pos); column_from->serializeValueIntoArena(3, arena, pos);
auto column_to = ColumnDynamic::create(255); auto column_to = ColumnDynamic::create(254);
pos = column_to->deserializeAndInsertFromArena(ref1.data); pos = column_to->deserializeAndInsertFromArena(ref1.data);
pos = column_to->deserializeAndInsertFromArena(pos); pos = column_to->deserializeAndInsertFromArena(pos);
pos = column_to->deserializeAndInsertFromArena(pos); pos = column_to->deserializeAndInsertFromArena(pos);
column_to->deserializeAndInsertFromArena(pos); column_to->deserializeAndInsertFromArena(pos);
ASSERT_EQ((*column_from)[column_from->size() - 4], 42); ASSERT_EQ((*column_to)[column_to->size() - 4], 42);
ASSERT_EQ((*column_from)[column_from->size() - 3], 42.42); ASSERT_EQ((*column_to)[column_to->size() - 3], 42.42);
ASSERT_EQ((*column_from)[column_from->size() - 2], "str"); ASSERT_EQ((*column_to)[column_to->size() - 2], "str");
ASSERT_EQ((*column_from)[column_from->size() - 1], Null()); ASSERT_EQ((*column_to)[column_to->size() - 1], Null());
ASSERT_EQ(column_to->getVariantInfo().variant_type->getName(), "Variant(Float64, Int8, String)"); ASSERT_EQ(column_to->getVariantInfo().variant_type->getName(), "Variant(Float64, Int8, SharedVariant, String)");
std::vector<String> expected_names = {"Float64", "Int8", "String"}; std::vector<String> expected_names = {"Float64", "Int8", "SharedVariant", "String"};
ASSERT_EQ(column_to->getVariantInfo().variant_names, expected_names); ASSERT_EQ(column_to->getVariantInfo().variant_names, expected_names);
std::unordered_map<String, UInt8> expected_variant_name_to_discriminator = {{"Float64", 0}, {"Int8", 1}, {"String", 2}}; std::unordered_map<String, UInt8> expected_variant_name_to_discriminator = {{"Float64", 0}, {"Int8", 1}, {"SharedVariant", 2}, {"String", 3}};
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator == expected_variant_name_to_discriminator); ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator == expected_variant_name_to_discriminator);
} }
TEST(ColumnDynamic, SerializeDeserializeFromArenaOverflow) TEST(ColumnDynamic, SerializeDeserializeFromArenaOverflow1)
{ {
auto column_from = ColumnDynamic::create(255); auto column_from = ColumnDynamic::create(254);
column_from->insert(Field(42)); column_from->insert(Field(42));
column_from->insert(Field(42.42)); column_from->insert(Field(42.42));
column_from->insert(Field("str")); column_from->insert(Field("str"));
@ -615,18 +819,56 @@ TEST(ColumnDynamic, SerializeDeserializeFromArenaOverflow)
pos = column_to->deserializeAndInsertFromArena(pos); pos = column_to->deserializeAndInsertFromArena(pos);
column_to->deserializeAndInsertFromArena(pos); column_to->deserializeAndInsertFromArena(pos);
ASSERT_EQ((*column_from)[column_from->size() - 4], 42); ASSERT_EQ((*column_to)[column_to->size() - 4], 42);
ASSERT_EQ((*column_from)[column_from->size() - 3], 42.42); ASSERT_EQ((*column_to)[column_to->size() - 3], 42.42);
ASSERT_EQ((*column_from)[column_from->size() - 2], "str"); ASSERT_EQ((*column_to)[column_to->size() - 2], "str");
ASSERT_EQ((*column_from)[column_from->size() - 1], Null()); ASSERT_EQ((*column_to)[column_to->size() - 1], Null());
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8")); ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String")); ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_EQ(column_to->getSharedVariant().size(), 2);
}
TEST(ColumnDynamic, SerializeDeserializeFromArenaOverflow2)
{
auto column_from = ColumnDynamic::create(2);
column_from->insert(Field(42));
column_from->insert(Field(42.42));
column_from->insert(Field("str"));
column_from->insert(Field(Null()));
column_from->insert(Field(Array({Field(42)})));
Arena arena;
const char * pos = nullptr;
auto ref1 = column_from->serializeValueIntoArena(0, arena, pos);
column_from->serializeValueIntoArena(1, arena, pos);
column_from->serializeValueIntoArena(2, arena, pos);
column_from->serializeValueIntoArena(3, arena, pos);
column_from->serializeValueIntoArena(4, arena, pos);
auto column_to = ColumnDynamic::create(2);
column_to->insert(Field(42.42));
pos = column_to->deserializeAndInsertFromArena(ref1.data);
pos = column_to->deserializeAndInsertFromArena(pos);
pos = column_to->deserializeAndInsertFromArena(pos);
pos = column_to->deserializeAndInsertFromArena(pos);
column_to->deserializeAndInsertFromArena(pos);
ASSERT_EQ((*column_to)[column_to->size() - 5], 42);
ASSERT_EQ((*column_to)[column_to->size() - 4], 42.42);
ASSERT_EQ((*column_to)[column_to->size() - 3], "str");
ASSERT_EQ((*column_to)[column_to->size() - 2], Null());
ASSERT_EQ((*column_to)[column_to->size() - 1], Field(Array({Field(42)})));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Int8"));
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Float64"));
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("String"));
ASSERT_FALSE(column_to->getVariantInfo().variant_name_to_discriminator.contains("Array(Int8)"));
ASSERT_EQ(column_to->getSharedVariant().size(), 2);
} }
TEST(ColumnDynamic, skipSerializedInArena) TEST(ColumnDynamic, skipSerializedInArena)
{ {
auto column_from = ColumnDynamic::create(255); auto column_from = ColumnDynamic::create(3);
column_from->insert(Field(42)); column_from->insert(Field(42));
column_from->insert(Field(42.42)); column_from->insert(Field(42.42));
column_from->insert(Field("str")); column_from->insert(Field("str"));
@ -640,13 +882,41 @@ TEST(ColumnDynamic, skipSerializedInArena)
auto ref4 = column_from->serializeValueIntoArena(3, arena, pos); auto ref4 = column_from->serializeValueIntoArena(3, arena, pos);
const char * end = ref4.data + ref4.size; const char * end = ref4.data + ref4.size;
auto column_to = ColumnDynamic::create(255); auto column_to = ColumnDynamic::create(254);
pos = column_to->skipSerializedInArena(ref1.data); pos = column_to->skipSerializedInArena(ref1.data);
pos = column_to->skipSerializedInArena(pos); pos = column_to->skipSerializedInArena(pos);
pos = column_to->skipSerializedInArena(pos); pos = column_to->skipSerializedInArena(pos);
pos = column_to->skipSerializedInArena(pos); pos = column_to->skipSerializedInArena(pos);
ASSERT_EQ(pos, end); ASSERT_EQ(pos, end);
ASSERT_TRUE(column_to->getVariantInfo().variant_name_to_discriminator.empty()); ASSERT_EQ(column_to->getVariantInfo().variant_name_to_discriminator.at("SharedVariant"), 0);
ASSERT_TRUE(column_to->getVariantInfo().variant_names.empty()); ASSERT_EQ(column_to->getVariantInfo().variant_names, Names{"SharedVariant"});
}
TEST(ColumnDynamic, compare)
{
auto column_from = ColumnDynamic::create(3);
column_from->insert(Field(42));
column_from->insert(Field(42.42));
column_from->insert(Field("str"));
column_from->insert(Field(Null()));
column_from->insert(Field(Array({Field(42)})));
ASSERT_EQ(column_from->compareAt(0, 0, *column_from, -1), 0);
ASSERT_EQ(column_from->compareAt(0, 1, *column_from, -1), 1);
ASSERT_EQ(column_from->compareAt(1, 1, *column_from, -1), 0);
ASSERT_EQ(column_from->compareAt(0, 2, *column_from, -1), -1);
ASSERT_EQ(column_from->compareAt(2, 0, *column_from, -1), 1);
ASSERT_EQ(column_from->compareAt(2, 4, *column_from, -1), 1);
ASSERT_EQ(column_from->compareAt(4, 2, *column_from, -1), -1);
ASSERT_EQ(column_from->compareAt(4, 4, *column_from, -1), 0);
ASSERT_EQ(column_from->compareAt(0, 3, *column_from, -1), 1);
ASSERT_EQ(column_from->compareAt(1, 3, *column_from, -1), 1);
ASSERT_EQ(column_from->compareAt(2, 3, *column_from, -1), 1);
ASSERT_EQ(column_from->compareAt(3, 3, *column_from, -1), 0);
ASSERT_EQ(column_from->compareAt(4, 3, *column_from, -1), 1);
ASSERT_EQ(column_from->compareAt(3, 0, *column_from, -1), -1);
ASSERT_EQ(column_from->compareAt(3, 1, *column_from, -1), -1);
ASSERT_EQ(column_from->compareAt(3, 2, *column_from, -1), -1);
ASSERT_EQ(column_from->compareAt(3, 4, *column_from, -1), -1);
} }

79
src/DataTypes/DataTypeDynamic.cpp
View File

@ -7,6 +7,7 @@
#include <DataTypes/NestedUtils.h> #include <DataTypes/NestedUtils.h>
#include <DataTypes/DataTypeNullable.h> #include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypesNumber.h> #include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypesBinaryEncoding.h>
#include <Columns/ColumnDynamic.h> #include <Columns/ColumnDynamic.h>
#include <Columns/ColumnVariant.h> #include <Columns/ColumnVariant.h>
#include <Core/Field.h> #include <Core/Field.h>
@ -14,6 +15,7 @@
#include <Parsers/ASTFunction.h> #include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h> #include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h> #include <Parsers/ASTLiteral.h>
#include <IO/ReadBufferFromMemory.h>
namespace DB namespace DB
{ {
@ -71,8 +73,8 @@ static DataTypePtr create(const ASTPtr & arguments)
auto * literal = argument->arguments->children[1]->as<ASTLiteral>(); auto * literal = argument->arguments->children[1]->as<ASTLiteral>();
if (!literal || literal->value.getType() != Field::Types::UInt64 || literal->value.safeGet<UInt64>() == 0 || literal->value.safeGet<UInt64>() > ColumnVariant::MAX_NESTED_COLUMNS) if (!literal || literal->value.getType() != Field::Types::UInt64 || literal->value.safeGet<UInt64>() > ColumnDynamic::MAX_DYNAMIC_TYPES_LIMIT)
throw Exception(ErrorCodes::UNEXPECTED_AST_STRUCTURE, "'max_types' argument for Dynamic type should be a positive integer between 1 and 255"); throw Exception(ErrorCodes::UNEXPECTED_AST_STRUCTURE, "'max_types' argument for Dynamic type should be a positive integer between 0 and {}", ColumnDynamic::MAX_DYNAMIC_TYPES_LIMIT);
return std::make_shared<DataTypeDynamic>(literal->value.safeGet<UInt64>()); return std::make_shared<DataTypeDynamic>(literal->value.safeGet<UInt64>());
} }
@ -84,30 +86,72 @@ void registerDataTypeDynamic(DataTypeFactory & factory)
std::unique_ptr<IDataType::SubstreamData> DataTypeDynamic::getDynamicSubcolumnData(std::string_view subcolumn_name, const DB::IDataType::SubstreamData & data, bool throw_if_null) const std::unique_ptr<IDataType::SubstreamData> DataTypeDynamic::getDynamicSubcolumnData(std::string_view subcolumn_name, const DB::IDataType::SubstreamData & data, bool throw_if_null) const
{ {
auto [subcolumn_type_name, subcolumn_nested_name] = Nested::splitName(subcolumn_name); auto [type_subcolumn_name, subcolumn_nested_name] = Nested::splitName(subcolumn_name);
/// Check if requested subcolumn is a valid data type. /// Check if requested subcolumn is a valid data type.
auto subcolumn_type = DataTypeFactory::instance().tryGet(String(subcolumn_type_name)); auto subcolumn_type = DataTypeFactory::instance().tryGet(String(type_subcolumn_name));
if (!subcolumn_type) if (!subcolumn_type)
{ {
if (throw_if_null) if (throw_if_null)
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Dynamic type doesn't have subcolumn '{}'", subcolumn_type_name); throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Dynamic type doesn't have subcolumn '{}'", type_subcolumn_name);
return nullptr; return nullptr;
} }
std::unique_ptr<SubstreamData> res = std::make_unique<SubstreamData>(subcolumn_type->getDefaultSerialization()); std::unique_ptr<SubstreamData> res = std::make_unique<SubstreamData>(subcolumn_type->getDefaultSerialization());
res->type = subcolumn_type; res->type = subcolumn_type;
std::optional<ColumnVariant::Discriminator> discriminator; std::optional<ColumnVariant::Discriminator> discriminator;
ColumnPtr null_map_for_variant_from_shared_variant;
if (data.column) if (data.column)
{ {
/// If column was provided, we should extract subcolumn from Dynamic column. /// If column was provided, we should extract subcolumn from Dynamic column.
const auto & dynamic_column = assert_cast<const ColumnDynamic &>(*data.column); const auto & dynamic_column = assert_cast<const ColumnDynamic &>(*data.column);
const auto & variant_info = dynamic_column.getVariantInfo(); const auto & variant_info = dynamic_column.getVariantInfo();
const auto & variant_column = dynamic_column.getVariantColumn();
const auto & shared_variant = dynamic_column.getSharedVariant();
/// Check if provided Dynamic column has subcolumn of this type. /// Check if provided Dynamic column has subcolumn of this type.
auto it = variant_info.variant_name_to_discriminator.find(subcolumn_type->getName()); String subcolumn_type_name = subcolumn_type->getName();
auto it = variant_info.variant_name_to_discriminator.find(subcolumn_type_name);
if (it != variant_info.variant_name_to_discriminator.end()) if (it != variant_info.variant_name_to_discriminator.end())
{ {
discriminator = it->second; discriminator = it->second;
res->column = dynamic_column.getVariantColumn().getVariantPtrByGlobalDiscriminator(*discriminator); res->column = variant_column.getVariantPtrByGlobalDiscriminator(*discriminator);
}
/// Otherwise if there is data in shared variant try to find requested type there.
else if (!shared_variant.empty())
{
/// Create null map for resulting subcolumn to make it Nullable.
auto null_map_column = ColumnUInt8::create();
NullMap & null_map = assert_cast<ColumnUInt8 &>(*null_map_column).getData();
null_map.reserve(variant_column.size());
auto subcolumn = subcolumn_type->createColumn();
auto shared_variant_local_discr = variant_column.localDiscriminatorByGlobal(dynamic_column.getSharedVariantDiscriminator());
const auto & local_discriminators = variant_column.getLocalDiscriminators();
const auto & offsets = variant_column.getOffsets();
const FormatSettings format_settings;
for (size_t i = 0; i != local_discriminators.size(); ++i)
{
if (local_discriminators[i] == shared_variant_local_discr)
{
auto value = shared_variant.getDataAt(offsets[i]);
ReadBufferFromMemory buf(value.data, value.size);
auto type = decodeDataType(buf);
if (type->getName() == subcolumn_type_name)
{
dynamic_column.getVariantSerialization(subcolumn_type, subcolumn_type_name)->deserializeBinary(*subcolumn, buf, format_settings);
null_map.push_back(0);
}
else
{
null_map.push_back(1);
}
}
else
{
null_map.push_back(1);
}
}
res->column = std::move(subcolumn);
null_map_for_variant_from_shared_variant = std::move(null_map_column);
} }
} }
@ -125,7 +169,7 @@ std::unique_ptr<IDataType::SubstreamData> DataTypeDynamic::getDynamicSubcolumnDa
return nullptr; return nullptr;
} }
res->serialization = std::make_shared<SerializationDynamicElement>(res->serialization, subcolumn_type->getName(), is_null_map_subcolumn); res->serialization = std::make_shared<SerializationDynamicElement>(res->serialization, subcolumn_type->getName(), String(subcolumn_nested_name), is_null_map_subcolumn);
/// Make resulting subcolumn Nullable only if type subcolumn can be inside Nullable or can be LowCardinality(Nullable()). /// Make resulting subcolumn Nullable only if type subcolumn can be inside Nullable or can be LowCardinality(Nullable()).
bool make_subcolumn_nullable = subcolumn_type->canBeInsideNullable() || subcolumn_type->lowCardinality(); bool make_subcolumn_nullable = subcolumn_type->canBeInsideNullable() || subcolumn_type->lowCardinality();
if (!is_null_map_subcolumn && make_subcolumn_nullable) if (!is_null_map_subcolumn && make_subcolumn_nullable)
@ -133,10 +177,10 @@ std::unique_ptr<IDataType::SubstreamData> DataTypeDynamic::getDynamicSubcolumnDa
if (data.column) if (data.column)
{ {
/// Check if provided Dynamic column has subcolumn of this type. In this case we should use VariantSubcolumnCreator/VariantNullMapSubcolumnCreator to
/// create full subcolumn from variant according to discriminators.
if (discriminator) if (discriminator)
{ {
/// Provided Dynamic column has subcolumn of this type, we should use VariantSubcolumnCreator/VariantNullMapSubcolumnCreator to
/// create full subcolumn from variant according to discriminators.
const auto & variant_column = assert_cast<const ColumnDynamic &>(*data.column).getVariantColumn(); const auto & variant_column = assert_cast<const ColumnDynamic &>(*data.column).getVariantColumn();
std::unique_ptr<ISerialization::ISubcolumnCreator> creator; std::unique_ptr<ISerialization::ISubcolumnCreator> creator;
if (is_null_map_subcolumn) if (is_null_map_subcolumn)
@ -154,6 +198,21 @@ std::unique_ptr<IDataType::SubstreamData> DataTypeDynamic::getDynamicSubcolumnDa
make_subcolumn_nullable); make_subcolumn_nullable);
res->column = creator->create(res->column); res->column = creator->create(res->column);
} }
/// Check if requested type was extracted from shared variant. In this case we should use
/// VariantSubcolumnCreator to create full subcolumn from variant according to created null map.
else if (null_map_for_variant_from_shared_variant)
{
if (is_null_map_subcolumn)
{
res->column = null_map_for_variant_from_shared_variant;
}
else
{
SerializationVariantElement::VariantSubcolumnCreator creator(
null_map_for_variant_from_shared_variant, "", 0, 0, make_subcolumn_nullable, null_map_for_variant_from_shared_variant);
res->column = creator.create(res->column);
}
}
/// Provided Dynamic column doesn't have subcolumn of this type, just create column filled with default values. /// Provided Dynamic column doesn't have subcolumn of this type, just create column filled with default values.
else if (is_null_map_subcolumn) else if (is_null_map_subcolumn)
{ {

6
src/DataTypes/DataTypeFactory.cpp
View File

@ -150,6 +150,12 @@ DataTypePtr DataTypeFactory::getCustom(DataTypeCustomDescPtr customization) cons
return type; return type;
} }
DataTypePtr DataTypeFactory::getCustom(const String & base_name, DataTypeCustomDescPtr customization) const
{
auto type = get(base_name);
type->setCustomization(std::move(customization));
return type;
}
void DataTypeFactory::registerDataType(const String & family_name, Value creator, Case case_sensitiveness) void DataTypeFactory::registerDataType(const String & family_name, Value creator, Case case_sensitiveness)
{ {

1
src/DataTypes/DataTypeFactory.h
View File

@ -34,6 +34,7 @@ public:
DataTypePtr get(const String & family_name, const ASTPtr & parameters) const; DataTypePtr get(const String & family_name, const ASTPtr & parameters) const;
DataTypePtr get(const ASTPtr & ast) const; DataTypePtr get(const ASTPtr & ast) const;
DataTypePtr getCustom(DataTypeCustomDescPtr customization) const; DataTypePtr getCustom(DataTypeCustomDescPtr customization) const;
DataTypePtr getCustom(const String & base_name, DataTypeCustomDescPtr customization) const;
/// Return nullptr in case of error. /// Return nullptr in case of error.
DataTypePtr tryGet(const String & full_name) const; DataTypePtr tryGet(const String & full_name) const;

13
src/DataTypes/DataTypeTuple.cpp
View File

@ -192,17 +192,12 @@ MutableColumnPtr DataTypeTuple::createColumn() const
MutableColumnPtr DataTypeTuple::createColumn(const ISerialization & serialization) const MutableColumnPtr DataTypeTuple::createColumn(const ISerialization & serialization) const
{ {
/// If we read Tuple as Variant subcolumn, it may be wrapped to SerializationVariantElement. /// If we read subcolumn of nested Tuple or this Tuple is a subcolumn, it may be wrapped to SerializationWrapper
/// Here we don't need it, so we drop this wrapper.
const auto * current_serialization = &serialization;
while (const auto * serialization_variant_element = typeid_cast<const SerializationVariantElement *>(current_serialization))
current_serialization = serialization_variant_element->getNested().get();
/// If we read subcolumn of nested Tuple, it may be wrapped to SerializationNamed
/// several times to allow to reconstruct the substream path name. /// several times to allow to reconstruct the substream path name.
/// Here we don't need substream path name, so we drop first several wrapper serializations. /// Here we don't need substream path name, so we drop first several wrapper serializations.
while (const auto * serialization_named = typeid_cast<const SerializationNamed *>(current_serialization)) const auto * current_serialization = &serialization;
current_serialization = serialization_named->getNested().get(); while (const auto * serialization_wrapper = dynamic_cast<const SerializationWrapper *>(current_serialization))
current_serialization = serialization_wrapper->getNested().get();
const auto * serialization_tuple = typeid_cast<const SerializationTuple *>(current_serialization); const auto * serialization_tuple = typeid_cast<const SerializationTuple *>(current_serialization);
if (!serialization_tuple) if (!serialization_tuple)

2
src/DataTypes/DataTypeVariant.cpp
View File

@ -117,7 +117,7 @@ bool DataTypeVariant::equals(const IDataType & rhs) const
/// The same data types with different custom names considered different. /// The same data types with different custom names considered different.
/// For example, UInt8 and Bool. /// For example, UInt8 and Bool.
if ((variants[i]->hasCustomName() || rhs_variant.variants[i]) && variants[i]->getName() != rhs_variant.variants[i]->getName()) if ((variants[i]->hasCustomName() || rhs_variant.variants[i]->hasCustomName()) && variants[i]->getName() != rhs_variant.variants[i]->getName())
return false; return false;
} }

2
src/DataTypes/DataTypesBinaryEncoding.cpp
View File

@ -444,7 +444,7 @@ void encodeDataType(const DataTypePtr & type, WriteBuffer & buf)
case BinaryTypeIndex::Dynamic: case BinaryTypeIndex::Dynamic:
{ {
const auto & dynamic_type = assert_cast<const DataTypeDynamic &>(*type); const auto & dynamic_type = assert_cast<const DataTypeDynamic &>(*type);
/// Maximum number of dynamic types is 255, we can write it as 1 byte. /// Maximum number of dynamic types is 254, we can write it as 1 byte.
writeBinary(UInt8(dynamic_type.getMaxDynamicTypes()), buf); writeBinary(UInt8(dynamic_type.getMaxDynamicTypes()), buf);
break; break;
} }

361
src/DataTypes/Serializations/SerializationDynamic.cpp
View File

@ -27,15 +27,21 @@ namespace ErrorCodes
struct SerializeBinaryBulkStateDynamic : public ISerialization::SerializeBinaryBulkState struct SerializeBinaryBulkStateDynamic : public ISerialization::SerializeBinaryBulkState
{ {
SerializationDynamic::DynamicStructureSerializationVersion structure_version; SerializationDynamic::DynamicStructureSerializationVersion structure_version;
size_t max_dynamic_types;
DataTypePtr variant_type; DataTypePtr variant_type;
Names variant_names; Names variant_names;
SerializationPtr variant_serialization; SerializationPtr variant_serialization;
ISerialization::SerializeBinaryBulkStatePtr variant_state; ISerialization::SerializeBinaryBulkStatePtr variant_state;
/// Variants statistics. Map (Variant name) -> (Variant size). /// Variants statistics.
ColumnDynamic::Statistics statistics = { .source = ColumnDynamic::Statistics::Source::READ, .data = {} }; ColumnDynamic::Statistics statistics;
/// If true, statistics will be recalculated during serialization.
bool recalculate_statistics = false;
explicit SerializeBinaryBulkStateDynamic(UInt64 structure_version_) : structure_version(structure_version_) {} explicit SerializeBinaryBulkStateDynamic(UInt64 structure_version_)
: structure_version(structure_version_), statistics(ColumnDynamic::Statistics::Source::READ)
{
}
}; };
struct DeserializeBinaryBulkStateDynamic : public ISerialization::DeserializeBinaryBulkState struct DeserializeBinaryBulkStateDynamic : public ISerialization::DeserializeBinaryBulkState
@ -106,20 +112,41 @@ void SerializationDynamic::serializeBinaryBulkStatePrefix(
writeBinaryLittleEndian(structure_version, *stream); writeBinaryLittleEndian(structure_version, *stream);
auto dynamic_state = std::make_shared<SerializeBinaryBulkStateDynamic>(structure_version); auto dynamic_state = std::make_shared<SerializeBinaryBulkStateDynamic>(structure_version);
dynamic_state->max_dynamic_types = column_dynamic.getMaxDynamicTypes();
/// Write max_dynamic_types parameter, because it can differ from the max_dynamic_types
/// that is specified in the Dynamic type (we could decrease it before merge).
writeBinaryLittleEndian(dynamic_state->max_dynamic_types, *stream);
dynamic_state->variant_type = variant_info.variant_type; dynamic_state->variant_type = variant_info.variant_type;
dynamic_state->variant_names = variant_info.variant_names; dynamic_state->variant_names = variant_info.variant_names;
const auto & variant_column = column_dynamic.getVariantColumn(); const auto & variant_column = column_dynamic.getVariantColumn();
/// Write internal Variant type name. /// Write information about variants.
size_t num_variants = dynamic_state->variant_names.size() - 1; /// Don't write shared variant, Dynamic column should always have it.
writeBinaryLittleEndian(num_variants, *stream);
if (settings.data_types_binary_encoding) if (settings.data_types_binary_encoding)
encodeDataType(dynamic_state->variant_type, *stream); {
const auto & variants = assert_cast<const DataTypeVariant &>(*dynamic_state->variant_type).getVariants();
for (const auto & variant: variants)
{
if (variant->getName() != ColumnDynamic::getSharedVariantTypeName())
encodeDataType(variant, *stream);
}
}
else else
writeStringBinary(dynamic_state->variant_type->getName(), *stream); {
for (const auto & name : dynamic_state->variant_names)
{
if (name != ColumnDynamic::getSharedVariantTypeName())
writeStringBinary(name, *stream);
}
}
/// Write statistics in prefix if needed. /// Write statistics in prefix if needed.
if (settings.dynamic_write_statistics == SerializeBinaryBulkSettings::DynamicStatisticsMode::PREFIX) if (settings.dynamic_write_statistics == SerializeBinaryBulkSettings::DynamicStatisticsMode::PREFIX)
{ {
const auto & statistics = column_dynamic.getStatistics(); const auto & statistics = column_dynamic.getStatistics();
/// First, write statistics for usual variants.
for (size_t i = 0; i != variant_info.variant_names.size(); ++i) for (size_t i = 0; i != variant_info.variant_names.size(); ++i)
{ {
size_t size = 0; size_t size = 0;
@ -129,13 +156,55 @@ void SerializationDynamic::serializeBinaryBulkStatePrefix(
/// - statistics read from the data part during deserialization of Dynamic column (Statistics::Source::READ). /// - statistics read from the data part during deserialization of Dynamic column (Statistics::Source::READ).
/// We can rely only on statistics calculated during the merge, because column with statistics that was read /// We can rely only on statistics calculated during the merge, because column with statistics that was read
/// during deserialization from some data part could be filtered/limited/transformed/etc and so the statistics can be outdated. /// during deserialization from some data part could be filtered/limited/transformed/etc and so the statistics can be outdated.
if (!statistics.data.empty() && statistics.source == ColumnDynamic::Statistics::Source::MERGE) if (statistics && statistics->source == ColumnDynamic::Statistics::Source::MERGE)
size = statistics.data.at(variant_info.variant_names[i]); size = statistics->variants_statistics.at(variant_info.variant_names[i]);
/// Otherwise we can use only variant sizes from current column. /// Otherwise we can use only variant sizes from current column.
else else
size = variant_column.getVariantByGlobalDiscriminator(i).size(); size = variant_column.getVariantByGlobalDiscriminator(i).size();
writeVarUInt(size, *stream); writeVarUInt(size, *stream);
} }
/// Second, write statistics for variants in shared variant.
/// Check if we have statistics calculated during merge of some data parts (Statistics::Source::MERGE).
if (statistics && statistics->source == ColumnDynamic::Statistics::Source::MERGE)
{
writeVarUInt(statistics->shared_variants_statistics.size(), *stream);
for (const auto & [variant_name, size] : statistics->shared_variants_statistics)
{
writeStringBinary(variant_name, *stream);
writeVarUInt(size, *stream);
}
}
/// If we don't have statistics for shared variants from merge, calculate it from the column.
else
{
std::unordered_map<String, size_t> shared_variants_statistics;
const auto & shared_variant = column_dynamic.getSharedVariant();
for (size_t i = 0; i != shared_variant.size(); ++i)
{
auto value = shared_variant.getDataAt(i);
ReadBufferFromMemory buf(value.data, value.size);
auto type = decodeDataType(buf);
auto type_name = type->getName();
if (auto it = shared_variants_statistics.find(type_name); it != shared_variants_statistics.end())
++it->second;
else if (shared_variants_statistics.size() < ColumnDynamic::Statistics::MAX_SHARED_VARIANT_STATISTICS_SIZE)
shared_variants_statistics.emplace(type_name, 1);
}
writeVarUInt(shared_variants_statistics.size(), *stream);
for (const auto & [variant_name, size] : shared_variants_statistics)
{
writeStringBinary(variant_name, *stream);
writeVarUInt(size, *stream);
}
}
}
/// Otherwise statistics will be written in the suffix, in this case we will recalculate
/// statistics during serialization to make it more precise.
else
{
dynamic_state->recalculate_statistics = true;
} }
dynamic_state->variant_serialization = dynamic_state->variant_type->getDefaultSerialization(); dynamic_state->variant_serialization = dynamic_state->variant_type->getDefaultSerialization();
@ -182,33 +251,58 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationDynamic::deserializeD
UInt64 structure_version; UInt64 structure_version;
readBinaryLittleEndian(structure_version, *structure_stream); readBinaryLittleEndian(structure_version, *structure_stream);
auto structure_state = std::make_shared<DeserializeBinaryBulkStateDynamicStructure>(structure_version); auto structure_state = std::make_shared<DeserializeBinaryBulkStateDynamicStructure>(structure_version);
/// Read internal Variant type name. /// Read max_dynamic_types parameter.
readBinaryLittleEndian(structure_state->max_dynamic_types, *structure_stream);
/// Read information about variants.
DataTypes variants;
size_t num_variants;
readBinaryLittleEndian(num_variants, *structure_stream);
variants.reserve(num_variants + 1); /// +1 for shared variant.
if (settings.data_types_binary_encoding) if (settings.data_types_binary_encoding)
{ {
structure_state->variant_type = decodeDataType(*structure_stream); for (size_t i = 0; i != num_variants; ++i)
variants.push_back(decodeDataType(*structure_stream));
} }
else else
{ {
String data_type_name; String data_type_name;
readStringBinary(data_type_name, *structure_stream); for (size_t i = 0; i != num_variants; ++i)
structure_state->variant_type = DataTypeFactory::instance().get(data_type_name); {
readStringBinary(data_type_name, *structure_stream);
variants.push_back(DataTypeFactory::instance().get(data_type_name));
}
} }
const auto * variant_type = typeid_cast<const DataTypeVariant *>(structure_state->variant_type.get()); /// Add shared variant, Dynamic column should always have it.
if (!variant_type) variants.push_back(ColumnDynamic::getSharedVariantDataType());
throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect type of Dynamic nested column, expected Variant, got {}", structure_state->variant_type->getName()); auto variant_type = std::make_shared<DataTypeVariant>(variants);
/// Read statistics. /// Read statistics.
if (settings.dynamic_read_statistics) if (settings.dynamic_read_statistics)
{ {
const auto & variants = variant_type->getVariants(); ColumnDynamic::Statistics statistics(ColumnDynamic::Statistics::Source::READ);
/// First, read statistics for usual variants.
size_t variant_size; size_t variant_size;
for (const auto & variant : variants) for (const auto & variant : variant_type->getVariants())
{ {
readVarUInt(variant_size, *structure_stream); readVarUInt(variant_size, *structure_stream);
structure_state->statistics.data[variant->getName()] = variant_size; statistics.variants_statistics[variant->getName()] = variant_size;
} }
/// Second, read statistics for shared variants.
size_t statistics_size;
readVarUInt(statistics_size, *structure_stream);
String variant_name;
for (size_t i = 0; i != statistics_size; ++i)
{
readStringBinary(variant_name, *structure_stream);
readVarUInt(variant_size, *structure_stream);
statistics.shared_variants_statistics[variant_name] = variant_size;
}
structure_state->statistics = std::make_shared<const ColumnDynamic::Statistics>(std::move(statistics));
} }
structure_state->variant_type = std::move(variant_type);
state = structure_state; state = structure_state;
addToSubstreamsDeserializeStatesCache(cache, settings.path, state); addToSubstreamsDeserializeStatesCache(cache, settings.path, state);
} }
@ -231,8 +325,16 @@ void SerializationDynamic::serializeBinaryBulkStateSuffix(
/// Write statistics in suffix if needed. /// Write statistics in suffix if needed.
if (settings.dynamic_write_statistics == SerializeBinaryBulkSettings::DynamicStatisticsMode::SUFFIX) if (settings.dynamic_write_statistics == SerializeBinaryBulkSettings::DynamicStatisticsMode::SUFFIX)
{ {
/// First, write statistics for usual variants.
for (const auto & variant_name : dynamic_state->variant_names) for (const auto & variant_name : dynamic_state->variant_names)
writeVarUInt(dynamic_state->statistics.data[variant_name], *stream); writeVarUInt(dynamic_state->statistics.variants_statistics[variant_name], *stream);
/// Second, write statistics for shared variants.
writeVarUInt(dynamic_state->statistics.shared_variants_statistics.size(), *stream);
for (const auto & [variant_name, size] : dynamic_state->statistics.shared_variants_statistics)
{
writeStringBinary(variant_name, *stream);
writeVarUInt(size, *stream);
}
} }
settings.path.push_back(Substream::DynamicData); settings.path.push_back(Substream::DynamicData);
@ -255,9 +357,42 @@ void SerializationDynamic::serializeBinaryBulkWithMultipleStreams(
if (!variant_info.variant_type->equals(*dynamic_state->variant_type)) if (!variant_info.variant_type->equals(*dynamic_state->variant_type))
throw Exception(ErrorCodes::LOGICAL_ERROR, "Mismatch of internal columns of Dynamic. Expected: {}, Got: {}", dynamic_state->variant_type->getName(), variant_info.variant_type->getName()); throw Exception(ErrorCodes::LOGICAL_ERROR, "Mismatch of internal columns of Dynamic. Expected: {}, Got: {}", dynamic_state->variant_type->getName(), variant_info.variant_type->getName());
if (column_dynamic.getMaxDynamicTypes() != dynamic_state->max_dynamic_types)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Mismatch of max_dynamic_types parameter of Dynamic. Expected: {}, Got: {}", dynamic_state->max_dynamic_types, column_dynamic.getMaxDynamicTypes());
settings.path.push_back(Substream::DynamicData); settings.path.push_back(Substream::DynamicData);
assert_cast<const SerializationVariant &>(*dynamic_state->variant_serialization) if (dynamic_state->recalculate_statistics)
.serializeBinaryBulkWithMultipleStreamsAndUpdateVariantStatistics(*variant_column, offset, limit, settings, dynamic_state->variant_state, dynamic_state->statistics.data); {
assert_cast<const SerializationVariant &>(*dynamic_state->variant_serialization)
.serializeBinaryBulkWithMultipleStreamsAndUpdateVariantStatistics(*variant_column, offset, limit, settings, dynamic_state->variant_state, dynamic_state->statistics.variants_statistics);
/// Calculate statistics for shared variants.
const auto & shared_variant = column_dynamic.getSharedVariant();
if (!shared_variant.empty())
{
const auto & local_discriminators = variant_column->getLocalDiscriminators();
const auto & offsets = variant_column->getOffsets();
const auto shared_variant_discr = variant_column->localDiscriminatorByGlobal(column_dynamic.getSharedVariantDiscriminator());
size_t end = limit == 0 || offset + limit > local_discriminators.size() ? local_discriminators.size() : offset + limit;
for (size_t i = offset; i != end; ++i)
{
if (local_discriminators[i] == shared_variant_discr)
{
auto value = shared_variant.getDataAt(offsets[i]);
ReadBufferFromMemory buf(value.data, value.size);
auto type = decodeDataType(buf);
auto type_name = type->getName();
if (auto it = dynamic_state->statistics.shared_variants_statistics.find(type_name); it != dynamic_state->statistics.shared_variants_statistics.end())
++it->second;
else if (dynamic_state->statistics.shared_variants_statistics.size() < ColumnDynamic::Statistics::MAX_SHARED_VARIANT_STATISTICS_SIZE)
dynamic_state->statistics.shared_variants_statistics.emplace(type_name, 1);
}
}
}
}
else
{
assert_cast<const SerializationVariant &>(*dynamic_state->variant_serialization).serializeBinaryBulkWithMultipleStreams(*variant_column, offset, limit, settings, dynamic_state->variant_state);
}
settings.path.pop_back(); settings.path.pop_back();
} }
@ -272,13 +407,17 @@ void SerializationDynamic::deserializeBinaryBulkWithMultipleStreams(
return; return;
auto mutable_column = column->assumeMutable(); auto mutable_column = column->assumeMutable();
auto & column_dynamic = assert_cast<ColumnDynamic &>(*mutable_column);
auto * dynamic_state = checkAndGetState<DeserializeBinaryBulkStateDynamic>(state); auto * dynamic_state = checkAndGetState<DeserializeBinaryBulkStateDynamic>(state);
auto * structure_state = checkAndGetState<DeserializeBinaryBulkStateDynamicStructure>(dynamic_state->structure_state); auto * structure_state = checkAndGetState<DeserializeBinaryBulkStateDynamicStructure>(dynamic_state->structure_state);
if (mutable_column->empty()) if (mutable_column->empty())
mutable_column = ColumnDynamic::create(structure_state->variant_type->createColumn(), structure_state->variant_type, max_dynamic_types, structure_state->statistics); {
column_dynamic.setMaxDynamicPaths(structure_state->max_dynamic_types);
column_dynamic.setVariantType(structure_state->variant_type);
column_dynamic.setStatistics(structure_state->statistics);
}
auto & column_dynamic = assert_cast<ColumnDynamic &>(*mutable_column);
const auto & variant_info = column_dynamic.getVariantInfo(); const auto & variant_info = column_dynamic.getVariantInfo();
if (!variant_info.variant_type->equals(*structure_state->variant_type)) if (!variant_info.variant_type->equals(*structure_state->variant_type))
throw Exception(ErrorCodes::LOGICAL_ERROR, "Mismatch of internal columns of Dynamic. Expected: {}, Got: {}", structure_state->variant_type->getName(), variant_info.variant_type->getName()); throw Exception(ErrorCodes::LOGICAL_ERROR, "Mismatch of internal columns of Dynamic. Expected: {}, Got: {}", structure_state->variant_type->getName(), variant_info.variant_type->getName());
@ -329,24 +468,42 @@ void SerializationDynamic::serializeBinary(const IColumn & column, size_t row_nu
encodeDataType(std::make_shared<DataTypeNothing>(), ostr); encodeDataType(std::make_shared<DataTypeNothing>(), ostr);
return; return;
} }
/// Check if this value is in shared variant. In this case it's already
/// in desired binary format.
else if (global_discr == dynamic_column.getSharedVariantDiscriminator())
{
auto value = dynamic_column.getSharedVariant().getDataAt(variant_column.offsetAt(row_num));
ostr.write(value.data, value.size);
return;
}
const auto & variant_type = assert_cast<const DataTypeVariant &>(*variant_info.variant_type).getVariant(global_discr); const auto & variant_type = assert_cast<const DataTypeVariant &>(*variant_info.variant_type).getVariant(global_discr);
const auto & variant_type_name = variant_info.variant_names[global_discr];
encodeDataType(variant_type, ostr); encodeDataType(variant_type, ostr);
variant_type->getDefaultSerialization()->serializeBinary(variant_column.getVariantByGlobalDiscriminator(global_discr), variant_column.offsetAt(row_num), ostr, settings); dynamic_column.getVariantSerialization(variant_type, variant_type_name)->serializeBinary(variant_column.getVariantByGlobalDiscriminator(global_discr), variant_column.offsetAt(row_num), ostr, settings);
} }
template <typename DeserializeFunc> template <typename ReturnType = void, typename DeserializeFunc>
static void deserializeVariant( static ReturnType deserializeVariant(
ColumnVariant & variant_column, ColumnVariant & variant_column,
const DataTypePtr & variant_type, const SerializationPtr & variant_serialization,
ColumnVariant::Discriminator global_discr, ColumnVariant::Discriminator global_discr,
ReadBuffer & istr, ReadBuffer & istr,
DeserializeFunc deserialize) DeserializeFunc deserialize)
{ {
auto & variant = variant_column.getVariantByGlobalDiscriminator(global_discr); auto & variant = variant_column.getVariantByGlobalDiscriminator(global_discr);
deserialize(*variant_type->getDefaultSerialization(), variant, istr); if constexpr (std::is_same_v<ReturnType, bool>)
{
if (!deserialize(*variant_serialization, variant, istr))
return ReturnType(false);
}
else
{
deserialize(*variant_serialization, variant, istr);
}
variant_column.getLocalDiscriminators().push_back(variant_column.localDiscriminatorByGlobal(global_discr)); variant_column.getLocalDiscriminators().push_back(variant_column.localDiscriminatorByGlobal(global_discr));
variant_column.getOffsets().push_back(variant.size() - 1); variant_column.getOffsets().push_back(variant.size() - 1);
return ReturnType(true);
} }
void SerializationDynamic::deserializeBinary(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const void SerializationDynamic::deserializeBinary(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
@ -360,11 +517,12 @@ void SerializationDynamic::deserializeBinary(IColumn & column, ReadBuffer & istr
} }
auto variant_type_name = variant_type->getName(); auto variant_type_name = variant_type->getName();
const auto & variant_serialization = dynamic_column.getVariantSerialization(variant_type, variant_type_name);
const auto & variant_info = dynamic_column.getVariantInfo(); const auto & variant_info = dynamic_column.getVariantInfo();
auto it = variant_info.variant_name_to_discriminator.find(variant_type_name); auto it = variant_info.variant_name_to_discriminator.find(variant_type_name);
if (it != variant_info.variant_name_to_discriminator.end()) if (it != variant_info.variant_name_to_discriminator.end())
{ {
deserializeVariant(dynamic_column.getVariantColumn(), variant_type, it->second, istr, [&settings](const ISerialization & serialization, IColumn & variant, ReadBuffer & buf){ serialization.deserializeBinary(variant, buf, settings); }); deserializeVariant(dynamic_column.getVariantColumn(), variant_serialization, it->second, istr, [&settings](const ISerialization & serialization, IColumn & variant, ReadBuffer & buf){ serialization.deserializeBinary(variant, buf, settings); });
return; return;
} }
@ -372,25 +530,15 @@ void SerializationDynamic::deserializeBinary(IColumn & column, ReadBuffer & istr
if (dynamic_column.addNewVariant(variant_type)) if (dynamic_column.addNewVariant(variant_type))
{ {
auto discr = variant_info.variant_name_to_discriminator.at(variant_type_name); auto discr = variant_info.variant_name_to_discriminator.at(variant_type_name);
deserializeVariant(dynamic_column.getVariantColumn(), variant_type, discr, istr, [&settings](const ISerialization & serialization, IColumn & variant, ReadBuffer & buf){ serialization.deserializeBinary(variant, buf, settings); }); deserializeVariant(dynamic_column.getVariantColumn(), variant_serialization, discr, istr, [&settings](const ISerialization & serialization, IColumn & variant, ReadBuffer & buf){ serialization.deserializeBinary(variant, buf, settings); });
return; return;
} }
/// We reached maximum number of variants and couldn't add new variant. /// We reached maximum number of variants and couldn't add new variant.
/// This case should be really rare in real use cases. /// In this case we insert this value into shared variant in binary form.
/// We should always be able to add String variant and insert value as String.
dynamic_column.addStringVariant();
auto tmp_variant_column = variant_type->createColumn(); auto tmp_variant_column = variant_type->createColumn();
variant_type->getDefaultSerialization()->deserializeBinary(*tmp_variant_column, istr, settings); variant_serialization->deserializeBinary(*tmp_variant_column, istr, settings);
auto string_column = castColumn(ColumnWithTypeAndName(tmp_variant_column->getPtr(), variant_type, ""), std::make_shared<DataTypeString>()); dynamic_column.insertValueIntoSharedVariant(*tmp_variant_column, variant_type, variant_type_name, 0);
auto & variant_column = dynamic_column.getVariantColumn();
variant_column.insertIntoVariantFrom(variant_info.variant_name_to_discriminator.at("String"), *string_column, 0);
}
void SerializationDynamic::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
const auto & dynamic_column = assert_cast<const ColumnDynamic &>(column);
dynamic_column.getVariantInfo().variant_type->getDefaultSerialization()->serializeTextCSV(dynamic_column.getVariantColumn(), row_num, ostr, settings);
} }
template <typename ReadFieldFunc, typename TryDeserializeVariantFunc, typename DeserializeVariant> template <typename ReadFieldFunc, typename TryDeserializeVariantFunc, typename DeserializeVariant>
@ -406,6 +554,7 @@ static void deserializeTextImpl(
auto & dynamic_column = assert_cast<ColumnDynamic &>(column); auto & dynamic_column = assert_cast<ColumnDynamic &>(column);
auto & variant_column = dynamic_column.getVariantColumn(); auto & variant_column = dynamic_column.getVariantColumn();
const auto & variant_info = dynamic_column.getVariantInfo(); const auto & variant_info = dynamic_column.getVariantInfo();
const auto & variant_types = assert_cast<const DataTypeVariant &>(*variant_info.variant_type).getVariants();
String field = read_field(istr); String field = read_field(istr);
auto field_buf = std::make_unique<ReadBufferFromString>(field); auto field_buf = std::make_unique<ReadBufferFromString>(field);
JSONInferenceInfo json_info; JSONInferenceInfo json_info;
@ -413,27 +562,81 @@ static void deserializeTextImpl(
if (escaping_rule == FormatSettings::EscapingRule::JSON) if (escaping_rule == FormatSettings::EscapingRule::JSON)
transformFinalInferredJSONTypeIfNeeded(variant_type, settings, &json_info); transformFinalInferredJSONTypeIfNeeded(variant_type, settings, &json_info);
if (checkIfTypeIsComplete(variant_type) && dynamic_column.addNewVariant(variant_type)) /// If inferred type is not complete, we cannot add it as a new variant.
/// Let's try to deserialize this field into existing variants.
/// If failed, insert this value as String.
if (!checkIfTypeIsComplete(variant_type))
{
size_t shared_variant_discr = dynamic_column.getSharedVariantDiscriminator();
for (size_t i = 0; i != variant_types.size(); ++i)
{
field_buf = std::make_unique<ReadBufferFromString>(field);
if (i != shared_variant_discr
&& deserializeVariant<bool>(
variant_column,
dynamic_column.getVariantSerialization(variant_types[i], variant_info.variant_names[i]),
i,
*field_buf,
try_deserialize_variant))
return;
}
variant_type = std::make_shared<DataTypeString>();
/// To be able to deserialize field as String with Quoted escaping rule, it should be quoted.
if (escaping_rule == FormatSettings::EscapingRule::Quoted && (field.size() < 2 || field.front() != '\'' || field.back() != '\''))
field = "'" + field + "'";
}
else if (dynamic_column.addNewVariant(variant_type, variant_type->getName()))
{ {
auto discr = variant_info.variant_name_to_discriminator.at(variant_type->getName()); auto discr = variant_info.variant_name_to_discriminator.at(variant_type->getName());
deserializeVariant(dynamic_column.getVariantColumn(), variant_type, discr, *field_buf, deserialize_variant); deserializeVariant(dynamic_column.getVariantColumn(), dynamic_column.getVariantSerialization(variant_type), discr, *field_buf, deserialize_variant);
return; return;
} }
/// We couldn't infer type or add new variant. Try to insert field into current variants. /// We couldn't infer type or add new variant. Insert it into shared variant.
auto tmp_variant_column = variant_type->createColumn();
field_buf = std::make_unique<ReadBufferFromString>(field); field_buf = std::make_unique<ReadBufferFromString>(field);
if (try_deserialize_variant(*variant_info.variant_type->getDefaultSerialization(), variant_column, *field_buf)) auto variant_type_name = variant_type->getName();
return; deserialize_variant(*dynamic_column.getVariantSerialization(variant_type, variant_type_name), *tmp_variant_column, *field_buf);
dynamic_column.insertValueIntoSharedVariant(*tmp_variant_column, variant_type, variant_type_name, 0);
}
/// We couldn't insert field into any existing variant, add String variant and read value as String. template <typename NestedSerialize>
dynamic_column.addStringVariant(); static void serializeTextImpl(
const IColumn & column,
size_t row_num,
WriteBuffer & ostr,
const FormatSettings & settings,
NestedSerialize nested_serialize)
{
const auto & dynamic_column = assert_cast<const ColumnDynamic &>(column);
const auto & variant_column = dynamic_column.getVariantColumn();
/// Check if this row has value in shared variant. In this case we should first deserialize it from binary format.
if (variant_column.globalDiscriminatorAt(row_num) == dynamic_column.getSharedVariantDiscriminator())
{
auto value = dynamic_column.getSharedVariant().getDataAt(variant_column.offsetAt(row_num));
ReadBufferFromMemory buf(value.data, value.size);
auto variant_type = decodeDataType(buf);
auto tmp_variant_column = variant_type->createColumn();
auto variant_serialization = dynamic_column.getVariantSerialization(variant_type);
variant_serialization->deserializeBinary(*tmp_variant_column, buf, settings);
nested_serialize(*variant_serialization, *tmp_variant_column, 0, ostr);
}
/// Otherwise just use serialization for Variant.
else
{
nested_serialize(*dynamic_column.getVariantInfo().variant_type->getDefaultSerialization(), variant_column, row_num, ostr);
}
}
if (escaping_rule == FormatSettings::EscapingRule::Quoted && (field.size() < 2 || field.front() != '\'' || field.back() != '\'')) void SerializationDynamic::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
field = "'" + field + "'"; {
auto nested_serialize = [&settings](const ISerialization & serialization, const IColumn & col, size_t row, WriteBuffer & buf)
{
serialization.serializeTextCSV(col, row, buf, settings);
};
field_buf = std::make_unique<ReadBufferFromString>(field); serializeTextImpl(column, row_num, ostr, settings, nested_serialize);
auto string_discr = variant_info.variant_name_to_discriminator.at("String");
deserializeVariant(dynamic_column.getVariantColumn(), std::make_shared<DataTypeString>(), string_discr, *field_buf, deserialize_variant);
} }
void SerializationDynamic::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const void SerializationDynamic::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
@ -466,8 +669,12 @@ bool SerializationDynamic::tryDeserializeTextCSV(DB::IColumn & column, DB::ReadB
void SerializationDynamic::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const void SerializationDynamic::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{ {
const auto & dynamic_column = assert_cast<const ColumnDynamic &>(column); auto nested_serialize = [&settings](const ISerialization & serialization, const IColumn & col, size_t row, WriteBuffer & buf)
dynamic_column.getVariantInfo().variant_type->getDefaultSerialization()->serializeTextEscaped(dynamic_column.getVariantColumn(), row_num, ostr, settings); {
serialization.serializeTextEscaped(col, row, buf, settings);
};
serializeTextImpl(column, row_num, ostr, settings, nested_serialize);
} }
void SerializationDynamic::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const void SerializationDynamic::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
@ -500,8 +707,12 @@ bool SerializationDynamic::tryDeserializeTextEscaped(DB::IColumn & column, DB::R
void SerializationDynamic::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const void SerializationDynamic::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{ {
const auto & dynamic_column = assert_cast<const ColumnDynamic &>(column); auto nested_serialize = [&settings](const ISerialization & serialization, const IColumn & col, size_t row, WriteBuffer & buf)
dynamic_column.getVariantInfo().variant_type->getDefaultSerialization()->serializeTextQuoted(dynamic_column.getVariantColumn(), row_num, ostr, settings); {
serialization.serializeTextQuoted(col, row, buf, settings);
};
serializeTextImpl(column, row_num, ostr, settings, nested_serialize);
} }
void SerializationDynamic::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const void SerializationDynamic::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
@ -534,8 +745,12 @@ bool SerializationDynamic::tryDeserializeTextQuoted(DB::IColumn & column, DB::Re
void SerializationDynamic::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const void SerializationDynamic::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{ {
const auto & dynamic_column = assert_cast<const ColumnDynamic &>(column); auto nested_serialize = [&settings](const ISerialization & serialization, const IColumn & col, size_t row, WriteBuffer & buf)
dynamic_column.getVariantInfo().variant_type->getDefaultSerialization()->serializeTextJSON(dynamic_column.getVariantColumn(), row_num, ostr, settings); {
serialization.serializeTextJSON(col, row, buf, settings);
};
serializeTextImpl(column, row_num, ostr, settings, nested_serialize);
} }
void SerializationDynamic::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const void SerializationDynamic::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
@ -568,8 +783,12 @@ bool SerializationDynamic::tryDeserializeTextJSON(DB::IColumn & column, DB::Read
void SerializationDynamic::serializeTextRaw(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const void SerializationDynamic::serializeTextRaw(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{ {
const auto & dynamic_column = assert_cast<const ColumnDynamic &>(column); auto nested_serialize = [&settings](const ISerialization & serialization, const IColumn & col, size_t row, WriteBuffer & buf)
dynamic_column.getVariantInfo().variant_type->getDefaultSerialization()->serializeTextRaw(dynamic_column.getVariantColumn(), row_num, ostr, settings); {
serialization.serializeTextRaw(col, row, buf, settings);
};
serializeTextImpl(column, row_num, ostr, settings, nested_serialize);
} }
void SerializationDynamic::deserializeTextRaw(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const void SerializationDynamic::deserializeTextRaw(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
@ -602,8 +821,12 @@ bool SerializationDynamic::tryDeserializeTextRaw(DB::IColumn & column, DB::ReadB
void SerializationDynamic::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const void SerializationDynamic::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{ {
const auto & dynamic_column = assert_cast<const ColumnDynamic &>(column); auto nested_serialize = [&settings](const ISerialization & serialization, const IColumn & col, size_t row, WriteBuffer & buf)
dynamic_column.getVariantInfo().variant_type->getDefaultSerialization()->serializeText(dynamic_column.getVariantColumn(), row_num, ostr, settings); {
serialization.serializeText(col, row, buf, settings);
};
serializeTextImpl(column, row_num, ostr, settings, nested_serialize);
} }
void SerializationDynamic::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const void SerializationDynamic::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
@ -636,8 +859,12 @@ bool SerializationDynamic::tryDeserializeWholeText(DB::IColumn & column, DB::Rea
void SerializationDynamic::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const void SerializationDynamic::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{ {
const auto & dynamic_column = assert_cast<const ColumnDynamic &>(column); auto nested_serialize = [&settings](const ISerialization & serialization, const IColumn & col, size_t row, WriteBuffer & buf)
dynamic_column.getVariantInfo().variant_type->getDefaultSerialization()->serializeTextXML(dynamic_column.getVariantColumn(), row_num, ostr, settings); {
serialization.serializeTextXML(col, row, buf, settings);
};
serializeTextImpl(column, row_num, ostr, settings, nested_serialize);
} }
} }

8
src/DataTypes/Serializations/SerializationDynamic.h
View File

@ -105,9 +105,13 @@ private:
{ {
DynamicStructureSerializationVersion structure_version; DynamicStructureSerializationVersion structure_version;
DataTypePtr variant_type; DataTypePtr variant_type;
ColumnDynamic::Statistics statistics = {.source = ColumnDynamic::Statistics::Source::READ, .data = {}}; size_t max_dynamic_types;
ColumnDynamic::StatisticsPtr statistics;
explicit DeserializeBinaryBulkStateDynamicStructure(UInt64 structure_version_) : structure_version(structure_version_) {} explicit DeserializeBinaryBulkStateDynamicStructure(UInt64 structure_version_)
: structure_version(structure_version_)
{
}
}; };
size_t max_dynamic_types; size_t max_dynamic_types;

127
src/DataTypes/Serializations/SerializationDynamicElement.cpp
View File

@ -4,7 +4,10 @@
#include <DataTypes/Serializations/SerializationDynamic.h> #include <DataTypes/Serializations/SerializationDynamic.h>
#include <DataTypes/DataTypeVariant.h> #include <DataTypes/DataTypeVariant.h>
#include <DataTypes/DataTypeFactory.h> #include <DataTypes/DataTypeFactory.h>
#include <DataTypes/DataTypesBinaryEncoding.h>
#include <Columns/ColumnDynamic.h> #include <Columns/ColumnDynamic.h>
#include <Columns/ColumnLowCardinality.h>
#include <Formats/FormatSettings.h>
#include <IO/ReadHelpers.h> #include <IO/ReadHelpers.h>
namespace DB namespace DB
@ -21,6 +24,8 @@ struct DeserializeBinaryBulkStateDynamicElement : public ISerialization::Deseria
ISerialization::DeserializeBinaryBulkStatePtr structure_state; ISerialization::DeserializeBinaryBulkStatePtr structure_state;
SerializationPtr variant_serialization; SerializationPtr variant_serialization;
ISerialization::DeserializeBinaryBulkStatePtr variant_element_state; ISerialization::DeserializeBinaryBulkStatePtr variant_element_state;
bool read_from_shared_variant;
ColumnPtr shared_variant;
}; };
void SerializationDynamicElement::enumerateStreams( void SerializationDynamicElement::enumerateStreams(
@ -73,9 +78,10 @@ void SerializationDynamicElement::deserializeBinaryBulkStatePrefix(
auto dynamic_element_state = std::make_shared<DeserializeBinaryBulkStateDynamicElement>(); auto dynamic_element_state = std::make_shared<DeserializeBinaryBulkStateDynamicElement>();
dynamic_element_state->structure_state = std::move(structure_state); dynamic_element_state->structure_state = std::move(structure_state);
const auto & variant_type = checkAndGetState<SerializationDynamic::DeserializeBinaryBulkStateDynamicStructure>(dynamic_element_state->structure_state)->variant_type; const auto & variant_type = assert_cast<const DataTypeVariant &>(
*checkAndGetState<SerializationDynamic::DeserializeBinaryBulkStateDynamicStructure>(dynamic_element_state->structure_state)->variant_type);
/// Check if we actually have required element in the Variant. /// Check if we actually have required element in the Variant.
if (auto global_discr = assert_cast<const DataTypeVariant &>(*variant_type).tryGetVariantDiscriminator(dynamic_element_name)) if (auto global_discr = variant_type.tryGetVariantDiscriminator(dynamic_element_name))
{ {
settings.path.push_back(Substream::DynamicData); settings.path.push_back(Substream::DynamicData);
if (is_null_map_subcolumn) if (is_null_map_subcolumn)
@ -83,6 +89,21 @@ void SerializationDynamicElement::deserializeBinaryBulkStatePrefix(
else else
dynamic_element_state->variant_serialization = std::make_shared<SerializationVariantElement>(nested_serialization, dynamic_element_name, *global_discr); dynamic_element_state->variant_serialization = std::make_shared<SerializationVariantElement>(nested_serialization, dynamic_element_name, *global_discr);
dynamic_element_state->variant_serialization->deserializeBinaryBulkStatePrefix(settings, dynamic_element_state->variant_element_state, cache); dynamic_element_state->variant_serialization->deserializeBinaryBulkStatePrefix(settings, dynamic_element_state->variant_element_state, cache);
dynamic_element_state->read_from_shared_variant = false;
settings.path.pop_back();
}
/// If we don't have this element in the Variant, we will read shared variant and try to find it there.
else
{
auto shared_variant_global_discr = variant_type.tryGetVariantDiscriminator(ColumnDynamic::getSharedVariantTypeName());
chassert(shared_variant_global_discr.has_value());
settings.path.push_back(Substream::DynamicData);
dynamic_element_state->variant_serialization = std::make_shared<SerializationVariantElement>(
ColumnDynamic::getSharedVariantDataType()->getDefaultSerialization(),
ColumnDynamic::getSharedVariantTypeName(),
*shared_variant_global_discr);
dynamic_element_state->variant_serialization->deserializeBinaryBulkStatePrefix(settings, dynamic_element_state->variant_element_state, cache);
dynamic_element_state->read_from_shared_variant = true;
settings.path.pop_back(); settings.path.pop_back();
} }
@ -115,23 +136,103 @@ void SerializationDynamicElement::deserializeBinaryBulkWithMultipleStreams(
auto * dynamic_element_state = checkAndGetState<DeserializeBinaryBulkStateDynamicElement>(state); auto * dynamic_element_state = checkAndGetState<DeserializeBinaryBulkStateDynamicElement>(state);
if (dynamic_element_state->variant_serialization) /// Check if this subcolumn should not be read from shared variant.
/// In this case just read data from the corresponding variant.
if (!dynamic_element_state->read_from_shared_variant)
{ {
settings.path.push_back(Substream::DynamicData); settings.path.push_back(Substream::DynamicData);
dynamic_element_state->variant_serialization->deserializeBinaryBulkWithMultipleStreams(result_column, limit, settings, dynamic_element_state->variant_element_state, cache); dynamic_element_state->variant_serialization->deserializeBinaryBulkWithMultipleStreams(
result_column, limit, settings, dynamic_element_state->variant_element_state, cache);
settings.path.pop_back(); settings.path.pop_back();
} }
else if (is_null_map_subcolumn) /// Otherwise, read the shared variant column and extract requested type from it.
{
auto mutable_column = result_column->assumeMutable();
auto & data = assert_cast<ColumnUInt8 &>(*mutable_column).getData();
data.resize_fill(data.size() + limit, 1);
}
else else
{ {
auto mutable_column = result_column->assumeMutable(); settings.path.push_back(Substream::DynamicData);
mutable_column->insertManyDefaults(limit); /// Initialize shared_variant column if needed.
result_column = std::move(mutable_column); if (result_column->empty())
dynamic_element_state->shared_variant = makeNullable(ColumnDynamic::getSharedVariantDataType()->createColumn());
size_t prev_size = result_column->size();
dynamic_element_state->variant_serialization->deserializeBinaryBulkWithMultipleStreams(
dynamic_element_state->shared_variant, limit, settings, dynamic_element_state->variant_element_state, cache);
settings.path.pop_back();
/// If we need to read a subcolumn from variant column, create an empty variant column, fill it and extract subcolumn.
auto variant_type = DataTypeFactory::instance().get(dynamic_element_name);
auto result_type = makeNullableOrLowCardinalityNullableSafe(variant_type);
MutableColumnPtr variant_column = nested_subcolumn.empty() || is_null_map_subcolumn ? result_column->assumeMutable() : result_type->createColumn();
variant_column->reserve(variant_column->size() + limit);
MutableColumnPtr non_nullable_variant_column = variant_column->assumeMutable();
NullMap * null_map = nullptr;
bool is_low_cardinality_nullable = isColumnLowCardinalityNullable(*variant_column);
/// Resulting subolumn can be Nullable, but value is serialized in shared variant as non-Nullable.
/// Extract non-nullable column and remember the null map to fill it during deserialization.
if (isColumnNullable(*variant_column))
{
auto & nullable_variant_column = assert_cast<ColumnNullable &>(*variant_column);
non_nullable_variant_column = nullable_variant_column.getNestedColumnPtr()->assumeMutable();
null_map = &nullable_variant_column.getNullMapData();
}
else if (is_null_map_subcolumn)
{
null_map = &assert_cast<ColumnUInt8 &>(*variant_column).getData();
}
auto variant_serialization = variant_type->getDefaultSerialization();
const auto & nullable_shared_variant = assert_cast<const ColumnNullable &>(*dynamic_element_state->shared_variant);
const auto & shared_null_map = nullable_shared_variant.getNullMapData();
const auto & shared_variant = assert_cast<const ColumnString &>(nullable_shared_variant.getNestedColumn());
const FormatSettings format_settings;
for (size_t i = prev_size; i != shared_variant.size(); ++i)
{
if (!shared_null_map[i])
{
auto value = shared_variant.getDataAt(i);
ReadBufferFromMemory buf(value.data, value.size);
auto type = decodeDataType(buf);
if (type->getName() == dynamic_element_name)
{
/// When requested type is LowCardinality the subcolumn type name will be LowCardinality(Nullable).
/// Value in shared variant is serialized as LowCardinality and we cannot simply deserialize it
/// inside LowCardinality(Nullable) column (it will try to deserialize null bit). In this case we
/// have to create temporary LowCardinality column, deserialize value into it and insert it into
/// resulting LowCardinality(Nullable) (insertion from LowCardinality column to LowCardinality(Nullable)
/// column is allowed).
if (is_low_cardinality_nullable)
{
auto tmp_column = variant_type->createColumn();
variant_serialization->deserializeBinary(*tmp_column, buf, format_settings);
non_nullable_variant_column->insertFrom(*tmp_column, 0);
}
else if (is_null_map_subcolumn)
{
null_map->push_back(0);
}
else
{
variant_serialization->deserializeBinary(*non_nullable_variant_column, buf, format_settings);
if (null_map)
null_map->push_back(0);
}
}
else
{
variant_column->insertDefault();
}
}
else
{
variant_column->insertDefault();
}
}
/// Extract nested subcolumn if needed.
if (!nested_subcolumn.empty() && !is_null_map_subcolumn)
{
auto subcolumn = result_type->getSubcolumn(nested_subcolumn, variant_column->getPtr());
result_column->assumeMutable()->insertRangeFrom(*subcolumn, 0, subcolumn->size());
}
} }
} }

8
src/DataTypes/Serializations/SerializationDynamicElement.h
View File

@ -13,11 +13,15 @@ private:
/// To be able to deserialize Dynamic element as a subcolumn /// To be able to deserialize Dynamic element as a subcolumn
/// we need its type name and global discriminator. /// we need its type name and global discriminator.
String dynamic_element_name; String dynamic_element_name;
/// Nested subcolumn of a type dynamic type. For example, for `Tuple(a UInt32)`.a
/// subcolumn dynamic_element_name = 'Tuple(a UInt32)' and nested_subcolumn = 'a'.
/// Needed to extract nested subcolumn from values in shared variant.
String nested_subcolumn;
bool is_null_map_subcolumn; bool is_null_map_subcolumn;
public: public:
SerializationDynamicElement(const SerializationPtr & nested_, const String & dynamic_element_name_, bool is_null_map_subcolumn_ = false) SerializationDynamicElement(const SerializationPtr & nested_, const String & dynamic_element_name_, const String & nested_subcolumn_, bool is_null_map_subcolumn_ = false)
: SerializationWrapper(nested_), dynamic_element_name(dynamic_element_name_), is_null_map_subcolumn(is_null_map_subcolumn_) : SerializationWrapper(nested_), dynamic_element_name(dynamic_element_name_), nested_subcolumn(nested_subcolumn_), is_null_map_subcolumn(is_null_map_subcolumn_)
{ {
} }

41
src/DataTypes/Serializations/SerializationVariantElement.cpp
View File

@ -305,8 +305,10 @@ SerializationVariantElement::VariantSubcolumnCreator::VariantSubcolumnCreator(
const String & variant_element_name_, const String & variant_element_name_,
ColumnVariant::Discriminator global_variant_discriminator_, ColumnVariant::Discriminator global_variant_discriminator_,
ColumnVariant::Discriminator local_variant_discriminator_, ColumnVariant::Discriminator local_variant_discriminator_,
bool make_nullable_) bool make_nullable_,
const ColumnPtr & null_map_)
: local_discriminators(local_discriminators_) : local_discriminators(local_discriminators_)
, null_map(null_map_)
, variant_element_name(variant_element_name_) , variant_element_name(variant_element_name_)
, global_variant_discriminator(global_variant_discriminator_) , global_variant_discriminator(global_variant_discriminator_)
, local_variant_discriminator(local_variant_discriminator_) , local_variant_discriminator(local_variant_discriminator_)
@ -314,12 +316,13 @@ SerializationVariantElement::VariantSubcolumnCreator::VariantSubcolumnCreator(
{ {
} }
DataTypePtr SerializationVariantElement::VariantSubcolumnCreator::create(const DB::DataTypePtr & prev) const
DataTypePtr SerializationVariantElement::VariantSubcolumnCreator::create(const DataTypePtr & prev) const
{ {
return make_nullable ? makeNullableOrLowCardinalityNullableSafe(prev) : prev; return make_nullable ? makeNullableOrLowCardinalityNullableSafe(prev) : prev;
} }
SerializationPtr SerializationVariantElement::VariantSubcolumnCreator::create(const DB::SerializationPtr & prev) const SerializationPtr SerializationVariantElement::VariantSubcolumnCreator::create(const SerializationPtr & prev) const
{ {
return std::make_shared<SerializationVariantElement>(prev, variant_element_name, global_variant_discriminator); return std::make_shared<SerializationVariantElement>(prev, variant_element_name, global_variant_discriminator);
} }
@ -339,12 +342,16 @@ ColumnPtr SerializationVariantElement::VariantSubcolumnCreator::create(const DB:
return res; return res;
} }
/// In general case we should iterate through discriminators and create null-map for our variant. /// In general case we should iterate through discriminators and create null-map for our variant if we don't already have it.
NullMap null_map; std::optional<NullMap> null_map_from_discriminators;
null_map.reserve(local_discriminators->size()); if (!null_map)
const auto & local_discriminators_data = assert_cast<const ColumnVariant::ColumnDiscriminators &>(*local_discriminators).getData(); {
for (auto local_discr : local_discriminators_data) null_map_from_discriminators = NullMap();
null_map.push_back(local_discr != local_variant_discriminator); null_map_from_discriminators->reserve(local_discriminators->size());
const auto & local_discriminators_data = assert_cast<const ColumnVariant::ColumnDiscriminators &>(*local_discriminators).getData();
for (auto local_discr : local_discriminators_data)
null_map_from_discriminators->push_back(local_discr != local_variant_discriminator);
}
/// Now we can create new column from null-map and variant column using IColumn::expand. /// Now we can create new column from null-map and variant column using IColumn::expand.
auto res_column = IColumn::mutate(prev); auto res_column = IColumn::mutate(prev);
@ -356,13 +363,21 @@ ColumnPtr SerializationVariantElement::VariantSubcolumnCreator::create(const DB:
if (make_nullable && prev->lowCardinality()) if (make_nullable && prev->lowCardinality())
res_column = assert_cast<ColumnLowCardinality &>(*res_column).cloneNullable(); res_column = assert_cast<ColumnLowCardinality &>(*res_column).cloneNullable();
res_column->expand(null_map, /*inverted = */ true); if (null_map_from_discriminators)
res_column->expand(*null_map_from_discriminators, /*inverted = */ true);
else
res_column->expand(assert_cast<const ColumnUInt8 &>(*null_map).getData(), /*inverted = */ true);
if (make_nullable && prev->canBeInsideNullable()) if (make_nullable && prev->canBeInsideNullable())
{ {
auto null_map_col = ColumnUInt8::create(); if (null_map_from_discriminators)
null_map_col->getData() = std::move(null_map); {
return ColumnNullable::create(std::move(res_column), std::move(null_map_col)); auto null_map_col = ColumnUInt8::create();
null_map_col->getData() = std::move(*null_map_from_discriminators);
return ColumnNullable::create(std::move(res_column), std::move(null_map_col));
}
return ColumnNullable::create(std::move(res_column), null_map->assumeMutable());
} }
return res_column; return res_column;

6
src/DataTypes/Serializations/SerializationVariantElement.h
View File

@ -63,18 +63,22 @@ public:
struct VariantSubcolumnCreator : public ISubcolumnCreator struct VariantSubcolumnCreator : public ISubcolumnCreator
{ {
private:
const ColumnPtr local_discriminators; const ColumnPtr local_discriminators;
const ColumnPtr null_map; /// optional
const String variant_element_name; const String variant_element_name;
const ColumnVariant::Discriminator global_variant_discriminator; const ColumnVariant::Discriminator global_variant_discriminator;
const ColumnVariant::Discriminator local_variant_discriminator; const ColumnVariant::Discriminator local_variant_discriminator;
bool make_nullable; bool make_nullable;
public:
VariantSubcolumnCreator( VariantSubcolumnCreator(
const ColumnPtr & local_discriminators_, const ColumnPtr & local_discriminators_,
const String & variant_element_name_, const String & variant_element_name_,
ColumnVariant::Discriminator global_variant_discriminator_, ColumnVariant::Discriminator global_variant_discriminator_,
ColumnVariant::Discriminator local_variant_discriminator_, ColumnVariant::Discriminator local_variant_discriminator_,
bool make_nullable_); bool make_nullable_,
const ColumnPtr & null_map_ = nullptr);
DataTypePtr create(const DataTypePtr & prev) const override; DataTypePtr create(const DataTypePtr & prev) const override;
ColumnPtr create(const ColumnPtr & prev) const override; ColumnPtr create(const ColumnPtr & prev) const override;

31
src/Formats/JSONExtractTree.cpp
View File

@ -1362,13 +1362,14 @@ public:
} }
auto & variant_column = column_dynamic.getVariantColumn(); auto & variant_column = column_dynamic.getVariantColumn();
auto variant_info = column_dynamic.getVariantInfo(); const auto & variant_info = column_dynamic.getVariantInfo();
/// Second, infer ClickHouse type for this element and add it as a new variant. /// Second, infer ClickHouse type for this element and add it as a new variant.
auto element_type = elementToDataType(element, format_settings); auto element_type = elementToDataType(element, format_settings);
if (column_dynamic.addNewVariant(element_type)) auto element_type_name = element_type->getName();
if (column_dynamic.addNewVariant(element_type, element_type_name))
{ {
auto node = buildJSONExtractTree<JSONParser>(element_type, "Dynamic inference"); auto node = buildJSONExtractTree<JSONParser>(element_type, "Dynamic inference");
auto global_discriminator = variant_info.variant_name_to_discriminator[element_type->getName()]; auto global_discriminator = variant_info.variant_name_to_discriminator.at(element_type_name);
auto & variant = variant_column.getVariantByGlobalDiscriminator(global_discriminator); auto & variant = variant_column.getVariantByGlobalDiscriminator(global_discriminator);
if (!node->insertResultToColumn(variant, element, insert_settings, format_settings, error)) if (!node->insertResultToColumn(variant, element, insert_settings, format_settings, error))
return false; return false;
@ -1377,29 +1378,15 @@ public:
return true; return true;
} }
/// We couldn't add new variant. Try to insert element into current variants. /// We couldn't add this variant, insert it into shared variant.
auto variant_node = buildJSONExtractTree<JSONParser>(variant_info.variant_type, "Dynamic inference"); auto tmp_variant_column = element_type->createColumn();
if (variant_node->insertResultToColumn(variant_column, element, insert_settings, format_settings, error)) auto node = buildJSONExtractTree<JSONParser>(element_type, "Dynamic inference");
return true; if (!node->insertResultToColumn(*tmp_variant_column, element, insert_settings, format_settings, error))
/// We couldn't insert element into any existing variant, add String variant and read value as String.
column_dynamic.addStringVariant();
auto string_global_discriminator = variant_info.variant_name_to_discriminator["String"];
auto & string_column = variant_column.getVariantByGlobalDiscriminator(string_global_discriminator);
if (!getStringNode()->insertResultToColumn(string_column, element, insert_settings, format_settings, error))
return false; return false;
variant_column.getLocalDiscriminators().push_back(variant_column.localDiscriminatorByGlobal(string_global_discriminator)); column_dynamic.insertValueIntoSharedVariant(*tmp_variant_column, element_type, element_type_name, 0);
variant_column.getOffsets().push_back(string_column.size() - 1);
return true; return true;
} }
static const std::unique_ptr<JSONExtractTreeNode<JSONParser>> & getStringNode()
{
static const std::unique_ptr<JSONExtractTreeNode<JSONParser>> string_node
= buildJSONExtractTree<JSONParser>(std::make_shared<DataTypeString>(), "Dynamic inference");
return string_node;
}
static DataTypePtr elementToDataType(const typename JSONParser::Element & element, const FormatSettings & format_settings) static DataTypePtr elementToDataType(const typename JSONParser::Element & element, const FormatSettings & format_settings)
{ {
JSONInferenceInfo json_inference_info; JSONInferenceInfo json_inference_info;

326
src/Functions/FunctionsConversion.cpp
View File

@ -43,6 +43,7 @@
#include <DataTypes/DataTypeDynamic.h> #include <DataTypes/DataTypeDynamic.h>
#include <DataTypes/DataTypesDecimal.h> #include <DataTypes/DataTypesDecimal.h>
#include <DataTypes/DataTypesNumber.h> #include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypesBinaryEncoding.h>
#include <DataTypes/ObjectUtils.h> #include <DataTypes/ObjectUtils.h>
#include <DataTypes/Serializations/SerializationDecimal.h> #include <DataTypes/Serializations/SerializationDecimal.h>
#include <Formats/FormatSettings.h> #include <Formats/FormatSettings.h>
@ -4287,13 +4288,98 @@ private:
WrapperType createDynamicToColumnWrapper(const DataTypePtr &) const WrapperType createDynamicToColumnWrapper(const DataTypePtr &) const
{ {
return [this] return [this]
(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable * col_nullable, size_t input_rows_count) -> ColumnPtr (ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable *, size_t input_rows_count) -> ColumnPtr
{ {
/// When casting Dynamic to regular column we should cast all variants from current Dynamic column
/// and construct the result based on discriminators.
const auto & column_dynamic = assert_cast<const ColumnDynamic &>(*arguments.front().column.get()); const auto & column_dynamic = assert_cast<const ColumnDynamic &>(*arguments.front().column.get());
const auto & variant_column = column_dynamic.getVariantColumn();
const auto & variant_info = column_dynamic.getVariantInfo(); const auto & variant_info = column_dynamic.getVariantInfo();
auto variant_wrapper = createVariantToColumnWrapper(assert_cast<const DataTypeVariant &>(*variant_info.variant_type), result_type);
ColumnsWithTypeAndName args = {ColumnWithTypeAndName(column_dynamic.getVariantColumnPtr(), variant_info.variant_type, "")}; /// First, cast usual variants to result type.
return variant_wrapper(args, result_type, col_nullable, input_rows_count); const auto & variant_types = assert_cast<const DataTypeVariant &>(*variant_info.variant_type).getVariants();
std::vector<ColumnPtr> casted_variant_columns;
casted_variant_columns.reserve(variant_types.size());
for (size_t i = 0; i != variant_types.size(); ++i)
{
const auto & variant_col = variant_column.getVariantPtrByGlobalDiscriminator(i);
ColumnsWithTypeAndName variant = {{variant_col, variant_types[i], ""}};
auto variant_wrapper = prepareUnpackDictionaries(variant_types[i], result_type);
casted_variant_columns.push_back(variant_wrapper(variant, result_type, nullptr, variant_col->size()));
}
/// Second, collect all variants stored in shared variant and cast them to result type.
std::vector<MutableColumnPtr> variant_columns_from_shared_variant;
DataTypes variant_types_from_shared_variant;
/// We will need to know what variant to use when we see discriminator of a shared variant.
/// To do it, we remember what variant was extracted from each row and what was it's offset.
PaddedPODArray<UInt64> shared_variant_indexes;
PaddedPODArray<UInt64> shared_variant_offsets;
std::unordered_map<String, UInt64> shared_variant_to_index;
const auto & shared_variant = column_dynamic.getSharedVariant();
const auto shared_variant_discr = column_dynamic.getSharedVariantDiscriminator();
const auto & local_discriminators = variant_column.getLocalDiscriminators();
const auto & offsets = variant_column.getOffsets();
if (!shared_variant.empty())
{
shared_variant_indexes.reserve(input_rows_count);
shared_variant_offsets.reserve(input_rows_count);
FormatSettings format_settings;
const auto shared_variant_local_discr = variant_column.localDiscriminatorByGlobal(shared_variant_discr);
for (size_t i = 0; i != input_rows_count; ++i)
{
if (local_discriminators[i] == shared_variant_local_discr)
{
auto value = shared_variant.getDataAt(offsets[i]);
ReadBufferFromMemory buf(value.data, value.size);
auto type = decodeDataType(buf);
auto type_name = type->getName();
auto it = shared_variant_to_index.find(type_name);
/// Check if we didn't create column for this variant yet.
if (it == shared_variant_to_index.end())
{
it = shared_variant_to_index.emplace(type_name, variant_columns_from_shared_variant.size()).first;
variant_columns_from_shared_variant.push_back(type->createColumn());
variant_types_from_shared_variant.push_back(type);
}
shared_variant_indexes.push_back(it->second);
shared_variant_offsets.push_back(variant_columns_from_shared_variant[it->second]->size());
type->getDefaultSerialization()->deserializeBinary(*variant_columns_from_shared_variant[it->second], buf, format_settings);
}
else
{
shared_variant_indexes.emplace_back();
shared_variant_offsets.emplace_back();
}
}
}
/// Cast all extracted variants into result type.
std::vector<ColumnPtr> casted_shared_variant_columns;
casted_shared_variant_columns.reserve(variant_types_from_shared_variant.size());
for (size_t i = 0; i != variant_types_from_shared_variant.size(); ++i)
{
ColumnsWithTypeAndName variant = {{variant_columns_from_shared_variant[i]->getPtr(), variant_types_from_shared_variant[i], ""}};
auto variant_wrapper = prepareUnpackDictionaries(variant_types_from_shared_variant[i], result_type);
casted_shared_variant_columns.push_back(variant_wrapper(variant, result_type, nullptr, variant_columns_from_shared_variant[i]->size()));
}
/// Construct result column from all casted variants.
auto res = result_type->createColumn();
res->reserve(input_rows_count);
for (size_t i = 0; i != input_rows_count; ++i)
{
auto global_discr = variant_column.globalDiscriminatorByLocal(local_discriminators[i]);
if (global_discr == ColumnVariant::NULL_DISCRIMINATOR)
res->insertDefault();
else if (global_discr == shared_variant_discr)
res->insertFrom(*casted_shared_variant_columns[shared_variant_indexes[i]], shared_variant_offsets[i]);
else
res->insertFrom(*casted_variant_columns[global_discr], offsets[i]);
}
return res;
}; };
} }
@ -4320,200 +4406,51 @@ private:
}; };
} }
std::pair<ColumnPtr, DataTypePtr> getReducedVariant( WrapperType createVariantToDynamicWrapper(const DataTypeVariant & from_variant_type, const DataTypeDynamic & dynamic_type) const
const ColumnVariant & variant_column,
const DataTypePtr & variant_type,
const std::unordered_map<String, ColumnVariant::Discriminator> & variant_name_to_discriminator,
size_t max_result_num_variants,
const ColumnDynamic::Statistics & statistics = {}) const
{ {
const auto & variant_types = assert_cast<const DataTypeVariant &>(*variant_type).getVariants(); /// First create extended Variant with shared variant type and cast this Variant to it.
/// First check if we don't exceed the limit in current Variant column. auto variants_for_dynamic = from_variant_type.getVariants();
if (variant_types.size() < max_result_num_variants || (variant_types.size() == max_result_num_variants && variant_name_to_discriminator.contains("String"))) size_t number_of_variants = variants_for_dynamic.size();
return {variant_column.getPtr(), variant_type}; variants_for_dynamic.push_back(ColumnDynamic::getSharedVariantDataType());
const auto & variant_type_for_dynamic = std::make_shared<DataTypeVariant>(variants_for_dynamic);
/// We want to keep the most frequent variants and convert to string the rarest. auto old_to_new_variant_wrapper = createVariantToVariantWrapper(from_variant_type, *variant_type_for_dynamic);
std::vector<std::pair<size_t, ColumnVariant::Discriminator>> variant_sizes; auto max_dynamic_types = dynamic_type.getMaxDynamicTypes();
variant_sizes.reserve(variant_types.size()); return [old_to_new_variant_wrapper, variant_type_for_dynamic, number_of_variants, max_dynamic_types]
std::optional<ColumnVariant::Discriminator> old_string_discriminator; (ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable * col_nullable, size_t input_rows_count) -> ColumnPtr
/// List of variants that should be converted to a single String variant.
std::vector<ColumnVariant::Discriminator> variants_to_convert_to_string;
for (size_t i = 0; i != variant_types.size(); ++i)
{ {
/// String variant won't be removed. auto variant_column_for_dynamic = old_to_new_variant_wrapper(arguments, result_type, col_nullable, input_rows_count);
String variant_name = variant_types[i]->getName(); /// If resulting Dynamic column can contain all variants from this Variant column, just create Dynamic column from it.
if (max_dynamic_types >= number_of_variants)
return ColumnDynamic::create(variant_column_for_dynamic, variant_type_for_dynamic, max_dynamic_types, max_dynamic_types);
if (variant_name == "String") /// Otherwise some variants should go to the shared variant. Create temporary Dynamic column from this Variant and insert
{ /// all data to the resulting Dynamic column, this insertion will do all the logic with shared variant.
old_string_discriminator = i; auto tmp_dynamic_column = ColumnDynamic::create(variant_column_for_dynamic, variant_type_for_dynamic, number_of_variants, number_of_variants);
/// For simplicity, add this variant to the list that will be converted to string, auto result_dynamic_column = ColumnDynamic::create(max_dynamic_types);
/// so we will process it with other variants when constructing the new String variant. result_dynamic_column->insertRangeFrom(*tmp_dynamic_column, 0, tmp_dynamic_column->size());
variants_to_convert_to_string.push_back(i); return result_dynamic_column;
}
else
{
size_t size = 0;
if (statistics.data.empty())
size = variant_column.getVariantByGlobalDiscriminator(i).size();
else
size = statistics.data.at(variant_name);
variant_sizes.emplace_back(size, i);
}
}
/// Sort variants by sizes, so we will keep the most frequent.
std::sort(variant_sizes.begin(), variant_sizes.end(), std::greater());
DataTypes remaining_variants;
remaining_variants.reserve(max_result_num_variants);
/// Add String variant in advance.
remaining_variants.push_back(std::make_shared<DataTypeString>());
for (auto [_, discr] : variant_sizes)
{
if (remaining_variants.size() != max_result_num_variants)
remaining_variants.push_back(variant_types[discr]);
else
variants_to_convert_to_string.push_back(discr);
}
auto reduced_variant = std::make_shared<DataTypeVariant>(remaining_variants);
const auto & new_variants = reduced_variant->getVariants();
/// To construct reduced variant column we will need mapping from old to new discriminators.
std::vector<ColumnVariant::Discriminator> old_to_new_discriminators_mapping;
old_to_new_discriminators_mapping.resize(variant_types.size());
ColumnVariant::Discriminator string_variant_discriminator = 0;
for (size_t i = 0; i != new_variants.size(); ++i)
{
String variant_name = new_variants[i]->getName();
if (variant_name == "String")
{
string_variant_discriminator = i;
for (auto discr : variants_to_convert_to_string)
old_to_new_discriminators_mapping[discr] = i;
}
else
{
auto old_discr = variant_name_to_discriminator.at(variant_name);
old_to_new_discriminators_mapping[old_discr] = i;
}
}
/// Convert all reduced variants to String.
std::unordered_map<ColumnVariant::Discriminator, ColumnPtr> variants_converted_to_string;
variants_converted_to_string.reserve(variants_to_convert_to_string.size());
size_t string_variant_size = 0;
for (auto discr : variants_to_convert_to_string)
{
auto string_type = std::make_shared<DataTypeString>();
auto string_wrapper = prepareUnpackDictionaries(variant_types[discr], string_type);
auto column_to_convert = ColumnWithTypeAndName(variant_column.getVariantPtrByGlobalDiscriminator(discr), variant_types[discr], "");
ColumnsWithTypeAndName args = {column_to_convert};
auto variant_string_column = string_wrapper(args, string_type, nullptr, column_to_convert.column->size());
string_variant_size += variant_string_column->size();
variants_converted_to_string[discr] = variant_string_column;
}
/// Create new discriminators and offsets and fill new String variant according to old discriminators.
auto string_variant = ColumnString::create();
string_variant->reserve(string_variant_size);
auto new_discriminators_column = variant_column.getLocalDiscriminatorsPtr()->cloneEmpty();
auto & new_discriminators_data = assert_cast<ColumnVariant::ColumnDiscriminators &>(*new_discriminators_column).getData();
new_discriminators_data.reserve(variant_column.size());
auto new_offsets = variant_column.getOffsetsPtr()->cloneEmpty();
auto & new_offsets_data = assert_cast<ColumnVariant::ColumnOffsets &>(*new_offsets).getData();
new_offsets_data.reserve(variant_column.size());
const auto & old_local_discriminators = variant_column.getLocalDiscriminators();
const auto & old_offsets = variant_column.getOffsets();
for (size_t i = 0; i != old_local_discriminators.size(); ++i)
{
auto old_discr = variant_column.globalDiscriminatorByLocal(old_local_discriminators[i]);
if (old_discr == ColumnVariant::NULL_DISCRIMINATOR)
{
new_discriminators_data.push_back(ColumnVariant::NULL_DISCRIMINATOR);
new_offsets_data.push_back(0);
continue;
}
auto new_discr = old_to_new_discriminators_mapping[old_discr];
new_discriminators_data.push_back(new_discr);
if (new_discr != string_variant_discriminator)
{
new_offsets_data.push_back(old_offsets[i]);
}
else
{
new_offsets_data.push_back(string_variant->size());
string_variant->insertFrom(*variants_converted_to_string[old_discr], old_offsets[i]);
}
}
/// Create new list of variant columns.
Columns new_variant_columns;
new_variant_columns.resize(new_variants.size());
for (size_t i = 0; i != variant_types.size(); ++i)
{
auto new_discr = old_to_new_discriminators_mapping[i];
if (new_discr != string_variant_discriminator)
new_variant_columns[new_discr] = variant_column.getVariantPtrByGlobalDiscriminator(i);
}
new_variant_columns[string_variant_discriminator] = std::move(string_variant);
return {ColumnVariant::create(std::move(new_discriminators_column), std::move(new_offsets), new_variant_columns), reduced_variant};
}
WrapperType createVariantToDynamicWrapper(const DataTypePtr & from_type, const DataTypeDynamic & dynamic_type) const
{
const auto & from_variant_type = assert_cast<const DataTypeVariant &>(*from_type);
size_t max_dynamic_types = dynamic_type.getMaxDynamicTypes();
const auto & variants = from_variant_type.getVariants();
std::unordered_map<String, ColumnVariant::Discriminator> variant_name_to_discriminator;
variant_name_to_discriminator.reserve(variants.size());
for (size_t i = 0; i != variants.size(); ++i)
variant_name_to_discriminator[variants[i]->getName()] = i;
return [from_type, max_dynamic_types, variant_name_to_discriminator, this]
(ColumnsWithTypeAndName & arguments, const DataTypePtr &, const ColumnNullable *, size_t) -> ColumnPtr
{
const auto & variant_column = assert_cast<const ColumnVariant &>(*arguments.front().column);
auto [reduced_variant_column, reduced_variant_type] = getReducedVariant(variant_column, from_type, variant_name_to_discriminator, max_dynamic_types);
return ColumnDynamic::create(reduced_variant_column, reduced_variant_type, max_dynamic_types);
}; };
} }
WrapperType createColumnToDynamicWrapper(const DataTypePtr & from_type, const DataTypeDynamic & dynamic_type) const WrapperType createColumnToDynamicWrapper(const DataTypePtr & from_type, const DataTypeDynamic & dynamic_type) const
{ {
if (const auto * variant_type = typeid_cast<const DataTypeVariant *>(from_type.get())) if (const auto * variant_type = typeid_cast<const DataTypeVariant *>(from_type.get()))
return createVariantToDynamicWrapper(from_type, dynamic_type); return createVariantToDynamicWrapper(*variant_type, dynamic_type);
if (dynamic_type.getMaxDynamicTypes() == 1)
{
DataTypePtr string_type = std::make_shared<DataTypeString>();
if (from_type->isNullable())
string_type = makeNullable(string_type);
auto string_wrapper = prepareUnpackDictionaries(from_type, string_type);
auto variant_type = std::make_shared<DataTypeVariant>(DataTypes{removeNullable(string_type)});
auto variant_wrapper = createColumnToVariantWrapper(string_type, *variant_type);
return [string_wrapper, variant_wrapper, string_type, variant_type, max_dynamic_types=dynamic_type.getMaxDynamicTypes()]
(ColumnsWithTypeAndName & arguments, const DataTypePtr &, const ColumnNullable * col_nullable, size_t input_rows_count) -> ColumnPtr
{
auto string_column = string_wrapper(arguments, string_type, col_nullable, input_rows_count);
auto column = ColumnWithTypeAndName(string_column, string_type, "");
ColumnsWithTypeAndName args = {column};
auto variant_column = variant_wrapper(args, variant_type, nullptr, string_column->size());
return ColumnDynamic::create(variant_column, variant_type, max_dynamic_types);
};
}
if (context && context->getSettingsRef().cast_string_to_dynamic_use_inference && isStringOrFixedString(removeNullable(removeLowCardinality(from_type)))) if (context && context->getSettingsRef().cast_string_to_dynamic_use_inference && isStringOrFixedString(removeNullable(removeLowCardinality(from_type))))
return createStringToDynamicThroughParsingWrapper(); return createStringToDynamicThroughParsingWrapper();
/// First, cast column to Variant with 2 variants - the type of the column we cast and shared variant type.
auto variant_type = std::make_shared<DataTypeVariant>(DataTypes{removeNullableOrLowCardinalityNullable(from_type)}); auto variant_type = std::make_shared<DataTypeVariant>(DataTypes{removeNullableOrLowCardinalityNullable(from_type)});
auto variant_wrapper = createColumnToVariantWrapper(from_type, *variant_type); auto column_to_variant_wrapper = createColumnToVariantWrapper(from_type, *variant_type);
return [variant_wrapper, variant_type, max_dynamic_types=dynamic_type.getMaxDynamicTypes()] /// Second, cast this Variant to Dynamic.
(ColumnsWithTypeAndName & arguments, const DataTypePtr &, const ColumnNullable * col_nullable, size_t input_rows_count) -> ColumnPtr auto variant_to_dynamic_wrapper = createVariantToDynamicWrapper(*variant_type, dynamic_type);
return [column_to_variant_wrapper, variant_to_dynamic_wrapper, variant_type]
(ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, const ColumnNullable * col_nullable, size_t input_rows_count) -> ColumnPtr
{ {
auto variant_res = variant_wrapper(arguments, variant_type, col_nullable, input_rows_count); auto variant_res = column_to_variant_wrapper(arguments, variant_type, col_nullable, input_rows_count);
return ColumnDynamic::create(variant_res, variant_type, max_dynamic_types); ColumnsWithTypeAndName args = {{variant_res, variant_type, ""}};
return variant_to_dynamic_wrapper(args, result_type, nullptr, input_rows_count);
}; };
} }
@ -4530,21 +4467,26 @@ private:
(ColumnsWithTypeAndName & arguments, const DataTypePtr &, const ColumnNullable *, size_t) -> ColumnPtr (ColumnsWithTypeAndName & arguments, const DataTypePtr &, const ColumnNullable *, size_t) -> ColumnPtr
{ {
const auto & column_dynamic = assert_cast<const ColumnDynamic &>(*arguments[0].column); const auto & column_dynamic = assert_cast<const ColumnDynamic &>(*arguments[0].column);
return ColumnDynamic::create(column_dynamic.getVariantColumnPtr(), column_dynamic.getVariantInfo(), to_max_types); /// We should use the same limit as already used in column and change only global limit.
/// It's needed because shared variant should contain values only when limit is exceeded,
/// so if there are already some data, we cannot increase the limit.
return ColumnDynamic::create(column_dynamic.getVariantColumnPtr(), column_dynamic.getVariantInfo(), column_dynamic.getMaxDynamicTypes(), to_max_types);
}; };
} }
return [to_max_types, this] return [to_max_types]
(ColumnsWithTypeAndName & arguments, const DataTypePtr &, const ColumnNullable *, size_t) -> ColumnPtr (ColumnsWithTypeAndName & arguments, const DataTypePtr &, const ColumnNullable *, size_t) -> ColumnPtr
{ {
const auto & column_dynamic = assert_cast<const ColumnDynamic &>(*arguments[0].column); const auto & column_dynamic = assert_cast<const ColumnDynamic &>(*arguments[0].column);
auto [reduced_variant_column, reduced_variant_type] = getReducedVariant( /// If real limit in the column is not greater than desired, just use the same variant column.
column_dynamic.getVariantColumn(), if (column_dynamic.getMaxDynamicTypes() <= to_max_types)
column_dynamic.getVariantInfo().variant_type, return ColumnDynamic::create(column_dynamic.getVariantColumnPtr(), column_dynamic.getVariantInfo(), column_dynamic.getMaxDynamicTypes(), to_max_types);
column_dynamic.getVariantInfo().variant_name_to_discriminator,
to_max_types, /// Otherwise some variants should go to the shared variant. In this case we can just insert all
column_dynamic.getStatistics()); /// the data into resulting column and it will do all the logic with shared variant.
return ColumnDynamic::create(reduced_variant_column, reduced_variant_type, to_max_types); auto result_dynamic_column = ColumnDynamic::create(to_max_types);
result_dynamic_column->insertRangeFrom(column_dynamic, 0, column_dynamic.size());
return result_dynamic_column;
}; };
} }

91
src/Functions/dynamicType.cpp
View File

@ -2,10 +2,14 @@
#include <Functions/FunctionFactory.h> #include <Functions/FunctionFactory.h>
#include <Functions/FunctionHelpers.h> #include <Functions/FunctionHelpers.h>
#include <DataTypes/IDataType.h> #include <DataTypes/IDataType.h>
#include <DataTypes/DataTypesBinaryEncoding.h>
#include <DataTypes/DataTypeFactory.h>
#include <DataTypes/DataTypeString.h> #include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeLowCardinality.h> #include <DataTypes/DataTypeLowCardinality.h>
#include <Columns/ColumnVariant.h> #include <Columns/ColumnVariant.h>
#include <Columns/ColumnDynamic.h> #include <Columns/ColumnDynamic.h>
#include <Columns/ColumnsNumber.h>
#include <IO/ReadBufferFromMemory.h>
#include <Common/assert_cast.h> #include <Common/assert_cast.h>
@ -65,11 +69,15 @@ public:
const auto & variant_column = dynamic_column->getVariantColumn(); const auto & variant_column = dynamic_column->getVariantColumn();
auto res = result_type->createColumn(); auto res = result_type->createColumn();
String element_type; String element_type;
auto shared_variant_discr = dynamic_column->getSharedVariantDiscriminator();
const auto & shared_variant = dynamic_column->getSharedVariant();
for (size_t i = 0; i != input_rows_count; ++i) for (size_t i = 0; i != input_rows_count; ++i)
{ {
auto global_discr = variant_column.globalDiscriminatorAt(i); auto global_discr = variant_column.globalDiscriminatorAt(i);
if (global_discr == ColumnVariant::NULL_DISCRIMINATOR) if (global_discr == ColumnVariant::NULL_DISCRIMINATOR)
element_type = name_for_null; element_type = name_for_null;
else if (global_discr == shared_variant_discr)
element_type = getTypeNameFromSharedVariantValue(shared_variant.getDataAt(variant_column.offsetAt(i)));
else else
element_type = variant_info.variant_names[global_discr]; element_type = variant_info.variant_names[global_discr];
@ -78,6 +86,63 @@ public:
return res; return res;
} }
String getTypeNameFromSharedVariantValue(StringRef value) const
{
ReadBufferFromMemory buf(value.data, value.size);
return decodeDataType(buf)->getName();
}
};
class FunctionIsDynamicElementInSharedData : public IFunction
{
public:
static constexpr auto name = "isDynamicElementInSharedData";
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionIsDynamicElementInSharedData>(); }
String getName() const override { return name; }
size_t getNumberOfArguments() const override { return 1; }
bool useDefaultImplementationForConstants() const override { return true; }
bool useDefaultImplementationForNulls() const override { return false; }
bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
if (arguments.empty() || arguments.size() > 1)
throw Exception(
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Number of arguments for function {} doesn't match: passed {}, should be 1",
getName(), arguments.empty());
if (!isDynamic(arguments[0].type.get()))
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"First argument for function {} must be Dynamic, got {} instead",
getName(), arguments[0].type->getName());
return DataTypeFactory::instance().get("Bool");
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{
const ColumnDynamic * dynamic_column = checkAndGetColumn<ColumnDynamic>(arguments[0].column.get());
if (!dynamic_column)
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"First argument for function {} must be Dynamic, got {} instead",
getName(), arguments[0].type->getName());
const auto & variant_column = dynamic_column->getVariantColumn();
const auto & local_discriminators = variant_column.getLocalDiscriminators();
auto res = result_type->createColumn();
auto & res_data = assert_cast<ColumnUInt8 &>(*res).getData();
res_data.reserve(dynamic_column->size());
auto shared_variant_local_discr = variant_column.localDiscriminatorByGlobal(dynamic_column->getSharedVariantDiscriminator());
for (size_t i = 0; i != input_rows_count; ++i)
res_data.push_back(local_discriminators[i] == shared_variant_local_discr);
return res;
}
}; };
} }
@ -88,7 +153,7 @@ REGISTER_FUNCTION(DynamicType)
.description = R"( .description = R"(
Returns the variant type name for each row of `Dynamic` column. If row contains NULL, it returns 'None' for it. Returns the variant type name for each row of `Dynamic` column. If row contains NULL, it returns 'None' for it.
)", )",
.syntax = {"dynamicType(variant)"}, .syntax = {"dynamicType(dynamic)"},
.arguments = {{"dynamic", "Dynamic column"}}, .arguments = {{"dynamic", "Dynamic column"}},
.examples = {{{ .examples = {{{
"Example", "Example",
@ -104,6 +169,30 @@ SELECT d, dynamicType(d) FROM test;
│ Hello, World! │ String │ │ Hello, World! │ String │
│ [1,2,3] │ Array(Int64) │ │ [1,2,3] │ Array(Int64) │
└───────────────┴────────────────┘ └───────────────┴────────────────┘
)"}}},
.categories{"Variant"},
});
factory.registerFunction<FunctionIsDynamicElementInSharedData>(FunctionDocumentation{
.description = R"(
Returns true for rows in Dynamic column that are not separated into subcolumns and stored inside shared variant in binary form.
)",
.syntax = {"isDynamicElementInSharedData(dynamic)"},
.arguments = {{"dynamic", "Dynamic column"}},
.examples = {{{
"Example",
R"(
CREATE TABLE test (d Dynamic(max_types=2)) ENGINE = Memory;
INSERT INTO test VALUES (NULL), (42), ('Hello, World!'), ([1, 2, 3]);
SELECT d, isDynamicElementInSharedData(d) FROM test;
)",
R"(
┌─d─────────────┬─isDynamicElementInSharedData(d)─┐
│ ᴺᵁᴸᴸ │ false │
│ 42 │ false │
│ Hello, World! │ true │
│ [1,2,3] │ true │
└───────────────┴────────────────────┘
)"}}}, )"}}},
.categories{"Variant"}, .categories{"Variant"},
}); });

26
tests/queries/0_stateless/03033_dynamic_text_serialization.reference
View File

@ -11,11 +11,11 @@ JSON
{"d":["1","str",["1","2","3"]],"dynamicType(d)":"Tuple(Int64, String, Array(Int64))"} {"d":["1","str",["1","2","3"]],"dynamicType(d)":"Tuple(Int64, String, Array(Int64))"}
{"d":null,"dynamicType(d)":"None"} {"d":null,"dynamicType(d)":"None"}
{"d":true,"dynamicType(d)":"Bool"} {"d":true,"dynamicType(d)":"Bool"}
{"d":"42","dynamicType(d)":"Int64"} {"d":"42","dynamicType(d)":"Int64","isDynamicElementInSharedData(d)":false}
{"d":"42.42","dynamicType(d)":"String"} {"d":42.42,"dynamicType(d)":"Float64","isDynamicElementInSharedData(d)":false}
{"d":"str","dynamicType(d)":"String"} {"d":"str","dynamicType(d)":"String","isDynamicElementInSharedData(d)":true}
{"d":null,"dynamicType(d)":"None"} {"d":null,"dynamicType(d)":"None","isDynamicElementInSharedData(d)":false}
{"d":"1","dynamicType(d)":"Int64"} {"d":true,"dynamicType(d)":"Bool","isDynamicElementInSharedData(d)":true}
CSV CSV
42,"Int64" 42,"Int64"
42.42,"Float64" 42.42,"Float64"
@ -44,12 +44,12 @@ Cast using parsing
[1,2,3] Array(Int64) [1,2,3] Array(Int64)
2020-01-01 Date 2020-01-01 Date
2020-01-01 10:00:00.000000000 DateTime64(9) 2020-01-01 10:00:00.000000000 DateTime64(9)
\N None NULL String
true Bool true Bool
42 Int64 42 Int64 false
42.42 Float64 42.42 Float64 false
[1, 2, 3] String [1,2,3] Array(Int64) false
2020-01-01 String 2020-01-01 Date true
2020-01-01 10:00:00 String 2020-01-01 10:00:00.000000000 DateTime64(9) true
\N None NULL String true
true String true Bool true

4
tests/queries/0_stateless/03033_dynamic_text_serialization.sql
View File

@ -16,7 +16,7 @@ select d, dynamicType(d) from format(JSONEachRow, 'd Dynamic', $$
{"d" : true} {"d" : true}
$$) format JSONEachRow; $$) format JSONEachRow;
select d, dynamicType(d) from format(JSONEachRow, 'd Dynamic(max_types=2)', $$ select d, dynamicType(d), isDynamicElementInSharedData(d) from format(JSONEachRow, 'd Dynamic(max_types=2)', $$
{"d" : 42} {"d" : 42}
{"d" : 42.42} {"d" : 42.42}
{"d" : "str"} {"d" : "str"}
@ -69,6 +69,6 @@ create table test (s String) engine=Memory;
insert into test values ('42'), ('42.42'), ('[1, 2, 3]'), ('2020-01-01'), ('2020-01-01 10:00:00'), ('NULL'), ('true'); insert into test values ('42'), ('42.42'), ('[1, 2, 3]'), ('2020-01-01'), ('2020-01-01 10:00:00'), ('NULL'), ('true');
set cast_string_to_dynamic_use_inference=1; set cast_string_to_dynamic_use_inference=1;
select s::Dynamic as d, dynamicType(d) from test; select s::Dynamic as d, dynamicType(d) from test;
select s::Dynamic(max_types=3) as d, dynamicType(d) from test; select s::Dynamic(max_types=3) as d, dynamicType(d), isDynamicElementInSharedData(d) from test;
drop table test; drop table test;

18
tests/queries/0_stateless/03034_dynamic_conversions.reference
View File

@ -1,9 +1,9 @@
0 UInt64 0 UInt64
1 UInt64 1 UInt64
2 UInt64 2 UInt64
0 String 0 UInt64
1 String 1 UInt64
2 String 2 UInt64
0 0
1 1
2 2
@ -25,15 +25,15 @@ str_1 String
\N None \N None
4 UInt64 4 UInt64
str_5 String str_5 String
0 String 0 UInt64
str_1 String str_1 String
[0,1] String [0,1] Array(UInt64)
\N None \N None
4 String 4 UInt64
str_5 String str_5 String
0 UInt64 0 UInt64
str_1 String str_1 String
[0,1] String [0,1] Array(UInt64)
\N None \N None
4 UInt64 4 UInt64
str_5 String str_5 String
@ -51,13 +51,13 @@ str_5 String
2 2
0 UInt64 0 UInt64
str_1 String str_1 String
[0,1] String [0,1] Array(UInt64)
\N None \N None
4 UInt64 4 UInt64
str_5 String str_5 String
0 UInt64 0 UInt64
1970-01-02 Date 1970-01-02 Date
[0,1] String [0,1] Array(UInt64)
\N None \N None
4 UInt64 4 UInt64
1970-01-06 Date 1970-01-06 Date

6
tests/queries/0_stateless/03034_dynamic_conversions.sql
View File

@ -3,7 +3,7 @@ set allow_experimental_variant_type=1;
set use_variant_as_common_type=1; set use_variant_as_common_type=1;
select number::Dynamic as d, dynamicType(d) from numbers(3); select number::Dynamic as d, dynamicType(d) from numbers(3);
select number::Dynamic(max_types=1) as d, dynamicType(d) from numbers(3); select number::Dynamic(max_types=0) as d, dynamicType(d) from numbers(3);
select number::Dynamic::UInt64 as v from numbers(3); select number::Dynamic::UInt64 as v from numbers(3);
select number::Dynamic::String as v from numbers(3); select number::Dynamic::String as v from numbers(3);
select number::Dynamic::Date as v from numbers(3); select number::Dynamic::Date as v from numbers(3);
@ -12,13 +12,13 @@ select number::Dynamic::Variant(UInt64, String) as v, variantType(v) from number
select (number % 2 ? NULL : number)::Dynamic as d, dynamicType(d) from numbers(3); select (number % 2 ? NULL : number)::Dynamic as d, dynamicType(d) from numbers(3);
select multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, range(number), NULL)::Dynamic as d, dynamicType(d) from numbers(6); select multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, range(number), NULL)::Dynamic as d, dynamicType(d) from numbers(6);
select multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, range(number), NULL)::Dynamic(max_types=0) as d, dynamicType(d) from numbers(6);
select multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, range(number), NULL)::Dynamic(max_types=1) as d, dynamicType(d) from numbers(6); select multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, range(number), NULL)::Dynamic(max_types=1) as d, dynamicType(d) from numbers(6);
select multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, range(number), NULL)::Dynamic(max_types=2) as d, dynamicType(d) from numbers(6); select multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, range(number), NULL)::Dynamic(max_types=2) as d, dynamicType(d) from numbers(6);
select multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, range(number), NULL)::Dynamic(max_types=3) as d, dynamicType(d) from numbers(6);
select number::Dynamic(max_types=2)::Dynamic(max_types=3) as d from numbers(3); select number::Dynamic(max_types=2)::Dynamic(max_types=3) as d from numbers(3);
select number::Dynamic(max_types=2)::Dynamic(max_types=1) as d from numbers(3); select number::Dynamic(max_types=2)::Dynamic(max_types=1) as d from numbers(3);
select multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, range(number), NULL)::Dynamic(max_types=3)::Dynamic(max_types=2) as d, dynamicType(d) from numbers(6); select multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, range(number), NULL)::Dynamic(max_types=2)::Dynamic(max_types=1) as d, dynamicType(d) from numbers(6);
select multiIf(number % 4 == 0, number, number % 4 == 1, toDate(number), number % 4 == 2, range(number), NULL)::Dynamic(max_types=4)::Dynamic(max_types=3) as d, dynamicType(d) from numbers(6); select multiIf(number % 4 == 0, number, number % 4 == 1, toDate(number), number % 4 == 2, range(number), NULL)::Dynamic(max_types=4)::Dynamic(max_types=3) as d, dynamicType(d) from numbers(6);

715
tests/queries/0_stateless/03035_dynamic_sorting.reference
View File

@ -1,299 +1,442 @@
order by d1 nulls first order by d1 nulls first
\N None \N None false
\N None \N None false
\N None \N None false
\N None \N None false
[1,2,3] Array(Int64) \N None false
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,4] Array(Int64) [1,2,3] Array(Int64) true
42 Int64 [1,2,3] Array(Int64) true
42 Int64 [1,2,4] Array(Int64) true
42 Int64 2020-01-01 Date true
42 Int64 2020-01-01 Date true
42 Int64 2020-01-01 Date true
43 Int64 2020-01-01 Date true
abc String 2020-01-01 Date true
abc String 2020-01-01 Date true
abc String 2020-01-02 Date true
abc String 42 Int64 false
abc String 42 Int64 false
abd String 42 Int64 false
42 Int64 false
42 Int64 false
42 Int64 false
43 Int64 false
abc String false
abc String false
abc String false
abc String false
abc String false
abc String false
abd String false
order by d1 nulls last order by d1 nulls last
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,4] Array(Int64) [1,2,3] Array(Int64) true
42 Int64 [1,2,4] Array(Int64) true
42 Int64 2020-01-01 Date true
42 Int64 2020-01-01 Date true
42 Int64 2020-01-01 Date true
42 Int64 2020-01-01 Date true
43 Int64 2020-01-01 Date true
abc String 2020-01-01 Date true
abc String 2020-01-02 Date true
abc String 42 Int64 false
abc String 42 Int64 false
abc String 42 Int64 false
abd String 42 Int64 false
\N None 42 Int64 false
\N None 42 Int64 false
\N None 43 Int64 false
\N None abc String false
abc String false
abc String false
abc String false
abc String false
abc String false
abd String false
\N None false
\N None false
\N None false
\N None false
\N None false
order by d2 nulls first order by d2 nulls first
\N None \N None false
\N None \N None false
\N None \N None false
\N None \N None false
[1,2,3] Array(Int64) \N None false
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,4] Array(Int64) [1,2,3] Array(Int64) true
42 Int64 [1,2,3] Array(Int64) true
42 Int64 [1,2,4] Array(Int64) true
42 Int64 2020-01-01 Date true
42 Int64 2020-01-01 Date true
42 Int64 2020-01-01 Date true
43 Int64 2020-01-01 Date true
abc String 2020-01-01 Date true
abc String 2020-01-01 Date true
abc String 2020-01-02 Date true
abc String 42 Int64 false
abc String 42 Int64 false
abd String 42 Int64 false
42 Int64 false
42 Int64 false
42 Int64 false
43 Int64 false
abc String false
abc String false
abc String false
abc String false
abc String false
abc String false
abd String false
order by d2 nulls last order by d2 nulls last
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,3] Array(Int64) [1,2,3] Array(Int64) true
[1,2,4] Array(Int64) [1,2,3] Array(Int64) true
42 Int64 [1,2,4] Array(Int64) true
42 Int64 2020-01-01 Date true
42 Int64 2020-01-01 Date true
42 Int64 2020-01-01 Date true
42 Int64 2020-01-01 Date true
43 Int64 2020-01-01 Date true
abc String 2020-01-01 Date true
abc String 2020-01-02 Date true
abc String 42 Int64 false
abc String 42 Int64 false
abc String 42 Int64 false
abd String 42 Int64 false
\N None 42 Int64 false
\N None 42 Int64 false
\N None 43 Int64 false
\N None abc String false
abc String false
abc String false
abc String false
abc String false
abc String false
abd String false
\N None false
\N None false
\N None false
\N None false
\N None false
order by d1, d2 nulls first order by d1, d2 nulls first
[1,2,3] \N Array(Int64) None [1,2,3] \N Array(Int64) true None false
[1,2,3] [1,2,3] Array(Int64) Array(Int64) [1,2,3] [1,2,3] Array(Int64) true Array(Int64) true
[1,2,3] [1,2,4] Array(Int64) Array(Int64) [1,2,3] [1,2,4] Array(Int64) true Array(Int64) true
[1,2,3] 42 Array(Int64) Int64 [1,2,3] 2020-01-01 Array(Int64) true Date true
[1,2,3] abc Array(Int64) String [1,2,3] 42 Array(Int64) true Int64 false
[1,2,4] [1,2,3] Array(Int64) Array(Int64) [1,2,3] abc Array(Int64) true String false
42 \N Int64 None [1,2,4] [1,2,3] Array(Int64) true Array(Int64) true
42 [1,2,3] Int64 Array(Int64) 2020-01-01 \N Date true None false
42 42 Int64 Int64 2020-01-01 [1,2,3] Date true Array(Int64) true
42 43 Int64 Int64 2020-01-01 2020-01-01 Date true Date true
42 abc Int64 String 2020-01-01 2020-01-02 Date true Date true
43 42 Int64 Int64 2020-01-01 42 Date true Int64 false
abc \N String None 2020-01-01 abc Date true String false
abc [1,2,3] String Array(Int64) 2020-01-02 2020-01-01 Date true Date true
abc 42 String Int64 42 \N Int64 false None false
abc abc String String 42 [1,2,3] Int64 false Array(Int64) true
abc abd String String 42 2020-01-01 Int64 false Date true
abd abc String String 42 42 Int64 false Int64 false
\N \N None None 42 43 Int64 false Int64 false
\N [1,2,3] None Array(Int64) 42 abc Int64 false String false
\N 42 None Int64 43 42 Int64 false Int64 false
\N abc None String abc \N String false None false
abc [1,2,3] String false Array(Int64) true
abc 2020-01-01 String false Date true
abc 42 String false Int64 false
abc abc String false String false
abc abd String false String false
abd abc String false String false
\N \N None false None false
\N [1,2,3] None false Array(Int64) true
\N 2020-01-01 None false Date true
\N 42 None false Int64 false
\N abc None false String false
order by d1, d2 nulls last order by d1, d2 nulls last
[1,2,3] [1,2,3] Array(Int64) Array(Int64) [1,2,3] [1,2,3] Array(Int64) true Array(Int64) true
[1,2,3] [1,2,4] Array(Int64) Array(Int64) [1,2,3] [1,2,4] Array(Int64) true Array(Int64) true
[1,2,3] 42 Array(Int64) Int64 [1,2,3] 2020-01-01 Array(Int64) true Date true
[1,2,3] abc Array(Int64) String [1,2,3] 42 Array(Int64) true Int64 false
[1,2,3] \N Array(Int64) None [1,2,3] abc Array(Int64) true String false
[1,2,4] [1,2,3] Array(Int64) Array(Int64) [1,2,3] \N Array(Int64) true None false
42 [1,2,3] Int64 Array(Int64) [1,2,4] [1,2,3] Array(Int64) true Array(Int64) true
42 42 Int64 Int64 2020-01-01 [1,2,3] Date true Array(Int64) true
42 43 Int64 Int64 2020-01-01 2020-01-01 Date true Date true
42 abc Int64 String 2020-01-01 2020-01-02 Date true Date true
42 \N Int64 None 2020-01-01 42 Date true Int64 false
43 42 Int64 Int64 2020-01-01 abc Date true String false
abc [1,2,3] String Array(Int64) 2020-01-01 \N Date true None false
abc 42 String Int64 2020-01-02 2020-01-01 Date true Date true
abc abc String String 42 [1,2,3] Int64 false Array(Int64) true
abc abd String String 42 2020-01-01 Int64 false Date true
abc \N String None 42 42 Int64 false Int64 false
abd abc String String 42 43 Int64 false Int64 false
\N [1,2,3] None Array(Int64) 42 abc Int64 false String false
\N 42 None Int64 42 \N Int64 false None false
\N abc None String 43 42 Int64 false Int64 false
\N \N None None abc [1,2,3] String false Array(Int64) true
abc 2020-01-01 String false Date true
abc 42 String false Int64 false
abc abc String false String false
abc abd String false String false
abc \N String false None false
abd abc String false String false
\N [1,2,3] None false Array(Int64) true
\N 2020-01-01 None false Date true
\N 42 None false Int64 false
\N abc None false String false
\N \N None false None false
order by d2, d1 nulls first order by d2, d1 nulls first
\N [1,2,3] None Array(Int64) \N [1,2,3] None false Array(Int64) true
[1,2,3] [1,2,3] Array(Int64) Array(Int64) [1,2,3] [1,2,3] Array(Int64) true Array(Int64) true
[1,2,4] [1,2,3] Array(Int64) Array(Int64) [1,2,4] [1,2,3] Array(Int64) true Array(Int64) true
42 [1,2,3] Int64 Array(Int64) 2020-01-01 [1,2,3] Date true Array(Int64) true
abc [1,2,3] String Array(Int64) 42 [1,2,3] Int64 false Array(Int64) true
[1,2,3] [1,2,4] Array(Int64) Array(Int64) abc [1,2,3] String false Array(Int64) true
\N 42 None Int64 [1,2,3] [1,2,4] Array(Int64) true Array(Int64) true
[1,2,3] 42 Array(Int64) Int64 \N 2020-01-01 None false Date true
42 42 Int64 Int64 [1,2,3] 2020-01-01 Array(Int64) true Date true
43 42 Int64 Int64 2020-01-01 2020-01-01 Date true Date true
abc 42 String Int64 2020-01-02 2020-01-01 Date true Date true
42 43 Int64 Int64 42 2020-01-01 Int64 false Date true
\N abc None String abc 2020-01-01 String false Date true
[1,2,3] abc Array(Int64) String 2020-01-01 2020-01-02 Date true Date true
42 abc Int64 String \N 42 None false Int64 false
abc abc String String [1,2,3] 42 Array(Int64) true Int64 false
abd abc String String 2020-01-01 42 Date true Int64 false
abc abd String String 42 42 Int64 false Int64 false
\N \N None None 43 42 Int64 false Int64 false
[1,2,3] \N Array(Int64) None abc 42 String false Int64 false
42 \N Int64 None 42 43 Int64 false Int64 false
abc \N String None \N abc None false String false
[1,2,3] abc Array(Int64) true String false
2020-01-01 abc Date true String false
42 abc Int64 false String false
abc abc String false String false
abd abc String false String false
abc abd String false String false
\N \N None false None false
[1,2,3] \N Array(Int64) true None false
2020-01-01 \N Date true None false
42 \N Int64 false None false
abc \N String false None false
order by d2, d1 nulls last order by d2, d1 nulls last
[1,2,3] [1,2,3] Array(Int64) Array(Int64) [1,2,3] [1,2,3] Array(Int64) true Array(Int64) true
[1,2,4] [1,2,3] Array(Int64) Array(Int64) [1,2,4] [1,2,3] Array(Int64) true Array(Int64) true
42 [1,2,3] Int64 Array(Int64) 2020-01-01 [1,2,3] Date true Array(Int64) true
abc [1,2,3] String Array(Int64) 42 [1,2,3] Int64 false Array(Int64) true
\N [1,2,3] None Array(Int64) abc [1,2,3] String false Array(Int64) true
[1,2,3] [1,2,4] Array(Int64) Array(Int64) \N [1,2,3] None false Array(Int64) true
[1,2,3] 42 Array(Int64) Int64 [1,2,3] [1,2,4] Array(Int64) true Array(Int64) true
42 42 Int64 Int64 [1,2,3] 2020-01-01 Array(Int64) true Date true
43 42 Int64 Int64 2020-01-01 2020-01-01 Date true Date true
abc 42 String Int64 2020-01-02 2020-01-01 Date true Date true
\N 42 None Int64 42 2020-01-01 Int64 false Date true
42 43 Int64 Int64 abc 2020-01-01 String false Date true
[1,2,3] abc Array(Int64) String \N 2020-01-01 None false Date true
42 abc Int64 String 2020-01-01 2020-01-02 Date true Date true
abc abc String String [1,2,3] 42 Array(Int64) true Int64 false
abd abc String String 2020-01-01 42 Date true Int64 false
\N abc None String 42 42 Int64 false Int64 false
abc abd String String 43 42 Int64 false Int64 false
[1,2,3] \N Array(Int64) None abc 42 String false Int64 false
42 \N Int64 None \N 42 None false Int64 false
abc \N String None 42 43 Int64 false Int64 false
\N \N None None [1,2,3] abc Array(Int64) true String false
2020-01-01 abc Date true String false
42 abc Int64 false String false
abc abc String false String false
abd abc String false String false
\N abc None false String false
abc abd String false String false
[1,2,3] \N Array(Int64) true None false
2020-01-01 \N Date true None false
42 \N Int64 false None false
abc \N String false None false
\N \N None false None false
d1 = d2 d1 = d2
[1,2,3] [1,2,3] 1 Array(Int64) Array(Int64) [1,2,3] [1,2,3] 1 Array(Int64) true Array(Int64) true
[1,2,3] [1,2,4] 0 Array(Int64) Array(Int64) [1,2,3] [1,2,4] 0 Array(Int64) true Array(Int64) true
[1,2,3] 42 0 Array(Int64) Int64 [1,2,3] 2020-01-01 0 Array(Int64) true Date true
[1,2,3] abc 0 Array(Int64) String [1,2,3] 42 0 Array(Int64) true Int64 false
[1,2,3] \N 0 Array(Int64) None [1,2,3] abc 0 Array(Int64) true String false
[1,2,4] [1,2,3] 0 Array(Int64) Array(Int64) [1,2,3] \N 0 Array(Int64) true None false
42 [1,2,3] 0 Int64 Array(Int64) [1,2,4] [1,2,3] 0 Array(Int64) true Array(Int64) true
42 42 1 Int64 Int64 2020-01-01 [1,2,3] 0 Date true Array(Int64) true
42 43 0 Int64 Int64 2020-01-01 2020-01-01 1 Date true Date true
42 abc 0 Int64 String 2020-01-01 2020-01-02 0 Date true Date true
42 \N 0 Int64 None 2020-01-01 42 0 Date true Int64 false
43 42 0 Int64 Int64 2020-01-01 abc 0 Date true String false
abc [1,2,3] 0 String Array(Int64) 2020-01-01 \N 0 Date true None false
abc 42 0 String Int64 2020-01-02 2020-01-01 0 Date true Date true
abc abc 1 String String 42 [1,2,3] 0 Int64 false Array(Int64) true
abc abd 0 String String 42 2020-01-01 0 Int64 false Date true
abc \N 0 String None 42 42 1 Int64 false Int64 false
abd abc 0 String String 42 43 0 Int64 false Int64 false
\N [1,2,3] 0 None Array(Int64) 42 abc 0 Int64 false String false
\N 42 0 None Int64 42 \N 0 Int64 false None false
\N abc 0 None String 43 42 0 Int64 false Int64 false
\N \N 1 None None abc [1,2,3] 0 String false Array(Int64) true
abc 2020-01-01 0 String false Date true
abc 42 0 String false Int64 false
abc abc 1 String false String false
abc abd 0 String false String false
abc \N 0 String false None false
abd abc 0 String false String false
\N [1,2,3] 0 None false Array(Int64) true
\N 2020-01-01 0 None false Date true
\N 42 0 None false Int64 false
\N abc 0 None false String false
\N \N 1 None false None false
d1 < d2 d1 < d2
[1,2,3] [1,2,3] 0 Array(Int64) Array(Int64) [1,2,3] [1,2,3] 0 Array(Int64) true Array(Int64) true
[1,2,3] [1,2,4] 1 Array(Int64) Array(Int64) [1,2,3] [1,2,4] 1 Array(Int64) true Array(Int64) true
[1,2,3] 42 1 Array(Int64) Int64 [1,2,3] 2020-01-01 1 Array(Int64) true Date true
[1,2,3] abc 1 Array(Int64) String [1,2,3] 42 1 Array(Int64) true Int64 false
[1,2,3] \N 1 Array(Int64) None [1,2,3] abc 1 Array(Int64) true String false
[1,2,4] [1,2,3] 0 Array(Int64) Array(Int64) [1,2,3] \N 1 Array(Int64) true None false
42 [1,2,3] 0 Int64 Array(Int64) [1,2,4] [1,2,3] 0 Array(Int64) true Array(Int64) true
42 42 0 Int64 Int64 2020-01-01 [1,2,3] 0 Date true Array(Int64) true
42 43 1 Int64 Int64 2020-01-01 2020-01-01 0 Date true Date true
42 abc 1 Int64 String 2020-01-01 2020-01-02 1 Date true Date true
42 \N 1 Int64 None 2020-01-01 42 1 Date true Int64 false
43 42 0 Int64 Int64 2020-01-01 abc 1 Date true String false
abc [1,2,3] 0 String Array(Int64) 2020-01-01 \N 1 Date true None false
abc 42 0 String Int64 2020-01-02 2020-01-01 0 Date true Date true
abc abc 0 String String 42 [1,2,3] 0 Int64 false Array(Int64) true
abc abd 1 String String 42 2020-01-01 0 Int64 false Date true
abc \N 1 String None 42 42 0 Int64 false Int64 false
abd abc 0 String String 42 43 1 Int64 false Int64 false
\N [1,2,3] 0 None Array(Int64) 42 abc 1 Int64 false String false
\N 42 0 None Int64 42 \N 1 Int64 false None false
\N abc 0 None String 43 42 0 Int64 false Int64 false
\N \N 0 None None abc [1,2,3] 0 String false Array(Int64) true
abc 2020-01-01 0 String false Date true
abc 42 0 String false Int64 false
abc abc 0 String false String false
abc abd 1 String false String false
abc \N 1 String false None false
abd abc 0 String false String false
\N [1,2,3] 0 None false Array(Int64) true
\N 2020-01-01 0 None false Date true
\N 42 0 None false Int64 false
\N abc 0 None false String false
\N \N 0 None false None false
d1 <= d2 d1 <= d2
[1,2,3] [1,2,3] 1 Array(Int64) Array(Int64) [1,2,3] [1,2,3] 1 Array(Int64) true Array(Int64) true
[1,2,3] [1,2,4] 1 Array(Int64) Array(Int64) [1,2,3] [1,2,4] 1 Array(Int64) true Array(Int64) true
[1,2,3] 42 1 Array(Int64) Int64 [1,2,3] 2020-01-01 1 Array(Int64) true Date true
[1,2,3] abc 1 Array(Int64) String [1,2,3] 42 1 Array(Int64) true Int64 false
[1,2,3] \N 1 Array(Int64) None [1,2,3] abc 1 Array(Int64) true String false
[1,2,4] [1,2,3] 0 Array(Int64) Array(Int64) [1,2,3] \N 1 Array(Int64) true None false
42 [1,2,3] 0 Int64 Array(Int64) [1,2,4] [1,2,3] 0 Array(Int64) true Array(Int64) true
42 42 1 Int64 Int64 2020-01-01 [1,2,3] 0 Date true Array(Int64) true
42 43 1 Int64 Int64 2020-01-01 2020-01-01 1 Date true Date true
42 abc 1 Int64 String 2020-01-01 2020-01-02 1 Date true Date true
42 \N 1 Int64 None 2020-01-01 42 1 Date true Int64 false
43 42 0 Int64 Int64 2020-01-01 abc 1 Date true String false
abc [1,2,3] 0 String Array(Int64) 2020-01-01 \N 1 Date true None false
abc 42 0 String Int64 2020-01-02 2020-01-01 0 Date true Date true
abc abc 1 String String 42 [1,2,3] 0 Int64 false Array(Int64) true
abc abd 1 String String 42 2020-01-01 0 Int64 false Date true
abc \N 1 String None 42 42 1 Int64 false Int64 false
abd abc 0 String String 42 43 1 Int64 false Int64 false
\N [1,2,3] 0 None Array(Int64) 42 abc 1 Int64 false String false
\N 42 0 None Int64 42 \N 1 Int64 false None false
\N abc 0 None String 43 42 0 Int64 false Int64 false
\N \N 1 None None abc [1,2,3] 0 String false Array(Int64) true
abc 2020-01-01 0 String false Date true
abc 42 0 String false Int64 false
abc abc 1 String false String false
abc abd 1 String false String false
abc \N 1 String false None false
abd abc 0 String false String false
\N [1,2,3] 0 None false Array(Int64) true
\N 2020-01-01 0 None false Date true
\N 42 0 None false Int64 false
\N abc 0 None false String false
\N \N 1 None false None false
d1 > d2 d1 > d2
[1,2,3] [1,2,3] 0 Array(Int64) Array(Int64) [1,2,3] [1,2,3] 0 Array(Int64) true Array(Int64) true
[1,2,3] [1,2,4] 0 Array(Int64) Array(Int64) [1,2,3] [1,2,4] 0 Array(Int64) true Array(Int64) true
[1,2,3] 42 0 Array(Int64) Int64 [1,2,3] 2020-01-01 0 Array(Int64) true Date true
[1,2,3] abc 0 Array(Int64) String [1,2,3] 42 0 Array(Int64) true Int64 false
[1,2,3] \N 0 Array(Int64) None [1,2,3] abc 0 Array(Int64) true String false
[1,2,4] [1,2,3] 1 Array(Int64) Array(Int64) [1,2,3] \N 0 Array(Int64) true None false
42 [1,2,3] 1 Int64 Array(Int64) [1,2,4] [1,2,3] 1 Array(Int64) true Array(Int64) true
42 42 0 Int64 Int64 2020-01-01 [1,2,3] 1 Date true Array(Int64) true
42 43 0 Int64 Int64 2020-01-01 2020-01-01 0 Date true Date true
42 abc 0 Int64 String 2020-01-01 2020-01-02 0 Date true Date true
42 \N 0 Int64 None 2020-01-01 42 0 Date true Int64 false
43 42 1 Int64 Int64 2020-01-01 abc 0 Date true String false
abc [1,2,3] 1 String Array(Int64) 2020-01-01 \N 0 Date true None false
abc 42 1 String Int64 2020-01-02 2020-01-01 1 Date true Date true
abc abc 0 String String 42 [1,2,3] 1 Int64 false Array(Int64) true
abc abd 0 String String 42 2020-01-01 1 Int64 false Date true
abc \N 0 String None 42 42 0 Int64 false Int64 false
abd abc 1 String String 42 43 0 Int64 false Int64 false
\N [1,2,3] 1 None Array(Int64) 42 abc 0 Int64 false String false
\N 42 1 None Int64 42 \N 0 Int64 false None false
\N abc 1 None String 43 42 1 Int64 false Int64 false
\N \N 0 None None abc [1,2,3] 1 String false Array(Int64) true
abc 2020-01-01 1 String false Date true
abc 42 1 String false Int64 false
abc abc 0 String false String false
abc abd 0 String false String false
abc \N 0 String false None false
abd abc 1 String false String false
\N [1,2,3] 1 None false Array(Int64) true
\N 2020-01-01 1 None false Date true
\N 42 1 None false Int64 false
\N abc 1 None false String false
\N \N 0 None false None false
d1 >= d2 d1 >= d2
[1,2,3] [1,2,3] 1 Array(Int64) Array(Int64) [1,2,3] [1,2,3] 1 Array(Int64) true Array(Int64) true
[1,2,3] [1,2,4] 1 Array(Int64) Array(Int64) [1,2,3] [1,2,4] 1 Array(Int64) true Array(Int64) true
[1,2,3] 42 1 Array(Int64) Int64 [1,2,3] 2020-01-01 1 Array(Int64) true Date true
[1,2,3] abc 1 Array(Int64) String [1,2,3] 42 1 Array(Int64) true Int64 false
[1,2,3] \N 1 Array(Int64) None [1,2,3] abc 1 Array(Int64) true String false
[1,2,4] [1,2,3] 1 Array(Int64) Array(Int64) [1,2,3] \N 1 Array(Int64) true None false
42 [1,2,3] 1 Int64 Array(Int64) [1,2,4] [1,2,3] 1 Array(Int64) true Array(Int64) true
42 42 1 Int64 Int64 2020-01-01 [1,2,3] 1 Date true Array(Int64) true
42 43 1 Int64 Int64 2020-01-01 2020-01-01 1 Date true Date true
42 abc 1 Int64 String 2020-01-01 2020-01-02 1 Date true Date true
42 \N 1 Int64 None 2020-01-01 42 1 Date true Int64 false
43 42 1 Int64 Int64 2020-01-01 abc 1 Date true String false
abc [1,2,3] 1 String Array(Int64) 2020-01-01 \N 1 Date true None false
abc 42 1 String Int64 2020-01-02 2020-01-01 1 Date true Date true
abc abc 1 String String 42 [1,2,3] 1 Int64 false Array(Int64) true
abc abd 1 String String 42 2020-01-01 1 Int64 false Date true
abc \N 1 String None 42 42 1 Int64 false Int64 false
abd abc 1 String String 42 43 1 Int64 false Int64 false
\N [1,2,3] 1 None Array(Int64) 42 abc 1 Int64 false String false
\N 42 1 None Int64 42 \N 1 Int64 false None false
\N abc 1 None String 43 42 1 Int64 false Int64 false
\N \N 1 None None abc [1,2,3] 1 String false Array(Int64) true
abc 2020-01-01 1 String false Date true
abc 42 1 String false Int64 false
abc abc 1 String false String false
abc abd 1 String false String false
abc \N 1 String false None false
abd abc 1 String false String false
\N [1,2,3] 1 None false Array(Int64) true
\N 2020-01-01 1 None false Date true
\N 42 1 None false Int64 false
\N abc 1 None false String false
\N \N 1 None false None false

67
tests/queries/0_stateless/03035_dynamic_sorting.sql
View File

@ -1,80 +1,55 @@
set allow_experimental_dynamic_type = 1; set allow_experimental_dynamic_type = 1;
drop table if exists test; drop table if exists test;
create table test (d1 Dynamic, d2 Dynamic) engine=Memory; create table test (d1 Dynamic(max_types=2), d2 Dynamic(max_types=2)) engine=Memory;
insert into test values (42, 42);
insert into test values (42, 43);
insert into test values (43, 42);
insert into test values ('abc', 'abc');
insert into test values ('abc', 'abd');
insert into test values ('abd', 'abc');
insert into test values ([1,2,3], [1,2,3]);
insert into test values ([1,2,3], [1,2,4]);
insert into test values ([1,2,4], [1,2,3]);
insert into test values (NULL, NULL);
insert into test values (42, 'abc');
insert into test values ('abc', 42);
insert into test values (42, [1,2,3]);
insert into test values ([1,2,3], 42);
insert into test values (42, NULL);
insert into test values (NULL, 42);
insert into test values ('abc', [1,2,3]);
insert into test values ([1,2,3], 'abc');
insert into test values ('abc', NULL);
insert into test values (NULL, 'abc');
insert into test values ([1,2,3], NULL);
insert into test values (NULL, [1,2,3]);
insert into test values (42, 42), (42, 43), (43, 42), ('abc', 'abc'), ('abc', 'abd'), ('abd', 'abc'),
([1,2,3], [1,2,3]), ([1,2,3], [1,2,4]), ([1,2,4], [1,2,3]),
('2020-01-01', '2020-01-01'), ('2020-01-01', '2020-01-02'), ('2020-01-02', '2020-01-01'),
(NULL, NULL), (42, 'abc'), ('abc', 42), (42, [1,2,3]), ([1,2,3], 42), (42, NULL), (NULL, 42),
('abc', [1,2,3]), ([1,2,3], 'abc'), ('abc', NULL), (NULL, 'abc'), ([1,2,3], NULL), (NULL, [1,2,3]),
(42, '2020-01-01'), ('2020-01-01', 42), ('2020-01-01', 'abc'), ('abc', '2020-01-01'),
('2020-01-01', [1,2,3]), ([1,2,3], '2020-01-01'), ('2020-01-01', NULL), (NULL, '2020-01-01');
select 'order by d1 nulls first'; select 'order by d1 nulls first';
select d1, dynamicType(d1) from test order by d1 nulls first; select d1, dynamicType(d1), isDynamicElementInSharedData(d1) from test order by d1 nulls first;
select 'order by d1 nulls last'; select 'order by d1 nulls last';
select d1, dynamicType(d1) from test order by d1 nulls last; select d1, dynamicType(d1), isDynamicElementInSharedData(d1) from test order by d1 nulls last;
select 'order by d2 nulls first'; select 'order by d2 nulls first';
select d2, dynamicType(d2) from test order by d2 nulls first; select d2, dynamicType(d2), isDynamicElementInSharedData(d2) from test order by d2 nulls first;
select 'order by d2 nulls last'; select 'order by d2 nulls last';
select d2, dynamicType(d2) from test order by d2 nulls last; select d2, dynamicType(d2), isDynamicElementInSharedData(d2) from test order by d2 nulls last;
select 'order by d1, d2 nulls first'; select 'order by d1, d2 nulls first';
select d1, d2, dynamicType(d1), dynamicType(d2) from test order by d1, d2 nulls first; select d1, d2, dynamicType(d1), isDynamicElementInSharedData(d1), dynamicType(d2), isDynamicElementInSharedData(d2) from test order by d1, d2 nulls first;
select 'order by d1, d2 nulls last'; select 'order by d1, d2 nulls last';
select d1, d2, dynamicType(d1), dynamicType(d2) from test order by d1, d2 nulls last; select d1, d2, dynamicType(d1), isDynamicElementInSharedData(d1), dynamicType(d2), isDynamicElementInSharedData(d2) from test order by d1, d2 nulls last;
select 'order by d2, d1 nulls first'; select 'order by d2, d1 nulls first';
select d1, d2, dynamicType(d1), dynamicType(d2) from test order by d2, d1 nulls first; select d1, d2, dynamicType(d1), isDynamicElementInSharedData(d1), dynamicType(d2), isDynamicElementInSharedData(d2) from test order by d2, d1 nulls first;
select 'order by d2, d1 nulls last'; select 'order by d2, d1 nulls last';
select d1, d2, dynamicType(d1), dynamicType(d2) from test order by d2, d1 nulls last; select d1, d2, dynamicType(d1), isDynamicElementInSharedData(d1), dynamicType(d2), isDynamicElementInSharedData(d2) from test order by d2, d1 nulls last;
select 'd1 = d2'; select 'd1 = d2';
select d1, d2, d1 = d2, dynamicType(d1), dynamicType(d2) from test order by d1, d2; select d1, d2, d1 = d2, dynamicType(d1), isDynamicElementInSharedData(d1), dynamicType(d2), isDynamicElementInSharedData(d2) from test order by d1, d2;
select 'd1 < d2'; select 'd1 < d2';
select d1, d2, d1 < d2, dynamicType(d1), dynamicType(d2) from test order by d1, d2; select d1, d2, d1 < d2, dynamicType(d1), isDynamicElementInSharedData(d1), dynamicType(d2), isDynamicElementInSharedData(d2) from test order by d1, d2;
select 'd1 <= d2'; select 'd1 <= d2';
select d1, d2, d1 <= d2, dynamicType(d1), dynamicType(d2) from test order by d1, d2; select d1, d2, d1 <= d2, dynamicType(d1), isDynamicElementInSharedData(d1), dynamicType(d2), isDynamicElementInSharedData(d2) from test order by d1, d2;
select 'd1 > d2'; select 'd1 > d2';
select d1, d2, d1 > d2, dynamicType(d1), dynamicType(d2) from test order by d1, d2; select d1, d2, d1 > d2, dynamicType(d1), isDynamicElementInSharedData(d1), dynamicType(d2), isDynamicElementInSharedData(d2) from test order by d1, d2;
select 'd1 >= d2'; select 'd1 >= d2';
select d1, d2, d2 >= d2, dynamicType(d1), dynamicType(d2) from test order by d1, d2; select d1, d2, d2 >= d2, dynamicType(d1), isDynamicElementInSharedData(d1), dynamicType(d2), isDynamicElementInSharedData(d2) from test order by d1, d2;
drop table test; drop table test;

20
tests/queries/0_stateless/03036_dynamic_read_shared_subcolumns_compact_merge_tree.reference Normal file
View File

@ -0,0 +1,20 @@
Array(Array(Dynamic))
Array(Variant(String, UInt64))
LowCardinality(String)
None
String
UInt64
360000
360000
200000
200000
0
0
20000
20000
200000
200000
20000
20000
200000
0

43
tests/queries/0_stateless/03036_dynamic_read_shared_subcolumns_compact_merge_tree.sql Normal file
View File

@ -0,0 +1,43 @@
-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
set allow_experimental_variant_type = 1;
set use_variant_as_common_type = 1;
set allow_experimental_dynamic_type = 1;
drop table if exists test;
create table test (id UInt64, d Dynamic(max_types=2)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;
insert into test select number, number from numbers(100000) settings min_insert_block_size_rows=50000;
insert into test select number, 'str_' || toString(number) from numbers(100000, 100000) settings min_insert_block_size_rows=50000;
insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(200000, 100000) settings min_insert_block_size_rows=50000;
insert into test select number, NULL from numbers(300000, 100000) settings min_insert_block_size_rows=50000;
insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(400000, 400000) settings min_insert_block_size_rows=50000;
insert into test select number, if (number % 5 == 1, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)), number) from numbers(100000, 100000) settings min_insert_block_size_rows=50000;
insert into test select number, if (number % 5 == 1, ('str_' || number)::LowCardinality(String)::Dynamic, number::Dynamic) from numbers(100000, 100000) settings min_insert_block_size_rows=50000;
select distinct dynamicType(d) as type from test order by type;
select count() from test where dynamicType(d) == 'UInt64';
select count() from test where d.UInt64 is not NULL;
select count() from test where dynamicType(d) == 'String';
select count() from test where d.String is not NULL;
select count() from test where dynamicType(d) == 'Date';
select count() from test where d.Date is not NULL;
select count() from test where dynamicType(d) == 'LowCardinality(String)';
select count() from test where d.`LowCardinality(String)` is not NULL;
select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))';
select count() from test where not empty(d.`Array(Variant(String, UInt64))`);
select count() from test where dynamicType(d) == 'Array(Array(Dynamic))';
select count() from test where not empty(d.`Array(Array(Dynamic))`);
select count() from test where d is NULL;
select count() from test where not empty(d.`Tuple(a Array(Dynamic))`.a.String);
select d, d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test format Null;
select d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test format Null;
select d.Int8, d.Date, d.`LowCardinality(String)`, d.`Array(String)` from test format Null;
select d, d.UInt64, d.Date, d.`LowCardinality(String)`, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
select d.UInt64, d.Date, d.`LowCardinality(String)`, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64, d.`Array(Variant(String, UInt64))`.String from test format Null;
select d, d.`Tuple(a UInt64, b String)`.a, d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
select d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Dynamic)`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
select d.`Array(Array(Dynamic))`.size1, d.`Array(Array(Dynamic))`.UInt64, d.`Array(Array(Dynamic))`.`Map(String, Tuple(a UInt64))`.values.a from test format Null;
drop table test;

20
tests/queries/0_stateless/03036_dynamic_read_shared_subcolumns_memory.reference Normal file
View File

@ -0,0 +1,20 @@
Array(Array(Dynamic))
Array(Variant(String, UInt64))
LowCardinality(String)
None
String
UInt64
360000
360000
200000
200000
0
0
20000
20000
200000
200000
20000
20000
200000
0

43
tests/queries/0_stateless/03036_dynamic_read_shared_subcolumns_memory.sql Normal file
View File

@ -0,0 +1,43 @@
-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
set allow_experimental_variant_type = 1;
set use_variant_as_common_type = 1;
set allow_experimental_dynamic_type = 1;
drop table if exists test;
create table test (id UInt64, d Dynamic(max_types=2)) engine=Memory;
insert into test select number, number from numbers(100000) settings min_insert_block_size_rows=50000;
insert into test select number, 'str_' || toString(number) from numbers(100000, 100000) settings min_insert_block_size_rows=50000;
insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(200000, 100000) settings min_insert_block_size_rows=50000;
insert into test select number, NULL from numbers(300000, 100000) settings min_insert_block_size_rows=50000;
insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(400000, 400000) settings min_insert_block_size_rows=50000;
insert into test select number, if (number % 5 == 1, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)), number) from numbers(100000, 100000) settings min_insert_block_size_rows=50000;
insert into test select number, if (number % 5 == 1, ('str_' || number)::LowCardinality(String)::Dynamic, number::Dynamic) from numbers(100000, 100000) settings min_insert_block_size_rows=50000;
select distinct dynamicType(d) as type from test order by type;
select count() from test where dynamicType(d) == 'UInt64';
select count() from test where d.UInt64 is not NULL;
select count() from test where dynamicType(d) == 'String';
select count() from test where d.String is not NULL;
select count() from test where dynamicType(d) == 'Date';
select count() from test where d.Date is not NULL;
select count() from test where dynamicType(d) == 'LowCardinality(String)';
select count() from test where d.`LowCardinality(String)` is not NULL;
select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))';
select count() from test where not empty(d.`Array(Variant(String, UInt64))`);
select count() from test where dynamicType(d) == 'Array(Array(Dynamic))';
select count() from test where not empty(d.`Array(Array(Dynamic))`);
select count() from test where d is NULL;
select count() from test where not empty(d.`Tuple(a Array(Dynamic))`.a.String);
select d, d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test format Null;
select d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test format Null;
select d.Int8, d.Date, d.`Array(String)` from test format Null;
select d, d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
select d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64, d.`Array(Variant(String, UInt64))`.String from test format Null;
select d, d.`Tuple(a UInt64, b String)`.a, d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
select d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Dynamic)`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
select d.`Array(Array(Dynamic))`.size1, d.`Array(Array(Dynamic))`.UInt64, d.`Array(Array(Dynamic))`.`Map(String, Tuple(a UInt64))`.values.a from test format Null;
drop table test;

2463
tests/queries/0_stateless/03036_dynamic_read_shared_subcolumns_small.reference.j2 Normal file
View File

File diff suppressed because it is too large Load Diff

47
tests/queries/0_stateless/03036_dynamic_read_shared_subcolumns_small.sql.j2 Normal file
View File

@ -0,0 +1,47 @@
set allow_experimental_variant_type = 1;
set use_variant_as_common_type = 1;
set allow_experimental_dynamic_type = 1;
drop table if exists test;
{% for engine in ['Memory', 'MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000', 'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1'] -%}
select '{{ engine }}';
create table test (id UInt64, d Dynamic(max_types=2)) engine={{ engine }};
insert into test select number, number from numbers(10);
insert into test select number, 'str_' || toString(number) from numbers(10, 10);
insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(20, 10);
insert into test select number, NULL from numbers(30, 10);
insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(40, 40);
insert into test select number, if(number % 5 == 1, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)), number) from numbers(10, 10);
insert into test select number, if(number % 5 == 1, ('str_' || number)::LowCardinality(String)::Dynamic, number::Dynamic) from numbers(10, 10);
select distinct dynamicType(d) as type from test order by type;
select count() from test where dynamicType(d) == 'UInt64';
select count() from test where d.UInt64 is not NULL;
select count() from test where dynamicType(d) == 'String';
select count() from test where d.String is not NULL;
select count() from test where dynamicType(d) == 'Date';
select count() from test where d.Date is not NULL;
select count() from test where dynamicType(d) == 'LowCardinality(String)';
select count() from test where d.`LowCardinality(String)` is not NULL;
select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))';
select count() from test where not empty(d.`Array(Variant(String, UInt64))`);
select count() from test where dynamicType(d) == 'Array(Array(Dynamic))';
select count() from test where not empty(d.`Array(Array(Dynamic))`);
select count() from test where d is NULL;
select count() from test where not empty(d.`Tuple(a Array(Dynamic))`.a.String);
select d, d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test order by id, d;
select d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test order by id, d;
select d.Int8, d.Date, d.`Array(String)` from test order by id, d;
select d, d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test order by id, d;
select d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64, d.`Array(Variant(String, UInt64))`.String from test order by id, d;
select d, d.`Tuple(a UInt64, b String)`.a, d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Variant(String, UInt64))`.UInt64 from test order by id, d;
select d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Dynamic)`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test order by id, d;
select d.`Array(Array(Dynamic))`.size1, d.`Array(Array(Dynamic))`.UInt64, d.`Array(Array(Dynamic))`.`Map(String, Tuple(a UInt64))`.values.a from test order by id, d;
drop table test;
{% endfor -%}

20
tests/queries/0_stateless/03036_dynamic_read_shared_subcolumns_wide_merge_tree.reference Normal file
View File

@ -0,0 +1,20 @@
Array(Array(Dynamic))
Array(Variant(String, UInt64))
LowCardinality(String)
None
String
UInt64
360000
360000
200000
200000
0
0
20000
20000
200000
200000
20000
20000
200000
0

43
tests/queries/0_stateless/03036_dynamic_read_shared_subcolumns_wide_merge_tree.sql Normal file
View File

@ -0,0 +1,43 @@
-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
set allow_experimental_variant_type = 1;
set use_variant_as_common_type = 1;
set allow_experimental_dynamic_type = 1;
drop table if exists test;
create table test (id UInt64, d Dynamic(max_types=2)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;
insert into test select number, number from numbers(100000) settings min_insert_block_size_rows=50000;
insert into test select number, 'str_' || toString(number) from numbers(100000, 100000) settings min_insert_block_size_rows=50000;
insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(200000, 100000) settings min_insert_block_size_rows=50000;
insert into test select number, NULL from numbers(300000, 100000) settings min_insert_block_size_rows=50000;
insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(400000, 400000) settings min_insert_block_size_rows=50000;
insert into test select number, if (number % 5 == 1, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)), number) from numbers(100000, 100000) settings min_insert_block_size_rows=50000;
insert into test select number, if (number % 5 == 1, ('str_' || number)::LowCardinality(String)::Dynamic, number::Dynamic) from numbers(100000, 100000) settings min_insert_block_size_rows=50000;
select distinct dynamicType(d) as type from test order by type;
select count() from test where dynamicType(d) == 'UInt64';
select count() from test where d.UInt64 is not NULL;
select count() from test where dynamicType(d) == 'String';
select count() from test where d.String is not NULL;
select count() from test where dynamicType(d) == 'Date';
select count() from test where d.Date is not NULL;
select count() from test where dynamicType(d) == 'LowCardinality(String)';
select count() from test where d.`LowCardinality(String)` is not NULL;
select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))';
select count() from test where not empty(d.`Array(Variant(String, UInt64))`);
select count() from test where dynamicType(d) == 'Array(Array(Dynamic))';
select count() from test where not empty(d.`Array(Array(Dynamic))`);
select count() from test where d is NULL;
select count() from test where not empty(d.`Tuple(a Array(Dynamic))`.a.String);
select d, d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test format Null;
select d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test format Null;
select d.Int8, d.Date, d.`Array(String)` from test format Null;
select d, d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
select d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64, d.`Array(Variant(String, UInt64))`.String from test format Null;
select d, d.`Tuple(a UInt64, b String)`.a, d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
select d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Dynamic)`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
select d.`Array(Array(Dynamic))`.size1, d.`Array(Array(Dynamic))`.UInt64, d.`Array(Array(Dynamic))`.`Map(String, Tuple(a UInt64))`.values.a from test format Null;
drop table test;

94
tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.reference
View File

@ -1,28 +1,66 @@
50000 DateTime 50000 DateTime false
60000 Date 60000 Date false
70000 Array(UInt16) 70000 Array(UInt16) false
80000 String 80000 String false
100000 None 100000 None false
100000 UInt64 100000 UInt64 false
70000 Array(UInt16) ---------------------
100000 None 50000 DateTime true
100000 UInt64 60000 Date true
190000 String 70000 Array(UInt16) true
70000 Array(UInt16) 80000 String false
100000 None 100000 None false
100000 UInt64 100000 UInt64 false
190000 String ---------------------
200000 Map(UInt64, UInt64) 50000 DateTime true
100000 None 60000 Date true
100000 UInt64 70000 Array(UInt16) true
200000 Map(UInt64, UInt64) 80000 String false
260000 String 100000 None false
10000 Tuple(UInt64, UInt64) 100000 UInt64 false
100000 None 200000 Map(UInt64, UInt64) false
100000 UInt64 ---------------------
200000 Map(UInt64, UInt64) 50000 DateTime true
260000 String 60000 Date true
100000 None 70000 Array(UInt16) true
100000 UInt64 80000 String true
200000 Map(UInt64, UInt64) 100000 None false
270000 String 100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) false
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
80000 String true
100000 None false
100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) true
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
80000 String true
100000 None false
100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) true
30000 String false
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
80000 String true
100000 None false
100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) true
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
100000 None false
100000 UInt64 true
110000 String false
200000 Map(UInt64, UInt64) false

29
tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.sql
View File

@ -2,7 +2,7 @@
set allow_experimental_dynamic_type=1; set allow_experimental_dynamic_type=1;
drop table if exists test; drop table if exists test;
create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_columns_to_activate=10, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760, lock_acquire_timeout_for_background_operations=600; create table test (id UInt64, d Dynamic(max_types=2)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_columns_to_activate=10, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760, lock_acquire_timeout_for_background_operations=600;
system stop merges test; system stop merges test;
insert into test select number, number from numbers(100000); insert into test select number, number from numbers(100000);
@ -12,22 +12,37 @@ insert into test select number, toDate(number) from numbers(60000);
insert into test select number, toDateTime(number) from numbers(50000); insert into test select number, toDateTime(number) from numbers(50000);
insert into test select number, NULL from numbers(100000); insert into test select number, NULL from numbers(100000);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test; optimize table test final;; system start merges test; optimize table test final;;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system stop merges test; system stop merges test;
insert into test select number, map(number, number) from numbers(200000); insert into test select number, map(number, number) from numbers(200000);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system stop merges test; system stop merges test;
insert into test select number, tuple(number, number) from numbers(10000); insert into test select number, tuple(number, number) from numbers(10000);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system stop merges test;
insert into test select number, 'str_' || number from numbers(30000);
select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test;
optimize table test final;
select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
drop table test; drop table test;

94
tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.reference
View File

@ -1,28 +1,66 @@
50000 DateTime 50000 DateTime false
60000 Date 60000 Date false
70000 Array(UInt16) 70000 Array(UInt16) false
80000 String 80000 String false
100000 None 100000 None false
100000 UInt64 100000 UInt64 false
70000 Array(UInt16) ---------------------
100000 None 50000 DateTime true
100000 UInt64 60000 Date true
190000 String 70000 Array(UInt16) true
70000 Array(UInt16) 80000 String false
100000 None 100000 None false
100000 UInt64 100000 UInt64 false
190000 String ---------------------
200000 Map(UInt64, UInt64) 50000 DateTime true
100000 None 60000 Date true
100000 UInt64 70000 Array(UInt16) true
200000 Map(UInt64, UInt64) 80000 String false
260000 String 100000 None false
10000 Tuple(UInt64, UInt64) 100000 UInt64 false
100000 None 200000 Map(UInt64, UInt64) false
100000 UInt64 ---------------------
200000 Map(UInt64, UInt64) 50000 DateTime true
260000 String 60000 Date true
100000 None 70000 Array(UInt16) true
100000 UInt64 80000 String true
200000 Map(UInt64, UInt64) 100000 None false
270000 String 100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) false
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
80000 String true
100000 None false
100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) true
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
80000 String true
100000 None false
100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) true
30000 String false
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
80000 String true
100000 None false
100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) true
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
100000 None false
100000 UInt64 true
110000 String false
200000 Map(UInt64, UInt64) false

28
tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.sql
View File

@ -2,7 +2,7 @@
set allow_experimental_dynamic_type=1; set allow_experimental_dynamic_type=1;
drop table if exists test; drop table if exists test;
create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_columns_to_activate=10, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760, lock_acquire_timeout_for_background_operations=600; create table test (id UInt64, d Dynamic(max_types=2)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_columns_to_activate=10, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760, lock_acquire_timeout_for_background_operations=600;
system stop merges test; system stop merges test;
insert into test select number, number from numbers(100000); insert into test select number, number from numbers(100000);
@ -12,22 +12,36 @@ insert into test select number, toDate(number) from numbers(60000);
insert into test select number, toDateTime(number) from numbers(50000); insert into test select number, toDateTime(number) from numbers(50000);
insert into test select number, NULL from numbers(100000); insert into test select number, NULL from numbers(100000);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test; optimize table test final;; system start merges test; optimize table test final;;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system stop merges test; system stop merges test;
insert into test select number, map(number, number) from numbers(200000); insert into test select number, map(number, number) from numbers(200000);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system stop merges test; system stop merges test;
insert into test select number, tuple(number, number) from numbers(10000); insert into test select number, tuple(number, number) from numbers(10000);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system stop merges test;
insert into test select number, 'str_' || number from numbers(30000);
select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test;
optimize table test final;
select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
drop table test; drop table test;

94
tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.reference
View File

@ -1,28 +1,66 @@
50000 DateTime 50000 DateTime false
60000 Date 60000 Date false
70000 Array(UInt16) 70000 Array(UInt16) false
80000 String 80000 String false
100000 None 100000 None false
100000 UInt64 100000 UInt64 false
70000 Array(UInt16) ---------------------
100000 None 50000 DateTime true
100000 UInt64 60000 Date true
190000 String 70000 Array(UInt16) true
70000 Array(UInt16) 80000 String false
100000 None 100000 None false
100000 UInt64 100000 UInt64 false
190000 String ---------------------
200000 Map(UInt64, UInt64) 50000 DateTime true
100000 None 60000 Date true
100000 UInt64 70000 Array(UInt16) true
200000 Map(UInt64, UInt64) 80000 String false
260000 String 100000 None false
10000 Tuple(UInt64, UInt64) 100000 UInt64 false
100000 None 200000 Map(UInt64, UInt64) false
100000 UInt64 ---------------------
200000 Map(UInt64, UInt64) 50000 DateTime true
260000 String 60000 Date true
100000 None 70000 Array(UInt16) true
100000 UInt64 80000 String true
200000 Map(UInt64, UInt64) 100000 None false
270000 String 100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) false
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
80000 String true
100000 None false
100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) true
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
80000 String true
100000 None false
100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) true
30000 String false
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
80000 String true
100000 None false
100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) true
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
100000 None false
100000 UInt64 true
110000 String false
200000 Map(UInt64, UInt64) false

31
tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.sql
View File

@ -2,7 +2,7 @@
set allow_experimental_dynamic_type=1; set allow_experimental_dynamic_type=1;
drop table if exists test; drop table if exists test;
create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760, lock_acquire_timeout_for_background_operations=600; create table test (id UInt64, d Dynamic(max_types=2)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760, lock_acquire_timeout_for_background_operations=600;
system stop merges test; system stop merges test;
insert into test select number, number from numbers(100000); insert into test select number, number from numbers(100000);
@ -12,23 +12,36 @@ insert into test select number, toDate(number) from numbers(60000);
insert into test select number, toDateTime(number) from numbers(50000); insert into test select number, toDateTime(number) from numbers(50000);
insert into test select number, NULL from numbers(100000); insert into test select number, NULL from numbers(100000);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test; system start merges test; optimize table test final;;
optimize table test final; select '---------------------';
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system stop merges test; system stop merges test;
insert into test select number, map(number, number) from numbers(200000); insert into test select number, map(number, number) from numbers(200000);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system stop merges test; system stop merges test;
insert into test select number, tuple(number, number) from numbers(10000); insert into test select number, tuple(number, number) from numbers(10000);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system stop merges test;
insert into test select number, 'str_' || number from numbers(30000);
select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test;
optimize table test final;
select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
drop table test; drop table test;

94
tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.reference
View File

@ -1,28 +1,66 @@
50000 DateTime 50000 DateTime false
60000 Date 60000 Date false
70000 Array(UInt16) 70000 Array(UInt16) false
80000 String 80000 String false
100000 None 100000 None false
100000 UInt64 100000 UInt64 false
70000 Array(UInt16) ---------------------
100000 None 50000 DateTime true
100000 UInt64 60000 Date true
190000 String 70000 Array(UInt16) true
70000 Array(UInt16) 80000 String false
100000 None 100000 None false
100000 UInt64 100000 UInt64 false
190000 String ---------------------
200000 Map(UInt64, UInt64) 50000 DateTime true
100000 None 60000 Date true
100000 UInt64 70000 Array(UInt16) true
200000 Map(UInt64, UInt64) 80000 String false
260000 String 100000 None false
10000 Tuple(UInt64, UInt64) 100000 UInt64 false
100000 None 200000 Map(UInt64, UInt64) false
100000 UInt64 ---------------------
200000 Map(UInt64, UInt64) 50000 DateTime true
260000 String 60000 Date true
100000 None 70000 Array(UInt16) true
100000 UInt64 80000 String true
200000 Map(UInt64, UInt64) 100000 None false
270000 String 100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) false
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
80000 String true
100000 None false
100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) true
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
80000 String true
100000 None false
100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) true
30000 String false
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
80000 String true
100000 None false
100000 UInt64 false
200000 Map(UInt64, UInt64) false
---------------------
10000 Tuple(UInt64, UInt64) true
50000 DateTime true
60000 Date true
70000 Array(UInt16) true
100000 None false
100000 UInt64 true
110000 String false
200000 Map(UInt64, UInt64) false

28
tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.sql
View File

@ -2,7 +2,7 @@
set allow_experimental_dynamic_type=1; set allow_experimental_dynamic_type=1;
drop table if exists test; drop table if exists test;
create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760, lock_acquire_timeout_for_background_operations=600; create table test (id UInt64, d Dynamic(max_types=2)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760, lock_acquire_timeout_for_background_operations=600;
system stop merges test; system stop merges test;
insert into test select number, number from numbers(100000); insert into test select number, number from numbers(100000);
@ -12,22 +12,36 @@ insert into test select number, toDate(number) from numbers(60000);
insert into test select number, toDateTime(number) from numbers(50000); insert into test select number, toDateTime(number) from numbers(50000);
insert into test select number, NULL from numbers(100000); insert into test select number, NULL from numbers(100000);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test; optimize table test final;; system start merges test; optimize table test final;;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system stop merges test; system stop merges test;
insert into test select number, map(number, number) from numbers(200000); insert into test select number, map(number, number) from numbers(200000);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system stop merges test; system stop merges test;
insert into test select number, tuple(number, number) from numbers(10000); insert into test select number, tuple(number, number) from numbers(10000);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system stop merges test;
insert into test select number, 'str_' || number from numbers(30000);
select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test;
optimize table test final;
select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
drop table test; drop table test;

376
tests/queries/0_stateless/03037_dynamic_merges_small.reference.j2
View File

@ -1,112 +1,264 @@
5 DateTime 5 DateTime false
6 Date 6 Date false
7 Array(UInt16) 7 Array(UInt16) false
8 String 8 String false
10 None 10 None false
10 UInt64 10 UInt64 false
7 Array(UInt16) ---------------------
10 None 5 DateTime true
10 UInt64 6 Date true
19 String 7 Array(UInt16) true
7 Array(UInt16) 8 String false
10 None 10 None false
10 UInt64 10 UInt64 false
19 String ---------------------
20 Map(UInt64, UInt64) 5 DateTime true
10 None 6 Date true
10 UInt64 7 Array(UInt16) true
20 Map(UInt64, UInt64) 8 String false
26 String 10 None false
1 Tuple(UInt64, UInt64) 10 UInt64 false
10 None 20 Map(UInt64, UInt64) false
10 UInt64 ---------------------
20 Map(UInt64, UInt64) 5 DateTime true
26 String 6 Date true
10 None 7 Array(UInt16) true
10 UInt64 8 String true
20 Map(UInt64, UInt64) 10 None false
27 String 10 UInt64 false
5 DateTime 20 Map(UInt64, UInt64) false
6 Date ---------------------
7 Array(UInt16) 1 Tuple(UInt64, UInt64) false
8 String 5 DateTime true
10 None 6 Date true
10 UInt64 7 Array(UInt16) true
7 Array(UInt16) 8 String true
10 None 10 None false
10 UInt64 10 UInt64 false
19 String 20 Map(UInt64, UInt64) false
7 Array(UInt16) ---------------------
10 None 1 Tuple(UInt64, UInt64) true
10 UInt64 5 DateTime true
19 String 6 Date true
20 Map(UInt64, UInt64) 7 Array(UInt16) true
10 None 8 String true
10 UInt64 10 None false
20 Map(UInt64, UInt64) 10 UInt64 false
26 String 20 Map(UInt64, UInt64) false
1 Tuple(UInt64, UInt64) ---------------------
10 None 1 Tuple(UInt64, UInt64) true
10 UInt64 3 String false
20 Map(UInt64, UInt64) 5 DateTime true
26 String 6 Date true
10 None 7 Array(UInt16) true
10 UInt64 8 String true
20 Map(UInt64, UInt64) 10 None false
27 String 10 UInt64 false
5 DateTime 20 Map(UInt64, UInt64) false
6 Date ---------------------
7 Array(UInt16) 1 Tuple(UInt64, UInt64) true
8 String 5 DateTime true
10 None 6 Date true
10 UInt64 7 Array(UInt16) true
7 Array(UInt16) 10 None false
10 None 10 UInt64 true
10 UInt64 11 String false
19 String 20 Map(UInt64, UInt64) false
7 Array(UInt16) 5 DateTime false
10 None 6 Date false
10 UInt64 7 Array(UInt16) false
19 String 8 String false
20 Map(UInt64, UInt64) 10 None false
10 None 10 UInt64 false
10 UInt64 ---------------------
20 Map(UInt64, UInt64) 5 DateTime true
26 String 6 Date true
1 Tuple(UInt64, UInt64) 7 Array(UInt16) true
10 None 8 String false
10 UInt64 10 None false
20 Map(UInt64, UInt64) 10 UInt64 false
26 String ---------------------
10 None 5 DateTime true
10 UInt64 6 Date true
20 Map(UInt64, UInt64) 7 Array(UInt16) true
27 String 8 String false
5 DateTime 10 None false
6 Date 10 UInt64 false
7 Array(UInt16) 20 Map(UInt64, UInt64) false
8 String ---------------------
10 None 5 DateTime true
10 UInt64 6 Date true
7 Array(UInt16) 7 Array(UInt16) true
10 None 8 String true
10 UInt64 10 None false
19 String 10 UInt64 false
7 Array(UInt16) 20 Map(UInt64, UInt64) false
10 None ---------------------
10 UInt64 1 Tuple(UInt64, UInt64) false
19 String 5 DateTime true
20 Map(UInt64, UInt64) 6 Date true
10 None 7 Array(UInt16) true
10 UInt64 8 String true
20 Map(UInt64, UInt64) 10 None false
26 String 10 UInt64 false
1 Tuple(UInt64, UInt64) 20 Map(UInt64, UInt64) false
10 None ---------------------
10 UInt64 1 Tuple(UInt64, UInt64) true
20 Map(UInt64, UInt64) 5 DateTime true
26 String 6 Date true
10 None 7 Array(UInt16) true
10 UInt64 8 String true
20 Map(UInt64, UInt64) 10 None false
27 String 10 UInt64 false
20 Map(UInt64, UInt64) false
---------------------
1 Tuple(UInt64, UInt64) true
3 String false
5 DateTime true
6 Date true
7 Array(UInt16) true
8 String true
10 None false
10 UInt64 false
20 Map(UInt64, UInt64) false
---------------------
1 Tuple(UInt64, UInt64) true
5 DateTime true
6 Date true
7 Array(UInt16) true
10 None false
10 UInt64 true
11 String false
20 Map(UInt64, UInt64) false
5 DateTime false
6 Date false
7 Array(UInt16) false
8 String false
10 None false
10 UInt64 false
---------------------
5 DateTime true
6 Date true
7 Array(UInt16) true
8 String false
10 None false
10 UInt64 false
---------------------
5 DateTime true
6 Date true
7 Array(UInt16) true
8 String false
10 None false
10 UInt64 false
20 Map(UInt64, UInt64) false
---------------------
5 DateTime true
6 Date true
7 Array(UInt16) true
8 String true
10 None false
10 UInt64 false
20 Map(UInt64, UInt64) false
---------------------
1 Tuple(UInt64, UInt64) false
5 DateTime true
6 Date true
7 Array(UInt16) true
8 String true
10 None false
10 UInt64 false
20 Map(UInt64, UInt64) false
---------------------
1 Tuple(UInt64, UInt64) true
5 DateTime true
6 Date true
7 Array(UInt16) true
8 String true
10 None false
10 UInt64 false
20 Map(UInt64, UInt64) false
---------------------
1 Tuple(UInt64, UInt64) true
3 String false
5 DateTime true
6 Date true
7 Array(UInt16) true
8 String true
10 None false
10 UInt64 false
20 Map(UInt64, UInt64) false
---------------------
1 Tuple(UInt64, UInt64) true
5 DateTime true
6 Date true
7 Array(UInt16) true
10 None false
10 UInt64 true
11 String false
20 Map(UInt64, UInt64) false
5 DateTime false
6 Date false
7 Array(UInt16) false
8 String false
10 None false
10 UInt64 false
---------------------
5 DateTime true
6 Date true
7 Array(UInt16) true
8 String false
10 None false
10 UInt64 false
---------------------
5 DateTime true
6 Date true
7 Array(UInt16) true
8 String false
10 None false
10 UInt64 false
20 Map(UInt64, UInt64) false
---------------------
5 DateTime true
6 Date true
7 Array(UInt16) true
8 String true
10 None false
10 UInt64 false
20 Map(UInt64, UInt64) false
---------------------
1 Tuple(UInt64, UInt64) false
5 DateTime true
6 Date true
7 Array(UInt16) true
8 String true
10 None false
10 UInt64 false
20 Map(UInt64, UInt64) false
---------------------
1 Tuple(UInt64, UInt64) true
5 DateTime true
6 Date true
7 Array(UInt16) true
8 String true
10 None false
10 UInt64 false
20 Map(UInt64, UInt64) false
---------------------
1 Tuple(UInt64, UInt64) true
3 String false
5 DateTime true
6 Date true
7 Array(UInt16) true
8 String true
10 None false
10 UInt64 false
20 Map(UInt64, UInt64) false
---------------------
1 Tuple(UInt64, UInt64) true
5 DateTime true
6 Date true
7 Array(UInt16) true
10 None false
10 UInt64 true
11 String false
20 Map(UInt64, UInt64) false

28
tests/queries/0_stateless/03037_dynamic_merges_small.sql.j2
View File

@ -9,7 +9,7 @@ drop table if exists test;
'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1', 'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1',
'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1'] -%} 'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1'] -%}
create table test (id UInt64, d Dynamic(max_types=3)) engine={{ engine }}; create table test (id UInt64, d Dynamic(max_types=2)) engine={{ engine }};
system stop merges test; system stop merges test;
insert into test select number, number from numbers(10); insert into test select number, number from numbers(10);
@ -19,23 +19,37 @@ insert into test select number, toDate(number) from numbers(6);
insert into test select number, toDateTime(number) from numbers(5); insert into test select number, toDateTime(number) from numbers(5);
insert into test select number, NULL from numbers(10); insert into test select number, NULL from numbers(10);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test; optimize table test final;; system start merges test; optimize table test final;;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system stop merges test; system stop merges test;
insert into test select number, map(number, number) from numbers(20); insert into test select number, map(number, number) from numbers(20);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system stop merges test; system stop merges test;
insert into test select number, tuple(number, number) from numbers(1); insert into test select number, tuple(number, number) from numbers(1);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system stop merges test;
insert into test select number, 'str_' || number from numbers(3);
select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
system start merges test;
optimize table test final;
select '---------------------';
select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
drop table test; drop table test;

84
tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.reference
View File

@ -1,21 +1,63 @@
16667 Tuple(a Dynamic(max_types=3)):Date 6667 Tuple(a Dynamic(max_types=2)):DateTime false
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8) 13333 Tuple(a Dynamic(max_types=2)):IPv4 false
50000 Tuple(a Dynamic(max_types=3)):String 16667 Tuple(a Dynamic(max_types=2)):Date false
50000 Tuple(a Dynamic(max_types=3)):UInt64 33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) false
100000 UInt64:None 33334 Tuple(a Dynamic(max_types=2)):UInt64 false
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8) 40000 String:None false
50000 Tuple(a Dynamic(max_types=3)):UInt64 66666 Tuple(a Dynamic(max_types=2)):String false
66667 Tuple(a Dynamic(max_types=3)):String 140000 UInt64:None false
100000 UInt64:None ---------------------
16667 Tuple(a Dynamic(max_types=3)):DateTime 6667 Tuple(a Dynamic(max_types=2)):DateTime true
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8) 13333 Tuple(a Dynamic(max_types=2)):IPv4 true
50000 Tuple(a Dynamic(max_types=3)):UInt64 16667 Tuple(a Dynamic(max_types=2)):Date true
66667 Tuple(a Dynamic(max_types=3)):String 33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) 33334 Tuple(a Dynamic(max_types=2)):UInt64 false
100000 UInt64:None 40000 String:None false
133333 Tuple(a Dynamic(max_types=3)):None 66666 Tuple(a Dynamic(max_types=2)):String false
50000 Tuple(a Dynamic(max_types=3)):UInt64 140000 UInt64:None false
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) ---------------------
100000 UInt64:None 6667 Tuple(a Dynamic(max_types=2)):DateTime true
116667 Tuple(a Dynamic(max_types=3)):String 13333 Tuple(a Dynamic(max_types=2)):IPv4 true
133333 Tuple(a Dynamic(max_types=3)):None 16667 Tuple(a Dynamic(max_types=2)):Date true
16667 Tuple(a Dynamic(max_types=2)):DateTime false
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 false
40000 String:None false
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false
---------------------
13333 Tuple(a Dynamic(max_types=2)):IPv4 true
16667 Tuple(a Dynamic(max_types=2)):Date true
23334 Tuple(a Dynamic(max_types=2)):DateTime true
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 true
40000 String:None false
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false
---------------------
13333 Tuple(a Dynamic(max_types=2)):IPv4 true
16667 Tuple(a Dynamic(max_types=2)):Date true
23334 Tuple(a Dynamic(max_types=2)):DateTime true
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 true
40000 String:None false
40000 Tuple(a Dynamic(max_types=2)):DateTime false
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false
---------------------
13333 Tuple(a Dynamic(max_types=2)):IPv4 true
16667 Tuple(a Dynamic(max_types=2)):Date true
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 true
40000 String:None false
63334 Tuple(a Dynamic(max_types=2)):DateTime true
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false

33
tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sql
View File

@ -6,24 +6,39 @@ set allow_experimental_dynamic_type = 1;
set enable_named_columns_in_function_tuple = 0; set enable_named_columns_in_function_tuple = 0;
drop table if exists test;; drop table if exists test;;
create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, lock_acquire_timeout_for_background_operations=600; create table test (id UInt64, d Dynamic(max_types=2)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, lock_acquire_timeout_for_background_operations=600;
system stop merges test; system stop merges test;
insert into test select number, number from numbers(100000); insert into test select number, number from numbers(100000);
insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000); insert into test select number, tuple(if(number % 3 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=2)) from numbers(100000);
insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000); insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=2)) from numbers(50000);
insert into test select number, multiIf(number % 5 == 0, tuple(if(number % 3 == 0, toDateTime(number), toIPv4(number)))::Tuple(a Dynamic(max_types=2)), number % 5 == 1 or number % 5 == 2, number, 'str_' || number) from numbers(100000);
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000); system stop merges test;
insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000); insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=2)) from numbers(50000);
insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=2)) from numbers(200000);
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system stop merges test;
insert into test select number, tuple(toDateTime(number))::Tuple(a Dynamic(max_types=2)) from numbers(40000);
select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system start merges test;
optimize table test final;
select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
drop table test; drop table test;

84
tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.reference
View File

@ -1,21 +1,63 @@
16667 Tuple(a Dynamic(max_types=3)):Date 6667 Tuple(a Dynamic(max_types=2)):DateTime false
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8) 13333 Tuple(a Dynamic(max_types=2)):IPv4 false
50000 Tuple(a Dynamic(max_types=3)):String 16667 Tuple(a Dynamic(max_types=2)):Date false
50000 Tuple(a Dynamic(max_types=3)):UInt64 33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) false
100000 UInt64:None 33334 Tuple(a Dynamic(max_types=2)):UInt64 false
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8) 40000 String:None false
50000 Tuple(a Dynamic(max_types=3)):UInt64 66666 Tuple(a Dynamic(max_types=2)):String false
66667 Tuple(a Dynamic(max_types=3)):String 140000 UInt64:None false
100000 UInt64:None ---------------------
16667 Tuple(a Dynamic(max_types=3)):DateTime 6667 Tuple(a Dynamic(max_types=2)):DateTime true
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8) 13333 Tuple(a Dynamic(max_types=2)):IPv4 true
50000 Tuple(a Dynamic(max_types=3)):UInt64 16667 Tuple(a Dynamic(max_types=2)):Date true
66667 Tuple(a Dynamic(max_types=3)):String 33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) 33334 Tuple(a Dynamic(max_types=2)):UInt64 false
100000 UInt64:None 40000 String:None false
133333 Tuple(a Dynamic(max_types=3)):None 66666 Tuple(a Dynamic(max_types=2)):String false
50000 Tuple(a Dynamic(max_types=3)):UInt64 140000 UInt64:None false
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) ---------------------
100000 UInt64:None 6667 Tuple(a Dynamic(max_types=2)):DateTime true
116667 Tuple(a Dynamic(max_types=3)):String 13333 Tuple(a Dynamic(max_types=2)):IPv4 true
133333 Tuple(a Dynamic(max_types=3)):None 16667 Tuple(a Dynamic(max_types=2)):Date true
16667 Tuple(a Dynamic(max_types=2)):DateTime false
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 false
40000 String:None false
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false
---------------------
13333 Tuple(a Dynamic(max_types=2)):IPv4 true
16667 Tuple(a Dynamic(max_types=2)):Date true
23334 Tuple(a Dynamic(max_types=2)):DateTime true
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 true
40000 String:None false
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false
---------------------
13333 Tuple(a Dynamic(max_types=2)):IPv4 true
16667 Tuple(a Dynamic(max_types=2)):Date true
23334 Tuple(a Dynamic(max_types=2)):DateTime true
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 true
40000 String:None false
40000 Tuple(a Dynamic(max_types=2)):DateTime false
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false
---------------------
13333 Tuple(a Dynamic(max_types=2)):IPv4 true
16667 Tuple(a Dynamic(max_types=2)):Date true
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 true
40000 String:None false
63334 Tuple(a Dynamic(max_types=2)):DateTime true
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false

33
tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sql
View File

@ -6,24 +6,39 @@ set allow_experimental_dynamic_type = 1;
set enable_named_columns_in_function_tuple = 0; set enable_named_columns_in_function_tuple = 0;
drop table if exists test;; drop table if exists test;;
create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, lock_acquire_timeout_for_background_operations=600; create table test (id UInt64, d Dynamic(max_types=2)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, lock_acquire_timeout_for_background_operations=600;
system stop merges test; system stop merges test;
insert into test select number, number from numbers(100000); insert into test select number, number from numbers(100000);
insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000); insert into test select number, tuple(if(number % 3 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=2)) from numbers(100000);
insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000); insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=2)) from numbers(50000);
insert into test select number, multiIf(number % 5 == 0, tuple(if(number % 3 == 0, toDateTime(number), toIPv4(number)))::Tuple(a Dynamic(max_types=2)), number % 5 == 1 or number % 5 == 2, number, 'str_' || number) from numbers(100000);
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000); system stop merges test;
insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000); insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=2)) from numbers(50000);
insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=2)) from numbers(200000);
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system stop merges test;
insert into test select number, tuple(toDateTime(number))::Tuple(a Dynamic(max_types=2)) from numbers(40000);
select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system start merges test;
optimize table test final;
select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
drop table test; drop table test;

340
tests/queries/0_stateless/03038_nested_dynamic_merges_small.reference.j2
View File

@ -1,84 +1,256 @@
2 Tuple(a Dynamic(max_types=3)):Date test
3 Tuple(a Dynamic(max_types=3)):Array(UInt8) 2 Tuple(a Dynamic(max_types=2)):DateTime false
5 Tuple(a Dynamic(max_types=3)):String 2 Tuple(a Dynamic(max_types=2)):IPv4 false
5 Tuple(a Dynamic(max_types=3)):UInt64 4 Tuple(a Dynamic(max_types=2)):Date false
10 UInt64:None 6 Tuple(a Dynamic(max_types=2)):Array(UInt8) false
3 Tuple(a Dynamic(max_types=3)):Array(UInt8) 7 Tuple(a Dynamic(max_types=2)):UInt64 false
5 Tuple(a Dynamic(max_types=3)):UInt64 8 String:None false
7 Tuple(a Dynamic(max_types=3)):String 13 Tuple(a Dynamic(max_types=2)):String false
10 UInt64:None 28 UInt64:None false
2 Tuple(a Dynamic(max_types=3)):DateTime ---------------------
3 Tuple(a Dynamic(max_types=3)):Array(UInt8) 2 Tuple(a Dynamic(max_types=2)):DateTime true
5 Tuple(a Dynamic(max_types=3)):UInt64 2 Tuple(a Dynamic(max_types=2)):IPv4 true
7 Tuple(a Dynamic(max_types=3)):String 4 Tuple(a Dynamic(max_types=2)):Date true
10 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) 6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
10 UInt64:None 7 Tuple(a Dynamic(max_types=2)):UInt64 false
13 Tuple(a Dynamic(max_types=3)):None 8 String:None false
5 Tuple(a Dynamic(max_types=3)):UInt64 13 Tuple(a Dynamic(max_types=2)):String false
10 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) 28 UInt64:None false
10 UInt64:None ---------------------
12 Tuple(a Dynamic(max_types=3)):String 2 Tuple(a Dynamic(max_types=2)):DateTime true
13 Tuple(a Dynamic(max_types=3)):None 2 Tuple(a Dynamic(max_types=2)):IPv4 true
2 Tuple(a Dynamic(max_types=3)):Date 4 Tuple(a Dynamic(max_types=2)):Date true
3 Tuple(a Dynamic(max_types=3)):Array(UInt8) 4 Tuple(a Dynamic(max_types=2)):DateTime false
5 Tuple(a Dynamic(max_types=3)):String 6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
5 Tuple(a Dynamic(max_types=3)):UInt64 7 Tuple(a Dynamic(max_types=2)):UInt64 false
10 UInt64:None 8 String:None false
3 Tuple(a Dynamic(max_types=3)):Array(UInt8) 13 Tuple(a Dynamic(max_types=2)):String false
5 Tuple(a Dynamic(max_types=3)):UInt64 20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
7 Tuple(a Dynamic(max_types=3)):String 26 Tuple(a Dynamic(max_types=2)):None false
10 UInt64:None 28 UInt64:None false
2 Tuple(a Dynamic(max_types=3)):DateTime ---------------------
3 Tuple(a Dynamic(max_types=3)):Array(UInt8) 2 Tuple(a Dynamic(max_types=2)):IPv4 true
5 Tuple(a Dynamic(max_types=3)):UInt64 4 Tuple(a Dynamic(max_types=2)):Date true
7 Tuple(a Dynamic(max_types=3)):String 6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
10 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) 6 Tuple(a Dynamic(max_types=2)):DateTime true
10 UInt64:None 7 Tuple(a Dynamic(max_types=2)):UInt64 true
13 Tuple(a Dynamic(max_types=3)):None 8 String:None false
5 Tuple(a Dynamic(max_types=3)):UInt64 13 Tuple(a Dynamic(max_types=2)):String false
10 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) 20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
10 UInt64:None 26 Tuple(a Dynamic(max_types=2)):None false
12 Tuple(a Dynamic(max_types=3)):String 28 UInt64:None false
13 Tuple(a Dynamic(max_types=3)):None ---------------------
2 Tuple(a Dynamic(max_types=3)):Date 2 Tuple(a Dynamic(max_types=2)):IPv4 true
3 Tuple(a Dynamic(max_types=3)):Array(UInt8) 4 Tuple(a Dynamic(max_types=2)):Date true
5 Tuple(a Dynamic(max_types=3)):String 6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
5 Tuple(a Dynamic(max_types=3)):UInt64 6 Tuple(a Dynamic(max_types=2)):DateTime true
10 UInt64:None 7 Tuple(a Dynamic(max_types=2)):UInt64 true
3 Tuple(a Dynamic(max_types=3)):Array(UInt8) 8 String:None false
5 Tuple(a Dynamic(max_types=3)):UInt64 8 Tuple(a Dynamic(max_types=2)):DateTime false
7 Tuple(a Dynamic(max_types=3)):String 13 Tuple(a Dynamic(max_types=2)):String false
10 UInt64:None 20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
2 Tuple(a Dynamic(max_types=3)):DateTime 26 Tuple(a Dynamic(max_types=2)):None false
3 Tuple(a Dynamic(max_types=3)):Array(UInt8) 28 UInt64:None false
5 Tuple(a Dynamic(max_types=3)):UInt64 ---------------------
7 Tuple(a Dynamic(max_types=3)):String 2 Tuple(a Dynamic(max_types=2)):IPv4 true
10 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) 4 Tuple(a Dynamic(max_types=2)):Date true
10 UInt64:None 6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
13 Tuple(a Dynamic(max_types=3)):None 7 Tuple(a Dynamic(max_types=2)):UInt64 true
5 Tuple(a Dynamic(max_types=3)):UInt64 8 String:None false
10 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) 13 Tuple(a Dynamic(max_types=2)):String false
10 UInt64:None 14 Tuple(a Dynamic(max_types=2)):DateTime true
12 Tuple(a Dynamic(max_types=3)):String 20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
13 Tuple(a Dynamic(max_types=3)):None 26 Tuple(a Dynamic(max_types=2)):None false
2 Tuple(a Dynamic(max_types=3)):Date 28 UInt64:None false
3 Tuple(a Dynamic(max_types=3)):Array(UInt8) test
5 Tuple(a Dynamic(max_types=3)):String 2 Tuple(a Dynamic(max_types=2)):DateTime false
5 Tuple(a Dynamic(max_types=3)):UInt64 2 Tuple(a Dynamic(max_types=2)):IPv4 false
10 UInt64:None 4 Tuple(a Dynamic(max_types=2)):Date false
3 Tuple(a Dynamic(max_types=3)):Array(UInt8) 6 Tuple(a Dynamic(max_types=2)):Array(UInt8) false
5 Tuple(a Dynamic(max_types=3)):UInt64 7 Tuple(a Dynamic(max_types=2)):UInt64 false
7 Tuple(a Dynamic(max_types=3)):String 8 String:None false
10 UInt64:None 13 Tuple(a Dynamic(max_types=2)):String false
2 Tuple(a Dynamic(max_types=3)):DateTime 28 UInt64:None false
3 Tuple(a Dynamic(max_types=3)):Array(UInt8) ---------------------
5 Tuple(a Dynamic(max_types=3)):UInt64 2 Tuple(a Dynamic(max_types=2)):DateTime true
7 Tuple(a Dynamic(max_types=3)):String 2 Tuple(a Dynamic(max_types=2)):IPv4 true
10 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) 4 Tuple(a Dynamic(max_types=2)):Date true
10 UInt64:None 6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
13 Tuple(a Dynamic(max_types=3)):None 7 Tuple(a Dynamic(max_types=2)):UInt64 false
5 Tuple(a Dynamic(max_types=3)):UInt64 8 String:None false
10 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) 13 Tuple(a Dynamic(max_types=2)):String false
10 UInt64:None 28 UInt64:None false
12 Tuple(a Dynamic(max_types=3)):String ---------------------
13 Tuple(a Dynamic(max_types=3)):None 2 Tuple(a Dynamic(max_types=2)):DateTime true
2 Tuple(a Dynamic(max_types=2)):IPv4 true
4 Tuple(a Dynamic(max_types=2)):Date true
4 Tuple(a Dynamic(max_types=2)):DateTime false
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
7 Tuple(a Dynamic(max_types=2)):UInt64 false
8 String:None false
13 Tuple(a Dynamic(max_types=2)):String false
20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
26 Tuple(a Dynamic(max_types=2)):None false
28 UInt64:None false
---------------------
2 Tuple(a Dynamic(max_types=2)):IPv4 true
4 Tuple(a Dynamic(max_types=2)):Date true
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
6 Tuple(a Dynamic(max_types=2)):DateTime true
7 Tuple(a Dynamic(max_types=2)):UInt64 true
8 String:None false
13 Tuple(a Dynamic(max_types=2)):String false
20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
26 Tuple(a Dynamic(max_types=2)):None false
28 UInt64:None false
---------------------
2 Tuple(a Dynamic(max_types=2)):IPv4 true
4 Tuple(a Dynamic(max_types=2)):Date true
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
6 Tuple(a Dynamic(max_types=2)):DateTime true
7 Tuple(a Dynamic(max_types=2)):UInt64 true
8 String:None false
8 Tuple(a Dynamic(max_types=2)):DateTime false
13 Tuple(a Dynamic(max_types=2)):String false
20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
26 Tuple(a Dynamic(max_types=2)):None false
28 UInt64:None false
---------------------
2 Tuple(a Dynamic(max_types=2)):IPv4 true
4 Tuple(a Dynamic(max_types=2)):Date true
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
7 Tuple(a Dynamic(max_types=2)):UInt64 true
8 String:None false
13 Tuple(a Dynamic(max_types=2)):String false
14 Tuple(a Dynamic(max_types=2)):DateTime true
20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
26 Tuple(a Dynamic(max_types=2)):None false
28 UInt64:None false
test
2 Tuple(a Dynamic(max_types=2)):DateTime false
2 Tuple(a Dynamic(max_types=2)):IPv4 false
4 Tuple(a Dynamic(max_types=2)):Date false
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) false
7 Tuple(a Dynamic(max_types=2)):UInt64 false
8 String:None false
13 Tuple(a Dynamic(max_types=2)):String false
28 UInt64:None false
---------------------
2 Tuple(a Dynamic(max_types=2)):DateTime true
2 Tuple(a Dynamic(max_types=2)):IPv4 true
4 Tuple(a Dynamic(max_types=2)):Date true
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
7 Tuple(a Dynamic(max_types=2)):UInt64 false
8 String:None false
13 Tuple(a Dynamic(max_types=2)):String false
28 UInt64:None false
---------------------
2 Tuple(a Dynamic(max_types=2)):DateTime true
2 Tuple(a Dynamic(max_types=2)):IPv4 true
4 Tuple(a Dynamic(max_types=2)):Date true
4 Tuple(a Dynamic(max_types=2)):DateTime false
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
7 Tuple(a Dynamic(max_types=2)):UInt64 false
8 String:None false
13 Tuple(a Dynamic(max_types=2)):String false
20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
26 Tuple(a Dynamic(max_types=2)):None false
28 UInt64:None false
---------------------
2 Tuple(a Dynamic(max_types=2)):IPv4 true
4 Tuple(a Dynamic(max_types=2)):Date true
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
6 Tuple(a Dynamic(max_types=2)):DateTime true
7 Tuple(a Dynamic(max_types=2)):UInt64 true
8 String:None false
13 Tuple(a Dynamic(max_types=2)):String false
20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
26 Tuple(a Dynamic(max_types=2)):None false
28 UInt64:None false
---------------------
2 Tuple(a Dynamic(max_types=2)):IPv4 true
4 Tuple(a Dynamic(max_types=2)):Date true
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
6 Tuple(a Dynamic(max_types=2)):DateTime true
7 Tuple(a Dynamic(max_types=2)):UInt64 true
8 String:None false
8 Tuple(a Dynamic(max_types=2)):DateTime false
13 Tuple(a Dynamic(max_types=2)):String false
20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
26 Tuple(a Dynamic(max_types=2)):None false
28 UInt64:None false
---------------------
2 Tuple(a Dynamic(max_types=2)):IPv4 true
4 Tuple(a Dynamic(max_types=2)):Date true
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
7 Tuple(a Dynamic(max_types=2)):UInt64 true
8 String:None false
13 Tuple(a Dynamic(max_types=2)):String true
14 Tuple(a Dynamic(max_types=2)):DateTime false
20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
26 Tuple(a Dynamic(max_types=2)):None false
28 UInt64:None false
test
2 Tuple(a Dynamic(max_types=2)):DateTime false
2 Tuple(a Dynamic(max_types=2)):IPv4 false
4 Tuple(a Dynamic(max_types=2)):Date false
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) false
7 Tuple(a Dynamic(max_types=2)):UInt64 false
8 String:None false
13 Tuple(a Dynamic(max_types=2)):String false
28 UInt64:None false
---------------------
2 Tuple(a Dynamic(max_types=2)):DateTime true
2 Tuple(a Dynamic(max_types=2)):IPv4 true
4 Tuple(a Dynamic(max_types=2)):Date true
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
7 Tuple(a Dynamic(max_types=2)):UInt64 false
8 String:None false
13 Tuple(a Dynamic(max_types=2)):String false
28 UInt64:None false
---------------------
2 Tuple(a Dynamic(max_types=2)):DateTime true
2 Tuple(a Dynamic(max_types=2)):IPv4 true
4 Tuple(a Dynamic(max_types=2)):Date true
4 Tuple(a Dynamic(max_types=2)):DateTime false
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
7 Tuple(a Dynamic(max_types=2)):UInt64 false
8 String:None false
13 Tuple(a Dynamic(max_types=2)):String false
20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
26 Tuple(a Dynamic(max_types=2)):None false
28 UInt64:None false
---------------------
2 Tuple(a Dynamic(max_types=2)):IPv4 true
4 Tuple(a Dynamic(max_types=2)):Date true
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
6 Tuple(a Dynamic(max_types=2)):DateTime true
7 Tuple(a Dynamic(max_types=2)):UInt64 true
8 String:None false
13 Tuple(a Dynamic(max_types=2)):String false
20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
26 Tuple(a Dynamic(max_types=2)):None false
28 UInt64:None false
---------------------
2 Tuple(a Dynamic(max_types=2)):IPv4 true
4 Tuple(a Dynamic(max_types=2)):Date true
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
6 Tuple(a Dynamic(max_types=2)):DateTime true
7 Tuple(a Dynamic(max_types=2)):UInt64 true
8 String:None false
8 Tuple(a Dynamic(max_types=2)):DateTime false
13 Tuple(a Dynamic(max_types=2)):String false
20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
26 Tuple(a Dynamic(max_types=2)):None false
28 UInt64:None false
---------------------
2 Tuple(a Dynamic(max_types=2)):IPv4 true
4 Tuple(a Dynamic(max_types=2)):Date true
6 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
7 Tuple(a Dynamic(max_types=2)):UInt64 true
8 String:None false
13 Tuple(a Dynamic(max_types=2)):String true
14 Tuple(a Dynamic(max_types=2)):DateTime false
20 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
26 Tuple(a Dynamic(max_types=2)):None false
28 UInt64:None false

36
tests/queries/0_stateless/03038_nested_dynamic_merges_small.sql.j2
View File

@ -10,25 +10,41 @@ drop table if exists test;
'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1', 'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1',
'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1'] -%} 'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1'] -%}
create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, lock_acquire_timeout_for_background_operations=600; select 'test';
create table test (id UInt64, d Dynamic(max_types=2)) engine={{ engine }};
system stop merges test; system stop merges test;
insert into test select number, number from numbers(10); insert into test select number, number from numbers(20);
insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(10); insert into test select number, tuple(if(number % 3 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=2)) from numbers(20);
insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(5); insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=2)) from numbers(10);
insert into test select number, multiIf(number % 5 == 0, tuple(if(number % 3 == 0, toDateTime(number), toIPv4(number)))::Tuple(a Dynamic(max_types=2)), number % 5 == 1 or number % 5 == 2, number, 'str_' || number) from numbers(20);
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(5); system stop merges test;
insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(20); insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=2)) from numbers(10);
insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=2)) from numbers(40);
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system stop merges test;
insert into test select number, tuple(toDateTime(number))::Tuple(a Dynamic(max_types=2)) from numbers(8);
select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system start merges test;
optimize table test final;
select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
drop table test; drop table test;

84
tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.reference
View File

@ -1,21 +1,63 @@
16667 Tuple(a Dynamic(max_types=3)):Date 6667 Tuple(a Dynamic(max_types=2)):DateTime false
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8) 13333 Tuple(a Dynamic(max_types=2)):IPv4 false
50000 Tuple(a Dynamic(max_types=3)):String 16667 Tuple(a Dynamic(max_types=2)):Date false
50000 Tuple(a Dynamic(max_types=3)):UInt64 33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) false
100000 UInt64:None 33334 Tuple(a Dynamic(max_types=2)):UInt64 false
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8) 40000 String:None false
50000 Tuple(a Dynamic(max_types=3)):UInt64 66666 Tuple(a Dynamic(max_types=2)):String false
66667 Tuple(a Dynamic(max_types=3)):String 140000 UInt64:None false
100000 UInt64:None ---------------------
16667 Tuple(a Dynamic(max_types=3)):DateTime 6667 Tuple(a Dynamic(max_types=2)):DateTime true
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8) 13333 Tuple(a Dynamic(max_types=2)):IPv4 true
50000 Tuple(a Dynamic(max_types=3)):UInt64 16667 Tuple(a Dynamic(max_types=2)):Date true
66667 Tuple(a Dynamic(max_types=3)):String 33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) 33334 Tuple(a Dynamic(max_types=2)):UInt64 false
100000 UInt64:None 40000 String:None false
133333 Tuple(a Dynamic(max_types=3)):None 66666 Tuple(a Dynamic(max_types=2)):String false
50000 Tuple(a Dynamic(max_types=3)):UInt64 140000 UInt64:None false
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) ---------------------
100000 UInt64:None 6667 Tuple(a Dynamic(max_types=2)):DateTime true
116667 Tuple(a Dynamic(max_types=3)):String 13333 Tuple(a Dynamic(max_types=2)):IPv4 true
133333 Tuple(a Dynamic(max_types=3)):None 16667 Tuple(a Dynamic(max_types=2)):Date true
16667 Tuple(a Dynamic(max_types=2)):DateTime false
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 false
40000 String:None false
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false
---------------------
13333 Tuple(a Dynamic(max_types=2)):IPv4 true
16667 Tuple(a Dynamic(max_types=2)):Date true
23334 Tuple(a Dynamic(max_types=2)):DateTime true
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 true
40000 String:None false
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false
---------------------
13333 Tuple(a Dynamic(max_types=2)):IPv4 true
16667 Tuple(a Dynamic(max_types=2)):Date true
23334 Tuple(a Dynamic(max_types=2)):DateTime true
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 true
40000 String:None false
40000 Tuple(a Dynamic(max_types=2)):DateTime false
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false
---------------------
13333 Tuple(a Dynamic(max_types=2)):IPv4 true
16667 Tuple(a Dynamic(max_types=2)):Date true
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 true
40000 String:None false
63334 Tuple(a Dynamic(max_types=2)):DateTime true
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false

33
tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sql
View File

@ -6,24 +6,39 @@ set allow_experimental_dynamic_type = 1;
set enable_named_columns_in_function_tuple = 0; set enable_named_columns_in_function_tuple = 0;
drop table if exists test;; drop table if exists test;;
create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, lock_acquire_timeout_for_background_operations=600; create table test (id UInt64, d Dynamic(max_types=2)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, lock_acquire_timeout_for_background_operations=600;
system stop merges test; system stop merges test;
insert into test select number, number from numbers(100000); insert into test select number, number from numbers(100000);
insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000); insert into test select number, tuple(if(number % 3 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=2)) from numbers(100000);
insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000); insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=2)) from numbers(50000);
insert into test select number, multiIf(number % 5 == 0, tuple(if(number % 3 == 0, toDateTime(number), toIPv4(number)))::Tuple(a Dynamic(max_types=2)), number % 5 == 1 or number % 5 == 2, number, 'str_' || number) from numbers(100000);
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000); system stop merges test;
insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000); insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=2)) from numbers(50000);
insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=2)) from numbers(200000);
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system stop merges test;
insert into test select number, tuple(toDateTime(number))::Tuple(a Dynamic(max_types=2)) from numbers(40000);
select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system start merges test;
optimize table test final;
select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
drop table test; drop table test;

84
tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.reference
View File

@ -1,21 +1,63 @@
16667 Tuple(a Dynamic(max_types=3)):Date 6667 Tuple(a Dynamic(max_types=2)):DateTime false
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8) 13333 Tuple(a Dynamic(max_types=2)):IPv4 false
50000 Tuple(a Dynamic(max_types=3)):String 16667 Tuple(a Dynamic(max_types=2)):Date false
50000 Tuple(a Dynamic(max_types=3)):UInt64 33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) false
100000 UInt64:None 33334 Tuple(a Dynamic(max_types=2)):UInt64 false
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8) 40000 String:None false
50000 Tuple(a Dynamic(max_types=3)):UInt64 66666 Tuple(a Dynamic(max_types=2)):String false
66667 Tuple(a Dynamic(max_types=3)):String 140000 UInt64:None false
100000 UInt64:None ---------------------
16667 Tuple(a Dynamic(max_types=3)):DateTime 6667 Tuple(a Dynamic(max_types=2)):DateTime true
33333 Tuple(a Dynamic(max_types=3)):Array(UInt8) 13333 Tuple(a Dynamic(max_types=2)):IPv4 true
50000 Tuple(a Dynamic(max_types=3)):UInt64 16667 Tuple(a Dynamic(max_types=2)):Date true
66667 Tuple(a Dynamic(max_types=3)):String 33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) 33334 Tuple(a Dynamic(max_types=2)):UInt64 false
100000 UInt64:None 40000 String:None false
133333 Tuple(a Dynamic(max_types=3)):None 66666 Tuple(a Dynamic(max_types=2)):String false
50000 Tuple(a Dynamic(max_types=3)):UInt64 140000 UInt64:None false
100000 Tuple(a Dynamic(max_types=3)):Tuple(UInt64) ---------------------
100000 UInt64:None 6667 Tuple(a Dynamic(max_types=2)):DateTime true
116667 Tuple(a Dynamic(max_types=3)):String 13333 Tuple(a Dynamic(max_types=2)):IPv4 true
133333 Tuple(a Dynamic(max_types=3)):None 16667 Tuple(a Dynamic(max_types=2)):Date true
16667 Tuple(a Dynamic(max_types=2)):DateTime false
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 false
40000 String:None false
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false
---------------------
13333 Tuple(a Dynamic(max_types=2)):IPv4 true
16667 Tuple(a Dynamic(max_types=2)):Date true
23334 Tuple(a Dynamic(max_types=2)):DateTime true
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 true
40000 String:None false
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false
---------------------
13333 Tuple(a Dynamic(max_types=2)):IPv4 true
16667 Tuple(a Dynamic(max_types=2)):Date true
23334 Tuple(a Dynamic(max_types=2)):DateTime true
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 true
40000 String:None false
40000 Tuple(a Dynamic(max_types=2)):DateTime false
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false
---------------------
13333 Tuple(a Dynamic(max_types=2)):IPv4 true
16667 Tuple(a Dynamic(max_types=2)):Date true
33333 Tuple(a Dynamic(max_types=2)):Array(UInt8) true
33334 Tuple(a Dynamic(max_types=2)):UInt64 true
40000 String:None false
63334 Tuple(a Dynamic(max_types=2)):DateTime true
66666 Tuple(a Dynamic(max_types=2)):String false
100000 Tuple(a Dynamic(max_types=2)):Tuple(UInt64) false
133333 Tuple(a Dynamic(max_types=2)):None false
140000 UInt64:None false

33
tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sql
View File

@ -6,24 +6,39 @@ set allow_experimental_dynamic_type = 1;
set enable_named_columns_in_function_tuple = 0; set enable_named_columns_in_function_tuple = 0;
drop table if exists test;; drop table if exists test;;
create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, lock_acquire_timeout_for_background_operations=600; create table test (id UInt64, d Dynamic(max_types=2)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, lock_acquire_timeout_for_background_operations=600;
system stop merges test; system stop merges test;
insert into test select number, number from numbers(100000); insert into test select number, number from numbers(100000);
insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000); insert into test select number, tuple(if(number % 3 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=2)) from numbers(100000);
insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000); insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=2)) from numbers(50000);
insert into test select number, multiIf(number % 5 == 0, tuple(if(number % 3 == 0, toDateTime(number), toIPv4(number)))::Tuple(a Dynamic(max_types=2)), number % 5 == 1 or number % 5 == 2, number, 'str_' || number) from numbers(100000);
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000); system stop merges test;
insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000); insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=2)) from numbers(50000);
insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=2)) from numbers(200000);
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system start merges test; system start merges test;
optimize table test final; optimize table test final;
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type; select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system stop merges test;
insert into test select number, tuple(toDateTime(number))::Tuple(a Dynamic(max_types=2)) from numbers(40000);
select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
system start merges test;
optimize table test final;
select '---------------------';
select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=2))`.a) as type, isDynamicElementInSharedData(d.`Tuple(a Dynamic(max_types=2))`.a) as flag from test group by type, flag order by count(), type;
drop table test; drop table test;

123
tests/queries/0_stateless/03040_dynamic_type_alters_1_compact_merge_tree.reference
View File

@ -1,13 +1,13 @@
initial insert initial insert
alter add column 1 alter add column 1
3 None 3 None false
0 0 \N \N \N 0 0 0 \N \N \N 0
1 1 \N \N \N 0 1 1 \N \N \N 0
2 2 \N \N \N 0 2 2 \N \N \N 0
insert after alter add column 1 insert after alter add column 1
4 String 4 String false
4 UInt64 4 UInt64 false
7 None 7 None false
0 0 \N \N \N \N 0 0 0 \N \N \N \N 0
1 1 \N \N \N \N 0 1 1 \N \N \N \N 0
2 2 \N \N \N \N 0 2 2 \N \N \N \N 0
@ -24,147 +24,158 @@ insert after alter add column 1
13 13 str_13 str_13 \N \N 0 13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0 14 14 \N \N \N \N 0
alter modify column 1 alter modify column 1
7 None 4 String true
8 String 4 UInt64 true
7 None false
0 0 \N \N \N \N 0 0 0 \N \N \N \N 0
1 1 \N \N \N \N 0 1 1 \N \N \N \N 0
2 2 \N \N \N \N 0 2 2 \N \N \N \N 0
3 3 3 3 \N \N 0 3 3 3 \N 3 \N 0
4 4 4 4 \N \N 0 4 4 4 \N 4 \N 0
5 5 5 5 \N \N 0 5 5 5 \N 5 \N 0
6 6 str_6 str_6 \N \N 0 6 6 str_6 str_6 \N \N 0
7 7 str_7 str_7 \N \N 0 7 7 str_7 str_7 \N \N 0
8 8 str_8 str_8 \N \N 0 8 8 str_8 str_8 \N \N 0
9 9 \N \N \N \N 0 9 9 \N \N \N \N 0
10 10 \N \N \N \N 0 10 10 \N \N \N \N 0
11 11 \N \N \N \N 0 11 11 \N \N \N \N 0
12 12 12 12 \N \N 0 12 12 12 \N 12 \N 0
13 13 str_13 str_13 \N \N 0 13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0 14 14 \N \N \N \N 0
insert after alter modify column 1 insert after alter modify column 1
8 None 1 Date true
11 String 5 String true
5 UInt64 true
8 None false
0 0 \N \N \N \N 0 0 0 \N \N \N \N 0
1 1 \N \N \N \N 0 1 1 \N \N \N \N 0
2 2 \N \N \N \N 0 2 2 \N \N \N \N 0
3 3 3 3 \N \N 0 3 3 3 \N 3 \N 0
4 4 4 4 \N \N 0 4 4 4 \N 4 \N 0
5 5 5 5 \N \N 0 5 5 5 \N 5 \N 0
6 6 str_6 str_6 \N \N 0 6 6 str_6 str_6 \N \N 0
7 7 str_7 str_7 \N \N 0 7 7 str_7 str_7 \N \N 0
8 8 str_8 str_8 \N \N 0 8 8 str_8 str_8 \N \N 0
9 9 \N \N \N \N 0 9 9 \N \N \N \N 0
10 10 \N \N \N \N 0 10 10 \N \N \N \N 0
11 11 \N \N \N \N 0 11 11 \N \N \N \N 0
12 12 12 12 \N \N 0 12 12 12 \N 12 \N 0
13 13 str_13 str_13 \N \N 0 13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0 14 14 \N \N \N \N 0
15 15 \N \N \N \N 0 15 15 \N \N \N \N 0
16 16 16 16 \N \N 0 16 16 16 \N 16 \N 0
17 17 str_17 str_17 \N \N 0 17 17 str_17 str_17 \N \N 0
18 18 1970-01-19 1970-01-19 \N \N 0 18 18 1970-01-19 \N \N 1970-01-19 0
alter modify column 2 alter modify column 2
8 None 1 Date true
11 String 5 String true
5 UInt64 true
8 None false
0 0 \N \N \N \N 0 0 0 \N \N \N \N 0
1 1 \N \N \N \N 0 1 1 \N \N \N \N 0
2 2 \N \N \N \N 0 2 2 \N \N \N \N 0
3 3 3 3 \N \N 0 3 3 3 \N 3 \N 0
4 4 4 4 \N \N 0 4 4 4 \N 4 \N 0
5 5 5 5 \N \N 0 5 5 5 \N 5 \N 0
6 6 str_6 str_6 \N \N 0 6 6 str_6 str_6 \N \N 0
7 7 str_7 str_7 \N \N 0 7 7 str_7 str_7 \N \N 0
8 8 str_8 str_8 \N \N 0 8 8 str_8 str_8 \N \N 0
9 9 \N \N \N \N 0 9 9 \N \N \N \N 0
10 10 \N \N \N \N 0 10 10 \N \N \N \N 0
11 11 \N \N \N \N 0 11 11 \N \N \N \N 0
12 12 12 12 \N \N 0 12 12 12 \N 12 \N 0
13 13 str_13 str_13 \N \N 0 13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0 14 14 \N \N \N \N 0
15 15 \N \N \N \N 0 15 15 \N \N \N \N 0
16 16 16 16 \N \N 0 16 16 16 \N 16 \N 0
17 17 str_17 str_17 \N \N 0 17 17 str_17 str_17 \N \N 0
18 18 1970-01-19 1970-01-19 \N \N 0 18 18 1970-01-19 \N \N 1970-01-19 0
insert after alter modify column 2 insert after alter modify column 2
1 Date 1 String false
1 UInt64 1 UInt64 false
9 None 2 Date true
12 String 5 String true
5 UInt64 true
9 None false
0 0 \N \N \N \N 0 0 0 \N \N \N \N 0
1 1 \N \N \N \N 0 1 1 \N \N \N \N 0
2 2 \N \N \N \N 0 2 2 \N \N \N \N 0
3 3 3 3 \N \N 0 3 3 3 \N 3 \N 0
4 4 4 4 \N \N 0 4 4 4 \N 4 \N 0
5 5 5 5 \N \N 0 5 5 5 \N 5 \N 0
6 6 str_6 str_6 \N \N 0 6 6 str_6 str_6 \N \N 0
7 7 str_7 str_7 \N \N 0 7 7 str_7 str_7 \N \N 0
8 8 str_8 str_8 \N \N 0 8 8 str_8 str_8 \N \N 0
9 9 \N \N \N \N 0 9 9 \N \N \N \N 0
10 10 \N \N \N \N 0 10 10 \N \N \N \N 0
11 11 \N \N \N \N 0 11 11 \N \N \N \N 0
12 12 12 12 \N \N 0 12 12 12 \N 12 \N 0
13 13 str_13 str_13 \N \N 0 13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0 14 14 \N \N \N \N 0
15 15 \N \N \N \N 0 15 15 \N \N \N \N 0
16 16 16 16 \N \N 0 16 16 16 \N 16 \N 0
17 17 str_17 str_17 \N \N 0 17 17 str_17 str_17 \N \N 0
18 18 1970-01-19 1970-01-19 \N \N 0 18 18 1970-01-19 \N \N 1970-01-19 0
19 19 \N \N \N \N 0 19 19 \N \N \N \N 0
20 20 20 \N 20 \N 0 20 20 20 \N 20 \N 0
21 21 str_21 str_21 \N \N 0 21 21 str_21 str_21 \N \N 0
22 22 1970-01-23 \N \N 1970-01-23 0 22 22 1970-01-23 \N \N 1970-01-23 0
alter modify column 3 alter modify column 3
1 Date 1 String false
1 UInt64 1 UInt64 false
9 None 2 Date true
12 String 5 String true
5 UInt64 true
9 None false
0 0 0 \N 0 \N \N \N 0 0 0 0 \N 0 \N \N \N 0
1 1 1 \N 0 \N \N \N 0 1 1 1 \N 0 \N \N \N 0
2 2 2 \N 0 \N \N \N 0 2 2 2 \N 0 \N \N \N 0
3 3 3 \N 0 3 \N \N 0 3 3 3 \N 0 \N 3 \N 0
4 4 4 \N 0 4 \N \N 0 4 4 4 \N 0 \N 4 \N 0
5 5 5 \N 0 5 \N \N 0 5 5 5 \N 0 \N 5 \N 0
6 6 6 \N 0 str_6 \N \N 0 6 6 6 \N 0 str_6 \N \N 0
7 7 7 \N 0 str_7 \N \N 0 7 7 7 \N 0 str_7 \N \N 0
8 8 8 \N 0 str_8 \N \N 0 8 8 8 \N 0 str_8 \N \N 0
9 9 9 \N 0 \N \N \N 0 9 9 9 \N 0 \N \N \N 0
10 10 10 \N 0 \N \N \N 0 10 10 10 \N 0 \N \N \N 0
11 11 11 \N 0 \N \N \N 0 11 11 11 \N 0 \N \N \N 0
12 12 12 \N 0 12 \N \N 0 12 12 12 \N 0 \N 12 \N 0
13 13 13 \N 0 str_13 \N \N 0 13 13 13 \N 0 str_13 \N \N 0
14 14 14 \N 0 \N \N \N 0 14 14 14 \N 0 \N \N \N 0
15 15 15 \N 0 \N \N \N 0 15 15 15 \N 0 \N \N \N 0
16 16 16 \N 0 16 \N \N 0 16 16 16 \N 0 \N 16 \N 0
17 17 17 \N 0 str_17 \N \N 0 17 17 17 \N 0 str_17 \N \N 0
18 18 18 \N 0 1970-01-19 \N \N 0 18 18 18 \N 0 \N \N 1970-01-19 0
19 19 19 \N 0 \N \N \N 0 19 19 19 \N 0 \N \N \N 0
20 20 20 \N 0 \N 20 \N 0 20 20 20 \N 0 \N 20 \N 0
21 21 21 \N 0 str_21 \N \N 0 21 21 21 \N 0 str_21 \N \N 0
22 22 22 \N 0 \N \N 1970-01-23 0 22 22 22 \N 0 \N \N 1970-01-23 0
insert after alter modify column 3 insert after alter modify column 3
1 Date 1 String false
1 UInt64 1 UInt64 false
12 None 2 Date true
12 String 5 String true
5 UInt64 true
12 None false
0 0 0 \N 0 \N \N \N 0 0 0 0 \N 0 \N \N \N 0
1 1 1 \N 0 \N \N \N 0 1 1 1 \N 0 \N \N \N 0
2 2 2 \N 0 \N \N \N 0 2 2 2 \N 0 \N \N \N 0
3 3 3 \N 0 3 \N \N 0 3 3 3 \N 0 \N 3 \N 0
4 4 4 \N 0 4 \N \N 0 4 4 4 \N 0 \N 4 \N 0
5 5 5 \N 0 5 \N \N 0 5 5 5 \N 0 \N 5 \N 0
6 6 6 \N 0 str_6 \N \N 0 6 6 6 \N 0 str_6 \N \N 0
7 7 7 \N 0 str_7 \N \N 0 7 7 7 \N 0 str_7 \N \N 0
8 8 8 \N 0 str_8 \N \N 0 8 8 8 \N 0 str_8 \N \N 0
9 9 9 \N 0 \N \N \N 0 9 9 9 \N 0 \N \N \N 0
10 10 10 \N 0 \N \N \N 0 10 10 10 \N 0 \N \N \N 0
11 11 11 \N 0 \N \N \N 0 11 11 11 \N 0 \N \N \N 0
12 12 12 \N 0 12 \N \N 0 12 12 12 \N 0 \N 12 \N 0
13 13 13 \N 0 str_13 \N \N 0 13 13 13 \N 0 str_13 \N \N 0
14 14 14 \N 0 \N \N \N 0 14 14 14 \N 0 \N \N \N 0
15 15 15 \N 0 \N \N \N 0 15 15 15 \N 0 \N \N \N 0
16 16 16 \N 0 16 \N \N 0 16 16 16 \N 0 \N 16 \N 0
17 17 17 \N 0 str_17 \N \N 0 17 17 17 \N 0 str_17 \N \N 0
18 18 18 \N 0 1970-01-19 \N \N 0 18 18 18 \N 0 \N \N 1970-01-19 0
19 19 19 \N 0 \N \N \N 0 19 19 19 \N 0 \N \N \N 0
20 20 20 \N 0 \N 20 \N 0 20 20 20 \N 0 \N 20 \N 0
21 21 21 \N 0 str_21 \N \N 0 21 21 21 \N 0 str_21 \N \N 0

20
tests/queries/0_stateless/03040_dynamic_type_alters_1_compact_merge_tree.sql
View File

@ -9,7 +9,7 @@ insert into test select number, number from numbers(3);
select 'alter add column 1'; select 'alter add column 1';
alter table test add column d Dynamic(max_types=3) settings mutations_sync=1; alter table test add column d Dynamic(max_types=3) settings mutations_sync=1;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
select x, y, d, d.String, d.UInt64, d.`Tuple(a UInt64)`.a from test order by x; select x, y, d, d.String, d.UInt64, d.`Tuple(a UInt64)`.a from test order by x;
select 'insert after alter add column 1'; select 'insert after alter add column 1';
@ -17,37 +17,37 @@ insert into test select number, number, number from numbers(3, 3);
insert into test select number, number, 'str_' || toString(number) from numbers(6, 3); insert into test select number, number, 'str_' || toString(number) from numbers(6, 3);
insert into test select number, number, NULL from numbers(9, 3); insert into test select number, number, NULL from numbers(9, 3);
insert into test select number, number, multiIf(number % 3 == 0, number, number % 3 == 1, 'str_' || toString(number), NULL) from numbers(12, 3); insert into test select number, number, multiIf(number % 3 == 0, number, number % 3 == 1, 'str_' || toString(number), NULL) from numbers(12, 3);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x; select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
select 'alter modify column 1'; select 'alter modify column 1';
alter table test modify column d Dynamic(max_types=1) settings mutations_sync=1; alter table test modify column d Dynamic(max_types=0) settings mutations_sync=1;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x; select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
select 'insert after alter modify column 1'; select 'insert after alter modify column 1';
insert into test select number, number, multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, toDate(number), NULL) from numbers(15, 4); insert into test select number, number, multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, toDate(number), NULL) from numbers(15, 4);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x; select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
select 'alter modify column 2'; select 'alter modify column 2';
alter table test modify column d Dynamic(max_types=3) settings mutations_sync=1; alter table test modify column d Dynamic(max_types=2) settings mutations_sync=1;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x; select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
select 'insert after alter modify column 2'; select 'insert after alter modify column 2';
insert into test select number, number, multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, toDate(number), NULL) from numbers(19, 4); insert into test select number, number, multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, toDate(number), NULL) from numbers(19, 4);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x; select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
select 'alter modify column 3'; select 'alter modify column 3';
alter table test modify column y Dynamic settings mutations_sync=1; alter table test modify column y Dynamic settings mutations_sync=1;
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
select x, y, y.UInt64, y.String, y.`Tuple(a UInt64)`.a, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x; select x, y, y.UInt64, y.String, y.`Tuple(a UInt64)`.a, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
select 'insert after alter modify column 3'; select 'insert after alter modify column 3';
insert into test select number, multiIf(number % 3 == 0, number, number % 3 == 1, 'str_' || toString(number), NULL), NULL from numbers(23, 3); insert into test select number, multiIf(number % 3 == 0, number, number % 3 == 1, 'str_' || toString(number), NULL), NULL from numbers(23, 3);
select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d); select count(), dynamicType(d), isDynamicElementInSharedData(d) from test group by dynamicType(d), isDynamicElementInSharedData(d) order by count(), dynamicType(d);
select x, y, y.UInt64, y.String, y.`Tuple(a UInt64)`.a, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x; select x, y, y.UInt64, y.String, y.`Tuple(a UInt64)`.a, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
drop table test; drop table test;

94
tests/queries/0_stateless/03040_dynamic_type_alters_1_memory.reference
View File

@ -24,48 +24,28 @@ insert after alter add column 1
13 13 str_13 str_13 \N \N 0 13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0 14 14 \N \N \N \N 0
alter modify column 1 alter modify column 1
4 String
4 UInt64
7 None 7 None
8 String
0 0 \N \N \N \N 0 0 0 \N \N \N \N 0
1 1 \N \N \N \N 0 1 1 \N \N \N \N 0
2 2 \N \N \N \N 0 2 2 \N \N \N \N 0
3 3 3 3 \N \N 0 3 3 3 \N 3 \N 0
4 4 4 4 \N \N 0 4 4 4 \N 4 \N 0
5 5 5 5 \N \N 0 5 5 5 \N 5 \N 0
6 6 str_6 str_6 \N \N 0 6 6 str_6 str_6 \N \N 0
7 7 str_7 str_7 \N \N 0 7 7 str_7 str_7 \N \N 0
8 8 str_8 str_8 \N \N 0 8 8 str_8 str_8 \N \N 0
9 9 \N \N \N \N 0 9 9 \N \N \N \N 0
10 10 \N \N \N \N 0 10 10 \N \N \N \N 0
11 11 \N \N \N \N 0 11 11 \N \N \N \N 0
12 12 12 12 \N \N 0 12 12 12 \N 12 \N 0
13 13 str_13 str_13 \N \N 0 13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0 14 14 \N \N \N \N 0
insert after alter modify column 1 insert after alter modify column 1
8 None 1 Date
11 String 5 String
0 0 \N \N \N \N 0 5 UInt64
1 1 \N \N \N \N 0
2 2 \N \N \N \N 0
3 3 3 3 \N \N 0
4 4 4 4 \N \N 0
5 5 5 5 \N \N 0
6 6 str_6 str_6 \N \N 0
7 7 str_7 str_7 \N \N 0
8 8 str_8 str_8 \N \N 0
9 9 \N \N \N \N 0
10 10 \N \N \N \N 0
11 11 \N \N \N \N 0
12 12 12 12 \N \N 0
13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0
15 15 \N \N \N \N 0
16 16 16 16 \N \N 0
17 17 str_17 str_17 \N \N 0
18 18 1970-01-19 1970-01-19 \N \N 0
alter modify column 2
4 UInt64
7 String
8 None 8 None
0 0 \N \N \N \N 0 0 0 \N \N \N \N 0
1 1 \N \N \N \N 0 1 1 \N \N \N \N 0
@ -83,13 +63,37 @@ alter modify column 2
13 13 str_13 str_13 \N \N 0 13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0 14 14 \N \N \N \N 0
15 15 \N \N \N \N 0 15 15 \N \N \N \N 0
16 16 16 16 \N \N 0 16 16 16 \N 16 \N 0
17 17 str_17 str_17 \N \N 0 17 17 str_17 str_17 \N \N 0
18 18 1970-01-19 1970-01-19 \N \N 0 18 18 1970-01-19 \N \N 1970-01-19 0
insert after alter modify column 2 alter modify column 2
1 Date 1 Date
5 String
5 UInt64 5 UInt64
8 String 8 None
0 0 \N \N \N \N 0
1 1 \N \N \N \N 0
2 2 \N \N \N \N 0
3 3 3 \N 3 \N 0
4 4 4 \N 4 \N 0
5 5 5 \N 5 \N 0
6 6 str_6 str_6 \N \N 0
7 7 str_7 str_7 \N \N 0
8 8 str_8 str_8 \N \N 0
9 9 \N \N \N \N 0
10 10 \N \N \N \N 0
11 11 \N \N \N \N 0
12 12 12 \N 12 \N 0
13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0
15 15 \N \N \N \N 0
16 16 16 \N 16 \N 0
17 17 str_17 str_17 \N \N 0
18 18 1970-01-19 \N \N 1970-01-19 0
insert after alter modify column 2
2 Date
6 String
6 UInt64
9 None 9 None
0 0 \N \N \N \N 0 0 0 \N \N \N \N 0
1 1 \N \N \N \N 0 1 1 \N \N \N \N 0
@ -107,17 +111,17 @@ insert after alter modify column 2
13 13 str_13 str_13 \N \N 0 13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0 14 14 \N \N \N \N 0
15 15 \N \N \N \N 0 15 15 \N \N \N \N 0
16 16 16 16 \N \N 0 16 16 16 \N 16 \N 0
17 17 str_17 str_17 \N \N 0 17 17 str_17 str_17 \N \N 0
18 18 1970-01-19 1970-01-19 \N \N 0 18 18 1970-01-19 \N \N 1970-01-19 0
19 19 \N \N \N \N 0 19 19 \N \N \N \N 0
20 20 20 \N 20 \N 0 20 20 20 \N 20 \N 0
21 21 str_21 str_21 \N \N 0 21 21 str_21 str_21 \N \N 0
22 22 1970-01-23 \N \N 1970-01-23 0 22 22 1970-01-23 \N \N 1970-01-23 0
alter modify column 3 alter modify column 3
1 Date 2 Date
5 UInt64 6 String
8 String 6 UInt64
9 None 9 None
0 0 0 \N 0 \N \N \N 0 0 0 0 \N 0 \N \N \N 0
1 1 1 \N 0 \N \N \N 0 1 1 1 \N 0 \N \N \N 0
@ -135,17 +139,17 @@ alter modify column 3
13 13 13 \N 0 str_13 \N \N 0 13 13 13 \N 0 str_13 \N \N 0
14 14 14 \N 0 \N \N \N 0 14 14 14 \N 0 \N \N \N 0
15 15 15 \N 0 \N \N \N 0 15 15 15 \N 0 \N \N \N 0
16 16 16 \N 0 16 \N \N 0 16 16 16 \N 0 \N 16 \N 0
17 17 17 \N 0 str_17 \N \N 0 17 17 17 \N 0 str_17 \N \N 0
18 18 18 \N 0 1970-01-19 \N \N 0 18 18 18 \N 0 \N \N 1970-01-19 0
19 19 19 \N 0 \N \N \N 0 19 19 19 \N 0 \N \N \N 0
20 20 20 \N 0 \N 20 \N 0 20 20 20 \N 0 \N 20 \N 0
21 21 21 \N 0 str_21 \N \N 0 21 21 21 \N 0 str_21 \N \N 0
22 22 22 \N 0 \N \N 1970-01-23 0 22 22 22 \N 0 \N \N 1970-01-23 0
insert after alter modify column 3 insert after alter modify column 3
1 Date 2 Date
5 UInt64 6 String
8 String 6 UInt64
12 None 12 None
0 0 0 \N 0 \N \N \N 0 0 0 0 \N 0 \N \N \N 0
1 1 1 \N 0 \N \N \N 0 1 1 1 \N 0 \N \N \N 0
@ -163,9 +167,9 @@ insert after alter modify column 3
13 13 13 \N 0 str_13 \N \N 0 13 13 13 \N 0 str_13 \N \N 0
14 14 14 \N 0 \N \N \N 0 14 14 14 \N 0 \N \N \N 0
15 15 15 \N 0 \N \N \N 0 15 15 15 \N 0 \N \N \N 0
16 16 16 \N 0 16 \N \N 0 16 16 16 \N 0 \N 16 \N 0
17 17 17 \N 0 str_17 \N \N 0 17 17 17 \N 0 str_17 \N \N 0
18 18 18 \N 0 1970-01-19 \N \N 0 18 18 18 \N 0 \N \N 1970-01-19 0
19 19 19 \N 0 \N \N \N 0 19 19 19 \N 0 \N \N \N 0
20 20 20 \N 0 \N 20 \N 0 20 20 20 \N 0 \N 20 \N 0
21 21 21 \N 0 str_21 \N \N 0 21 21 21 \N 0 str_21 \N \N 0

157
tests/queries/0_stateless/03040_dynamic_type_alters_1_wide_merge_tree.reference
View File

@ -24,147 +24,152 @@ insert after alter add column 1
13 13 str_13 str_13 \N \N 0 13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0 14 14 \N \N \N \N 0
alter modify column 1 alter modify column 1
4 String
4 UInt64
7 None 7 None
8 String
0 0 \N \N \N \N 0 0 0 \N \N \N \N 0
1 1 \N \N \N \N 0 1 1 \N \N \N \N 0
2 2 \N \N \N \N 0 2 2 \N \N \N \N 0
3 3 3 3 \N \N 0 3 3 3 \N 3 \N 0
4 4 4 4 \N \N 0 4 4 4 \N 4 \N 0
5 5 5 5 \N \N 0 5 5 5 \N 5 \N 0
6 6 str_6 str_6 \N \N 0 6 6 str_6 str_6 \N \N 0
7 7 str_7 str_7 \N \N 0 7 7 str_7 str_7 \N \N 0
8 8 str_8 str_8 \N \N 0 8 8 str_8 str_8 \N \N 0
9 9 \N \N \N \N 0 9 9 \N \N \N \N 0
10 10 \N \N \N \N 0 10 10 \N \N \N \N 0
11 11 \N \N \N \N 0 11 11 \N \N \N \N 0
12 12 12 12 \N \N 0 12 12 12 \N 12 \N 0
13 13 str_13 str_13 \N \N 0 13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0 14 14 \N \N \N \N 0
insert after alter modify column 1 insert after alter modify column 1
8 None
11 String
0 0 \N \N \N \N 0
1 1 \N \N \N \N 0
2 2 \N \N \N \N 0
3 3 3 3 \N \N 0
4 4 4 4 \N \N 0
5 5 5 5 \N \N 0
6 6 str_6 str_6 \N \N 0
7 7 str_7 str_7 \N \N 0
8 8 str_8 str_8 \N \N 0
9 9 \N \N \N \N 0
10 10 \N \N \N \N 0
11 11 \N \N \N \N 0
12 12 12 12 \N \N 0
13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0
15 15 \N \N \N \N 0
16 16 16 16 \N \N 0
17 17 str_17 str_17 \N \N 0
18 18 1970-01-19 1970-01-19 \N \N 0
alter modify column 2
8 None
11 String
0 0 \N \N \N \N 0
1 1 \N \N \N \N 0
2 2 \N \N \N \N 0
3 3 3 3 \N \N 0
4 4 4 4 \N \N 0
5 5 5 5 \N \N 0
6 6 str_6 str_6 \N \N 0
7 7 str_7 str_7 \N \N 0
8 8 str_8 str_8 \N \N 0
9 9 \N \N \N \N 0
10 10 \N \N \N \N 0
11 11 \N \N \N \N 0
12 12 12 12 \N \N 0
13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0
15 15 \N \N \N \N 0
16 16 16 16 \N \N 0
17 17 str_17 str_17 \N \N 0
18 18 1970-01-19 1970-01-19 \N \N 0
insert after alter modify column 2
1 Date 1 Date
1 UInt64 5 String
9 None 5 UInt64
12 String 8 None
0 0 \N \N \N \N 0 0 0 \N \N \N \N 0
1 1 \N \N \N \N 0 1 1 \N \N \N \N 0
2 2 \N \N \N \N 0 2 2 \N \N \N \N 0
3 3 3 3 \N \N 0 3 3 3 \N 3 \N 0
4 4 4 4 \N \N 0 4 4 4 \N 4 \N 0
5 5 5 5 \N \N 0 5 5 5 \N 5 \N 0
6 6 str_6 str_6 \N \N 0 6 6 str_6 str_6 \N \N 0
7 7 str_7 str_7 \N \N 0 7 7 str_7 str_7 \N \N 0
8 8 str_8 str_8 \N \N 0 8 8 str_8 str_8 \N \N 0
9 9 \N \N \N \N 0 9 9 \N \N \N \N 0
10 10 \N \N \N \N 0 10 10 \N \N \N \N 0
11 11 \N \N \N \N 0 11 11 \N \N \N \N 0
12 12 12 12 \N \N 0 12 12 12 \N 12 \N 0
13 13 str_13 str_13 \N \N 0 13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0 14 14 \N \N \N \N 0
15 15 \N \N \N \N 0 15 15 \N \N \N \N 0
16 16 16 16 \N \N 0 16 16 16 \N 16 \N 0
17 17 str_17 str_17 \N \N 0 17 17 str_17 str_17 \N \N 0
18 18 1970-01-19 1970-01-19 \N \N 0 18 18 1970-01-19 \N \N 1970-01-19 0
alter modify column 2
1 Date
5 String
5 UInt64
8 None
0 0 \N \N \N \N 0
1 1 \N \N \N \N 0
2 2 \N \N \N \N 0
3 3 3 \N 3 \N 0
4 4 4 \N 4 \N 0
5 5 5 \N 5 \N 0
6 6 str_6 str_6 \N \N 0
7 7 str_7 str_7 \N \N 0
8 8 str_8 str_8 \N \N 0
9 9 \N \N \N \N 0
10 10 \N \N \N \N 0
11 11 \N \N \N \N 0
12 12 12 \N 12 \N 0
13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0
15 15 \N \N \N \N 0
16 16 16 \N 16 \N 0
17 17 str_17 str_17 \N \N 0
18 18 1970-01-19 \N \N 1970-01-19 0
insert after alter modify column 2
2 Date
6 String
6 UInt64
9 None
0 0 \N \N \N \N 0
1 1 \N \N \N \N 0
2 2 \N \N \N \N 0
3 3 3 \N 3 \N 0
4 4 4 \N 4 \N 0
5 5 5 \N 5 \N 0
6 6 str_6 str_6 \N \N 0
7 7 str_7 str_7 \N \N 0
8 8 str_8 str_8 \N \N 0
9 9 \N \N \N \N 0
10 10 \N \N \N \N 0
11 11 \N \N \N \N 0
12 12 12 \N 12 \N 0
13 13 str_13 str_13 \N \N 0
14 14 \N \N \N \N 0
15 15 \N \N \N \N 0
16 16 16 \N 16 \N 0
17 17 str_17 str_17 \N \N 0
18 18 1970-01-19 \N \N 1970-01-19 0
19 19 \N \N \N \N 0 19 19 \N \N \N \N 0
20 20 20 \N 20 \N 0 20 20 20 \N 20 \N 0
21 21 str_21 str_21 \N \N 0 21 21 str_21 str_21 \N \N 0
22 22 1970-01-23 \N \N 1970-01-23 0 22 22 1970-01-23 \N \N 1970-01-23 0
alter modify column 3 alter modify column 3
1 Date 2 Date
1 UInt64 6 String
6 UInt64
9 None 9 None
12 String
0 0 0 \N 0 \N \N \N 0 0 0 0 \N 0 \N \N \N 0
1 1 1 \N 0 \N \N \N 0 1 1 1 \N 0 \N \N \N 0
2 2 2 \N 0 \N \N \N 0 2 2 2 \N 0 \N \N \N 0
3 3 3 \N 0 3 \N \N 0 3 3 3 \N 0 \N 3 \N 0
4 4 4 \N 0 4 \N \N 0 4 4 4 \N 0 \N 4 \N 0
5 5 5 \N 0 5 \N \N 0 5 5 5 \N 0 \N 5 \N 0
6 6 6 \N 0 str_6 \N \N 0 6 6 6 \N 0 str_6 \N \N 0
7 7 7 \N 0 str_7 \N \N 0 7 7 7 \N 0 str_7 \N \N 0
8 8 8 \N 0 str_8 \N \N 0 8 8 8 \N 0 str_8 \N \N 0
9 9 9 \N 0 \N \N \N 0 9 9 9 \N 0 \N \N \N 0
10 10 10 \N 0 \N \N \N 0 10 10 10 \N 0 \N \N \N 0
11 11 11 \N 0 \N \N \N 0 11 11 11 \N 0 \N \N \N 0
12 12 12 \N 0 12 \N \N 0 12 12 12 \N 0 \N 12 \N 0
13 13 13 \N 0 str_13 \N \N 0 13 13 13 \N 0 str_13 \N \N 0
14 14 14 \N 0 \N \N \N 0 14 14 14 \N 0 \N \N \N 0
15 15 15 \N 0 \N \N \N 0 15 15 15 \N 0 \N \N \N 0
16 16 16 \N 0 16 \N \N 0 16 16 16 \N 0 \N 16 \N 0
17 17 17 \N 0 str_17 \N \N 0 17 17 17 \N 0 str_17 \N \N 0
18 18 18 \N 0 1970-01-19 \N \N 0 18 18 18 \N 0 \N \N 1970-01-19 0
19 19 19 \N 0 \N \N \N 0 19 19 19 \N 0 \N \N \N 0
20 20 20 \N 0 \N 20 \N 0 20 20 20 \N 0 \N 20 \N 0
21 21 21 \N 0 str_21 \N \N 0 21 21 21 \N 0 str_21 \N \N 0
22 22 22 \N 0 \N \N 1970-01-23 0 22 22 22 \N 0 \N \N 1970-01-23 0
insert after alter modify column 3 insert after alter modify column 3
1 Date 2 Date
1 UInt64 6 String
6 UInt64
12 None 12 None
12 String
0 0 0 \N 0 \N \N \N 0 0 0 0 \N 0 \N \N \N 0
1 1 1 \N 0 \N \N \N 0 1 1 1 \N 0 \N \N \N 0
2 2 2 \N 0 \N \N \N 0 2 2 2 \N 0 \N \N \N 0
3 3 3 \N 0 3 \N \N 0 3 3 3 \N 0 \N 3 \N 0
4 4 4 \N 0 4 \N \N 0 4 4 4 \N 0 \N 4 \N 0
5 5 5 \N 0 5 \N \N 0 5 5 5 \N 0 \N 5 \N 0
6 6 6 \N 0 str_6 \N \N 0 6 6 6 \N 0 str_6 \N \N 0
7 7 7 \N 0 str_7 \N \N 0 7 7 7 \N 0 str_7 \N \N 0
8 8 8 \N 0 str_8 \N \N 0 8 8 8 \N 0 str_8 \N \N 0
9 9 9 \N 0 \N \N \N 0 9 9 9 \N 0 \N \N \N 0
10 10 10 \N 0 \N \N \N 0 10 10 10 \N 0 \N \N \N 0
11 11 11 \N 0 \N \N \N 0 11 11 11 \N 0 \N \N \N 0
12 12 12 \N 0 12 \N \N 0 12 12 12 \N 0 \N 12 \N 0
13 13 13 \N 0 str_13 \N \N 0 13 13 13 \N 0 str_13 \N \N 0
14 14 14 \N 0 \N \N \N 0 14 14 14 \N 0 \N \N \N 0
15 15 15 \N 0 \N \N \N 0 15 15 15 \N 0 \N \N \N 0
16 16 16 \N 0 16 \N \N 0 16 16 16 \N 0 \N 16 \N 0
17 17 17 \N 0 str_17 \N \N 0 17 17 17 \N 0 str_17 \N \N 0
18 18 18 \N 0 1970-01-19 \N \N 0 18 18 18 \N 0 \N \N 1970-01-19 0
19 19 19 \N 0 \N \N \N 0 19 19 19 \N 0 \N \N \N 0
20 20 20 \N 0 \N 20 \N 0 20 20 20 \N 0 \N 20 \N 0
21 21 21 \N 0 str_21 \N \N 0 21 21 21 \N 0 str_21 \N \N 0

2
tests/queries/0_stateless/03041_dynamic_type_check_table.sh
View File

@ -13,7 +13,7 @@ function run()
$CH_CLIENT -q "insert into test select number, number from numbers(3)" $CH_CLIENT -q "insert into test select number, number from numbers(3)"
echo "alter add column" echo "alter add column"
$CH_CLIENT -q "alter table test add column d Dynamic(max_types=3) settings mutations_sync=1" $CH_CLIENT -q "alter table test add column d Dynamic(max_types=2) settings mutations_sync=1"
$CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
$CH_CLIENT -q "select x, y, d, d.String, d.UInt64, d.\`Tuple(a UInt64)\`.a from test order by x" $CH_CLIENT -q "select x, y, d, d.String, d.UInt64, d.\`Tuple(a UInt64)\`.a from test order by x"

50
tests/queries/0_stateless/03150_dynamic_type_mv_insert.reference
View File

@ -1,35 +1,35 @@
1 2024-01-01 Date 1 2024-01-01 Date
2 1704056400 Decimal(18, 3) 2 1704056400 Decimal(18, 3)
3 1 String 3 1 Float32
4 2 String 4 2 Float64
1 2024-01-01 Date 1 2024-01-01 Date
1 2024-01-01 Date 1 2024-01-01 Date
2 1704056400 Decimal(18, 3) 2 1704056400 Decimal(18, 3)
2 1704056400 Decimal(18, 3) 2 1704056400 Decimal(18, 3)
3 1 String 3 1 Float32
3 1 String 3 1 Float32
4 2 String 4 2 Float64
4 2 String 4 2 Float64
1 2024-01-01 String
1 2024-01-01 String
2 1704056400 String
2 1704056400 String
3 1 String
3 1 String
4 2 String
4 2 String
1 2024-01-01 Date 1 2024-01-01 Date
1 2024-01-01 String 1 2024-01-01 Date
1 2024-01-01 String
2 1704056400 Decimal(18, 3) 2 1704056400 Decimal(18, 3)
2 1704056400 String 2 1704056400 Decimal(18, 3)
2 1704056400 String 3 1 Float32
3 1 String 3 1 Float32
3 1 String 4 2 Float64
3 1 String 4 2 Float64
4 2 String
4 2 String 1 2024-01-01 Date
4 2 String 1 2024-01-01 Date
1 2024-01-01 Date
2 1704056400 Decimal(18, 3)
2 1704056400 Decimal(18, 3)
2 1704056400 Decimal(18, 3)
3 1 Float32
3 1 Float32
3 1 Float32
4 2 Float64
4 2 Float64
4 2 Float64

7
tests/queries/0_stateless/03150_dynamic_type_mv_insert.sql
View File

@ -1,5 +1,6 @@
SET allow_experimental_dynamic_type=1; SET allow_experimental_dynamic_type=1;
DROP TABLE IF EXISTS null_table;
CREATE TABLE null_table CREATE TABLE null_table
( (
n1 UInt8, n1 UInt8,
@ -7,9 +8,11 @@ CREATE TABLE null_table
) )
ENGINE = Null; ENGINE = Null;
DROP VIEW IF EXISTS dummy_rmv;
CREATE MATERIALIZED VIEW dummy_rmv TO to_table CREATE MATERIALIZED VIEW dummy_rmv TO to_table
AS SELECT * FROM null_table; AS SELECT * FROM null_table;
DROP TABLE IF EXISTS to_table;
CREATE TABLE to_table CREATE TABLE to_table
( (
n1 UInt8, n1 UInt8,
@ -32,3 +35,7 @@ select '';
ALTER TABLE to_table MODIFY COLUMN n2 Dynamic(max_types=10); ALTER TABLE to_table MODIFY COLUMN n2 Dynamic(max_types=10);
INSERT INTO null_table ( n1, n2 ) VALUES (1, '2024-01-01'), (2, toDateTime64('2024-01-01', 3, 'Asia/Istanbul')), (3, toFloat32(1)), (4, toFloat64(2)); INSERT INTO null_table ( n1, n2 ) VALUES (1, '2024-01-01'), (2, toDateTime64('2024-01-01', 3, 'Asia/Istanbul')), (3, toFloat32(1)), (4, toFloat64(2));
SELECT *, dynamicType(n2) FROM to_table ORDER BY ALL; SELECT *, dynamicType(n2) FROM to_table ORDER BY ALL;
DROP TABLE null_table;
DROP VIEW dummy_rmv;
DROP TABLE to_table;

48
tests/queries/0_stateless/03151_dynamic_type_scale_max_types.reference
View File

@ -1,26 +1,26 @@
1 2024-01-01 Date 1 2024-01-01 Date false
2 1704056400 String 2 1704056400 Decimal(18, 3) false
3 1 String 3 1 Float32 true
4 2 String 4 2 Float64 true
1 2024-01-01 Date 1 2024-01-01 Date false
1 2024-01-01 Date 1 2024-01-01 Date false
2 1704056400 Decimal(18, 3) 2 1704056400 Decimal(18, 3) false
2 1704056400 String 2 1704056400 Decimal(18, 3) false
3 1 Float32 3 1 Float32 false
3 1 String 3 1 Float32 false
4 2 Float64 4 2 Float64 false
4 2 String 4 2 Float64 false
1 2024-01-01 String 1 2024-01-01 Date true
1 2024-01-01 String 1 2024-01-01 Date true
1 2024-01-01 String 1 2024-01-01 Date true
2 1704056400 String 2 1704056400 Decimal(18, 3) true
2 1704056400 String 2 1704056400 Decimal(18, 3) true
2 1704056400 String 2 1704056400 Decimal(18, 3) true
3 1 String 3 1 Float32 true
3 1 String 3 1 Float32 true
3 1 String 3 1 Float32 true
4 2 String 4 2 Float64 true
4 2 String 4 2 Float64 true
4 2 String 4 2 Float64 true

8
tests/queries/0_stateless/03151_dynamic_type_scale_max_types.sql
View File

@ -11,16 +11,16 @@ CREATE TABLE to_table
ENGINE = MergeTree ORDER BY n1; ENGINE = MergeTree ORDER BY n1;
INSERT INTO to_table ( n1, n2 ) VALUES (1, '2024-01-01'), (2, toDateTime64('2024-01-01', 3, 'Asia/Istanbul')), (3, toFloat32(1)), (4, toFloat64(2)); INSERT INTO to_table ( n1, n2 ) VALUES (1, '2024-01-01'), (2, toDateTime64('2024-01-01', 3, 'Asia/Istanbul')), (3, toFloat32(1)), (4, toFloat64(2));
SELECT *, dynamicType(n2) FROM to_table ORDER BY ALL; SELECT *, dynamicType(n2), isDynamicElementInSharedData(n2) FROM to_table ORDER BY ALL;
select ''; select '';
ALTER TABLE to_table MODIFY COLUMN n2 Dynamic(max_types=5); ALTER TABLE to_table MODIFY COLUMN n2 Dynamic(max_types=5);
INSERT INTO to_table ( n1, n2 ) VALUES (1, '2024-01-01'), (2, toDateTime64('2024-01-01', 3, 'Asia/Istanbul')), (3, toFloat32(1)), (4, toFloat64(2)); INSERT INTO to_table ( n1, n2 ) VALUES (1, '2024-01-01'), (2, toDateTime64('2024-01-01', 3, 'Asia/Istanbul')), (3, toFloat32(1)), (4, toFloat64(2));
SELECT *, dynamicType(n2) FROM to_table ORDER BY ALL; SELECT *, dynamicType(n2), isDynamicElementInSharedData(n2) FROM to_table ORDER BY ALL;
select ''; select '';
ALTER TABLE to_table MODIFY COLUMN n2 Dynamic(max_types=1); ALTER TABLE to_table MODIFY COLUMN n2 Dynamic(max_types=0);
INSERT INTO to_table ( n1, n2 ) VALUES (1, '2024-01-01'), (2, toDateTime64('2024-01-01', 3, 'Asia/Istanbul')), (3, toFloat32(1)), (4, toFloat64(2)); INSERT INTO to_table ( n1, n2 ) VALUES (1, '2024-01-01'), (2, toDateTime64('2024-01-01', 3, 'Asia/Istanbul')), (3, toFloat32(1)), (4, toFloat64(2));
SELECT *, dynamicType(n2) FROM to_table ORDER BY ALL; SELECT *, dynamicType(n2), isDynamicElementInSharedData(n2) FROM to_table ORDER BY ALL;
ALTER TABLE to_table MODIFY COLUMN n2 Dynamic(max_types=500); -- { serverError UNEXPECTED_AST_STRUCTURE } ALTER TABLE to_table MODIFY COLUMN n2 Dynamic(max_types=500); -- { serverError UNEXPECTED_AST_STRUCTURE }

2
tests/queries/0_stateless/03152_dynamic_type_simple.reference
View File

@ -7,7 +7,7 @@ string2 String
\N None 42 Int64 \N None 42 Int64
42 Int64 string String 42 Int64 string String
string String [1, 2] String string String [1,2] Array(Int64)
[1,2] Array(Int64) \N None [1,2] Array(Int64) \N None
┌─d────────────────────────┬─dynamicType(d)─┬─d.Int64─┬─d.String─┬─────d.Date─┬─d.Float64─┬──────────d.DateTime─┬─d.Array(Int64)─┬─d.Array(String)──────────┐ ┌─d────────────────────────┬─dynamicType(d)─┬─d.Int64─┬─d.String─┬─────d.Date─┬─d.Float64─┬──────────d.DateTime─┬─d.Array(Int64)─┬─d.Array(String)──────────┐
1. │ 42 │ Int64 │ 42 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ [] │ [] │ 1. │ 42 │ Int64 │ 42 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ [] │ [] │

8
tests/queries/0_stateless/03152_dynamic_type_simple.sql
View File

@ -1,14 +1,17 @@
SET allow_experimental_dynamic_type=1; SET allow_experimental_dynamic_type=1;
DROP TABLE IF EXISTS test_max_types;
CREATE TABLE test_max_types (d Dynamic(max_types=5)) ENGINE = Memory; CREATE TABLE test_max_types (d Dynamic(max_types=5)) ENGINE = Memory;
INSERT INTO test_max_types VALUES ('string1'), (42), (3.14), ([1, 2]), (toDate('2021-01-01')), ('string2'); INSERT INTO test_max_types VALUES ('string1'), (42), (3.14), ([1, 2]), (toDate('2021-01-01')), ('string2');
SELECT d, dynamicType(d) FROM test_max_types; SELECT d, dynamicType(d) FROM test_max_types;
SELECT ''; SELECT '';
DROP TABLE IF EXISTS test_nested_dynamic;
CREATE TABLE test_nested_dynamic (d1 Dynamic, d2 Dynamic(max_types=2)) ENGINE = Memory; CREATE TABLE test_nested_dynamic (d1 Dynamic, d2 Dynamic(max_types=2)) ENGINE = Memory;
INSERT INTO test_nested_dynamic VALUES (NULL, 42), (42, 'string'), ('string', [1, 2]), ([1, 2], NULL); INSERT INTO test_nested_dynamic VALUES (NULL, 42), (42, 'string'), ('string', [1, 2]), ([1, 2], NULL);
SELECT d1, dynamicType(d1), d2, dynamicType(d2) FROM test_nested_dynamic; SELECT d1, dynamicType(d1), d2, dynamicType(d2) FROM test_nested_dynamic;
DROP TABLE IF EXISTS test_rapid_schema;
CREATE TABLE test_rapid_schema (d Dynamic) ENGINE = Memory; CREATE TABLE test_rapid_schema (d Dynamic) ENGINE = Memory;
INSERT INTO test_rapid_schema VALUES (42), ('string1'), (toDate('2021-01-01')), ([1, 2, 3]), (3.14), ('string2'), (toDateTime('2021-01-01 12:00:00')), (['array', 'of', 'strings']), (NULL), (toFloat64(42.42)); INSERT INTO test_rapid_schema VALUES (42), ('string1'), (toDate('2021-01-01')), ([1, 2, 3]), (3.14), ('string2'), (toDateTime('2021-01-01 12:00:00')), (['array', 'of', 'strings']), (NULL), (toFloat64(42.42));
@ -27,3 +30,8 @@ FROM
FROM numbers(10000) FROM numbers(10000)
) )
); );
DROP TABLE test_max_types;
DROP TABLE test_nested_dynamic;
DROP TABLE test_rapid_schema;

2
tests/queries/0_stateless/03153_dynamic_type_empty.sql
View File

@ -1,5 +1,7 @@
SET allow_experimental_dynamic_type=1; SET allow_experimental_dynamic_type=1;
DROP TABLE IF EXISTS test_null_empty;
CREATE TABLE test_null_empty (d Dynamic) ENGINE = Memory; CREATE TABLE test_null_empty (d Dynamic) ENGINE = Memory;
INSERT INTO test_null_empty VALUES ([]), ([1]), ([]), (['1']), ([]), (()),((1)), (()), (('1')), (()), ({}), ({1:2}), ({}), ({'1':'2'}), ({}); INSERT INTO test_null_empty VALUES ([]), ([1]), ([]), (['1']), ([]), (()),((1)), (()), (('1')), (()), ({}), ({1:2}), ({}), ({'1':'2'}), ({});
SELECT d, dynamicType(d) FROM test_null_empty; SELECT d, dynamicType(d) FROM test_null_empty;
DROP TABLE test_null_empty;

4
tests/queries/0_stateless/03159_dynamic_type_all_types.sql
View File

@ -6,7 +6,7 @@ SET allow_experimental_variant_type=1;
SET allow_suspicious_low_cardinality_types=1; SET allow_suspicious_low_cardinality_types=1;
CREATE TABLE t (d Dynamic(max_types=255)) ENGINE = Memory; CREATE TABLE t (d Dynamic(max_types=254)) ENGINE = Memory;
-- Integer types: signed and unsigned integers (UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256) -- Integer types: signed and unsigned integers (UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256)
INSERT INTO t VALUES (-128::Int8), (-127::Int8), (-1::Int8), (0::Int8), (1::Int8), (126::Int8), (127::Int8); INSERT INTO t VALUES (-128::Int8), (-127::Int8), (-1::Int8), (0::Int8), (1::Int8), (126::Int8), (127::Int8);
INSERT INTO t VALUES (-128::Int8), (-127::Int8), (-1::Int8), (0::Int8), (1::Int8), (126::Int8), (127::Int8); INSERT INTO t VALUES (-128::Int8), (-127::Int8), (-1::Int8), (0::Int8), (1::Int8), (126::Int8), (127::Int8);
@ -84,7 +84,7 @@ INSERT INTO t VALUES ([(1, (2, ['aa', 'bb']), [(3, 'cc'), (4, 'dd')]), (5, (6, [
SELECT dynamicType(d), d FROM t ORDER BY substring(dynamicType(d),1,1), length(dynamicType(d)), d; SELECT dynamicType(d), d FROM t ORDER BY substring(dynamicType(d),1,1), length(dynamicType(d)), d;
CREATE TABLE t2 (d Dynamic(max_types=255)) ENGINE = Memory; CREATE TABLE t2 (d Dynamic(max_types=254)) ENGINE = Memory;
INSERT INTO t2 SELECT * FROM t; INSERT INTO t2 SELECT * FROM t;
SELECT ''; SELECT '';

4
tests/queries/0_stateless/03172_dynamic_binary_serialization.sh
View File

@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/../shell_config.sh . "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -q "drop table if exists test" $CLICKHOUSE_CLIENT -q "drop table if exists test"
$CLICKHOUSE_CLIENT --allow_experimental_dynamic_type=1 -q "create table test (id UInt64, d Dynamic(max_types=255)) engine=Memory" $CLICKHOUSE_CLIENT --allow_experimental_dynamic_type=1 -q "create table test (id UInt64, d Dynamic(max_types=254)) engine=Memory"
$CLICKHOUSE_CLIENT -q "insert into test select 0, NULL" $CLICKHOUSE_CLIENT -q "insert into test select 0, NULL"
$CLICKHOUSE_CLIENT -q "insert into test select 1, materialize(42)::UInt8" $CLICKHOUSE_CLIENT -q "insert into test select 1, materialize(42)::UInt8"
@ -58,6 +58,6 @@ $CLICKHOUSE_CLIENT -q "insert into test select 47, materialize([[(20, 20), (50,
$CLICKHOUSE_CLIENT -q "insert into test select 48, materialize([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]])::MultiPolygon" $CLICKHOUSE_CLIENT -q "insert into test select 48, materialize([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]])::MultiPolygon"
$CLICKHOUSE_CLIENT -q "insert into test select 49, materialize([map(42, tuple(1, [tuple(2, map(1, 2))]))])" $CLICKHOUSE_CLIENT -q "insert into test select 49, materialize([map(42, tuple(1, [tuple(2, map(1, 2))]))])"
$CLICKHOUSE_CLIENT -q "select * from test format RowBinary" | $CLICKHOUSE_LOCAL --allow_experimental_dynamic_type=1 --input-format RowBinary --structure 'id UInt64, d Dynamic(max_types=255)' -q "select d, dynamicType(d) from table order by id" $CLICKHOUSE_CLIENT -q "select * from test format RowBinary" | $CLICKHOUSE_LOCAL --allow_experimental_dynamic_type=1 --input-format RowBinary --structure 'id UInt64, d Dynamic(max_types=254)' -q "select d, dynamicType(d) from table order by id"
$CLICKHOUSE_CLIENT -q "drop table test" $CLICKHOUSE_CLIENT -q "drop table test"

4
tests/queries/0_stateless/03173_row_binary_and_native_with_binary_encoded_types.reference
View File

@ -110,5 +110,5 @@ true Bool
[{42:(1,[(2,{1:2})])}] Dynamic [{42:(1,[(2,{1:2})])}] Dynamic
[{42:(1,[(2,{1:2})])}] Dynamic(max_types=10) [{42:(1,[(2,{1:2})])}] Dynamic(max_types=10)
[{42:(1,[(2,{1:2})])}] Dynamic(max_types=10) [{42:(1,[(2,{1:2})])}] Dynamic(max_types=10)
[{42:(1,[(2,{1:2})])}] Dynamic(max_types=255) [{42:(1,[(2,{1:2})])}] Dynamic(max_types=254)
[{42:(1,[(2,{1:2})])}] Dynamic(max_types=255) [{42:(1,[(2,{1:2})])}] Dynamic(max_types=254)

6
tests/queries/0_stateless/03173_row_binary_and_native_with_binary_encoded_types.sh
View File

@ -6,8 +6,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
function test function test
{ {
$CLICKHOUSE_LOCAL --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --output_format_binary_encode_types_in_binary_format=1 -q "select $1 as value format RowBinaryWithNamesAndTypes" | $CLICKHOUSE_LOCAL --input-format RowBinaryWithNamesAndTypes --input_format_binary_decode_types_in_binary_format=1 -q "select value, toTypeName(value) from table" $CLICKHOUSE_LOCAL --stacktrace --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --output_format_binary_encode_types_in_binary_format=1 -q "select $1 as value format RowBinaryWithNamesAndTypes" | $CLICKHOUSE_LOCAL --input-format RowBinaryWithNamesAndTypes --input_format_binary_decode_types_in_binary_format=1 -q "select value, toTypeName(value) from table"
$CLICKHOUSE_LOCAL --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --output_format_native_encode_types_in_binary_format=1 -q "select $1 as value format Native" | $CLICKHOUSE_LOCAL --input-format Native --input_format_native_decode_types_in_binary_format=1 -q "select value, toTypeName(value) from table" $CLICKHOUSE_LOCAL --stacktrace --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --output_format_native_encode_types_in_binary_format=1 -q "select $1 as value format Native" | $CLICKHOUSE_LOCAL --input-format Native --input_format_native_decode_types_in_binary_format=1 -q "select value, toTypeName(value) from table"
} }
test "materialize(42)::UInt8" test "materialize(42)::UInt8"
@ -66,4 +66,4 @@ test "materialize([map(42, tuple(1, [tuple(2, map(1, 2))]))])"
test "materialize(42::UInt32)::Variant(UInt32, String, Tuple(a UInt32, b Array(Map(String, String))))" test "materialize(42::UInt32)::Variant(UInt32, String, Tuple(a UInt32, b Array(Map(String, String))))"
test "materialize([map(42, tuple(1, [tuple(2, map(1, 2))]))])::Dynamic" test "materialize([map(42, tuple(1, [tuple(2, map(1, 2))]))])::Dynamic"
test "materialize([map(42, tuple(1, [tuple(2, map(1, 2))]))])::Dynamic(max_types=10)" test "materialize([map(42, tuple(1, [tuple(2, map(1, 2))]))])::Dynamic(max_types=10)"
test "materialize([map(42, tuple(1, [tuple(2, map(1, 2))]))])::Dynamic(max_types=255)" test "materialize([map(42, tuple(1, [tuple(2, map(1, 2))]))])::Dynamic(max_types=254)"

20
tests/queries/0_stateless/03200_memory_engine_alter_dynamic.reference
View File

@ -1,10 +1,10 @@
\N 0
\N 1
\N 2
\N 3
\N 4
\N 0
\N 1
\N 2
\N 3
\N 4

4
tests/queries/0_stateless/03200_memory_engine_alter_dynamic.sql
View File

@ -1,6 +1,8 @@
set allow_experimental_dynamic_type=1; set allow_experimental_dynamic_type=1;
drop table if exists test;
create table test (d Dynamic) engine=Memory; create table test (d Dynamic) engine=Memory;
insert into table test select * from numbers(5); insert into table test select * from numbers(5);
alter table test modify column d Dynamic(max_types=1); alter table test modify column d Dynamic(max_types=0);
select d.UInt64 from test settings enable_analyzer=1; select d.UInt64 from test settings enable_analyzer=1;
select d.UInt64 from test settings enable_analyzer=0; select d.UInt64 from test settings enable_analyzer=0;
drop table test;

20
tests/queries/0_stateless/03210_dynamic_squashing.reference
View File

@ -1,8 +1,12 @@
Array(UInt8) 1
None Array(UInt8) true
UInt64 None false
None UInt64 false
String 2
UInt64 Array(UInt8) true
String None false
UInt64 UInt64 false
3
Array(UInt8) true
String false
UInt64 true

19
tests/queries/0_stateless/03210_dynamic_squashing.sql
View File

@ -1,20 +1,25 @@
-- Tags: long
set allow_experimental_dynamic_type = 1; set allow_experimental_dynamic_type = 1;
set max_block_size = 1000; set max_block_size = 1000;
drop table if exists test; drop table if exists test;
create table test (d Dynamic) engine=MergeTree order by tuple(); create table test (d Dynamic) engine=MergeTree order by tuple();
insert into test select multiIf(number < 1000, NULL::Dynamic(max_types=2), number < 3000, range(number % 5)::Dynamic(max_types=2), number::Dynamic(max_types=2)) from numbers(1000000); insert into test select multiIf(number < 1000, NULL::Dynamic(max_types=1), number < 3000, range(number % 5)::Dynamic(max_types=1), number::Dynamic(max_types=1)) from numbers(100000);
select distinct dynamicType(d) as type from test order by type; select '1';
select distinct dynamicType(d) as type, isDynamicElementInSharedData(d) as flag from test order by type;
drop table test; drop table test;
create table test (d Dynamic(max_types=2)) engine=MergeTree order by tuple(); create table test (d Dynamic(max_types=1)) engine=MergeTree order by tuple();
insert into test select multiIf(number < 1000, NULL::Dynamic(max_types=2), number < 3000, range(number % 5)::Dynamic(max_types=2), number::Dynamic(max_types=2)) from numbers(1000000); insert into test select multiIf(number < 1000, NULL::Dynamic(max_types=1), number < 3000, range(number % 5)::Dynamic(max_types=1), number::Dynamic(max_types=1)) from numbers(100000);
select distinct dynamicType(d) as type from test order by type; select '2';
select distinct dynamicType(d) as type, isDynamicElementInSharedData(d) as flag from test order by type;
truncate table test; truncate table test;
insert into test select multiIf(number < 1000, 'Str'::Dynamic(max_types=2), number < 3000, range(number % 5)::Dynamic(max_types=2), number::Dynamic(max_types=2)) from numbers(1000000); insert into test select multiIf(number < 1000, 'Str'::Dynamic(max_types=1), number < 3000, range(number % 5)::Dynamic(max_types=1), number::Dynamic(max_types=1)) from numbers(100000);
select distinct dynamicType(d) as type from test order by type; select '3';
select distinct dynamicType(d) as type, isDynamicElementInSharedData(d) as flag from test order by type;
drop table test; drop table test;