thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into tighten-limits-functional-tests

Browse Source
This commit is contained in:
Alexey Milovidov 2024-08-16 00:47:45 +02:00
commit 60b09ecaf2
101 changed files with 1404 additions and 430 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
docs/en/interfaces/schema-inference.md
View File

@ -359,13 +359,14 @@ DESC format(JSONEachRow, '{"int" : 42, "float" : 42.42, "string" : "Hello, World
Dates, DateTimes:
```sql
DESC format(JSONEachRow, '{"date" : "2022-01-01", "datetime" : "2022-01-01 00:00:00"}')
DESC format(JSONEachRow, '{"date" : "2022-01-01", "datetime" : "2022-01-01 00:00:00", "datetime64" : "2022-01-01 00:00:00.000"}')
```
```response
┌─name─────┬─type────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
│ date │ Nullable(Date) │ │ │ │ │ │
│ datetime │ Nullable(DateTime64(9)) │ │ │ │ │ │
└──────────┴─────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
┌─name───────┬─type────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
│ date │ Nullable(Date) │ │ │ │ │ │
│ datetime │ Nullable(DateTime) │ │ │ │ │ │
│ datetime64 │ Nullable(DateTime64(9)) │ │ │ │ │ │
└────────────┴─────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
```
Arrays:
@ -759,12 +760,13 @@ DESC format(CSV, 'Hello world!,World hello!')
Dates, DateTimes:
```sql
DESC format(CSV, '"2020-01-01","2020-01-01 00:00:00"')
DESC format(CSV, '"2020-01-01","2020-01-01 00:00:00","2022-01-01 00:00:00.000"')
```
```response
┌─name─┬─type────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
│ c1 │ Nullable(Date) │ │ │ │ │ │
│ c2 │ Nullable(DateTime64(9)) │ │ │ │ │ │
│ c2 │ Nullable(DateTime) │ │ │ │ │ │
│ c3 │ Nullable(DateTime64(9)) │ │ │ │ │ │
└──────┴─────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
```
@ -956,12 +958,13 @@ DESC format(TSKV, 'int=42 float=42.42 bool=true string=Hello,World!\n')
Dates, DateTimes:
```sql
DESC format(TSV, '2020-01-01 2020-01-01 00:00:00')
DESC format(TSV, '2020-01-01 2020-01-01 00:00:00 2022-01-01 00:00:00.000')
```
```response
┌─name─┬─type────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
│ c1 │ Nullable(Date) │ │ │ │ │ │
│ c2 │ Nullable(DateTime64(9)) │ │ │ │ │ │
│ c2 │ Nullable(DateTime) │ │ │ │ │ │
│ c3 │ Nullable(DateTime64(9)) │ │ │ │ │ │
└──────┴─────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
```
@ -1126,12 +1129,13 @@ DESC format(Values, $$(42, 42.42, true, 'Hello,World!')$$)
Dates, DateTimes:
```sql
DESC format(Values, $$('2020-01-01', '2020-01-01 00:00:00')$$)
```
DESC format(Values, $$('2020-01-01', '2020-01-01 00:00:00', '2022-01-01 00:00:00.000')$$)
```
```response
┌─name─┬─type────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
│ c1 │ Nullable(Date) │ │ │ │ │ │
│ c2 │ Nullable(DateTime64(9)) │ │ │ │ │ │
│ c2 │ Nullable(DateTime) │ │ │ │ │ │
│ c3 │ Nullable(DateTime64(9)) │ │ │ │ │ │
└──────┴─────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
```
@ -1504,8 +1508,8 @@ DESC format(JSONEachRow, $$
#### input_format_try_infer_datetimes
If enabled, ClickHouse will try to infer type `DateTime64` from string fields in schema inference for text formats.
If all fields from a column in sample data were successfully parsed as datetimes, the result type will be `DateTime64(9)`,
If enabled, ClickHouse will try to infer type `DateTime` or `DateTime64` from string fields in schema inference for text formats.
If all fields from a column in sample data were successfully parsed as datetimes, the result type will be `DateTime` or `DateTime64(9)` (if any datetime had fractional part),
if at least one field was not parsed as datetime, the result type will be `String`.
Enabled by default.
@ -1513,39 +1517,66 @@ Enabled by default.
**Examples**
```sql
SET input_format_try_infer_datetimes = 0
SET input_format_try_infer_datetimes = 0;
DESC format(JSONEachRow, $$
{"datetime" : "2021-01-01 00:00:00.000"}
{"datetime" : "2022-01-01 00:00:00.000"}
{"datetime" : "2021-01-01 00:00:00", "datetime64" : "2021-01-01 00:00:00.000"}
{"datetime" : "2022-01-01 00:00:00", "datetime64" : "2022-01-01 00:00:00.000"}
$$)
```
```response
┌─name─────┬─type─────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
│ datetime │ Nullable(String) │ │ │ │ │ │
└──────────┴──────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
┌─name───────┬─type─────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
│ datetime │ Nullable(String) │ │ │ │ │ │
│ datetime64 │ Nullable(String) │ │ │ │ │ │
└────────────┴──────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
```
```sql
SET input_format_try_infer_datetimes = 1
SET input_format_try_infer_datetimes = 1;
DESC format(JSONEachRow, $$
{"datetime" : "2021-01-01 00:00:00.000"}
{"datetime" : "2022-01-01 00:00:00.000"}
{"datetime" : "2021-01-01 00:00:00", "datetime64" : "2021-01-01 00:00:00.000"}
{"datetime" : "2022-01-01 00:00:00", "datetime64" : "2022-01-01 00:00:00.000"}
$$)
```
```response
┌─name─────┬─type────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
│ datetime │ Nullable(DateTime64(9)) │ │ │ │ │ │
└──────────┴─────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
┌─name───────┬─type────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
│ datetime │ Nullable(DateTime) │ │ │ │ │ │
│ datetime64 │ Nullable(DateTime64(9)) │ │ │ │ │ │
└────────────┴─────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
```
```sql
DESC format(JSONEachRow, $$
{"datetime" : "2021-01-01 00:00:00.000"}
{"datetime" : "unknown"}
{"datetime" : "2021-01-01 00:00:00", "datetime64" : "2021-01-01 00:00:00.000"}
{"datetime" : "unknown", "datetime64" : "unknown"}
$$)
```
```response
┌─name─────┬─type─────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
│ datetime │ Nullable(String) │ │ │ │ │ │
└──────────┴──────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
┌─name───────┬─type─────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
│ datetime │ Nullable(String) │ │ │ │ │ │
│ datetime64 │ Nullable(String) │ │ │ │ │ │
└────────────┴──────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
```
#### input_format_try_infer_datetimes_only_datetime64
If enabled, ClickHouse will always infer `DateTime64(9)` when `input_format_try_infer_datetimes` is enabled even if datetime values don't contain fractional part.
Disabled by default.
**Examples**
```sql
SET input_format_try_infer_datetimes = 1;
SET input_format_try_infer_datetimes_only_datetime64 = 1;
DESC format(JSONEachRow, $$
{"datetime" : "2021-01-01 00:00:00", "datetime64" : "2021-01-01 00:00:00.000"}
{"datetime" : "2022-01-01 00:00:00", "datetime64" : "2022-01-01 00:00:00.000"}
$$)
```
```text
┌─name───────┬─type────────────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
│ datetime │ Nullable(DateTime64(9)) │ │ │ │ │ │
│ datetime64 │ Nullable(DateTime64(9)) │ │ │ │ │ │
└────────────┴─────────────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
```
Note: Parsing datetimes during schema inference respect setting [date_time_input_format](/docs/en/operations/settings/settings-formats.md#date_time_input_format)

6
docs/en/operations/settings/settings.md
View File

@ -5654,3 +5654,9 @@ Possible values:
- 1 — the [TimeSeries](../../engines/table-engines/integrations/time-series.md) table engine is enabled.
Default value: `0`.
## create_if_not_exists
Enable `IF NOT EXISTS` for `CREATE` statement by default. If either this setting or `IF NOT EXISTS` is specified and a table with the provided name already exists, no exception will be thrown.
Default value: `false`.

3
src/Analyzer/FunctionNode.cpp
View File

@ -242,7 +242,8 @@ ASTPtr FunctionNode::toASTImpl(const ConvertToASTOptions & options) const
/// Avoid cast for `IN tuple(...)` expression.
/// Tuples could be quite big, and adding a type may significantly increase query size.
/// It should be safe because set type for `column IN tuple` is deduced from `column` type.
if (isNameOfInFunction(function_name) && argument_nodes.size() > 1 && argument_nodes[1]->getNodeType() == QueryTreeNodeType::CONSTANT)
if (isNameOfInFunction(function_name) && argument_nodes.size() > 1 && argument_nodes[1]->getNodeType() == QueryTreeNodeType::CONSTANT
&& !static_cast<const ConstantNode *>(argument_nodes[1].get())->hasSourceExpression())
new_options.add_cast_for_constants = false;
const auto & parameters = getParameters();

4
src/Columns/ColumnLowCardinality.h
View File

@ -46,8 +46,8 @@ public:
return Base::create(std::move(column_unique), std::move(indexes), is_shared);
}
std::string getName() const override { return "ColumnLowCardinality"; }
const char * getFamilyName() const override { return "ColumnLowCardinality"; }
std::string getName() const override { return "LowCardinality(" + getDictionary().getNestedColumn()->getName() + ")"; }
const char * getFamilyName() const override { return "LowCardinality"; }
TypeIndex getDataType() const override { return TypeIndex::LowCardinality; }
ColumnPtr convertToFullColumn() const { return getDictionary().getNestedColumn()->index(getIndexes(), 0); }

2
src/Columns/ColumnUnique.h
View File

@ -48,6 +48,8 @@ private:
ColumnUnique(const ColumnUnique & other);
public:
std::string getName() const override { return "Unique(" + getNestedColumn()->getName() + ")"; }
MutableColumnPtr cloneEmpty() const override;
const ColumnPtr & getNestedColumn() const override;

2
src/Columns/IColumnUnique.h
View File

@ -73,7 +73,7 @@ public:
/// Returns dictionary hash which is SipHash is applied to each row of nested column.
virtual UInt128 getHash() const = 0;
const char * getFamilyName() const override { return "ColumnUnique"; }
const char * getFamilyName() const override { return "Unique"; }
TypeIndex getDataType() const override { return getNestedColumn()->getDataType(); }
void insert(const Field &) override

2
src/Columns/tests/gtest_column_dump_structure.cpp
View File

@ -10,7 +10,7 @@ TEST(IColumn, dumpStructure)
{
auto type_lc = std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>());
ColumnPtr column_lc = type_lc->createColumn();
String expected_structure = "ColumnLowCardinality(size = 0, UInt8(size = 0), ColumnUnique(size = 1, String(size = 1)))";
String expected_structure = "LowCardinality(size = 0, UInt8(size = 0), Unique(size = 1, String(size = 1)))";
std::vector<std::thread> threads;
for (size_t i = 0; i < 6; ++i)

42
src/Common/OptimizedRegularExpression.cpp
View File

@ -244,33 +244,43 @@ const char * analyzeImpl(
is_trivial = false;
if (!in_square_braces)
{
/// Check for case-insensitive flag.
if (pos + 1 < end && pos[1] == '?')
/// it means flag negation
/// there are various possible flags
/// actually only imsU are supported by re2
auto is_flag_char = [](char x)
{
for (size_t offset = 2; pos + offset < end; ++offset)
return x == '-' || x == 'i' || x == 'm' || x == 's' || x == 'U' || x == 'u';
};
/// Check for case-insensitive flag.
if (pos + 2 < end && pos[1] == '?' && is_flag_char(pos[2]))
{
size_t offset = 2;
for (; pos + offset < end; ++offset)
{
if (pos[offset] == '-' /// it means flag negation
/// various possible flags, actually only imsU are supported by re2
|| (pos[offset] >= 'a' && pos[offset] <= 'z')
|| (pos[offset] >= 'A' && pos[offset] <= 'Z'))
if (pos[offset] == 'i')
{
if (pos[offset] == 'i')
{
/// Actually it can be negated case-insensitive flag. But we don't care.
has_case_insensitive_flag = true;
break;
}
/// Actually it can be negated case-insensitive flag. But we don't care.
has_case_insensitive_flag = true;
}
else
else if (!is_flag_char(pos[offset]))
break;
}
pos += offset;
if (pos == end)
return pos;
/// if this group only contains flags, we have nothing to do.
if (*pos == ')')
{
++pos;
break;
}
}
/// (?:regex) means non-capturing parentheses group
if (pos + 2 < end && pos[1] == '?' && pos[2] == ':')
else if (pos + 2 < end && pos[1] == '?' && pos[2] == ':')
{
pos += 2;
}
if (pos + 3 < end && pos[1] == '?' && (pos[2] == '<' || pos[2] == '\'' || (pos[2] == 'P' && pos[3] == '<')))
else if (pos + 3 < end && pos[1] == '?' && (pos[2] == '<' || pos[2] == '\'' || (pos[2] == 'P' && pos[3] == '<')))
{
pos = skipNameCapturingGroup(pos, pos[2] == 'P' ? 3: 2, end);
}

3
src/Common/tests/gtest_optimize_re.cpp
View File

@ -19,6 +19,9 @@ TEST(OptimizeRE, analyze)
};
test_f("abc", "abc", {}, true, true);
test_f("c([^k]*)de", "");
test_f("(?-s)bob", "bob", {}, false, true);
test_f("(?s)bob", "bob", {}, false, true);
test_f("(?ssss", "");
test_f("abc(de)fg", "abcdefg", {}, false, true);
test_f("abc(de|xyz)fg", "abc", {"abcdefg", "abcxyzfg"}, false, true);
test_f("abc(de?f|xyz)fg", "abc", {"abcd", "abcxyzfg"}, false, true);

2
src/Core/Settings.h
View File

@ -897,6 +897,7 @@ class IColumn;
M(UInt64, extract_key_value_pairs_max_pairs_per_row, 1000, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory.", 0) ALIAS(extract_kvp_max_pairs_per_row) \
M(Bool, restore_replace_external_engines_to_null, false, "Replace all the external table engines to Null on restore. Useful for testing purposes", 0) \
M(Bool, restore_replace_external_table_functions_to_null, false, "Replace all table functions to Null on restore. Useful for testing purposes", 0) \
M(Bool, create_if_not_exists, false, "Enable IF NOT EXISTS for CREATE statements by default", 0) \
\
\
/* ###################################### */ \
@ -1136,6 +1137,7 @@ class IColumn;
M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \
M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \
M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \
M(Bool, input_format_try_infer_datetimes_only_datetime64, false, "When input_format_try_infer_datetimes is enabled, infer only DateTime64 but not DateTime types", 0) \
M(Bool, input_format_try_infer_exponent_floats, false, "Try to infer floats in exponential notation while schema inference in text formats (except JSON, where exponent numbers are always inferred)", 0) \
M(Bool, output_format_markdown_escape_special_characters, false, "Escape special characters in Markdown", 0) \
M(Bool, input_format_protobuf_flatten_google_wrappers, false, "Enable Google wrappers for regular non-nested columns, e.g. google.protobuf.StringValue 'str' for String column 'str'. For Nullable columns empty wrappers are recognized as defaults, and missing as nulls", 0) \

2
src/Core/SettingsChangesHistory.cpp
View File

@ -75,6 +75,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
},
{"24.8",
{
{"create_if_not_exists", false, false, "New setting."},
{"rows_before_aggregation", true, true, "Provide exact value for rows_before_aggregation statistic, represents the number of rows read before aggregation"},
{"restore_replace_external_table_functions_to_null", false, false, "New setting."},
{"restore_replace_external_engines_to_null", false, false, "New setting."},
@ -89,6 +90,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
{"optimize_functions_to_subcolumns", false, true, "Enabled settings by default"},
{"join_output_by_rowlist_perkey_rows_threshold", 0, 5, "The lower limit of per-key average rows in the right table to determine whether to output by row list in hash join."},
{"allow_experimental_vector_similarity_index", false, false, "Added new setting to allow experimental vector similarity indexes"},
{"input_format_try_infer_datetimes_only_datetime64", true, false, "Allow to infer DateTime instead of DateTime64 in data formats"}
}
},
{"24.7",

2
src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
View File

@ -1031,7 +1031,7 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep()
}
else
LOG_TRACE(log, "No space left in cache to reserve {} bytes, reason: {}, "
"will continue without cache download", failure_reason, size);
"will continue without cache download", size, failure_reason);
if (!success)
{

2
src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
View File

@ -63,7 +63,7 @@ void throwIfError(const Aws::Utils::Outcome<Result, Error> & response)
{
const auto & err = response.GetError();
throw S3Exception(
fmt::format("{} (Code: {}, s3 exception: {})",
fmt::format("{} (Code: {}, S3 exception: '{}')",
err.GetMessage(), static_cast<size_t>(err.GetErrorType()), err.GetExceptionName()),
err.GetErrorType());
}

5
src/Formats/EscapingRuleUtils.cpp
View File

@ -419,10 +419,11 @@ String getAdditionalFormatInfoByEscapingRule(const FormatSettings & settings, Fo
String result = getAdditionalFormatInfoForAllRowBasedFormats(settings);
/// First, settings that are common for all text formats:
result += fmt::format(
", try_infer_integers={}, try_infer_dates={}, try_infer_datetimes={}",
", try_infer_integers={}, try_infer_dates={}, try_infer_datetimes={}, try_infer_datetimes_only_datetime64={}",
settings.try_infer_integers,
settings.try_infer_dates,
settings.try_infer_datetimes);
settings.try_infer_datetimes,
settings.try_infer_datetimes_only_datetime64);
/// Second, format-specific settings:
switch (escaping_rule)

1
src/Formats/FormatFactory.cpp
View File

@ -266,6 +266,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
format_settings.try_infer_integers = settings.input_format_try_infer_integers;
format_settings.try_infer_dates = settings.input_format_try_infer_dates;
format_settings.try_infer_datetimes = settings.input_format_try_infer_datetimes;
format_settings.try_infer_datetimes_only_datetime64 = settings.input_format_try_infer_datetimes_only_datetime64;
format_settings.try_infer_exponent_floats = settings.input_format_try_infer_exponent_floats;
format_settings.markdown.escape_special_characters = settings.output_format_markdown_escape_special_characters;
format_settings.bson.output_string_as_string = settings.output_format_bson_string_as_string;

1
src/Formats/FormatSettings.h
View File

@ -46,6 +46,7 @@ struct FormatSettings
bool try_infer_integers = true;
bool try_infer_dates = true;
bool try_infer_datetimes = true;
bool try_infer_datetimes_only_datetime64 = false;
bool try_infer_exponent_floats = false;
enum class DateTimeInputFormat : uint8_t

119
src/Formats/SchemaInferenceUtils.cpp
View File

@ -306,37 +306,45 @@ namespace
type_indexes.erase(TypeIndex::UInt64);
}
/// If we have only Date and DateTime types, convert Date to DateTime,
/// otherwise, convert all Date and DateTime to String.
/// If we have only date/datetimes types (Date/DateTime/DateTime64), convert all of them to the common type,
/// otherwise, convert all Date, DateTime and DateTime64 to String.
void transformDatesAndDateTimes(DataTypes & data_types, TypeIndexesSet & type_indexes)
{
bool have_dates = type_indexes.contains(TypeIndex::Date);
bool have_datetimes = type_indexes.contains(TypeIndex::DateTime64);
bool all_dates_or_datetimes = (type_indexes.size() == (static_cast<size_t>(have_dates) + static_cast<size_t>(have_datetimes)));
bool have_datetimes = type_indexes.contains(TypeIndex::DateTime);
bool have_datetimes64 = type_indexes.contains(TypeIndex::DateTime64);
bool all_dates_or_datetimes = (type_indexes.size() == (static_cast<size_t>(have_dates) + static_cast<size_t>(have_datetimes) + static_cast<size_t>(have_datetimes64)));
if (!all_dates_or_datetimes && (have_dates || have_datetimes))
if (!all_dates_or_datetimes && (have_dates || have_datetimes || have_datetimes64))
{
for (auto & type : data_types)
{
if (isDate(type) || isDateTime64(type))
if (isDate(type) || isDateTime(type) || isDateTime64(type))
type = std::make_shared<DataTypeString>();
}
type_indexes.erase(TypeIndex::Date);
type_indexes.erase(TypeIndex::DateTime);
type_indexes.erase(TypeIndex::DateTime64);
type_indexes.insert(TypeIndex::String);
return;
}
if (have_dates && have_datetimes)
for (auto & type : data_types)
{
for (auto & type : data_types)
if (isDate(type) && (have_datetimes || have_datetimes64))
{
if (isDate(type))
if (have_datetimes64)
type = std::make_shared<DataTypeDateTime64>(9);
else
type = std::make_shared<DataTypeDateTime>();
type_indexes.erase(TypeIndex::Date);
}
else if (isDateTime(type) && have_datetimes64)
{
type = std::make_shared<DataTypeDateTime64>(9);
type_indexes.erase(TypeIndex::DateTime);
}
type_indexes.erase(TypeIndex::Date);
}
}
@ -697,55 +705,87 @@ namespace
bool tryInferDate(std::string_view field)
{
if (field.empty())
/// Minimum length of Date text representation is 8 (YYYY-M-D) and maximum is 10 (YYYY-MM-DD)
if (field.size() < 8 || field.size() > 10)
return false;
ReadBufferFromString buf(field);
Float64 tmp_float;
/// Check if it's just a number, and if so, don't try to infer Date from it,
/// because we can interpret this number as a Date (for example 20000101 will be 2000-01-01)
/// and it will lead to inferring Date instead of simple Int64/UInt64 in some cases.
if (tryReadFloatText(tmp_float, buf) && buf.eof())
return false;
buf.seek(0, SEEK_SET); /// Return position to the beginning
DayNum tmp;
return tryReadDateText(tmp, buf) && buf.eof();
}
bool tryInferDateTime(std::string_view field, const FormatSettings & settings)
{
if (field.empty())
if (std::all_of(field.begin(), field.end(), isNumericASCII))
return false;
ReadBufferFromString buf(field);
Float64 tmp_float;
DayNum tmp;
return tryReadDateText(tmp, buf, DateLUT::instance(), /*allowed_delimiters=*/"-/:") && buf.eof();
}
DataTypePtr tryInferDateTimeOrDateTime64(std::string_view field, const FormatSettings & settings)
{
/// Don't try to infer DateTime if string is too long.
/// It's difficult to say what is the real maximum length of
/// DateTime we can parse using BestEffort approach.
/// 50 symbols is more or less valid limit for date times that makes sense.
if (field.empty() || field.size() > 50)
return nullptr;
/// Check that we have at least one digit, don't infer datetime form strings like "Apr"/"May"/etc.
if (!std::any_of(field.begin(), field.end(), isNumericASCII))
return nullptr;
/// Check if it's just a number, and if so, don't try to infer DateTime from it,
/// because we can interpret this number as a timestamp and it will lead to
/// inferring DateTime instead of simple Int64/Float64 in some cases.
/// inferring DateTime instead of simple Int64 in some cases.
if (std::all_of(field.begin(), field.end(), isNumericASCII))
return nullptr;
ReadBufferFromString buf(field);
Float64 tmp_float;
/// Check if it's a float value, and if so, don't try to infer DateTime from it,
/// because it will lead to inferring DateTime instead of simple Float64 in some cases.
if (tryReadFloatText(tmp_float, buf) && buf.eof())
return false;
return nullptr;
buf.seek(0, SEEK_SET); /// Return position to the beginning
if (!settings.try_infer_datetimes_only_datetime64)
{
time_t tmp;
switch (settings.date_time_input_format)
{
case FormatSettings::DateTimeInputFormat::Basic:
if (tryReadDateTimeText(tmp, buf, DateLUT::instance(), /*allowed_date_delimiters=*/"-/:", /*allowed_time_delimiters=*/":") && buf.eof())
return std::make_shared<DataTypeDateTime>();
break;
case FormatSettings::DateTimeInputFormat::BestEffort:
if (tryParseDateTimeBestEffortStrict(tmp, buf, DateLUT::instance(), DateLUT::instance("UTC"), /*allowed_date_delimiters=*/"-/:") && buf.eof())
return std::make_shared<DataTypeDateTime>();
break;
case FormatSettings::DateTimeInputFormat::BestEffortUS:
if (tryParseDateTimeBestEffortUSStrict(tmp, buf, DateLUT::instance(), DateLUT::instance("UTC"), /*allowed_date_delimiters=*/"-/:") && buf.eof())
return std::make_shared<DataTypeDateTime>();
break;
}
}
buf.seek(0, SEEK_SET); /// Return position to the beginning
DateTime64 tmp;
switch (settings.date_time_input_format)
{
case FormatSettings::DateTimeInputFormat::Basic:
if (tryReadDateTime64Text(tmp, 9, buf) && buf.eof())
return true;
if (tryReadDateTime64Text(tmp, 9, buf, DateLUT::instance(), /*allowed_date_delimiters=*/"-/:", /*allowed_time_delimiters=*/":") && buf.eof())
return std::make_shared<DataTypeDateTime64>(9);
break;
case FormatSettings::DateTimeInputFormat::BestEffort:
if (tryParseDateTime64BestEffort(tmp, 9, buf, DateLUT::instance(), DateLUT::instance("UTC")) && buf.eof())
return true;
if (tryParseDateTime64BestEffortStrict(tmp, 9, buf, DateLUT::instance(), DateLUT::instance("UTC"), /*allowed_date_delimiters=*/"-/:") && buf.eof())
return std::make_shared<DataTypeDateTime64>(9);
break;
case FormatSettings::DateTimeInputFormat::BestEffortUS:
if (tryParseDateTime64BestEffortUS(tmp, 9, buf, DateLUT::instance(), DateLUT::instance("UTC")) && buf.eof())
return true;
if (tryParseDateTime64BestEffortUSStrict(tmp, 9, buf, DateLUT::instance(), DateLUT::instance("UTC"), /*allowed_date_delimiters=*/"-/:") && buf.eof())
return std::make_shared<DataTypeDateTime64>(9);
break;
}
return false;
return nullptr;
}
template <bool is_json>
@ -1439,8 +1479,11 @@ DataTypePtr tryInferDateOrDateTimeFromString(std::string_view field, const Forma
if (settings.try_infer_dates && tryInferDate(field))
return std::make_shared<DataTypeDate>();
if (settings.try_infer_datetimes && tryInferDateTime(field, settings))
return std::make_shared<DataTypeDateTime64>(9);
if (settings.try_infer_datetimes)
{
if (auto type = tryInferDateTimeOrDateTime64(field, settings))
return type;
}
return nullptr;
}

29
src/IO/ReadHelpers.cpp
View File

@ -1271,7 +1271,7 @@ template void readJSONArrayInto<PaddedPODArray<UInt8>, void>(PaddedPODArray<UInt
template bool readJSONArrayInto<PaddedPODArray<UInt8>, bool>(PaddedPODArray<UInt8> & s, ReadBuffer & buf);
template <typename ReturnType>
ReturnType readDateTextFallback(LocalDate & date, ReadBuffer & buf)
ReturnType readDateTextFallback(LocalDate & date, ReadBuffer & buf, const char * allowed_delimiters)
{
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
@ -1318,6 +1318,9 @@ ReturnType readDateTextFallback(LocalDate & date, ReadBuffer & buf)
}
else
{
if (!isSymbolIn(*buf.position(), allowed_delimiters))
return error();
++buf.position();
if (!append_digit(month))
@ -1325,7 +1328,11 @@ ReturnType readDateTextFallback(LocalDate & date, ReadBuffer & buf)
append_digit(month);
if (!buf.eof() && !isNumericASCII(*buf.position()))
{
if (!isSymbolIn(*buf.position(), allowed_delimiters))
return error();
++buf.position();
}
else
return error();
@ -1338,12 +1345,12 @@ ReturnType readDateTextFallback(LocalDate & date, ReadBuffer & buf)
return ReturnType(true);
}
template void readDateTextFallback<void>(LocalDate &, ReadBuffer &);
template bool readDateTextFallback<bool>(LocalDate &, ReadBuffer &);
template void readDateTextFallback<void>(LocalDate &, ReadBuffer &, const char * allowed_delimiters);
template bool readDateTextFallback<bool>(LocalDate &, ReadBuffer &, const char * allowed_delimiters);
template <typename ReturnType, bool dt64_mode>
ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut)
ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters, const char * allowed_time_delimiters)
{
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
@ -1413,6 +1420,9 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D
if (!isNumericASCII(s[0]) || !isNumericASCII(s[1]) || !isNumericASCII(s[2]) || !isNumericASCII(s[3])
|| !isNumericASCII(s[5]) || !isNumericASCII(s[6]) || !isNumericASCII(s[8]) || !isNumericASCII(s[9]))
return false;
if (!isSymbolIn(s[4], allowed_date_delimiters) || !isSymbolIn(s[7], allowed_date_delimiters))
return false;
}
UInt16 year = (s[0] - '0') * 1000 + (s[1] - '0') * 100 + (s[2] - '0') * 10 + (s[3] - '0');
@ -1443,6 +1453,9 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D
if (!isNumericASCII(s[0]) || !isNumericASCII(s[1]) || !isNumericASCII(s[3]) || !isNumericASCII(s[4])
|| !isNumericASCII(s[6]) || !isNumericASCII(s[7]))
return false;
if (!isSymbolIn(s[2], allowed_time_delimiters) || !isSymbolIn(s[5], allowed_time_delimiters))
return false;
}
hour = (s[0] - '0') * 10 + (s[1] - '0');
@ -1488,10 +1501,10 @@ ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const D
return ReturnType(true);
}
template void readDateTimeTextFallback<void, false>(time_t &, ReadBuffer &, const DateLUTImpl &);
template void readDateTimeTextFallback<void, true>(time_t &, ReadBuffer &, const DateLUTImpl &);
template bool readDateTimeTextFallback<bool, false>(time_t &, ReadBuffer &, const DateLUTImpl &);
template bool readDateTimeTextFallback<bool, true>(time_t &, ReadBuffer &, const DateLUTImpl &);
template void readDateTimeTextFallback<void, false>(time_t &, ReadBuffer &, const DateLUTImpl &, const char *, const char *);
template void readDateTimeTextFallback<void, true>(time_t &, ReadBuffer &, const DateLUTImpl &, const char *, const char *);
template bool readDateTimeTextFallback<bool, false>(time_t &, ReadBuffer &, const DateLUTImpl &, const char *, const char *);
template bool readDateTimeTextFallback<bool, true>(time_t &, ReadBuffer &, const DateLUTImpl &, const char *, const char *);
template <typename ReturnType>

77
src/IO/ReadHelpers.h
View File

@ -703,13 +703,28 @@ struct NullOutput
};
template <typename ReturnType>
ReturnType readDateTextFallback(LocalDate & date, ReadBuffer & buf);
ReturnType readDateTextFallback(LocalDate & date, ReadBuffer & buf, const char * allowed_delimiters);
inline bool isSymbolIn(char symbol, const char * symbols)
{
if (symbols == nullptr)
return true;
const char * pos = symbols;
while (*pos)
{
if (*pos == symbol)
return true;
++pos;
}
return false;
}
/// In YYYY-MM-DD format.
/// For convenience, Month and Day parts can have single digit instead of two digits.
/// Any separators other than '-' are supported.
template <typename ReturnType = void>
inline ReturnType readDateTextImpl(LocalDate & date, ReadBuffer & buf)
inline ReturnType readDateTextImpl(LocalDate & date, ReadBuffer & buf, const char * allowed_delimiters = nullptr)
{
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
@ -753,6 +768,9 @@ inline ReturnType readDateTextImpl(LocalDate & date, ReadBuffer & buf)
}
else
{
if (!isSymbolIn(pos[-1], allowed_delimiters))
return error();
if (!isNumericASCII(pos[0]))
return error();
@ -768,6 +786,9 @@ inline ReturnType readDateTextImpl(LocalDate & date, ReadBuffer & buf)
if (isNumericASCII(pos[-1]) || !isNumericASCII(pos[0]))
return error();
if (!isSymbolIn(pos[-1], allowed_delimiters))
return error();
day = pos[0] - '0';
if (isNumericASCII(pos[1]))
{
@ -783,7 +804,7 @@ inline ReturnType readDateTextImpl(LocalDate & date, ReadBuffer & buf)
return ReturnType(true);
}
else
return readDateTextFallback<ReturnType>(date, buf);
return readDateTextFallback<ReturnType>(date, buf, allowed_delimiters);
}
inline void convertToDayNum(DayNum & date, ExtendedDayNum & from)
@ -797,15 +818,15 @@ inline void convertToDayNum(DayNum & date, ExtendedDayNum & from)
}
template <typename ReturnType = void>
inline ReturnType readDateTextImpl(DayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut)
inline ReturnType readDateTextImpl(DayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_delimiters = nullptr)
{
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
LocalDate local_date;
if constexpr (throw_exception)
readDateTextImpl<ReturnType>(local_date, buf);
else if (!readDateTextImpl<ReturnType>(local_date, buf))
readDateTextImpl<ReturnType>(local_date, buf, allowed_delimiters);
else if (!readDateTextImpl<ReturnType>(local_date, buf, allowed_delimiters))
return false;
ExtendedDayNum ret = date_lut.makeDayNum(local_date.year(), local_date.month(), local_date.day());
@ -814,15 +835,15 @@ inline ReturnType readDateTextImpl(DayNum & date, ReadBuffer & buf, const DateLU
}
template <typename ReturnType = void>
inline ReturnType readDateTextImpl(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut)
inline ReturnType readDateTextImpl(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_delimiters = nullptr)
{
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
LocalDate local_date;
if constexpr (throw_exception)
readDateTextImpl<ReturnType>(local_date, buf);
else if (!readDateTextImpl<ReturnType>(local_date, buf))
readDateTextImpl<ReturnType>(local_date, buf, allowed_delimiters);
else if (!readDateTextImpl<ReturnType>(local_date, buf, allowed_delimiters))
return false;
/// When the parameter is out of rule or out of range, Date32 uses 1925-01-01 as the default value (-DateLUT::instance().getDayNumOffsetEpoch(), -16436) and Date uses 1970-01-01.
@ -846,19 +867,19 @@ inline void readDateText(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTI
readDateTextImpl<void>(date, buf, date_lut);
}
inline bool tryReadDateText(LocalDate & date, ReadBuffer & buf)
inline bool tryReadDateText(LocalDate & date, ReadBuffer & buf, const char * allowed_delimiters = nullptr)
{
return readDateTextImpl<bool>(date, buf);
return readDateTextImpl<bool>(date, buf, allowed_delimiters);
}
inline bool tryReadDateText(DayNum & date, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance())
inline bool tryReadDateText(DayNum & date, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), const char * allowed_delimiters = nullptr)
{
return readDateTextImpl<bool>(date, buf, time_zone);
return readDateTextImpl<bool>(date, buf, time_zone, allowed_delimiters);
}
inline bool tryReadDateText(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance())
inline bool tryReadDateText(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), const char * allowed_delimiters = nullptr)
{
return readDateTextImpl<bool>(date, buf, time_zone);
return readDateTextImpl<bool>(date, buf, time_zone, allowed_delimiters);
}
UUID parseUUID(std::span<const UInt8> src);
@ -975,13 +996,13 @@ inline T parseFromString(std::string_view str)
template <typename ReturnType = void, bool dt64_mode = false>
ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut);
ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr);
/** In YYYY-MM-DD hh:mm:ss or YYYY-MM-DD format, according to specified time zone.
* As an exception, also supported parsing of unix timestamp in form of decimal number.
*/
template <typename ReturnType = void, bool dt64_mode = false>
inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut)
inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr)
{
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
@ -1014,6 +1035,9 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons
if (!isNumericASCII(s[0]) || !isNumericASCII(s[1]) || !isNumericASCII(s[2]) || !isNumericASCII(s[3])
|| !isNumericASCII(s[5]) || !isNumericASCII(s[6]) || !isNumericASCII(s[8]) || !isNumericASCII(s[9]))
return ReturnType(false);
if (!isSymbolIn(s[4], allowed_date_delimiters) || !isSymbolIn(s[7], allowed_date_delimiters))
return ReturnType(false);
}
UInt16 year = (s[0] - '0') * 1000 + (s[1] - '0') * 100 + (s[2] - '0') * 10 + (s[3] - '0');
@ -1033,6 +1057,9 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons
if (!isNumericASCII(s[11]) || !isNumericASCII(s[12]) || !isNumericASCII(s[14]) || !isNumericASCII(s[15])
|| !isNumericASCII(s[17]) || !isNumericASCII(s[18]))
return ReturnType(false);
if (!isSymbolIn(s[13], allowed_time_delimiters) || !isSymbolIn(s[16], allowed_time_delimiters))
return ReturnType(false);
}
hour = (s[11] - '0') * 10 + (s[12] - '0');
@ -1057,11 +1084,11 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons
return readIntTextImpl<time_t, ReturnType, ReadIntTextCheckOverflow::CHECK_OVERFLOW>(datetime, buf);
}
else
return readDateTimeTextFallback<ReturnType, dt64_mode>(datetime, buf, date_lut);
return readDateTimeTextFallback<ReturnType, dt64_mode>(datetime, buf, date_lut, allowed_date_delimiters, allowed_time_delimiters);
}
template <typename ReturnType>
inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut)
inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr)
{
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
@ -1075,7 +1102,7 @@ inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, Re
{
try
{
readDateTimeTextImpl<ReturnType, true>(whole, buf, date_lut);
readDateTimeTextImpl<ReturnType, true>(whole, buf, date_lut, allowed_date_delimiters, allowed_time_delimiters);
}
catch (const DB::Exception &)
{
@ -1085,7 +1112,7 @@ inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, Re
}
else
{
auto ok = readDateTimeTextImpl<ReturnType, true>(whole, buf, date_lut);
auto ok = readDateTimeTextImpl<ReturnType, true>(whole, buf, date_lut, allowed_date_delimiters, allowed_time_delimiters);
if (!ok && (buf.eof() || *buf.position() != '.'))
return ReturnType(false);
}
@ -1168,14 +1195,14 @@ inline void readDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer
readDateTimeTextImpl<void>(datetime64, scale, buf, date_lut);
}
inline bool tryReadDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance())
inline bool tryReadDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr)
{
return readDateTimeTextImpl<bool>(datetime, buf, time_zone);
return readDateTimeTextImpl<bool>(datetime, buf, time_zone, allowed_date_delimiters, allowed_time_delimiters);
}
inline bool tryReadDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance())
inline bool tryReadDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance(), const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr)
{
return readDateTimeTextImpl<bool>(datetime64, scale, buf, date_lut);
return readDateTimeTextImpl<bool>(datetime64, scale, buf, date_lut, allowed_date_delimiters, allowed_time_delimiters);
}
inline void readDateTimeText(LocalDateTime & datetime, ReadBuffer & buf)

16
src/IO/S3/Credentials.cpp
View File

@ -145,12 +145,16 @@ Aws::String AWSEC2MetadataClient::getDefaultCredentialsSecurely() const
{
String user_agent_string = awsComputeUserAgentString();
auto [new_token, response_code] = getEC2MetadataToken(user_agent_string);
if (response_code == Aws::Http::HttpResponseCode::BAD_REQUEST)
if (response_code == Aws::Http::HttpResponseCode::BAD_REQUEST
|| response_code == Aws::Http::HttpResponseCode::REQUEST_NOT_MADE)
{
/// At least the host should be available and reply, otherwise neither IMDSv2 nor IMDSv1 are usable.
return {};
}
else if (response_code != Aws::Http::HttpResponseCode::OK || new_token.empty())
{
LOG_TRACE(logger, "Calling EC2MetadataService to get token failed, "
"falling back to less secure way. HTTP response code: {}", response_code);
"falling back to a less secure way. HTTP response code: {}", response_code);
return getDefaultCredentials();
}
@ -247,7 +251,7 @@ static Aws::String getAWSMetadataEndpoint()
return ec2_metadata_service_endpoint;
}
std::shared_ptr<AWSEC2MetadataClient> InitEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration)
std::shared_ptr<AWSEC2MetadataClient> createEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration)
{
auto endpoint = getAWSMetadataEndpoint();
return std::make_shared<AWSEC2MetadataClient>(client_configuration, endpoint.c_str());
@ -781,11 +785,13 @@ S3CredentialsProviderChain::S3CredentialsProviderChain(
/// EC2MetadataService throttles by delaying the response so the service client should set a large read timeout.
/// EC2MetadataService delay is in order of seconds so it only make sense to retry after a couple of seconds.
aws_client_configuration.connectTimeoutMs = 1000;
/// But the connection timeout should be small because there is the case when there is no IMDS at all,
/// like outside of the cloud, on your own machines.
aws_client_configuration.connectTimeoutMs = 10;
aws_client_configuration.requestTimeoutMs = 1000;
aws_client_configuration.retryStrategy = std::make_shared<Aws::Client::DefaultRetryStrategy>(1, 1000);
auto ec2_metadata_client = InitEC2MetadataClient(aws_client_configuration);
auto ec2_metadata_client = createEC2MetadataClient(aws_client_configuration);
auto config_loader = std::make_shared<AWSEC2InstanceProfileConfigLoader>(ec2_metadata_client, !credentials_configuration.use_insecure_imds_request);
AddProvider(std::make_shared<AWSInstanceProfileCredentialsProvider>(config_loader));

2
src/IO/S3/Credentials.h
View File

@ -70,7 +70,7 @@ private:
LoggerPtr logger;
};
std::shared_ptr<AWSEC2MetadataClient> InitEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration);
std::shared_ptr<AWSEC2MetadataClient> createEC2MetadataClient(const Aws::Client::ClientConfiguration & client_configuration);
class AWSEC2InstanceProfileConfigLoader : public Aws::Config::AWSProfileConfigLoader
{

2
src/IO/S3/PocoHTTPClient.cpp
View File

@ -128,7 +128,7 @@ void PocoHTTPClientConfiguration::updateSchemeAndRegion()
}
else
{
/// In global mode AWS C++ SDK send `us-east-1` but accept switching to another one if being suggested.
/// In global mode AWS C++ SDK sends `us-east-1` but accepts switching to another one if being suggested.
region = Aws::Region::AWS_GLOBAL;
}
}

49
src/IO/S3/URI.cpp
View File

@ -1,8 +1,8 @@
#include <IO/S3/URI.h>
#include <Interpreters/Context.h>
#include <Storages/NamedCollectionsHelpers.h>
#include "Common/Macros.h"
#if USE_AWS_S3
#include <Interpreters/Context.h>
#include <Common/Macros.h>
#include <Common/Exception.h>
#include <Common/quoteString.h>
#include <Common/re2.h>
@ -10,6 +10,7 @@
#include <boost/algorithm/string/case_conv.hpp>
namespace DB
{
@ -40,21 +41,13 @@ URI::URI(const std::string & uri_, bool allow_archive_path_syntax)
/// Case when AWS Private Link Interface is being used
/// E.g. (bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com/bucket-name/key)
/// https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html
static const RE2 aws_private_link_style_pattern(R"(bucket\.vpce\-([a-z0-9\-.]+)\.vpce.amazonaws.com(:\d{1,5})?)");
static const RE2 aws_private_link_style_pattern(R"(bucket\.vpce\-([a-z0-9\-.]+)\.vpce\.amazonaws\.com(:\d{1,5})?)");
/// Case when bucket name and key represented in path of S3 URL.
/// Case when bucket name and key represented in the path of S3 URL.
/// E.g. (https://s3.region.amazonaws.com/bucket-name/key)
/// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#path-style-access
static const RE2 path_style_pattern("^/([^/]*)/(.*)");
static constexpr auto S3 = "S3";
static constexpr auto S3EXPRESS = "S3EXPRESS";
static constexpr auto COSN = "COSN";
static constexpr auto COS = "COS";
static constexpr auto OBS = "OBS";
static constexpr auto OSS = "OSS";
static constexpr auto EOS = "EOS";
if (allow_archive_path_syntax)
std::tie(uri_str, archive_pattern) = getURIAndArchivePattern(uri_);
else
@ -85,7 +78,7 @@ URI::URI(const std::string & uri_, bool allow_archive_path_syntax)
URIConverter::modifyURI(uri, mapper);
}
storage_name = S3;
storage_name = "S3";
if (uri.getHost().empty())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Host is empty in S3 URI.");
@ -93,11 +86,13 @@ URI::URI(const std::string & uri_, bool allow_archive_path_syntax)
/// Extract object version ID from query string.
bool has_version_id = false;
for (const auto & [query_key, query_value] : uri.getQueryParameters())
{
if (query_key == "versionId")
{
version_id = query_value;
has_version_id = true;
}
}
/// Poco::URI will ignore '?' when parsing the path, but if there is a versionId in the http parameter,
/// '?' can not be used as a wildcard, otherwise it will be ambiguous.
@ -129,15 +124,8 @@ URI::URI(const std::string & uri_, bool allow_archive_path_syntax)
}
boost::to_upper(name);
/// For S3Express it will look like s3express-eun1-az1, i.e. contain region and AZ info
if (name != S3 && !name.starts_with(S3EXPRESS) && name != COS && name != OBS && name != OSS && name != EOS)
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Object storage system name is unrecognized in virtual hosted style S3 URI: {}",
quoteString(name));
if (name == COS)
storage_name = COSN;
if (name == "COS")
storage_name = "COSN";
else
storage_name = name;
}
@ -148,13 +136,22 @@ URI::URI(const std::string & uri_, bool allow_archive_path_syntax)
validateBucket(bucket, uri);
}
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bucket or key name are invalid in S3 URI.");
{
/// Custom endpoint, e.g. a public domain of Cloudflare R2,
/// which could be served by a custom server-side code.
storage_name = "S3";
bucket = "default";
is_virtual_hosted_style = false;
endpoint = uri.getScheme() + "://" + uri.getAuthority();
if (!uri.getPath().empty())
key = uri.getPath().substr(1);
}
}
void URI::addRegionToURI(const std::string &region)
{
if (auto pos = endpoint.find("amazonaws.com"); pos != std::string::npos)
endpoint = endpoint.substr(0, pos) + region + "." + endpoint.substr(pos);
if (auto pos = endpoint.find(".amazonaws.com"); pos != std::string::npos)
endpoint = endpoint.substr(0, pos) + "." + region + endpoint.substr(pos);
}
void URI::validateBucket(const String & bucket, const Poco::URI & uri)

8
src/IO/S3/URI.h
View File

@ -1,14 +1,14 @@
#pragma once
#include <optional>
#include <string>
#include "config.h"
#if USE_AWS_S3
#include <optional>
#include <string>
#include <Poco/URI.h>
namespace DB::S3
{
@ -23,7 +23,7 @@ namespace DB::S3
struct URI
{
Poco::URI uri;
// Custom endpoint if URI scheme is not S3.
// Custom endpoint if URI scheme, if not S3.
std::string endpoint;
std::string bucket;
std::string key;

91
src/IO/parseDateTimeBestEffort.cpp
View File

@ -82,13 +82,14 @@ struct DateTimeSubsecondPart
UInt8 digits;
};
template <typename ReturnType, bool is_us_style>
template <typename ReturnType, bool is_us_style, bool strict = false, bool is_64 = false>
ReturnType parseDateTimeBestEffortImpl(
time_t & res,
ReadBuffer & in,
const DateLUTImpl & local_time_zone,
const DateLUTImpl & utc_time_zone,
DateTimeSubsecondPart * fractional)
DateTimeSubsecondPart * fractional,
const char * allowed_date_delimiters = nullptr)
{
auto on_error = [&]<typename... FmtArgs>(int error_code [[maybe_unused]],
FormatStringHelper<FmtArgs...> fmt_string [[maybe_unused]],
@ -170,22 +171,36 @@ ReturnType parseDateTimeBestEffortImpl(
fractional->digits = 3;
readDecimalNumber<3>(fractional->value, digits + 10);
}
else if constexpr (strict)
{
/// Fractional part is not allowed.
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: unexpected fractional part");
}
return ReturnType(true);
}
else if (num_digits == 10 && !year && !has_time)
{
if (strict && month)
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: month component is duplicated");
/// This is unix timestamp.
readDecimalNumber<10>(res, digits);
return ReturnType(true);
}
else if (num_digits == 9 && !year && !has_time)
{
if (strict && month)
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: month component is duplicated");
/// This is unix timestamp.
readDecimalNumber<9>(res, digits);
return ReturnType(true);
}
else if (num_digits == 14 && !year && !has_time)
{
if (strict && month)
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: month component is duplicated");
/// This is YYYYMMDDhhmmss
readDecimalNumber<4>(year, digits);
readDecimalNumber<2>(month, digits + 4);
@ -197,6 +212,9 @@ ReturnType parseDateTimeBestEffortImpl(
}
else if (num_digits == 8 && !year)
{
if (strict && month)
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: month component is duplicated");
/// This is YYYYMMDD
readDecimalNumber<4>(year, digits);
readDecimalNumber<2>(month, digits + 4);
@ -272,6 +290,9 @@ ReturnType parseDateTimeBestEffortImpl(
else
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: unexpected number of decimal digits after year and month: {}", num_digits);
}
if (!isSymbolIn(delimiter_after_year, allowed_date_delimiters))
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: '{}' delimiter between date parts is not allowed", delimiter_after_year);
}
}
else if (num_digits == 2 || num_digits == 1)
@ -403,9 +424,16 @@ ReturnType parseDateTimeBestEffortImpl(
else
{
if (day_of_month)
{
if (strict && hour)
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: hour component is duplicated");
hour = hour_or_day_of_month_or_month;
}
else
{
day_of_month = hour_or_day_of_month_or_month;
}
}
}
else if (num_digits != 0)
@ -446,6 +474,11 @@ ReturnType parseDateTimeBestEffortImpl(
fractional->digits = num_digits;
readDecimalNumber(fractional->value, num_digits, digits);
}
else if (strict)
{
/// Fractional part is not allowed.
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: unexpected fractional part");
}
}
else if (c == '+' || c == '-')
{
@ -582,12 +615,24 @@ ReturnType parseDateTimeBestEffortImpl(
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: neither Date nor Time was parsed successfully");
if (!day_of_month)
{
if constexpr (strict)
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: day of month is required");
day_of_month = 1;
}
if (!month)
{
if constexpr (strict)
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: month is required");
month = 1;
}
if (!year)
{
if constexpr (strict)
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: year is required");
/// If year is not specified, it will be the current year if the date is unknown or not greater than today,
/// otherwise it will be the previous year.
/// This convoluted logic is needed to parse the syslog format, which looks as follows: "Mar 3 01:33:48".
@ -641,6 +686,20 @@ ReturnType parseDateTimeBestEffortImpl(
}
};
if constexpr (strict)
{
if constexpr (is_64)
{
if (year < 1900)
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime64: year {} is less than minimum supported year 1900", year);
}
else
{
if (year < 1970)
return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: year {} is less than minimum supported year 1970", year);
}
}
if (has_time_zone_offset)
{
res = utc_time_zone.makeDateTime(year, month, day_of_month, hour, minute, second);
@ -654,20 +713,20 @@ ReturnType parseDateTimeBestEffortImpl(
return ReturnType(true);
}
template <typename ReturnType, bool is_us_style>
ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone)
template <typename ReturnType, bool is_us_style, bool strict = false>
ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, const char * allowed_date_delimiters = nullptr)
{
time_t whole;
DateTimeSubsecondPart subsecond = {0, 0}; // needs to be explicitly initialized sine it could be missing from input string
if constexpr (std::is_same_v<ReturnType, bool>)
{
if (!parseDateTimeBestEffortImpl<bool, is_us_style>(whole, in, local_time_zone, utc_time_zone, &subsecond))
if (!parseDateTimeBestEffortImpl<bool, is_us_style, strict, true>(whole, in, local_time_zone, utc_time_zone, &subsecond, allowed_date_delimiters))
return false;
}
else
{
parseDateTimeBestEffortImpl<ReturnType, is_us_style>(whole, in, local_time_zone, utc_time_zone, &subsecond);
parseDateTimeBestEffortImpl<ReturnType, is_us_style, strict, true>(whole, in, local_time_zone, utc_time_zone, &subsecond, allowed_date_delimiters);
}
@ -730,4 +789,24 @@ bool tryParseDateTime64BestEffortUS(DateTime64 & res, UInt32 scale, ReadBuffer &
return parseDateTime64BestEffortImpl<bool, true>(res, scale, in, local_time_zone, utc_time_zone);
}
bool tryParseDateTimeBestEffortStrict(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, const char * allowed_date_delimiters)
{
return parseDateTimeBestEffortImpl<bool, false, true>(res, in, local_time_zone, utc_time_zone, nullptr, allowed_date_delimiters);
}
bool tryParseDateTimeBestEffortUSStrict(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, const char * allowed_date_delimiters)
{
return parseDateTimeBestEffortImpl<bool, true, true>(res, in, local_time_zone, utc_time_zone, nullptr, allowed_date_delimiters);
}
bool tryParseDateTime64BestEffortStrict(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, const char * allowed_date_delimiters)
{
return parseDateTime64BestEffortImpl<bool, false, true>(res, scale, in, local_time_zone, utc_time_zone, allowed_date_delimiters);
}
bool tryParseDateTime64BestEffortUSStrict(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, const char * allowed_date_delimiters)
{
return parseDateTime64BestEffortImpl<bool, true, true>(res, scale, in, local_time_zone, utc_time_zone, allowed_date_delimiters);
}
}

8
src/IO/parseDateTimeBestEffort.h
View File

@ -63,4 +63,12 @@ void parseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in,
bool tryParseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone);
void parseDateTime64BestEffortUS(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone);
bool tryParseDateTime64BestEffortUS(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone);
/// More strict version of best effort parsing. Requires day, month and year to be present, checks for allowed
/// delimiters between date components, makes additional correctness checks. Used in schema inference if date times.
bool tryParseDateTimeBestEffortStrict(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, const char * allowed_date_delimiters);
bool tryParseDateTimeBestEffortUSStrict(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, const char * allowed_date_delimiters);
bool tryParseDateTime64BestEffortStrict(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, const char * allowed_date_delimiters);
bool tryParseDateTime64BestEffortUSStrict(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, const char * allowed_date_delimiters);
}

19
src/IO/tests/gtest_s3_uri.cpp
View File

@ -206,11 +206,6 @@ TEST(S3UriTest, validPatterns)
}
}
TEST_P(S3UriTest, invalidPatterns)
{
ASSERT_ANY_THROW(S3::URI new_uri(GetParam()));
}
TEST(S3UriTest, versionIdChecks)
{
for (const auto& test_case : TestCases)
@ -223,19 +218,5 @@ TEST(S3UriTest, versionIdChecks)
}
}
INSTANTIATE_TEST_SUITE_P(
S3,
S3UriTest,
testing::Values(
"https:///",
"https://.s3.amazonaws.com/key",
"https://s3.amazonaws.com/key",
"https://jokserfn.s3amazonaws.com/key",
"https://s3.amazonaws.com//",
"https://amazonaws.com/",
"https://amazonaws.com//",
"https://amazonaws.com//key"));
}
#endif

2
src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -1944,6 +1944,8 @@ BlockIO InterpreterCreateQuery::execute()
FunctionNameNormalizer::visit(query_ptr.get());
auto & create = query_ptr->as<ASTCreateQuery &>();
create.if_not_exists |= getContext()->getSettingsRef().create_if_not_exists;
bool is_create_database = create.database && !create.table;
if (!create.cluster.empty() && !maybeRemoveOnCluster(query_ptr, getContext()))
{

20
src/Processors/QueryPlan/ReadFromMergeTree.cpp
View File

@ -352,7 +352,15 @@ Pipe ReadFromMergeTree::readFromPoolParallelReplicas(
/// We have a special logic for local replica. It has to read less data, because in some cases it should
/// merge states of aggregate functions or do some other important stuff other than reading from Disk.
const auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier;
auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier;
const auto min_marks_for_concurrent_read_limit = std::numeric_limits<Int64>::max() >> 1;
if (pool_settings.min_marks_for_concurrent_read > min_marks_for_concurrent_read_limit)
{
/// limit min marks to read in case it's big, happened in test since due to settings randomzation
pool_settings.min_marks_for_concurrent_read = min_marks_for_concurrent_read_limit;
multiplier = 1.0f;
}
if (auto result = pool_settings.min_marks_for_concurrent_read * multiplier; canConvertTo<size_t>(result))
pool_settings.min_marks_for_concurrent_read = static_cast<size_t>(result);
else
@ -521,7 +529,15 @@ Pipe ReadFromMergeTree::readInOrder(
.number_of_current_replica = client_info.number_of_current_replica,
};
const auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier;
auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier;
const auto min_marks_for_concurrent_read_limit = std::numeric_limits<Int64>::max() >> 1;
if (pool_settings.min_marks_for_concurrent_read > min_marks_for_concurrent_read_limit)
{
/// limit min marks to read in case it's big, happened in test since due to settings randomzation
pool_settings.min_marks_for_concurrent_read = min_marks_for_concurrent_read_limit;
multiplier = 1.0f;
}
if (auto result = pool_settings.min_marks_for_concurrent_read * multiplier; canConvertTo<size_t>(result))
pool_settings.min_marks_for_concurrent_read = static_cast<size_t>(result);
else

15
src/Storages/MergeTree/MergeTreeIndexGranularity.cpp
View File

@ -103,8 +103,19 @@ size_t MergeTreeIndexGranularity::countMarksForRows(size_t from_mark, size_t num
/// This is a heuristic to respect min_marks_to_read which is ignored by MergeTreeReadPool in case of remote disk.
/// See comment in IMergeTreeSelectAlgorithm.
if (min_marks_to_read && from_mark + 2 * min_marks_to_read <= to_mark)
to_mark = from_mark + min_marks_to_read;
if (min_marks_to_read)
{
// check overflow
size_t min_marks_to_read_2 = 0;
bool overflow = common::mulOverflow(min_marks_to_read, 2, min_marks_to_read_2);
size_t to_mark_overwrite = 0;
if (!overflow)
overflow = common::addOverflow(from_mark, min_marks_to_read_2, to_mark_overwrite);
if (!overflow && to_mark_overwrite < to_mark)
to_mark = to_mark_overwrite;
}
return getRowsCountInRange(from_mark, std::max(1UL, to_mark)) - offset_in_rows;
}

2
src/Storages/StorageMergeTree.cpp
View File

@ -895,7 +895,7 @@ void StorageMergeTree::loadDeduplicationLog()
std::string path = fs::path(relative_data_path) / "deduplication_logs";
/// If either there is already a deduplication log, or we will be able to use it.
if (disk->exists(path) || !disk->isReadOnly())
if (!disk->isReadOnly() || disk->exists(path))
{
deduplication_log = std::make_unique<MergeTreeDeduplicationLog>(path, settings->non_replicated_deduplication_window, format_version, disk);
deduplication_log->load();

44
tests/integration/test_odbc_interaction/test.py
View File

@ -51,9 +51,9 @@ create_table_sql_nullable_template = """
"""
def skip_test_msan(instance):
if instance.is_built_with_memory_sanitizer():
pytest.skip("Memory Sanitizer cannot work with third-party shared libraries")
def skip_test_sanitizers(instance):
if instance.is_built_with_sanitizer():
pytest.skip("Sanitizers cannot work with third-party shared libraries")
def get_mysql_conn():
@ -208,7 +208,7 @@ def started_cluster():
def test_mysql_odbc_select_nullable(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
mysql_setup = node1.odbc_drivers["MySQL"]
table_name = "test_insert_nullable_select"
@ -248,7 +248,7 @@ def test_mysql_odbc_select_nullable(started_cluster):
def test_mysql_simple_select_works(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
mysql_setup = node1.odbc_drivers["MySQL"]
@ -331,7 +331,7 @@ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32, column_x Nulla
def test_mysql_insert(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
mysql_setup = node1.odbc_drivers["MySQL"]
table_name = "test_insert"
@ -374,7 +374,7 @@ def test_mysql_insert(started_cluster):
def test_sqlite_simple_select_function_works(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
sqlite_setup = node1.odbc_drivers["SQLite3"]
sqlite_db = sqlite_setup["Database"]
@ -438,7 +438,7 @@ def test_sqlite_simple_select_function_works(started_cluster):
def test_sqlite_table_function(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
sqlite_setup = node1.odbc_drivers["SQLite3"]
sqlite_db = sqlite_setup["Database"]
@ -470,7 +470,7 @@ def test_sqlite_table_function(started_cluster):
def test_sqlite_simple_select_storage_works(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
sqlite_setup = node1.odbc_drivers["SQLite3"]
sqlite_db = sqlite_setup["Database"]
@ -503,7 +503,7 @@ def test_sqlite_simple_select_storage_works(started_cluster):
def test_sqlite_odbc_hashed_dictionary(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
sqlite_db = node1.odbc_drivers["SQLite3"]["Database"]
node1.exec_in_container(
@ -586,7 +586,7 @@ def test_sqlite_odbc_hashed_dictionary(started_cluster):
def test_sqlite_odbc_cached_dictionary(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
sqlite_db = node1.odbc_drivers["SQLite3"]["Database"]
node1.exec_in_container(
@ -635,7 +635,7 @@ def test_sqlite_odbc_cached_dictionary(started_cluster):
def test_postgres_odbc_hashed_dictionary_with_schema(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
try:
conn = get_postgres_conn(started_cluster)
@ -663,7 +663,7 @@ def test_postgres_odbc_hashed_dictionary_with_schema(started_cluster):
def test_postgres_odbc_hashed_dictionary_no_tty_pipe_overflow(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
try:
conn = get_postgres_conn(started_cluster)
@ -685,7 +685,7 @@ def test_postgres_odbc_hashed_dictionary_no_tty_pipe_overflow(started_cluster):
def test_no_connection_pooling(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
try:
conn = get_postgres_conn(started_cluster)
@ -717,7 +717,7 @@ def test_no_connection_pooling(started_cluster):
def test_postgres_insert(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
conn = get_postgres_conn(started_cluster)
@ -754,7 +754,7 @@ def test_postgres_insert(started_cluster):
def test_odbc_postgres_date_data_type(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
try:
conn = get_postgres_conn(started_cluster)
@ -783,7 +783,7 @@ def test_odbc_postgres_date_data_type(started_cluster):
def test_odbc_postgres_conversions(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
try:
conn = get_postgres_conn(started_cluster)
@ -841,7 +841,7 @@ def test_odbc_postgres_conversions(started_cluster):
def test_odbc_cyrillic_with_varchar(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
conn = get_postgres_conn(started_cluster)
cursor = conn.cursor()
@ -868,7 +868,7 @@ def test_odbc_cyrillic_with_varchar(started_cluster):
def test_many_connections(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
conn = get_postgres_conn(started_cluster)
cursor = conn.cursor()
@ -894,7 +894,7 @@ def test_many_connections(started_cluster):
def test_concurrent_queries(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
conn = get_postgres_conn(started_cluster)
cursor = conn.cursor()
@ -948,7 +948,7 @@ def test_concurrent_queries(started_cluster):
def test_odbc_long_column_names(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
conn = get_postgres_conn(started_cluster)
cursor = conn.cursor()
@ -986,7 +986,7 @@ def test_odbc_long_column_names(started_cluster):
def test_odbc_long_text(started_cluster):
skip_test_msan(node1)
skip_test_sanitizers(node1)
conn = get_postgres_conn(started_cluster)
cursor = conn.cursor()

2
tests/integration/test_s3_imds/test_simple.py
View File

@ -56,7 +56,7 @@ def test_credentials_from_metadata():
)
expected_logs = [
"Calling EC2MetadataService to get token failed, falling back to less secure way",
"Calling EC2MetadataService to get token failed, falling back to a less secure way",
"Getting default credentials for ec2 instance from resolver:8080",
"Calling EC2MetadataService resource, /latest/meta-data/iam/security-credentials returned credential string myrole",
"Calling EC2MetadataService resource /latest/meta-data/iam/security-credentials/myrole",

30
tests/performance/parquet_read_with_index.xml Normal file
View File

@ -0,0 +1,30 @@
<test>
<fill_query>
INSERT INTO FUNCTION file('test_pq_index', Parquet) SELECT * FROM generateRandom('int64_column Nullable(Int64), tuple_column Tuple(a Nullable(String), b Nullable(Float64), c Tuple(i UInt32, j UInt32)),array_tuple_column Array(Tuple(a Nullable(String), b Nullable(Float64), c Nullable(Int64))), map_tuple_column Map(String, Tuple(a Nullable(String), b Nullable(Float64), c Nullable(Int64)))') limit 1000000 SETTINGS output_format_parquet_use_custom_encoder=false, output_format_parquet_write_page_index=true
</fill_query>
<query>
SELECT * FROM file('test_pq_index', Parquet, 'tuple_column Tuple(a Nullable(String))') Format Null
</query>
<query>
SELECT tuple_column.a FROM file('test_pq_index', Parquet) Format Null
</query>
<query>
SELECT tuple_column.a FROM file('test_pq_index', Parquet, 'tuple_column Tuple(a Nullable(String))') Format Null
</query>
<query>
SELECT tuple_column.c.i FROM file('test_pq_index', Parquet) Format Null
</query>
<query>
SELECT * FROM file('test_pq_index', Parquet, 'array_tuple_column Array (Tuple(a Nullable(String)))') Format Null
</query>
<query>
SELECT * FROM file('test_pq_index', Parquet, 'map_tuple_column Map(String, Tuple(a Nullable(String)))') Format Null
</query>
</test>

20
tests/queries/0_stateless/00366_multi_statements.sh
View File

@ -14,22 +14,22 @@ $CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2"
$CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2;"
$CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2; SELECT" 2>&1 | grep -o 'Syntax error'
$CLICKHOUSE_CLIENT -n --query="SELECT 1; S" 2>&1 | grep -o 'Syntax error'
$CLICKHOUSE_CLIENT -n --query="SELECT 1; SELECT 2"
$CLICKHOUSE_CLIENT -n --query="SELECT 1; SELECT 2;"
$CLICKHOUSE_CLIENT -n --query="SELECT 1; SELECT 2; SELECT" 2>&1 | grep -o 'Syntax error'
$CLICKHOUSE_CLIENT --query="SELECT 1; S" 2>&1 | grep -o 'Syntax error'
$CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2"
$CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2;"
$CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2; SELECT" 2>&1 | grep -o 'Syntax error'
$CLICKHOUSE_CLIENT -n --query="DROP TABLE IF EXISTS t_00366; CREATE TABLE t_00366 (x UInt64) ENGINE = TinyLog;"
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS t_00366; CREATE TABLE t_00366 (x UInt64) ENGINE = TinyLog;"
$CLICKHOUSE_CLIENT --query="INSERT INTO t_00366 VALUES (1),(2),(3);"
$CLICKHOUSE_CLIENT --query="SELECT * FROM t_00366"
$CLICKHOUSE_CLIENT --query="INSERT INTO t_00366 VALUES" <<< "(4),(5),(6)"
$CLICKHOUSE_CLIENT --query="SELECT * FROM t_00366"
$CLICKHOUSE_CLIENT -n --query="INSERT INTO t_00366 VALUES (1),(2),(3);"
$CLICKHOUSE_CLIENT -n --query="SELECT * FROM t_00366"
$CLICKHOUSE_CLIENT -n --query="INSERT INTO t_00366 VALUES" <<< "(4),(5),(6)"
$CLICKHOUSE_CLIENT -n --query="SELECT * FROM t_00366"
$CLICKHOUSE_CLIENT --query="INSERT INTO t_00366 VALUES (1),(2),(3);"
$CLICKHOUSE_CLIENT --query="SELECT * FROM t_00366"
$CLICKHOUSE_CLIENT --query="INSERT INTO t_00366 VALUES" <<< "(4),(5),(6)"
$CLICKHOUSE_CLIENT --query="SELECT * FROM t_00366"
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT 1"
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT 1;"
@ -48,4 +48,4 @@ $CLICKHOUSE_CLIENT --query="SELECT * FROM t_00366"
${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_00366+VALUES" -d "(7),(8),(9)"
$CLICKHOUSE_CLIENT --query="SELECT * FROM t_00366"
$CLICKHOUSE_CLIENT -n --query="DROP TABLE t_00366;"
$CLICKHOUSE_CLIENT --query="DROP TABLE t_00366;"

4
tests/queries/0_stateless/00443_preferred_block_size_bytes.sh
View File

@ -43,10 +43,10 @@ popd > /dev/null
#SCRIPTDIR=`dirname "$SCRIPTPATH"`
SCRIPTDIR=$SCRIPTPATH
cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=10 -n > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout
cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=10 > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout
cmp "$SCRIPTDIR"/00282_merging.reference "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout && echo PASSED || echo FAILED
cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=20 -n > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout
cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=20 > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout
cmp "$SCRIPTDIR"/00282_merging.reference "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout && echo PASSED || echo FAILED
rm "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout

20
tests/queries/0_stateless/00543_access_to_temporary_table_in_readonly_mode.sh
View File

@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
DROP TABLE IF EXISTS test_readonly;
CREATE TABLE test_readonly (
ID Int
@ -16,7 +16,7 @@ $CLICKHOUSE_CLIENT -n --query="
################
# Try to create temporary table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 1;
CREATE TEMPORARY TABLE readonly (
ID Int
@ -26,7 +26,7 @@ CODE=$?;
[ "$CODE" -ne "164" ] && [ "$CODE" -ne "0" ] && echo "Fail" && exit $CODE;
# Try to insert into exists (non temporary) table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 1;
INSERT INTO test_readonly (ID) VALUES (1);
" 2> /dev/null;
@ -34,7 +34,7 @@ CODE=$?;
[ "$CODE" -ne "164" ] && [ "$CODE" -ne "0" ] && echo "Fail" && exit $CODE;
# Try to drop exists (non temporary) table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 1;
DROP TABLE test_readonly;
" 2> /dev/null;
@ -46,7 +46,7 @@ CODE=$?;
################
# Try to create temporary table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 2;
CREATE TEMPORARY TABLE readonly (
ID Int
@ -58,7 +58,7 @@ CODE=$?;
[ "$CODE" -ne "0" ] && echo "Fail" && exit $CODE;
# Try to insert into exists (non temporary) table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 2;
INSERT INTO test_readonly (ID) VALUES (1);
" 2> /dev/null;
@ -66,7 +66,7 @@ CODE=$?;
[ "$CODE" -ne "164" ] && [ "$CODE" -ne "0" ] && echo "Fail" && exit $CODE;
# Try to drop exists (non temporary) table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 2;
DROP TABLE test_readonly;
" 2> /dev/null;
@ -78,7 +78,7 @@ CODE=$?;
################
# Try to create temporary table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 0;
CREATE TEMPORARY TABLE readonly (
ID Int
@ -90,7 +90,7 @@ CODE=$?;
[ "$CODE" -ne "0" ] && echo "Fail" && exit $CODE;
# Try to insert into exists (non temporary) table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 0;
INSERT INTO test_readonly (ID) VALUES (1);
" 2> /dev/null;
@ -98,7 +98,7 @@ CODE=$?;
[ "$CODE" -ne "0" ] && echo "Fail" && exit $CODE;
# Try to drop exists (non temporary) table
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
SET readonly = 0;
DROP TABLE test_readonly;
" 2> /dev/null;

4
tests/queries/0_stateless/00634_performance_introspection_and_logging.sh
View File

@ -19,13 +19,13 @@ settings="$server_logs --log_queries=1 --log_query_threads=1 --log_profile_event
# Test insert logging on each block and checkPacket() method
$CLICKHOUSE_CLIENT $settings -n -q "
$CLICKHOUSE_CLIENT $settings -q "
DROP TABLE IF EXISTS null_00634;
CREATE TABLE null_00634 (i UInt8) ENGINE = MergeTree PARTITION BY tuple() ORDER BY tuple();"
head -c 1000 /dev/zero | $CLICKHOUSE_CLIENT $settings --max_insert_block_size=10 --min_insert_block_size_rows=1 --min_insert_block_size_bytes=1 -q "INSERT INTO null_00634 FORMAT RowBinary"
$CLICKHOUSE_CLIENT $settings -n -q "
$CLICKHOUSE_CLIENT $settings -q "
SELECT count() FROM null_00634;
DROP TABLE null_00634;"

6
tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh
View File

@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/mergetree_mutations.lib
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
DROP TABLE IF EXISTS fetches_r1 SYNC;
DROP TABLE IF EXISTS fetches_r2 SYNC"
@ -17,7 +17,7 @@ ${CLICKHOUSE_CLIENT} --query="CREATE TABLE fetches_r2(x UInt32) ENGINE Replicate
SETTINGS prefer_fetch_merged_part_time_threshold=0, \
prefer_fetch_merged_part_size_threshold=0"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET insert_keeper_fault_injection_probability=0;
INSERT INTO fetches_r1 VALUES (1);
INSERT INTO fetches_r1 VALUES (2);
@ -51,6 +51,6 @@ ${CLICKHOUSE_CLIENT} --query="SYSTEM SYNC REPLICA fetches_r2"
${CLICKHOUSE_CLIENT} --query="SELECT '*** Check data after fetch/clone of mutated part ***'"
${CLICKHOUSE_CLIENT} --query="SELECT _part, * FROM fetches_r2 ORDER BY x"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
DROP TABLE fetches_r1 SYNC;
DROP TABLE fetches_r2 SYNC"

30
tests/queries/0_stateless/00754_distributed_optimize_skip_select_on_unused_shards.sh
View File

@ -25,83 +25,83 @@ ${CLICKHOUSE_CLIENT} --query "SELECT count(*) FROM distributed WHERE a = 0 AND b
| grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
# Should pass now
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b = 0;
"
# Should still fail because of matching unavailable shard
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 2 AND b = 2;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
# Try more complext expressions for constant folding - all should pass.
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 1 AND a = 0 AND b = 0;
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a IN (0, 1) AND b IN (0, 1);
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b = 0 OR a = 1 AND b = 1;
"
# TODO: should pass one day.
#${CLICKHOUSE_CLIENT} -n --query="
#${CLICKHOUSE_CLIENT} --query="
# SET optimize_skip_unused_shards = 1;
# SELECT count(*) FROM distributed WHERE a = 0 AND b >= 0 AND b <= 1;
#"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b = 0 AND c = 0;
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b = 0 AND c != 10;
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b = 0 AND (a+b)*b != 12;
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE (a = 0 OR a = 1) AND (b = 0 OR b = 1);
"
# These ones should fail.
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b <= 1;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND c = 0;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 OR a = 1 AND b = 0;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b = 0 OR a = 2 AND b = 2;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed WHERE a = 0 AND b = 0 OR c = 0;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'

26
tests/queries/0_stateless/00754_distributed_optimize_skip_select_on_unused_shards_with_prewhere.sh
View File

@ -30,73 +30,73 @@ ${CLICKHOUSE_CLIENT} --query "SELECT count(*) FROM distributed_00754 PREWHERE a
| grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
# Should pass now
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 0 AND b = 0;
"
# Should still fail because of matching unavailable shard
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 2 AND b = 2;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
# Try more complex expressions for constant folding - all should pass.
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 1 AND a = 0 WHERE b = 0;
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 1 WHERE b = 1 AND length(c) = 5;
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a IN (0, 1) AND b IN (0, 1) WHERE c LIKE '%l%';
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a IN (0, 1) WHERE b IN (0, 1) AND c LIKE '%l%';
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 0 AND b = 0 OR a = 1 AND b = 1 WHERE c LIKE '%l%';
"
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE (a = 0 OR a = 1) WHERE (b = 0 OR b = 1);
"
# These should fail.
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 0 AND b <= 1;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 0 WHERE c LIKE '%l%';
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 0 OR a = 1 AND b = 0;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 0 AND b = 0 OR a = 2 AND b = 2;
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'
${CLICKHOUSE_CLIENT} -n --query="
${CLICKHOUSE_CLIENT} --query="
SET optimize_skip_unused_shards = 1;
SELECT count(*) FROM distributed_00754 PREWHERE a = 0 AND b = 0 OR c LIKE '%l%';
" 2>&1 \ | grep -F -q "All connection tries failed" && echo 'OK' || echo 'FAIL'

4
tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sh
View File

@ -10,7 +10,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
SHARD=$($CLICKHOUSE_CLIENT --query "Select getMacro('shard')")
REPLICA=$($CLICKHOUSE_CLIENT --query "Select getMacro('replica')")
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
DROP TABLE IF EXISTS part_header_r1;
DROP TABLE IF EXISTS part_header_r2;
@ -62,7 +62,7 @@ do
[[ $count1 == 1 && $count2 == 1 ]] && break
done
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
SELECT '*** Test part removal ***';
SELECT '*** replica 1 ***';

2
tests/queries/0_stateless/00837_minmax_index.sh
View File

@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE minmax_idx
(
u64 UInt64,

2
tests/queries/0_stateless/00838_unique_index.sh
View File

@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS set_idx;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE set_idx
(
u64 UInt64,

2
tests/queries/0_stateless/00907_set_index_max_rows.sh
View File

@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS set_idx;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE set_idx
(
u64 UInt64,

10
tests/queries/0_stateless/00908_bloom_filter_index.sh
View File

@ -12,7 +12,7 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS bloom_filter_idx3;"
# NGRAM BF
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE bloom_filter_idx
(
k UInt64,
@ -22,7 +22,7 @@ CREATE TABLE bloom_filter_idx
ORDER BY k
SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi';"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE bloom_filter_idx2
(
k UInt64,
@ -109,7 +109,7 @@ $CLICKHOUSE_CLIENT --optimize_or_like_chain 0 --query="SELECT count() FROM bloom
# TOKEN BF
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE bloom_filter_idx3
(
k UInt64,
@ -147,7 +147,7 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE bloom_filter_idx2"
$CLICKHOUSE_CLIENT --query="DROP TABLE bloom_filter_idx3"
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS bloom_filter_idx_na;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE bloom_filter_idx_na
(
na Array(Array(String)),
@ -156,7 +156,7 @@ CREATE TABLE bloom_filter_idx_na
ORDER BY na" 2>&1 | grep -c 'DB::Exception: Unexpected type Array(Array(String)) of bloom filter index'
# NGRAM BF with IPv6
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE bloom_filter_ipv6_idx
(
foo IPv6,

2
tests/queries/0_stateless/00942_mutate_index.sh
View File

@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE minmax_idx
(
u64 UInt64,

4
tests/queries/0_stateless/00943_materialize_index.sh
View File

@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE minmax_idx
(
u64 UInt64,
@ -34,7 +34,7 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO minmax_idx VALUES
$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0;"
$CLICKHOUSE_CLIENT --query="SELECT count() FROM minmax_idx WHERE i64 = 2 SETTINGS merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = 0.0 FORMAT JSON" | grep "rows_read"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
ALTER TABLE minmax_idx ADD INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1 SETTINGS mutations_sync = 2;"
$CLICKHOUSE_CLIENT --query="ALTER TABLE minmax_idx MATERIALIZE INDEX idx IN PARTITION 1 SETTINGS mutations_sync = 2;"

2
tests/queries/0_stateless/00944_clear_index_in_partition.sh
View File

@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS minmax_idx;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE minmax_idx
(
u64 UInt64,

2
tests/queries/0_stateless/00964_bloom_index_string_functions.sh
View File

@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS bloom_filter_idx;"
# NGRAM BF
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE bloom_filter_idx
(
k UInt64,

2
tests/queries/0_stateless/00965_set_index_string_functions.sh
View File

@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS set_idx;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
CREATE TABLE set_idx
(
k UInt64,

4
tests/queries/0_stateless/00974_primary_key_for_lowCardinality.sh
View File

@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS lowString;"
$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS string;"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
create table lowString
(
a LowCardinality(String),
@ -18,7 +18,7 @@ ENGINE = MergeTree()
PARTITION BY toYYYYMM(b)
ORDER BY (a)"
$CLICKHOUSE_CLIENT -n --query="
$CLICKHOUSE_CLIENT --query="
create table string
(
a String,

2
tests/queries/0_stateless/00990_hasToken.sh
View File

@ -6,4 +6,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# We should have correct env vars from shell_config.sh to run this test
python3 "$CURDIR"/00990_hasToken.python | ${CLICKHOUSE_CLIENT} --max_query_size 1048576 -nm
python3 "$CURDIR"/00990_hasToken.python | ${CLICKHOUSE_CLIENT} --max_query_size 1048576 -m

8
tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference
View File

@ -49,7 +49,7 @@ order by query;
tuple(2)
select 'optimize_skip_unused_shards_rewrite_in(2,)';
optimize_skip_unused_shards_rewrite_in(2,)
with (select currentDatabase()) as id_2 select *, ignore(id_2) from dist_01756 where dummy in (2,);
with (select currentDatabase()) as id_2 select *, ignore(id_2) from dist_01756 where dummy in (2);
system flush logs;
select splitByString('IN', query)[-1] from system.query_log where
event_date >= yesterday() and
@ -59,10 +59,10 @@ select splitByString('IN', query)[-1] from system.query_log where
query like concat('%', currentDatabase(), '%AS%id_2%') and
type = 'QueryFinish'
order by query;
tuple(2)
(2)
select 'optimize_skip_unused_shards_rewrite_in(0,)';
optimize_skip_unused_shards_rewrite_in(0,)
with (select currentDatabase()) as id_00 select *, ignore(id_00) from dist_01756 where dummy in (0,);
with (select currentDatabase()) as id_00 select *, ignore(id_00) from dist_01756 where dummy in (0);
0 0
system flush logs;
select splitByString('IN', query)[-1] from system.query_log where
@ -73,7 +73,7 @@ select splitByString('IN', query)[-1] from system.query_log where
query like concat('%', currentDatabase(), '%AS%id_00%') and
type = 'QueryFinish'
order by query;
tuple(0)
(0)
-- signed column
select 'signed column';
signed column

4
tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql
View File

@ -63,7 +63,7 @@ select splitByString('IN', query)[-1] from system.query_log where
order by query;
select 'optimize_skip_unused_shards_rewrite_in(2,)';
with (select currentDatabase()) as id_2 select *, ignore(id_2) from dist_01756 where dummy in (2,);
with (select currentDatabase()) as id_2 select *, ignore(id_2) from dist_01756 where dummy in (2);
system flush logs;
select splitByString('IN', query)[-1] from system.query_log where
event_date >= yesterday() and
@ -75,7 +75,7 @@ select splitByString('IN', query)[-1] from system.query_log where
order by query;
select 'optimize_skip_unused_shards_rewrite_in(0,)';
with (select currentDatabase()) as id_00 select *, ignore(id_00) from dist_01756 where dummy in (0,);
with (select currentDatabase()) as id_00 select *, ignore(id_00) from dist_01756 where dummy in (0);
system flush logs;
select splitByString('IN', query)[-1] from system.query_log where
event_date >= yesterday() and

32
tests/queries/0_stateless/02003_memory_limit_in_client.sh
View File

@ -4,21 +4,21 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT --max_result_bytes 0 --max_memory_usage_in_client=1 -n -q "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000) -- { clientError MEMORY_LIMIT_EXCEEDED }"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client=0 -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_result_bytes 0 --max_memory_usage_in_client=1 -q "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000) -- { clientError MEMORY_LIMIT_EXCEEDED }"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client=0 -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_result_bytes 0 --max_memory_usage_in_client='5K' -n -q "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000) -- { clientError MEMORY_LIMIT_EXCEEDED }"
$CLICKHOUSE_CLIENT --max_result_bytes 0 --max_memory_usage_in_client='5k' -n -q "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000) -- { clientError MEMORY_LIMIT_EXCEEDED }"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='1M' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='23G' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='11T' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_result_bytes 0 --max_memory_usage_in_client='5K' -q "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000) -- { clientError MEMORY_LIMIT_EXCEEDED }"
$CLICKHOUSE_CLIENT --max_result_bytes 0 --max_memory_usage_in_client='5k' -q "SELECT arrayMap(x -> range(x), range(number)) FROM numbers(1000) -- { clientError MEMORY_LIMIT_EXCEEDED }"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='1M' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='23G' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='11T' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='2P' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='2.1p' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='10E' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='10.2e' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='-1.1T' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_NUMBER"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='-1' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_NUMBER"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='1m' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='14g' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='11t' -n -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='2P' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='2.1p' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='10E' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='10.2e' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='-1.1T' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_NUMBER"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='-1' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_NUMBER"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='1m' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='14g' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"
$CLICKHOUSE_CLIENT --max_memory_usage_in_client='11t' -q "SELECT * FROM (SELECT * FROM system.numbers LIMIT 600000) as num WHERE num.number=60000" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED"

2
tests/queries/0_stateless/02021_create_database_with_comment.sh
View File

@ -20,7 +20,7 @@ function test_db_comments()
local ENGINE_NAME="$1"
echo "engine : ${ENGINE_NAME}"
$CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -nm <<EOF
$CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -m <<EOF
DROP DATABASE IF EXISTS ${DB_NAME};
CREATE DATABASE ${DB_NAME} ENGINE = ${ENGINE_NAME} COMMENT 'Test DB with comment';
EOF

8
tests/queries/0_stateless/02050_client_profile_events.sh
View File

@ -11,14 +11,14 @@ echo 'print only last (and also number of rows to provide more info in case of f
$CLICKHOUSE_CLIENT --max_block_size=65505 --print-profile-events --profile-events-delay-ms=-1 -q 'select * from numbers(1e5)' |& grep -o -e '\[ 0 \] SelectedRows: .*$' -e Exception
echo 'regression test for incorrect filtering out snapshots'
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -n -q 'select 1; select 1' >& /dev/null
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q 'select 1; select 1' >& /dev/null
echo $?
echo 'regression test for overlap profile events snapshots between queries'
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -n -q 'select 1; select 1' |& grep -F -o '[ 0 ] SelectedRows: 1 (increment)'
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q 'select 1; select 1' |& grep -F -o '[ 0 ] SelectedRows: 1 (increment)'
echo 'regression test for overlap profile events snapshots between queries (clickhouse-local)'
$CLICKHOUSE_LOCAL --print-profile-events --profile-events-delay-ms=-1 -n -q 'select 1; select 1' |& grep -F -o '[ 0 ] SelectedRows: 1 (increment)'
$CLICKHOUSE_LOCAL --print-profile-events --profile-events-delay-ms=-1 -q 'select 1; select 1' |& grep -F -o '[ 0 ] SelectedRows: 1 (increment)'
echo 'print everything'
profile_events="$(
@ -35,5 +35,5 @@ profile_events="$(
test "$profile_events" -gt 1 && echo OK || echo "FAIL ($profile_events)"
echo 'check that ProfileEvents is new for each query'
sleep_function_calls=$($CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -n -q 'select sleep(1); select 1' |& grep -c 'SleepFunctionCalls')
sleep_function_calls=$($CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q 'select sleep(1); select 1' |& grep -c 'SleepFunctionCalls')
test "$sleep_function_calls" -eq 1 && echo OK || echo "FAIL ($sleep_function_calls)"

2
tests/queries/0_stateless/02210_toColumnTypeName_toLowCardinality_const.reference
View File

@ -1 +1 @@
Const(ColumnLowCardinality)
Const(LowCardinality(UInt8))

2
tests/queries/0_stateless/02221_parallel_replicas_bug.sh
View File

@ -4,4 +4,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} --allow_experimental_parallel_reading_from_replicas=1 --parallel_replicas_for_non_replicated_merge_tree=1 -nm < "$CURDIR"/01099_parallel_distributed_insert_select.sql > /dev/null
${CLICKHOUSE_CLIENT} --allow_experimental_parallel_reading_from_replicas=1 --parallel_replicas_for_non_replicated_merge_tree=1 -m < "$CURDIR"/01099_parallel_distributed_insert_select.sql > /dev/null

4
tests/queries/0_stateless/02221_system_zookeeper_unrestricted.sh
View File

@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS sample_table"
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS sample_table_2"
${CLICKHOUSE_CLIENT} -n -q"
${CLICKHOUSE_CLIENT} -q"
CREATE TABLE sample_table (
key UInt64
)
@ -16,7 +16,7 @@ ENGINE ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/02221_
ORDER BY tuple();
"
${CLICKHOUSE_CLIENT} -n -q"
${CLICKHOUSE_CLIENT} -q"
CREATE TABLE sample_table_2 (
key UInt64
)

4
tests/queries/0_stateless/02221_system_zookeeper_unrestricted_like.sh
View File

@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS sample_table;"
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS sample_table_2;"
${CLICKHOUSE_CLIENT} -n --query="CREATE TABLE sample_table (
${CLICKHOUSE_CLIENT} --query="CREATE TABLE sample_table (
key UInt64
)
ENGINE ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/02221_system_zookeeper_unrestricted_like', '1')
@ -16,7 +16,7 @@ ORDER BY tuple();
DROP TABLE IF EXISTS sample_table SYNC;"
${CLICKHOUSE_CLIENT} -n --query "CREATE TABLE sample_table_2 (
${CLICKHOUSE_CLIENT} --query "CREATE TABLE sample_table_2 (
key UInt64
)
ENGINE ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/02221_system_zookeeper_unrestricted_like_2', '1')

6
tests/queries/0_stateless/02225_parallel_distributed_insert_select_view.sh
View File

@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
drop table if exists dst_02225;
drop table if exists src_02225;
create table dst_02225 (key Int) engine=Memory();
@ -14,7 +14,7 @@ create table src_02225 (key Int) engine=Memory();
insert into src_02225 values (1);
"
$CLICKHOUSE_CLIENT --param_database=$CLICKHOUSE_DATABASE -nm -q "
$CLICKHOUSE_CLIENT --param_database=$CLICKHOUSE_DATABASE -m -q "
truncate table dst_02225;
insert into function remote('127.{1,2}', currentDatabase(), dst_02225, key)
select * from remote('127.{1,2}', view(select * from {database:Identifier}.src_02225), key)
@ -29,7 +29,7 @@ settings parallel_distributed_insert_select=2, max_distributed_depth=1;
select * from dst_02225;
"
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
drop table src_02225;
drop table dst_02225;
"

4
tests/queries/0_stateless/02226_parallel_reading_from_replicas_benchmark.sh
View File

@ -4,7 +4,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
drop table if exists data_02226;
create table data_02226 (key Int) engine=MergeTree() order by key
as select * from numbers(1);
@ -24,7 +24,7 @@ opts=(
$CLICKHOUSE_BENCHMARK --query "select * from remote('127.1', $CLICKHOUSE_DATABASE, data_02226)" "${opts[@]}" >& /dev/null
ret=$?
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
drop table data_02226;
"

2
tests/queries/0_stateless/02228_unquoted_dates_in_csv_schema_inference.reference
View File

@ -1 +1 @@
c1 Nullable(DateTime64(9))
c1 Nullable(DateTime)

6
tests/queries/0_stateless/02232_allow_only_replicated_engine.sh
View File

@ -12,9 +12,9 @@ ${CLICKHOUSE_CLIENT} -q "GRANT CREATE TABLE ON ${CLICKHOUSE_DATABASE}_db.* TO us
${CLICKHOUSE_CLIENT} -q "GRANT TABLE ENGINE ON Memory, TABLE ENGINE ON MergeTree, TABLE ENGINE ON ReplicatedMergeTree TO user_${CLICKHOUSE_DATABASE}"
${CLICKHOUSE_CLIENT} -q "CREATE DATABASE ${CLICKHOUSE_DATABASE}_db engine = Replicated('/clickhouse/databases/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_DATABASE}_db', '{shard}', '{replica}')"
${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_memory (x UInt32) engine = Memory;"
${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" -n --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_mt (x UInt32) engine = MergeTree order by x;" 2>&1 | grep -o "Only tables with a Replicated engine"
${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none -n --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_mt (x UInt32) engine = MergeTree order by x;"
${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" -n --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_rmt (x UInt32) engine = ReplicatedMergeTree order by x;"
${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_mt (x UInt32) engine = MergeTree order by x;" 2>&1 | grep -o "Only tables with a Replicated engine"
${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_mt (x UInt32) engine = MergeTree order by x;"
${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --user "user_${CLICKHOUSE_DATABASE}" --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.tab_rmt (x UInt32) engine = ReplicatedMergeTree order by x;"
${CLICKHOUSE_CLIENT} --query "DROP DATABASE ${CLICKHOUSE_DATABASE}_db"
${CLICKHOUSE_CLIENT} -q "DROP USER user_${CLICKHOUSE_DATABASE}"

4
tests/queries/0_stateless/02250_ON_CLUSTER_grant.sh
View File

@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
function cleanup()
{
$CLICKHOUSE_CLIENT -nmq "
$CLICKHOUSE_CLIENT -mq "
DROP USER IF EXISTS with_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME;
DROP USER IF EXISTS without_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME;
DROP DATABASE IF EXISTS db_with_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME;
@ -15,7 +15,7 @@ function cleanup()
cleanup
trap cleanup EXIT
$CLICKHOUSE_CLIENT -nmq "
$CLICKHOUSE_CLIENT -mq "
CREATE USER with_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME;
CREATE USER without_on_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME;

4
tests/queries/0_stateless/02262_column_ttl.sh
View File

@ -14,7 +14,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# note, that this should be written in .sh since we need $CLICKHOUSE_DATABASE
# not 'default' to catch text_log
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
drop table if exists ttl_02262;
drop table if exists this_text_log;
@ -31,7 +31,7 @@ $CLICKHOUSE_CLIENT -nm -q "
ttl_02262_uuid=$($CLICKHOUSE_CLIENT -q "select uuid from system.tables where database = '$CLICKHOUSE_DATABASE' and name = 'ttl_02262'")
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
-- OPTIMIZE TABLE x FINAL will be done in background
-- attach to it's log, via table UUID in query_id (see merger/mutator code).
create materialized view this_text_log engine=Memory() as

4
tests/queries/0_stateless/02286_parallel_final.sh
View File

@ -9,7 +9,7 @@ echo "Test intersecting ranges"
test_random_values() {
layers=$1
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
drop table if exists tbl_8parts_${layers}granules_rnd;
create table tbl_8parts_${layers}granules_rnd (key1 UInt32, sign Int8) engine = CollapsingMergeTree(sign) order by (key1) partition by (key1 % 8);
insert into tbl_8parts_${layers}granules_rnd select number, 1 from numbers_mt($((layers * 8 * 8192)));
@ -29,7 +29,7 @@ echo "Test non intersecting ranges"
test_sequential_values() {
layers=$1
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
drop table if exists tbl_8parts_${layers}granules_seq;
create table tbl_8parts_${layers}granules_seq (key1 UInt32, sign Int8) engine = CollapsingMergeTree(sign) order by (key1) partition by (key1 / $((layers * 8192)))::UInt64;
insert into tbl_8parts_${layers}granules_seq select number, 1 from numbers_mt($((layers * 8 * 8192)));

2
tests/queries/0_stateless/02313_dump_column_structure_low_cardinality.reference
View File

@ -1 +1 @@
Array(LowCardinality(String)), Const(size = 1, Array(size = 1, UInt64(size = 1), ColumnLowCardinality(size = 2, UInt8(size = 2), ColumnUnique(size = 3, String(size = 3)))))
Array(LowCardinality(String)), Const(size = 1, Array(size = 1, UInt64(size = 1), LowCardinality(size = 2, UInt8(size = 2), Unique(size = 3, String(size = 3)))))

66
tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.sh
View File

@ -23,99 +23,99 @@ $CLICKHOUSE_CLIENT -q "insert into distinct_in_order_explain select number % num
$CLICKHOUSE_CLIENT -q "select '-- disable optimize_distinct_in_order'"
$CLICKHOUSE_CLIENT -q "select '-- distinct all primary key columns -> ordinary distinct'"
$CLICKHOUSE_CLIENT -nq "$DISABLE_OPTIMIZATION;explain pipeline select distinct * from distinct_in_order_explain" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "$DISABLE_OPTIMIZATION;explain pipeline select distinct * from distinct_in_order_explain" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "select '-- enable optimize_distinct_in_order'"
$CLICKHOUSE_CLIENT -q "select '-- distinct with all primary key columns -> pre-distinct optimization only'"
$CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct * from distinct_in_order_explain" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct * from distinct_in_order_explain" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "select '-- distinct with primary key prefix -> pre-distinct optimization only'"
$CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, c from distinct_in_order_explain" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, c from distinct_in_order_explain" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "select '-- distinct with primary key prefix and order by column in distinct -> pre-distinct and final distinct optimization'"
$CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, c from distinct_in_order_explain order by c" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, c from distinct_in_order_explain order by c" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "select '-- distinct with primary key prefix and order by the same columns -> pre-distinct and final distinct optimization'"
$CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, b from distinct_in_order_explain order by a, b" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, b from distinct_in_order_explain order by a, b" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "select '-- distinct with primary key prefix and order by columns are prefix of distinct columns -> pre-distinct and final distinct optimization'"
$CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, b from distinct_in_order_explain order by a" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, b from distinct_in_order_explain order by a" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "select '-- distinct with primary key prefix and order by column in distinct but non-primary key prefix -> pre-distinct and final distinct optimization'"
$CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, b, c from distinct_in_order_explain order by c" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, b, c from distinct_in_order_explain order by c" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "select '-- distinct with primary key prefix and order by column _not_ in distinct -> pre-distinct optimization only'"
$CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, c from distinct_in_order_explain order by b" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, c from distinct_in_order_explain order by b" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "select '-- distinct with non-primary key prefix -> ordinary distinct'"
$CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct b, c from distinct_in_order_explain" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct b, c from distinct_in_order_explain" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "select '-- distinct with non-primary key prefix and order by column in distinct -> final distinct optimization only'"
$CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct b, c from distinct_in_order_explain order by b" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct b, c from distinct_in_order_explain order by b" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "select '-- distinct with non-primary key prefix and order by column _not_ in distinct -> ordinary distinct'"
$CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct b, c from distinct_in_order_explain order by a" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct b, c from distinct_in_order_explain order by a" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "select '-- distinct with non-primary key prefix and order by _const_ column in distinct -> ordinary distinct'"
$CLICKHOUSE_CLIENT -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct b, 1 as x from distinct_in_order_explain order by x" | eval $FIND_DISTINCT
$CLICKHOUSE_CLIENT -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct b, 1 as x from distinct_in_order_explain order by x" | eval $FIND_DISTINCT
echo "-- Check reading in order for distinct"
echo "-- disabled, distinct columns match sorting key"
$CLICKHOUSE_CLIENT --max_threads=0 -nq "$DISABLE_OPTIMIZATION;explain pipeline select distinct a, b from distinct_in_order_explain" | eval $FIND_READING_DEFAULT
$CLICKHOUSE_CLIENT --max_threads=0 -q "$DISABLE_OPTIMIZATION;explain pipeline select distinct a, b from distinct_in_order_explain" | eval $FIND_READING_DEFAULT
echo "-- enabled, distinct columns match sorting key"
# read_in_order_two_level_merge_threshold is set here to avoid repeating MergeTreeInOrder in output
$CLICKHOUSE_CLIENT --read_in_order_two_level_merge_threshold=2 -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, b from distinct_in_order_explain" | eval $FIND_READING_IN_ORDER
$CLICKHOUSE_CLIENT --read_in_order_two_level_merge_threshold=2 -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, b from distinct_in_order_explain" | eval $FIND_READING_IN_ORDER
echo "-- enabled, distinct columns form prefix of sorting key"
$CLICKHOUSE_CLIENT --read_in_order_two_level_merge_threshold=2 -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, b from distinct_in_order_explain" | eval $FIND_READING_IN_ORDER
$CLICKHOUSE_CLIENT --read_in_order_two_level_merge_threshold=2 -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, b from distinct_in_order_explain" | eval $FIND_READING_IN_ORDER
echo "-- enabled, distinct columns DON't form prefix of sorting key"
$CLICKHOUSE_CLIENT --max_threads=0 -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct b from distinct_in_order_explain" | eval $FIND_READING_DEFAULT
$CLICKHOUSE_CLIENT --max_threads=0 -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct b from distinct_in_order_explain" | eval $FIND_READING_DEFAULT
echo "-- enabled, distinct columns contains constant columns, non-const columns form prefix of sorting key"
$CLICKHOUSE_CLIENT --read_in_order_two_level_merge_threshold=2 -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct 1, a from distinct_in_order_explain" | eval $FIND_READING_IN_ORDER
$CLICKHOUSE_CLIENT --read_in_order_two_level_merge_threshold=2 -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct 1, a from distinct_in_order_explain" | eval $FIND_READING_IN_ORDER
echo "-- enabled, distinct columns contains constant columns, non-const columns match prefix of sorting key"
$CLICKHOUSE_CLIENT --read_in_order_two_level_merge_threshold=2 -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct 1, b, a from distinct_in_order_explain" | eval $FIND_READING_IN_ORDER
$CLICKHOUSE_CLIENT --read_in_order_two_level_merge_threshold=2 -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct 1, b, a from distinct_in_order_explain" | eval $FIND_READING_IN_ORDER
echo "-- enabled, only part of distinct columns form prefix of sorting key"
$CLICKHOUSE_CLIENT --max_threads=0 -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, c from distinct_in_order_explain" | eval $FIND_READING_DEFAULT
$CLICKHOUSE_CLIENT --max_threads=0 -q "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, c from distinct_in_order_explain" | eval $FIND_READING_DEFAULT
echo "=== disable new analyzer ==="
DISABLE_ANALYZER="set enable_analyzer=0"
echo "-- enabled, check that sorting properties are propagated from ReadFromMergeTree till preliminary distinct"
$CLICKHOUSE_CLIENT -nq "$DISABLE_ANALYZER;$ENABLE_OPTIMIZATION;explain plan sorting=1 select distinct b, a from distinct_in_order_explain where a > 0" | eval $FIND_SORTING_PROPERTIES
$CLICKHOUSE_CLIENT -q "$DISABLE_ANALYZER;$ENABLE_OPTIMIZATION;explain plan sorting=1 select distinct b, a from distinct_in_order_explain where a > 0" | eval $FIND_SORTING_PROPERTIES
echo "-- check that reading in order optimization for ORDER BY and DISTINCT applied correctly in the same query"
ENABLE_READ_IN_ORDER="set optimize_read_in_order=1"
echo "-- disabled, check that sorting description for ReadFromMergeTree match ORDER BY columns"
$CLICKHOUSE_CLIENT -nq "$DISABLE_ANALYZER;$DISABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a" | eval $FIND_SORTING_PROPERTIES
$CLICKHOUSE_CLIENT -q "$DISABLE_ANALYZER;$DISABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a" | eval $FIND_SORTING_PROPERTIES
echo "-- enabled, check that ReadFromMergeTree sorting description is overwritten by DISTINCT optimization i.e. it contains columns from DISTINCT clause"
$CLICKHOUSE_CLIENT -nq "$DISABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a" | eval $FIND_SORTING_PROPERTIES
$CLICKHOUSE_CLIENT -q "$DISABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a" | eval $FIND_SORTING_PROPERTIES
echo "-- enabled, check that ReadFromMergeTree sorting description is overwritten by DISTINCT optimization, but direction used from ORDER BY clause"
$CLICKHOUSE_CLIENT -nq "$DISABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a DESC" | eval $FIND_SORTING_PROPERTIES
$CLICKHOUSE_CLIENT -q "$DISABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a DESC" | eval $FIND_SORTING_PROPERTIES
echo "-- enabled, check that ReadFromMergeTree sorting description is NOT overwritten by DISTINCT optimization (1), - it contains columns from ORDER BY clause"
$CLICKHOUSE_CLIENT -nq "$DISABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct a from distinct_in_order_explain order by a, b" | eval $FIND_SORTING_PROPERTIES
$CLICKHOUSE_CLIENT -q "$DISABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct a from distinct_in_order_explain order by a, b" | eval $FIND_SORTING_PROPERTIES
echo "-- enabled, check that ReadFromMergeTree sorting description is NOT overwritten by DISTINCT optimization (2), - direction used from ORDER BY clause"
$CLICKHOUSE_CLIENT -nq "$DISABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a DESC, b DESC" | eval $FIND_SORTING_PROPERTIES
$CLICKHOUSE_CLIENT -q "$DISABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a DESC, b DESC" | eval $FIND_SORTING_PROPERTIES
echo "-- enabled, check that disabling other 'read in order' optimizations do not disable distinct in order optimization"
$CLICKHOUSE_CLIENT -nq "$DISABLE_ANALYZER;$ENABLE_OPTIMIZATION;set optimize_read_in_order=0;set optimize_aggregation_in_order=0;set optimize_read_in_window_order=0;explain plan sorting=1 select distinct a,b from distinct_in_order_explain" | eval $FIND_SORTING_PROPERTIES
$CLICKHOUSE_CLIENT -q "$DISABLE_ANALYZER;$ENABLE_OPTIMIZATION;set optimize_read_in_order=0;set optimize_aggregation_in_order=0;set optimize_read_in_window_order=0;explain plan sorting=1 select distinct a,b from distinct_in_order_explain" | eval $FIND_SORTING_PROPERTIES
echo "=== enable new analyzer ==="
ENABLE_ANALYZER="set enable_analyzer=1"
echo "-- enabled, check that sorting properties are propagated from ReadFromMergeTree till preliminary distinct"
$CLICKHOUSE_CLIENT -nq "$ENABLE_ANALYZER;$ENABLE_OPTIMIZATION;explain plan sorting=1 select distinct b, a from distinct_in_order_explain where a > 0 settings optimize_move_to_prewhere=1" | eval $FIND_SORTING_PROPERTIES
$CLICKHOUSE_CLIENT -q "$ENABLE_ANALYZER;$ENABLE_OPTIMIZATION;explain plan sorting=1 select distinct b, a from distinct_in_order_explain where a > 0 settings optimize_move_to_prewhere=1" | eval $FIND_SORTING_PROPERTIES
echo "-- disabled, check that sorting description for ReadFromMergeTree match ORDER BY columns"
$CLICKHOUSE_CLIENT -nq "$ENABLE_ANALYZER;$DISABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a" | eval $FIND_SORTING_PROPERTIES
$CLICKHOUSE_CLIENT -q "$ENABLE_ANALYZER;$DISABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a" | eval $FIND_SORTING_PROPERTIES
echo "-- enabled, check that ReadFromMergeTree sorting description is overwritten by DISTINCT optimization i.e. it contains columns from DISTINCT clause"
$CLICKHOUSE_CLIENT -nq "$ENABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a" | eval $FIND_SORTING_PROPERTIES
$CLICKHOUSE_CLIENT -q "$ENABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a" | eval $FIND_SORTING_PROPERTIES
echo "-- enabled, check that ReadFromMergeTree sorting description is overwritten by DISTINCT optimization, but direction used from ORDER BY clause"
$CLICKHOUSE_CLIENT -nq "$ENABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a DESC" | eval $FIND_SORTING_PROPERTIES
$CLICKHOUSE_CLIENT -q "$ENABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a DESC" | eval $FIND_SORTING_PROPERTIES
echo "-- enabled, check that ReadFromMergeTree sorting description is NOT overwritten by DISTINCT optimization (1), - it contains columns from ORDER BY clause"
$CLICKHOUSE_CLIENT -nq "$ENABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct a from distinct_in_order_explain order by a, b" | eval $FIND_SORTING_PROPERTIES
$CLICKHOUSE_CLIENT -q "$ENABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct a from distinct_in_order_explain order by a, b" | eval $FIND_SORTING_PROPERTIES
echo "-- enabled, check that ReadFromMergeTree sorting description is NOT overwritten by DISTINCT optimization (2), - direction used from ORDER BY clause"
$CLICKHOUSE_CLIENT -nq "$ENABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a DESC, b DESC" | eval $FIND_SORTING_PROPERTIES
$CLICKHOUSE_CLIENT -q "$ENABLE_ANALYZER;$ENABLE_OPTIMIZATION;$ENABLE_READ_IN_ORDER;explain plan sorting=1 select distinct b, a from distinct_in_order_explain order by a DESC, b DESC" | eval $FIND_SORTING_PROPERTIES
echo "-- enabled, check that disabling other 'read in order' optimizations do not disable distinct in order optimization"
$CLICKHOUSE_CLIENT -nq "$ENABLE_ANALYZER;$ENABLE_OPTIMIZATION;set optimize_read_in_order=0;set optimize_aggregation_in_order=0;set optimize_read_in_window_order=0;explain plan sorting=1 select distinct a,b from distinct_in_order_explain" | eval $FIND_SORTING_PROPERTIES
$CLICKHOUSE_CLIENT -q "$ENABLE_ANALYZER;$ENABLE_OPTIMIZATION;set optimize_read_in_order=0;set optimize_aggregation_in_order=0;set optimize_read_in_window_order=0;explain plan sorting=1 select distinct a,b from distinct_in_order_explain" | eval $FIND_SORTING_PROPERTIES
$CLICKHOUSE_CLIENT -q "drop table if exists distinct_in_order_explain sync"

58
tests/queries/0_stateless/02325_dates_schema_inference.reference
View File

@ -1,29 +1,29 @@
JSONEachRow
x Nullable(Date)
x Nullable(DateTime64(9))
x Nullable(DateTime64(9))
x Nullable(DateTime)
x Array(Nullable(Date))
x Array(Nullable(DateTime64(9)))
x Array(Nullable(DateTime64(9)))
x Tuple(\n date1 Nullable(DateTime64(9)),\n date2 Nullable(Date))
x Array(Nullable(DateTime64(9)))
x Array(Nullable(DateTime64(9)))
x Nullable(DateTime64(9))
x Array(Nullable(DateTime))
x Array(Nullable(DateTime))
x Tuple(\n date1 Nullable(DateTime),\n date2 Nullable(Date))
x Array(Nullable(DateTime))
x Array(Nullable(DateTime))
x Nullable(DateTime)
x Array(Nullable(String))
x Nullable(String)
x Array(Nullable(String))
x Tuple(\n key1 Array(Array(Nullable(DateTime64(9)))),\n key2 Array(Array(Nullable(String))))
x Tuple(\n key1 Array(Array(Nullable(DateTime))),\n key2 Array(Array(Nullable(String))))
CSV
c1 Nullable(Date)
c1 Nullable(DateTime64(9))
c1 Nullable(DateTime64(9))
c1 Nullable(DateTime)
c1 Array(Nullable(Date))
c1 Array(Nullable(DateTime64(9)))
c1 Array(Nullable(DateTime64(9)))
c1 Map(String, Nullable(DateTime64(9)))
c1 Array(Nullable(DateTime64(9)))
c1 Array(Nullable(DateTime64(9)))
c1 Nullable(DateTime64(9))
c1 Array(Nullable(DateTime))
c1 Array(Nullable(DateTime))
c1 Map(String, Nullable(DateTime))
c1 Array(Nullable(DateTime))
c1 Array(Nullable(DateTime))
c1 Nullable(DateTime)
c1 Array(Nullable(String))
c1 Nullable(String)
c1 Array(Nullable(String))
@ -31,14 +31,14 @@ c1 Map(String, Array(Array(Nullable(String))))
TSV
c1 Nullable(Date)
c1 Nullable(DateTime64(9))
c1 Nullable(DateTime64(9))
c1 Nullable(DateTime)
c1 Array(Nullable(Date))
c1 Array(Nullable(DateTime64(9)))
c1 Array(Nullable(DateTime64(9)))
c1 Map(String, Nullable(DateTime64(9)))
c1 Array(Nullable(DateTime64(9)))
c1 Array(Nullable(DateTime64(9)))
c1 Nullable(DateTime64(9))
c1 Array(Nullable(DateTime))
c1 Array(Nullable(DateTime))
c1 Map(String, Nullable(DateTime))
c1 Array(Nullable(DateTime))
c1 Array(Nullable(DateTime))
c1 Nullable(DateTime)
c1 Array(Nullable(String))
c1 Nullable(String)
c1 Array(Nullable(String))
@ -46,14 +46,14 @@ c1 Map(String, Array(Array(Nullable(String))))
Values
c1 Nullable(Date)
c1 Nullable(DateTime64(9))
c1 Nullable(DateTime64(9))
c1 Nullable(DateTime)
c1 Array(Nullable(Date))
c1 Array(Nullable(DateTime64(9)))
c1 Array(Nullable(DateTime64(9)))
c1 Map(String, Nullable(DateTime64(9)))
c1 Array(Nullable(DateTime64(9)))
c1 Array(Nullable(DateTime64(9)))
c1 Nullable(DateTime64(9))
c1 Array(Nullable(DateTime))
c1 Array(Nullable(DateTime))
c1 Map(String, Nullable(DateTime))
c1 Array(Nullable(DateTime))
c1 Array(Nullable(DateTime))
c1 Nullable(DateTime)
c1 Array(Nullable(String))
c1 Nullable(String)
c1 Array(Nullable(String))

2
tests/queries/0_stateless/02335_column_ttl_expired_column_optimization.sh
View File

@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
data_path="$CLICKHOUSE_TMP/local"
$CLICKHOUSE_LOCAL --path "$data_path" -nm -q "
$CLICKHOUSE_LOCAL --path "$data_path" -m -q "
create table ttl_02335 (
date Date,
key Int,

2
tests/queries/0_stateless/02355_column_type_name_lc.reference
View File

@ -1 +1 @@
ColumnLowCardinality
LowCardinality(String)

4
tests/queries/0_stateless/02361_fsync_profile_events.sh
View File

@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
$CLICKHOUSE_CLIENT -m -q "
drop table if exists data_fsync_pe;
create table data_fsync_pe (key Int) engine=MergeTree()
@ -27,7 +27,7 @@ for i in {1..100}; do
$CLICKHOUSE_CLIENT --query_id "$query_id" -q "insert into data_fsync_pe values (1)"
read -r FileSync FileOpen DirectorySync FileSyncElapsedMicroseconds DirectorySyncElapsedMicroseconds <<<"$(
$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "
$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q "
system flush logs;
select

10
tests/queries/0_stateless/02377_extend_protocol_with_query_parameters.sh
View File

@ -24,7 +24,7 @@ $CLICKHOUSE_CLIENT \
table_name="t_02377_extend_protocol_with_query_parameters_$RANDOM$RANDOM"
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
create table $table_name(
id Int64,
arr Array(UInt8),
@ -57,17 +57,17 @@ $CLICKHOUSE_CLIENT \
# it is possible to set parameter for the current session
$CLICKHOUSE_CLIENT -n -q "set param_n = 42; select {n: UInt8}"
$CLICKHOUSE_CLIENT -q "set param_n = 42; select {n: UInt8}"
# and it will not be visible to other sessions
$CLICKHOUSE_CLIENT -n -q "select {n: UInt8} -- { serverError 456 }"
$CLICKHOUSE_CLIENT -q "select {n: UInt8} -- { serverError 456 }"
# the same parameter could be set multiple times within one session (new value overrides the previous one)
$CLICKHOUSE_CLIENT -n -q "set param_n = 12; set param_n = 13; select {n: UInt8}"
$CLICKHOUSE_CLIENT -q "set param_n = 12; set param_n = 13; select {n: UInt8}"
# multiple different parameters could be defined within each session
$CLICKHOUSE_CLIENT -n -q "
$CLICKHOUSE_CLIENT -q "
set param_a = 13, param_b = 'str';
set param_c = '2022-08-04 18:30:53';
set param_d = '{\'10\': [11, 12], \'13\': [14, 15]}';

2
tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh
View File

@ -15,7 +15,7 @@ FIND_SORTMODE="$GREP_SORTMODE | $TRIM_LEADING_SPACES"
function explain_sorting {
echo "-- QUERY: "$1
$CLICKHOUSE_CLIENT --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0 -nq "$1" | eval $FIND_SORTING
$CLICKHOUSE_CLIENT --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0 -q "$1" | eval $FIND_SORTING
}
function explain_sortmode {

10
tests/queries/0_stateless/02404_data.CSV Normal file
View File

@ -0,0 +1,10 @@
0,"1970-01-01"
1,"1970-01-02"
2,"1970-01-03"
3,"1970-01-04"
4,"1970-01-05"
5,"1970-01-06"
6,"1970-01-07"
7,"1970-01-08"
8,"1970-01-09"
9,"1970-01-10"
1 0 1970-01-01
2 1 1970-01-02
3 2 1970-01-03
4 3 1970-01-04
5 4 1970-01-05
6 5 1970-01-06
7 6 1970-01-07
8 7 1970-01-08
9 8 1970-01-09
10 9 1970-01-10

11
tests/queries/0_stateless/02404_data.CSVWithNames Normal file
View File

@ -0,0 +1,11 @@
"number","toDate(number)"
0,"1970-01-01"
1,"1970-01-02"
2,"1970-01-03"
3,"1970-01-04"
4,"1970-01-05"
5,"1970-01-06"
6,"1970-01-07"
7,"1970-01-08"
8,"1970-01-09"
9,"1970-01-10"

10
tests/queries/0_stateless/02404_data.CustomSeparated Normal file
View File

@ -0,0 +1,10 @@
0 1970-01-01
1 1970-01-02
2 1970-01-03
3 1970-01-04
4 1970-01-05
5 1970-01-06
6 1970-01-07
7 1970-01-08
8 1970-01-09
9 1970-01-10

10
tests/queries/0_stateless/02404_data.JSONCompactEachRow Normal file
View File

@ -0,0 +1,10 @@
["0", "1970-01-01"]
["1", "1970-01-02"]
["2", "1970-01-03"]
["3", "1970-01-04"]
["4", "1970-01-05"]
["5", "1970-01-06"]
["6", "1970-01-07"]
["7", "1970-01-08"]
["8", "1970-01-09"]
["9", "1970-01-10"]

10
tests/queries/0_stateless/02404_data.JSONEachRow Normal file
View File

@ -0,0 +1,10 @@
{"number":"0","toDate(number)":"1970-01-01"}
{"number":"1","toDate(number)":"1970-01-02"}
{"number":"2","toDate(number)":"1970-01-03"}
{"number":"3","toDate(number)":"1970-01-04"}
{"number":"4","toDate(number)":"1970-01-05"}
{"number":"5","toDate(number)":"1970-01-06"}
{"number":"6","toDate(number)":"1970-01-07"}
{"number":"7","toDate(number)":"1970-01-08"}
{"number":"8","toDate(number)":"1970-01-09"}
{"number":"9","toDate(number)":"1970-01-10"}

10
tests/queries/0_stateless/02404_data.TSKV Normal file
View File

@ -0,0 +1,10 @@
number=0 toDate(number)=1970-01-01
number=1 toDate(number)=1970-01-02
number=2 toDate(number)=1970-01-03
number=3 toDate(number)=1970-01-04
number=4 toDate(number)=1970-01-05
number=5 toDate(number)=1970-01-06
number=6 toDate(number)=1970-01-07
number=7 toDate(number)=1970-01-08
number=8 toDate(number)=1970-01-09
number=9 toDate(number)=1970-01-10

10
tests/queries/0_stateless/02404_data.TSV Normal file
View File

@ -0,0 +1,10 @@
0 1970-01-01
1 1970-01-02
2 1970-01-03
3 1970-01-04
4 1970-01-05
5 1970-01-06
6 1970-01-07
7 1970-01-08
8 1970-01-09
9 1970-01-10
1 0 1970-01-01
2 1 1970-01-02
3 2 1970-01-03
4 3 1970-01-04
5 4 1970-01-05
6 5 1970-01-06
7 6 1970-01-07
8 7 1970-01-08
9 8 1970-01-09
10 9 1970-01-10

11
tests/queries/0_stateless/02404_data.TSVWithNames Normal file
View File

@ -0,0 +1,11 @@
number toDate(number)
0 1970-01-01
1 1970-01-02
2 1970-01-03
3 1970-01-04
4 1970-01-05
5 1970-01-06
6 1970-01-07
7 1970-01-08
8 1970-01-09
9 1970-01-10

1
tests/queries/0_stateless/02404_data.Values Normal file
View File

@ -0,0 +1 @@
(0,'1970-01-01'),(1,'1970-01-02'),(2,'1970-01-03'),(3,'1970-01-04'),(4,'1970-01-05'),(5,'1970-01-06'),(6,'1970-01-07'),(7,'1970-01-08'),(8,'1970-01-09'),(9,'1970-01-10')

18
tests/queries/0_stateless/02404_schema_inference_cache_respect_format_settings.reference
View File

@ -4,7 +4,7 @@ c2 Nullable(Date)
c1 Nullable(Float64)
c2 Nullable(Date)
c1 Nullable(Int64)
c2 Nullable(DateTime64(9))
c2 Nullable(DateTime)
c1 UInt8
c2 Nullable(Date)
4
@ -14,7 +14,7 @@ toDate(number) Nullable(Date)
number Nullable(Float64)
toDate(number) Nullable(Date)
number Nullable(Int64)
toDate(number) Nullable(DateTime64(9))
toDate(number) Nullable(DateTime)
number Nullable(Int64)
toDate(number) Nullable(Date)
4
@ -24,7 +24,7 @@ c2 Nullable(Date)
c1 Nullable(Float64)
c2 Nullable(Date)
c1 Nullable(Int64)
c2 Nullable(DateTime64(9))
c2 Nullable(DateTime)
c1 UInt8
c2 Nullable(Date)
4
@ -34,7 +34,7 @@ toDate(number) Nullable(Date)
number Nullable(Float64)
toDate(number) Nullable(Date)
number Nullable(Int64)
toDate(number) Nullable(DateTime64(9))
toDate(number) Nullable(DateTime)
number Nullable(Int64)
toDate(number) Nullable(Date)
4
@ -44,7 +44,7 @@ toDate(number) Nullable(Date)
number Nullable(Float64)
toDate(number) Nullable(Date)
number Nullable(Int64)
toDate(number) Nullable(DateTime64(9))
toDate(number) Nullable(DateTime)
number Nullable(Int64)
toDate(number) Nullable(Date)
4
@ -54,7 +54,7 @@ c2 Nullable(Date)
c1 Nullable(Float64)
c2 Nullable(Date)
c1 Nullable(Int64)
c2 Nullable(DateTime64(9))
c2 Nullable(DateTime)
c1 UInt8
c2 Nullable(Date)
4
@ -64,7 +64,7 @@ toDate(number) Nullable(Date)
number Nullable(Float64)
toDate(number) Nullable(Date)
number Nullable(Int64)
toDate(number) Nullable(DateTime64(9))
toDate(number) Nullable(DateTime)
number Nullable(Int64)
toDate(number) Nullable(Date)
4
@ -74,7 +74,7 @@ c2 Nullable(Date)
c1 Nullable(Float64)
c2 Nullable(Date)
c1 Nullable(Int64)
c2 Nullable(DateTime64(9))
c2 Nullable(DateTime)
c1 UInt8
c2 Nullable(Date)
4
@ -84,7 +84,7 @@ c2 Nullable(Date)
c1 Nullable(Float64)
c2 Nullable(Date)
c1 Nullable(Int64)
c2 Nullable(DateTime64(9))
c2 Nullable(DateTime)
c1 UInt8
c2 Nullable(Date)
4

2
tests/queries/0_stateless/02417_load_marks_async.sh
View File

@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS test;"
${CLICKHOUSE_CLIENT} -n -q "
${CLICKHOUSE_CLIENT} -q "
CREATE TABLE test
(
n0 UInt64,

12
tests/queries/0_stateless/03033_dynamic_text_serialization.reference
View File

@ -4,7 +4,7 @@ JSON
{"d":"str","dynamicType(d)":"String"}
{"d":["1","2","3"],"dynamicType(d)":"Array(Int64)"}
{"d":"2020-01-01","dynamicType(d)":"Date"}
{"d":"2020-01-01 10:00:00.000000000","dynamicType(d)":"DateTime64(9)"}
{"d":"2020-01-01 10:00:00","dynamicType(d)":"DateTime"}
{"d":{"a":"42","b":"str"},"dynamicType(d)":"Tuple(a Int64, b String)"}
{"d":{"a":"43"},"dynamicType(d)":"Tuple(a Int64)"}
{"d":{"a":"44","c":["1","2","3"]},"dynamicType(d)":"Tuple(a Int64, c Array(Int64))"}
@ -22,7 +22,7 @@ CSV
"str","String"
"[1,2,3]","Array(Int64)"
"2020-01-01","Date"
"2020-01-01 10:00:00.000000000","DateTime64(9)"
"2020-01-01 10:00:00","DateTime"
"[1, 'str', [1, 2, 3]]","String"
\N,"None"
true,"Bool"
@ -32,24 +32,24 @@ TSV
str String
[1,2,3] Array(Int64)
2020-01-01 Date
2020-01-01 10:00:00.000000000 DateTime64(9)
2020-01-01 10:00:00 DateTime
[1, \'str\', [1, 2, 3]] String
\N None
true Bool
Values
(42,'Int64'),(42.42,'Float64'),('str','String'),([1,2,3],'Array(Int64)'),('2020-01-01','Date'),('2020-01-01 10:00:00.000000000','DateTime64(9)'),(NULL,'None'),(true,'Bool')
(42,'Int64'),(42.42,'Float64'),('str','String'),([1,2,3],'Array(Int64)'),('2020-01-01','Date'),('2020-01-01 10:00:00','DateTime'),(NULL,'None'),(true,'Bool')
Cast using parsing
42 Int64
42.42 Float64
[1,2,3] Array(Int64)
2020-01-01 Date
2020-01-01 10:00:00.000000000 DateTime64(9)
2020-01-01 10:00:00 DateTime
NULL String
true Bool
42 Int64 false
42.42 Float64 false
[1,2,3] Array(Int64) false
2020-01-01 Date true
2020-01-01 10:00:00.000000000 DateTime64(9) true
2020-01-01 10:00:00 DateTime true
NULL String true
true Bool true

2
tests/queries/0_stateless/03199_json_extract_dynamic.reference
View File

@ -12,7 +12,7 @@ Hello String
[1,2,3] Array(Nullable(Int64))
['str1','str2','str3'] Array(Nullable(String))
[[[1],[2,3,4]],[[5,6],[7]]] Array(Array(Array(Nullable(Int64))))
['2020-01-01 00:00:00.000000000','2020-01-01 00:00:00.000000000'] Array(Nullable(DateTime64(9)))
['2020-01-01 00:00:00','2020-01-01 00:00:00'] Array(Nullable(DateTime))
['2020-01-01','2020-01-01 date'] Array(Nullable(String))
['2020-01-01','2020-01-01 00:00:00','str'] Array(Nullable(String))
['2020-01-01','2020-01-01 00:00:00','42'] Array(Nullable(String))

0
tests/queries/0_stateless/03209_parallel_replicas_lost_decimal_conversion.reference Normal file
View File

11
tests/queries/0_stateless/03209_parallel_replicas_lost_decimal_conversion.sql Normal file
View File

@ -0,0 +1,11 @@
DROP TABLE IF EXISTS t_03209 SYNC;
CREATE TABLE t_03209 ( `a` Decimal(18, 0), `b` Decimal(18, 1), `c` Decimal(36, 0) ) ENGINE = ReplicatedMergeTree('/clickhouse/{database}/test_03209', 'r1') ORDER BY tuple();
INSERT INTO t_03209 VALUES ('33', '44.4', '35');
SET max_parallel_replicas = 2, cluster_for_parallel_replicas='parallel_replicas';
SELECT * FROM t_03209 WHERE a IN toDecimal32('33.3000', 4) SETTINGS allow_experimental_parallel_reading_from_replicas=0;
SELECT * FROM t_03209 WHERE a IN toDecimal32('33.3000', 4) SETTINGS allow_experimental_parallel_reading_from_replicas=1;
DROP TABLE t_03209 SYNC;

2
tests/queries/0_stateless/03212_max_bytes_to_read_for_schema_inference_in_cache.reference
View File

@ -1,2 +1,2 @@
x Nullable(Int64)
schema_inference_hints=, max_rows_to_read_for_schema_inference=25000, max_bytes_to_read_for_schema_inference=1000, schema_inference_make_columns_nullable=true, try_infer_integers=true, try_infer_dates=true, try_infer_datetimes=true, try_infer_numbers_from_strings=false, read_bools_as_numbers=true, read_bools_as_strings=true, read_objects_as_strings=true, read_numbers_as_strings=true, read_arrays_as_strings=true, try_infer_objects_as_tuples=true, infer_incomplete_types_as_strings=true, try_infer_objects=false, use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects=false
schema_inference_hints=, max_rows_to_read_for_schema_inference=25000, max_bytes_to_read_for_schema_inference=1000, schema_inference_make_columns_nullable=true, try_infer_integers=true, try_infer_dates=true, try_infer_datetimes=true, try_infer_datetimes_only_datetime64=false, try_infer_numbers_from_strings=false, read_bools_as_numbers=true, read_bools_as_strings=true, read_objects_as_strings=true, read_numbers_as_strings=true, read_arrays_as_strings=true, try_infer_objects_as_tuples=true, infer_incomplete_types_as_strings=true, try_infer_objects=false, use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects=false

0
tests/queries/0_stateless/03221_create_if_not_exists_setting.reference Normal file
View File

24
tests/queries/0_stateless/03221_create_if_not_exists_setting.sql Normal file
View File

@ -0,0 +1,24 @@
-- Tags: no-parallel
SET create_if_not_exists=0; -- Default
DROP TABLE IF EXISTS example_table;
CREATE TABLE example_table (id UInt32) ENGINE=MergeTree() ORDER BY id;
CREATE TABLE example_table (id UInt32) ENGINE=MergeTree() ORDER BY id; -- { serverError TABLE_ALREADY_EXISTS }
DROP DATABASE IF EXISTS example_database;
CREATE DATABASE example_database;
CREATE DATABASE example_database; -- { serverError DATABASE_ALREADY_EXISTS }
SET create_if_not_exists=1;
DROP TABLE IF EXISTS example_table;
CREATE TABLE example_table (id UInt32) ENGINE=MergeTree() ORDER BY id;
CREATE TABLE example_table (id UInt32) ENGINE=MergeTree() ORDER BY id;
DROP DATABASE IF EXISTS example_database;
CREATE DATABASE example_database;
CREATE DATABASE example_database;
DROP DATABASE IF EXISTS example_database;
DROP TABLE IF EXISTS example_table;

1
tests/queries/0_stateless/03221_s3_imds_decent_timeout.reference Normal file
View File

@ -0,0 +1 @@
1

16
tests/queries/0_stateless/03221_s3_imds_decent_timeout.sh Executable file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env bash
# Tags: no-fasttest, no-asan, no-msan, no-tsan
# ^ requires S3
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
# Inaccessible IMDS should not introduce large delays, so this query should reply quickly at least sometimes:
while true
do
# This host (likely) drops packets sent to it (does not reply), so it is good for testing timeouts.
# At the same time, we expect that the clickhouse host does not drop packets and quickly replies with 4xx, which is a non-retriable error for S3.
AWS_EC2_METADATA_SERVICE_ENDPOINT='https://10.255.255.255/' ${CLICKHOUSE_LOCAL} --time --query "SELECT * FROM s3('${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/nonexistent')" |& grep -v -F 404 |
${CLICKHOUSE_LOCAL} --input-format TSV "SELECT c1::Float64 < 1 FROM table" | grep 1 && break
done

253
tests/queries/0_stateless/03222_date_time_inference.reference Normal file
View File

@ -0,0 +1,253 @@
Date
2020-01-01 Date
2020-01-01 Date
2020-01-01 Date
2020-01-01 Date
2020-01-01 Date
2020-01-01 Date
2020-01-01 Date
2020-01-01 Date
2020-01-01 Date
2020-01-01 Date
2020-01-01 Date
2020-01-01 Date
String
2020_01_01 String
2020_1_01 String
2020_01_1 String
2020_1_1 String
2020a01a01 String
2020a1a01 String
2020a01a1 String
2020a1a1 String
20200101 String
DateTime
2020-01-02 18:42:42 DateTime
2020-01-02 18:42:42 DateTime
2020-01-02 18:42:42 DateTime
String
2020_01_01 42:42:42 String
2020a01a01 42:42:42 String
2020-01-01 42.42.42 String
2020-01-01 42 42 42 String
2020-01-01 42a42a42 String
DateTime64
2020-01-02 18:42:42.424200000 DateTime64(9)
2020-01-02 18:42:42.424200000 DateTime64(9)
2020-01-02 18:42:42.424200000 DateTime64(9)
String
2020_01_01 42:42:42.4242 String
2020a01a01 42:42:42.4242 String
2020-01-01 42.42.42.4242 String
2020-01-01 42 42 42.4242 String
2020-01-01 42a42a42.4242 String
DateTime/DateTime64 best effort
2000-01-01 00:00:00 DateTime
2000-01-01 01:00:00 DateTime
2000-01-01 01:00:00.000000000 DateTime64(9)
2017-01-01 22:02:03 DateTime
2017-01-01 22:02:03.000000000 DateTime64(9)
2017-01-01 21:02:03 DateTime
2017-01-01 21:02:03.000000000 DateTime64(9)
2017-01-01 22:02:03 DateTime
2017-01-01 22:02:03.000000000 DateTime64(9)
2017-01-02 01:02:03 DateTime
2017-01-02 01:02:03.000000000 DateTime64(9)
1970-01-02 01:02:03 DateTime
1970-01-02 01:02:03.000000000 DateTime64(9)
1970-01-02 01:02:03 DateTime
1970-01-02 01:02:03.000000000 DateTime64(9)
2018-02-11 03:40:50 DateTime
2018-02-11 03:40:50.000000000 DateTime64(9)
2000-04-17 01:02:03 DateTime
2000-04-17 01:02:03.000000000 DateTime64(9)
1970-01-02 01:00:00 DateTime
1970-01-02 01:00:00.000000000 DateTime64(9)
1970-01-02 01:02:03 DateTime
1970-01-02 01:02:03.000000000 DateTime64(9)
1970-01-02 01:02:03 DateTime
1970-01-02 01:02:03.000000000 DateTime64(9)
2015-12-31 20:00:00 DateTime
2015-12-31 20:00:00 DateTime
2016-01-01 00:00:00 DateTime
2016-01-01 00:00:00 DateTime
2017-01-01 22:02:03 DateTime
2017-01-01 22:02:03.000000000 DateTime64(9)
2017-01-02 03:04:05 DateTime
2017-01-02 03:04:05.000000000 DateTime64(9)
2017-01-02 03:04:05 DateTime
2017-01-02 03:04:05.000000000 DateTime64(9)
2017-01-02 03:04:05 DateTime
2017-01-02 03:04:05.000000000 DateTime64(9)
2017-01-02 03:04:05 DateTime
2017-01-02 03:04:05.000000000 DateTime64(9)
2017-01-02 04:04:05 DateTime
2017-01-02 04:04:05.000000000 DateTime64(9)
2017-01-02 02:34:05 DateTime
2017-01-02 02:34:05.000000000 DateTime64(9)
2017-01-02 00:04:05 DateTime
2017-01-02 00:04:05.000000000 DateTime64(9)
2017-01-02 02:04:05 DateTime
2017-01-02 02:04:05.000000000 DateTime64(9)
2017-01-02 00:04:05 DateTime
2017-01-02 00:04:05.000000000 DateTime64(9)
2017-01-01 18:04:05 DateTime
2017-01-01 18:04:05.000000000 DateTime64(9)
2017-01-02 03:04:05 DateTime
2017-01-02 03:04:05.000000000 DateTime64(9)
2017-01-01 23:04:05 DateTime
2017-01-01 23:04:05.000000000 DateTime64(9)
2017-02-01 23:04:05 DateTime
2017-02-01 23:04:05.000000000 DateTime64(9)
2017-06-01 23:04:05 DateTime
2017-06-01 23:04:05.000000000 DateTime64(9)
2017-01-02 00:04:05 DateTime
2017-01-02 00:04:05.000000000 DateTime64(9)
2017-01-02 03:04:05 DateTime
2017-01-02 03:04:05.000000000 DateTime64(9)
2017-01-02 03:04:05 DateTime
2017-01-02 03:04:05.000000000 DateTime64(9)
2017-01-02 04:04:05 DateTime
2017-01-02 04:04:05.000000000 DateTime64(9)
2017-01-02 04:04:05 DateTime
2017-01-02 04:04:05.000000000 DateTime64(9)
2017-01-02 02:04:05 DateTime
2017-01-02 02:04:05.000000000 DateTime64(9)
2017-01-02 03:04:05 DateTime
2017-01-02 03:04:05.000000000 DateTime64(9)
2017-04-01 11:22:33 DateTime
2017-04-01 11:22:33.000000000 DateTime64(9)
2017-04-01 22:02:03 DateTime
2017-04-01 22:02:03.000000000 DateTime64(9)
2017-04-01 22:02:03 DateTime
2017-04-01 22:02:03.000000000 DateTime64(9)
2017-04-02 01:02:03 DateTime
2017-04-02 01:02:03.000000000 DateTime64(9)
2017-04-02 11:22:33 DateTime
2017-04-02 11:22:33.000000000 DateTime64(9)
2017-04-02 01:02:03 DateTime
2017-04-02 01:02:03.000000000 DateTime64(9)
2017-04-02 01:22:33 DateTime
2017-04-02 01:22:33.000000000 DateTime64(9)
2017-04-02 01:02:03 DateTime
2017-04-02 01:02:03.000000000 DateTime64(9)
2017-04-02 01:02:33 DateTime
2017-04-02 01:02:33.000000000 DateTime64(9)
2017-04-01 22:02:03 DateTime
2017-04-01 22:02:03.000000000 DateTime64(9)
2017-04-02 01:02:03 DateTime
2017-04-02 01:02:03.000000000 DateTime64(9)
2017-04-01 22:02:03 DateTime
2017-04-01 22:02:03.000000000 DateTime64(9)
2017-04-01 21:02:03 DateTime
2017-04-01 21:02:03.000000000 DateTime64(9)
2017-04-02 01:02:03 DateTime
2017-04-02 01:02:03.000000000 DateTime64(9)
2017-01-01 22:02:03 DateTime
2017-01-01 22:02:03.000000000 DateTime64(9)
2017-04-25 01:02:03 DateTime
2017-04-25 01:02:03.000000000 DateTime64(9)
2017-04-25 01:02:03 DateTime
2017-04-25 01:02:03.000000000 DateTime64(9)
2017-01-25 01:02:03 DateTime
2017-01-25 01:02:03.000000000 DateTime64(9)
2017-01-24 22:02:03 DateTime
2017-01-24 22:02:03.000000000 DateTime64(9)
2017-01-25 13:02:03 DateTime
2017-01-25 13:02:03.000000000 DateTime64(9)
2017-01-25 01:02:03 DateTime
2017-01-25 01:02:03.000000000 DateTime64(9)
2017-01-25 01:02:03 DateTime
2017-01-25 01:02:03.000000000 DateTime64(9)
2017-01-24 22:02:03 DateTime
2017-01-24 22:02:03.000000000 DateTime64(9)
2017-01-24 22:02:03 DateTime
2017-01-24 22:02:03.000000000 DateTime64(9)
2017-01-25 10:02:03 DateTime
2017-01-25 10:02:03.000000000 DateTime64(9)
2017-01-25 10:02:03 DateTime
2017-01-25 10:02:03.000000000 DateTime64(9)
2017-01-25 10:02:03 DateTime
2017-01-25 10:02:03.000000000 DateTime64(9)
2017-01-25 09:32:03 DateTime
2017-01-25 09:32:03.000000000 DateTime64(9)
2017-01-25 01:02:03 DateTime
2017-01-25 01:02:03.000000000 DateTime64(9)
2017-01-25 13:02:03 DateTime
2017-01-25 13:02:03.000000000 DateTime64(9)
2017-01-25 13:02:03 DateTime
2017-01-25 13:02:03.000000000 DateTime64(9)
2017-01-25 10:02:03 DateTime
2017-01-25 10:02:03.000000000 DateTime64(9)
2018-02-11 03:40:50 DateTime
2018-02-11 03:40:50.000000000 DateTime64(9)
2018-02-11 03:40:50 DateTime
2018-02-11 03:40:50.000000000 DateTime64(9)
String
2 String
20 String
200 String
2000 String
20000 String
200001 String
2000010 String
20000101 String
200001010 String
2000010101 String
20000101010 String
200001010101 String
2000010101010 String
20000101010101 String
2.1 String
20.1 String
200.1 String
2000.1 String
20000.1 String
200001.1 String
2000010.1 String
20000101.1 String
200001010.1 String
2000010101.1 String
20000101010.1 String
200001010101.1 String
2000010101010.1 String
20000101010101.1 String
Mar String
Mar1 String
Mar 1 String
Mar01 String
Mar 01 String
Mar2020 String
Mar 2020 String
Mar012020 String
Mar 012020 String
Mar01012020 String
Mar 01012020 String
Mar0101202001 String
Mar 0101202001 String
Mar010120200101 String
Mar 010120200101 String
Mar01012020010101 String
Mar 01012020010101 String
Mar01012020010101.000 String
Mar 0101202001010101.000 String
2000 01 01 01:00:00 String
2000 01 01 01:00:00.000 String
2000a01a01 01:00:00 String
2000a01a01 01:00:00.000 String
2000-01-01 01 00 00 String
2000-01-01 01 00 00.000 String
2000-01-01 01-00-00 String
2000-01-01 01-00-00.000 String
2000-01-01 01a00a00 String
2000-01-01 01a00a00.000 String
2000-01 01:00:00 String
2000-01 01:00:00.000 String
2000 01 String
2000-01 String
Mar 2000 00:00:00 String
Mar 2000 00:00:00.000 String
2000 00:00:00 String
2000 00:00:00.000 String
Mar 2000-01-01 00:00:00 String
Mar 2000-01-01 00:00:00.000 String

269
tests/queries/0_stateless/03222_date_time_inference.sql Normal file
View File

@ -0,0 +1,269 @@
set input_format_try_infer_datetimes = 1;
set input_format_try_infer_dates = 1;
set schema_inference_make_columns_nullable = 0;
set input_format_json_try_infer_numbers_from_strings = 0;
set session_timezone = 'UTC';
select 'Date';
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020:01:01"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020:1:01"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020:01:1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020:1:1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020-01-01"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020-1-01"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020-01-1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020-1-1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020/01/01"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020/1/01"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020/01/1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020/1/1"}');
select 'String';
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020_01_01"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020_1_01"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020_01_1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020_1_1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020a01a01"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020a1a01"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020a01a1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020a1a1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "20200101"}');
select 'DateTime';
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020:01:01 42:42:42"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020/01/01 42:42:42"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020-01-01 42:42:42"}');
select 'String';
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020_01_01 42:42:42"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020a01a01 42:42:42"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020-01-01 42.42.42"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020-01-01 42 42 42"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020-01-01 42a42a42"}');
select 'DateTime64';
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020:01:01 42:42:42.4242"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020/01/01 42:42:42.4242"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020-01-01 42:42:42.4242"}');
select 'String';
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020_01_01 42:42:42.4242"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020a01a01 42:42:42.4242"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020-01-01 42.42.42.4242"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020-01-01 42 42 42.4242"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2020-01-01 42a42a42.4242"}');
set date_time_input_format='best_effort';
select 'DateTime/DateTime64 best effort';
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000-01-01 00:00:00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000-01-01 01:00:00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000-01-01 01:00:00.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "02/01/17 010203 MSK"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "02/01/17 010203.000 MSK"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "02/01/17 010203 MSK+0100"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "02/01/17 010203.000 MSK+0100"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "02/01/17 010203 UTC+0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "02/01/17 010203.000 UTC+0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "02/01/17 010203Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "02/01/17 010203.000Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "02/01/1970 010203Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "02/01/1970 010203.000Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "02/01/70 010203Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "02/01/70 010203.000Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "11 Feb 2018 06:40:50 +0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "11 Feb 2018 06:40:50.000 +0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "17 Apr 2000 2 1:2:3"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "17 Apr 2000 2 1:2:3.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "19700102 01:00:00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "19700102 01:00:00.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "19700102010203Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "19700102010203Z.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "1970/01/02 010203Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "1970/01/02 010203.000Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2016-01-01MSD"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2016-01-01 MSD"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2016-01-01UTC"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2016-01-01Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "201701 02 010203 UTC+0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "201701 02 010203.000 UTC+0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05+0"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05.000+0"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05+00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05.000+00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05+0000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05.000+0000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05 -0100"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05.000 -0100"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05+030"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05.000+030"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05+0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05.000+0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05+1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05.000+1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05+300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05.000+300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05+900"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05.000+900"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05GMT"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05.000GMT"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05 MSD"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05.000 MSD"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05 MSD Feb"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05.000 MSD Feb"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05 MSD Jun"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05.000 MSD Jun"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05 MSK"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02 03:04:05.000 MSK"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02T03:04:05"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02T03:04:05.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02T03:04:05+00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02T03:04:05.000+00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02T03:04:05 -0100"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02T03:04:05.000 -0100"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02T03:04:05-0100"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02T03:04:05.000-0100"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02T03:04:05+0100"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02T03:04:05.000+0100"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02T03:04:05Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017-01-02T03:04:05.000Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 01 11:22:33"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 01 11:22:33.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 010203 UTC+0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 010203.000 UTC+0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 01:2:3 UTC+0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 01:2:3.000 UTC+0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:02:3"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:02:3.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 11:22:33"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 11:22:33.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:2:03"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:2:03.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:22:33"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:22:33.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:2:3"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:2:3.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:2:33"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:2:33.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:2:3 MSK"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:2:3.000 MSK"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:2:3 UTC+0000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:2:3.000 UTC+0000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:2:3 UTC+0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:2:3.000 UTC+0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:2:3 UTC+0400"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 02 1:2:3.000 UTC+0400"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 2 1:2:3"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Apr 2 1:2:3.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Jan 02 010203 UTC+0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2017 Jan 02 010203.000 UTC+0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Apr 2017 01:02:03"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Apr 2017 01:02:03.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Apr 2017 1:2:3"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Apr 2017 1:2:3.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3 MSK"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3.000 MSK"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3 PM"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3.000 PM"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3.000Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3 Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3.000 Z"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3 Z +0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3.000 Z +0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3 Z+03:00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3.000 Z+03:00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3 Z +03:00 PM"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3.000 Z +03:00 PM"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3 Z +0300 PM"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3.000 Z +0300 PM"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3 Z+03:00 PM"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3.000 Z+03:00 PM"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3 Z +03:30 PM"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3.000 Z +03:30 PM"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3Z Mon"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3.000Z Mon"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3 Z PM"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3.000 Z PM"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3Z PM"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3.000Z PM"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3 Z PM +03:00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "25 Jan 2017 1:2:3.000 Z PM +03:00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Sun 11 Feb 2018 06:40:50 +0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Sun 11 Feb 2018 06:40:50.000 +0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Sun, 11 Feb 2018 06:40:50 +0300"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Sun, 11 Feb 2018 06:40:50.000 +0300"}');
select 'String';
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "20"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "200"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "20000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "200001"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000010"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "20000101"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "200001010"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000010101"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "20000101010"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "200001010101"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000010101010"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "20000101010101"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2.1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "20.1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "200.1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000.1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "20000.1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "200001.1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000010.1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "20000101.1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "200001010.1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000010101.1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "20000101010.1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "200001010101.1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000010101010.1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "20000101010101.1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar 1"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar01"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar 01"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar2020"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar 2020"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar012020"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar 012020"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar01012020"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar 01012020"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar0101202001"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar 0101202001"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar010120200101"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar 010120200101"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar01012020010101"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar 01012020010101"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar01012020010101.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar 0101202001010101.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000 01 01 01:00:00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000 01 01 01:00:00.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000a01a01 01:00:00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000a01a01 01:00:00.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000-01-01 01 00 00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000-01-01 01 00 00.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000-01-01 01-00-00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000-01-01 01-00-00.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000-01-01 01a00a00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000-01-01 01a00a00.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000-01 01:00:00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000-01 01:00:00.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000 01"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000-01"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar 2000 00:00:00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar 2000 00:00:00.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000 00:00:00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "2000 00:00:00.000"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar 2000-01-01 00:00:00"}');
select x, toTypeName(x) from format(JSONEachRow, '{"x" : "Mar 2000-01-01 00:00:00.000"}');

21
tests/queries/0_stateless/03222_parallel_replicas_min_marks_to_read_overflow.reference Normal file
View File

@ -0,0 +1,21 @@
1006
1007
1008
1009
101
1010
1011
1012
1013
1014
---
100 100
101 101
102 102
103 103
104 104
105 105
106 106
107 107
108 108
109 109

Some files were not shown because too many files have changed in this diff Show More