mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Merge branch 'master' into support-distinct-on
This commit is contained in:
mergify[bot]
GitHub
commit
d9195c6f4a
@ -1178,11 +1178,11 @@ create view right_async_metric_log as
|
||||
-- Use the right log as time reference because it may have higher precision.
|
||||
create table metrics engine File(TSV, 'metrics/metrics.tsv') as
|
||||
with (select min(event_time) from right_async_metric_log) as min_time
|
||||
select name metric, r.event_time - min_time event_time, l.value as left, r.value as right
|
||||
select metric, r.event_time - min_time event_time, l.value as left, r.value as right
|
||||
from right_async_metric_log r
|
||||
asof join file('left-async-metric-log.tsv', TSVWithNamesAndTypes,
|
||||
'$(cat left-async-metric-log.tsv.columns)') l
|
||||
on l.name = r.name and r.event_time <= l.event_time
|
||||
on l.metric = r.metric and r.event_time <= l.event_time
|
||||
order by metric, event_time
|
||||
;
|
||||
|
||||
|
||||
@ -159,7 +159,7 @@ Configuration fields:
|
||||
| Tag | Description | Required |
|
||||
|------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|
|
||||
| `name` | Column name. | Yes |
|
||||
| `type` | ClickHouse data type: [UInt8](../../../sql-reference/data-types/int-uint.md), [UInt16](../../../sql-reference/data-types/int-uint.md), [UInt32](../../../sql-reference/data-types/int-uint.md), [UInt64](../../../sql-reference/data-types/int-uint.md), [Int8](../../../sql-reference/data-types/int-uint.md), [Int16](../../../sql-reference/data-types/int-uint.md), [Int32](../../../sql-reference/data-types/int-uint.md), [Int64](../../../sql-reference/data-types/int-uint.md), [Float32](../../../sql-reference/data-types/float.md), [Float64](../../../sql-reference/data-types/float.md), [UUID](../../../sql-reference/data-types/uuid.md), [Decimal32](../../../sql-reference/data-types/decimal.md), [Decimal64](../../../sql-reference/data-types/decimal.md), [Decimal128](../../../sql-reference/data-types/decimal.md), [Decimal256](../../../sql-reference/data-types/decimal.md), [String](../../../sql-reference/data-types/string.md).<br/>ClickHouse tries to cast value from dictionary to the specified data type. For example, for MySQL, the field might be `TEXT`, `VARCHAR`, or `BLOB` in the MySQL source table, but it can be uploaded as `String` in ClickHouse.<br/>[Nullable](../../../sql-reference/data-types/nullable.md) is currently supported for [Flat](external-dicts-dict-layout.md#flat), [Hashed](external-dicts-dict-layout.md#dicts-external_dicts_dict_layout-hashed), [ComplexKeyHashed](external-dicts-dict-layout.md#complex-key-hashed), [Direct](external-dicts-dict-layout.md#direct), [ComplexKeyDirect](external-dicts-dict-layout.md#complex-key-direct), [RangeHashed](external-dicts-dict-layout.md#range-hashed), [Polygon](external-dicts-dict-polygon.md), [Cache](external-dicts-dict-layout.md#cache), [ComplexKeyCache](external-dicts-dict-layout.md#complex-key-cache), [SSDCache](external-dicts-dict-layout.md#ssd-cache), [SSDComplexKeyCache](external-dicts-dict-layout.md#complex-key-ssd-cache) dictionaries. In [IPTrie](external-dicts-dict-layout.md#ip-trie) dictionaries `Nullable` types are not supported. | Yes |
|
||||
| `type` | ClickHouse data type: [UInt8](../../../sql-reference/data-types/int-uint.md), [UInt16](../../../sql-reference/data-types/int-uint.md), [UInt32](../../../sql-reference/data-types/int-uint.md), [UInt64](../../../sql-reference/data-types/int-uint.md), [Int8](../../../sql-reference/data-types/int-uint.md), [Int16](../../../sql-reference/data-types/int-uint.md), [Int32](../../../sql-reference/data-types/int-uint.md), [Int64](../../../sql-reference/data-types/int-uint.md), [Float32](../../../sql-reference/data-types/float.md), [Float64](../../../sql-reference/data-types/float.md), [UUID](../../../sql-reference/data-types/uuid.md), [Decimal32](../../../sql-reference/data-types/decimal.md), [Decimal64](../../../sql-reference/data-types/decimal.md), [Decimal128](../../../sql-reference/data-types/decimal.md), [Decimal256](../../../sql-reference/data-types/decimal.md), [String](../../../sql-reference/data-types/string.md), [Array](../../../sql-reference/data-types/array.md).<br/>ClickHouse tries to cast value from dictionary to the specified data type. For example, for MySQL, the field might be `TEXT`, `VARCHAR`, or `BLOB` in the MySQL source table, but it can be uploaded as `String` in ClickHouse.<br/>[Nullable](../../../sql-reference/data-types/nullable.md) is currently supported for [Flat](external-dicts-dict-layout.md#flat), [Hashed](external-dicts-dict-layout.md#dicts-external_dicts_dict_layout-hashed), [ComplexKeyHashed](external-dicts-dict-layout.md#complex-key-hashed), [Direct](external-dicts-dict-layout.md#direct), [ComplexKeyDirect](external-dicts-dict-layout.md#complex-key-direct), [RangeHashed](external-dicts-dict-layout.md#range-hashed), [Polygon](external-dicts-dict-polygon.md), [Cache](external-dicts-dict-layout.md#cache), [ComplexKeyCache](external-dicts-dict-layout.md#complex-key-cache), [SSDCache](external-dicts-dict-layout.md#ssd-cache), [SSDComplexKeyCache](external-dicts-dict-layout.md#complex-key-ssd-cache) dictionaries. In [IPTrie](external-dicts-dict-layout.md#ip-trie) dictionaries `Nullable` types are not supported. | Yes |
|
||||
| `null_value` | Default value for a non-existing element.<br/>In the example, it is an empty string. [NULL](../../syntax.md#null-literal) value can be used only for the `Nullable` types (see the previous line with types description). | Yes |
|
||||
| `expression` | [Expression](../../../sql-reference/syntax.md#syntax-expressions) that ClickHouse executes on the value.<br/>The expression can be a column name in the remote SQL database. Thus, you can use it to create an alias for the remote column.<br/><br/>Default value: no expression. | No |
|
||||
| <a name="hierarchical-dict-attr"></a> `hierarchical` | If `true`, the attribute contains the value of a parent key for the current key. See [Hierarchical Dictionaries](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md).<br/><br/>Default value: `false`. | No |
|
||||
|
||||
@ -159,7 +159,7 @@ CREATE DICTIONARY somename (
|
||||
| Тег | Описание | Обязательный |
|
||||
|------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------|
|
||||
| `name` | Имя столбца. | Да |
|
||||
| `type` | Тип данных ClickHouse: [UInt8](../../../sql-reference/data-types/int-uint.md), [UInt16](../../../sql-reference/data-types/int-uint.md), [UInt32](../../../sql-reference/data-types/int-uint.md), [UInt64](../../../sql-reference/data-types/int-uint.md), [Int8](../../../sql-reference/data-types/int-uint.md), [Int16](../../../sql-reference/data-types/int-uint.md), [Int32](../../../sql-reference/data-types/int-uint.md), [Int64](../../../sql-reference/data-types/int-uint.md), [Float32](../../../sql-reference/data-types/float.md), [Float64](../../../sql-reference/data-types/float.md), [UUID](../../../sql-reference/data-types/uuid.md), [Decimal32](../../../sql-reference/data-types/decimal.md), [Decimal64](../../../sql-reference/data-types/decimal.md), [Decimal128](../../../sql-reference/data-types/decimal.md), [Decimal256](../../../sql-reference/data-types/decimal.md), [String](../../../sql-reference/data-types/string.md).<br/>ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`. <br/>[Nullable](../../../sql-reference/data-types/nullable.md) в настоящее время поддерживается для словарей [Flat](external-dicts-dict-layout.md#flat), [Hashed](external-dicts-dict-layout.md#dicts-external_dicts_dict_layout-hashed), [ComplexKeyHashed](external-dicts-dict-layout.md#complex-key-hashed), [Direct](external-dicts-dict-layout.md#direct), [ComplexKeyDirect](external-dicts-dict-layout.md#complex-key-direct), [RangeHashed](external-dicts-dict-layout.md#range-hashed), [Polygon](external-dicts-dict-polygon.md), [Cache](external-dicts-dict-layout.md#cache), [ComplexKeyCache](external-dicts-dict-layout.md#complex-key-cache), [SSDCache](external-dicts-dict-layout.md#ssd-cache), [SSDComplexKeyCache](external-dicts-dict-layout.md#complex-key-ssd-cache). Для словарей [IPTrie](external-dicts-dict-layout.md#ip-trie) `Nullable`-типы не поддерживаются. | Да |
|
||||
| `type` | Тип данных ClickHouse: [UInt8](../../../sql-reference/data-types/int-uint.md), [UInt16](../../../sql-reference/data-types/int-uint.md), [UInt32](../../../sql-reference/data-types/int-uint.md), [UInt64](../../../sql-reference/data-types/int-uint.md), [Int8](../../../sql-reference/data-types/int-uint.md), [Int16](../../../sql-reference/data-types/int-uint.md), [Int32](../../../sql-reference/data-types/int-uint.md), [Int64](../../../sql-reference/data-types/int-uint.md), [Float32](../../../sql-reference/data-types/float.md), [Float64](../../../sql-reference/data-types/float.md), [UUID](../../../sql-reference/data-types/uuid.md), [Decimal32](../../../sql-reference/data-types/decimal.md), [Decimal64](../../../sql-reference/data-types/decimal.md), [Decimal128](../../../sql-reference/data-types/decimal.md), [Decimal256](../../../sql-reference/data-types/decimal.md), [String](../../../sql-reference/data-types/string.md), [Array](../../../sql-reference/data-types/array.md).<br/>ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`. <br/>[Nullable](../../../sql-reference/data-types/nullable.md) в настоящее время поддерживается для словарей [Flat](external-dicts-dict-layout.md#flat), [Hashed](external-dicts-dict-layout.md#dicts-external_dicts_dict_layout-hashed), [ComplexKeyHashed](external-dicts-dict-layout.md#complex-key-hashed), [Direct](external-dicts-dict-layout.md#direct), [ComplexKeyDirect](external-dicts-dict-layout.md#complex-key-direct), [RangeHashed](external-dicts-dict-layout.md#range-hashed), [Polygon](external-dicts-dict-polygon.md), [Cache](external-dicts-dict-layout.md#cache), [ComplexKeyCache](external-dicts-dict-layout.md#complex-key-cache), [SSDCache](external-dicts-dict-layout.md#ssd-cache), [SSDComplexKeyCache](external-dicts-dict-layout.md#complex-key-ssd-cache). Для словарей [IPTrie](external-dicts-dict-layout.md#ip-trie) `Nullable`-типы не поддерживаются. | Да |
|
||||
| `null_value` | Значение по умолчанию для несуществующего элемента.<br/>В примере это пустая строка. Значение [NULL](../../syntax.md#null-literal) можно указывать только для типов `Nullable` (см. предыдущую строку с описанием типов). | Да |
|
||||
| `expression` | [Выражение](../../syntax.md#syntax-expressions), которое ClickHouse выполняет со значением.<br/>Выражением может быть имя столбца в удаленной SQL базе. Таким образом, вы можете использовать его для создания псевдонима удаленного столбца.<br/><br/>Значение по умолчанию: нет выражения. | Нет |
|
||||
| <a name="hierarchical-dict-attr"></a> `hierarchical` | Если `true`, то атрибут содержит ключ предка для текущего элемента. Смотрите [Иерархические словари](external-dicts-dict-hierarchical.md).<br/><br/>Значение по умолчанию: `false`. | Нет |
|
||||
|
||||
@ -1159,7 +1159,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
{
|
||||
/// This object will periodically calculate some metrics.
|
||||
AsynchronousMetrics async_metrics(
|
||||
global_context, config().getUInt("asynchronous_metrics_update_period_s", 60), servers_to_start_before_tables, servers);
|
||||
global_context, config().getUInt("asynchronous_metrics_update_period_s", 1), servers_to_start_before_tables, servers);
|
||||
attachSystemTablesAsync(*DatabaseCatalog::instance().getSystemDatabase(), async_metrics);
|
||||
|
||||
for (const auto & listen_host : listen_hosts)
|
||||
|
||||
@ -583,7 +583,7 @@
|
||||
<port>9019</port>
|
||||
</jdbc_bridge>
|
||||
-->
|
||||
|
||||
|
||||
<!-- Configuration of clusters that could be used in Distributed tables.
|
||||
https://clickhouse.tech/docs/en/operations/table_engines/distributed/
|
||||
-->
|
||||
@ -917,7 +917,7 @@
|
||||
Asynchronous metrics are updated once a minute, so there is
|
||||
no need to flush more often.
|
||||
-->
|
||||
<flush_interval_milliseconds>60000</flush_interval_milliseconds>
|
||||
<flush_interval_milliseconds>7000</flush_interval_milliseconds>
|
||||
</asynchronous_metric_log>
|
||||
|
||||
<!--
|
||||
|
||||
@ -557,6 +557,7 @@
|
||||
M(587, CONCURRENT_ACCESS_NOT_SUPPORTED) \
|
||||
M(588, DISTRIBUTED_BROKEN_BATCH_INFO) \
|
||||
M(589, DISTRIBUTED_BROKEN_BATCH_FILES) \
|
||||
M(590, CANNOT_SYSCONF) \
|
||||
\
|
||||
M(998, POSTGRESQL_CONNECTION_FAILURE) \
|
||||
M(999, KEEPER_EXCEPTION) \
|
||||
|
||||
@ -56,3 +56,37 @@ const char * const hex_char_to_digit_table =
|
||||
"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
|
||||
"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
|
||||
"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff";
|
||||
|
||||
const char * const bin_byte_to_char_table =
|
||||
"0000000000000001000000100000001100000100000001010000011000000111"
|
||||
"0000100000001001000010100000101100001100000011010000111000001111"
|
||||
"0001000000010001000100100001001100010100000101010001011000010111"
|
||||
"0001100000011001000110100001101100011100000111010001111000011111"
|
||||
"0010000000100001001000100010001100100100001001010010011000100111"
|
||||
"0010100000101001001010100010101100101100001011010010111000101111"
|
||||
"0011000000110001001100100011001100110100001101010011011000110111"
|
||||
"0011100000111001001110100011101100111100001111010011111000111111"
|
||||
"0100000001000001010000100100001101000100010001010100011001000111"
|
||||
"0100100001001001010010100100101101001100010011010100111001001111"
|
||||
"0101000001010001010100100101001101010100010101010101011001010111"
|
||||
"0101100001011001010110100101101101011100010111010101111001011111"
|
||||
"0110000001100001011000100110001101100100011001010110011001100111"
|
||||
"0110100001101001011010100110101101101100011011010110111001101111"
|
||||
"0111000001110001011100100111001101110100011101010111011001110111"
|
||||
"0111100001111001011110100111101101111100011111010111111001111111"
|
||||
"1000000010000001100000101000001110000100100001011000011010000111"
|
||||
"1000100010001001100010101000101110001100100011011000111010001111"
|
||||
"1001000010010001100100101001001110010100100101011001011010010111"
|
||||
"1001100010011001100110101001101110011100100111011001111010011111"
|
||||
"1010000010100001101000101010001110100100101001011010011010100111"
|
||||
"1010100010101001101010101010101110101100101011011010111010101111"
|
||||
"1011000010110001101100101011001110110100101101011011011010110111"
|
||||
"1011100010111001101110101011101110111100101111011011111010111111"
|
||||
"1100000011000001110000101100001111000100110001011100011011000111"
|
||||
"1100100011001001110010101100101111001100110011011100111011001111"
|
||||
"1101000011010001110100101101001111010100110101011101011011010111"
|
||||
"1101100011011001110110101101101111011100110111011101111011011111"
|
||||
"1110000011100001111000101110001111100100111001011110011011100111"
|
||||
"1110100011101001111010101110101111101100111011011110111011101111"
|
||||
"1111000011110001111100101111001111110100111101011111011011110111"
|
||||
"1111100011111001111110101111101111111100111111011111111011111111";
|
||||
|
||||
@ -39,6 +39,12 @@ inline void writeHexByteLowercase(UInt8 byte, void * out)
|
||||
memcpy(out, &hex_byte_to_char_lowercase_table[static_cast<size_t>(byte) * 2], 2);
|
||||
}
|
||||
|
||||
extern const char * const bin_byte_to_char_table;
|
||||
|
||||
inline void writeBinByte(UInt8 byte, void * out)
|
||||
{
|
||||
memcpy(out, &bin_byte_to_char_table[static_cast<size_t>(byte) * 8], 8);
|
||||
}
|
||||
|
||||
/// Produces hex representation of an unsigned int with leading zeros (for checksums)
|
||||
template <typename TUInt>
|
||||
|
||||
@ -21,6 +21,8 @@ void registerFunctionsCoding(FunctionFactory & factory)
|
||||
factory.registerFunction<FunctionUUIDStringToNum>();
|
||||
factory.registerFunction<FunctionHex>(FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionUnhex>(FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionBin>(FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionUnbin>(FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionChar>(FunctionFactory::CaseInsensitive);
|
||||
factory.registerFunction<FunctionBitmaskToArray>();
|
||||
factory.registerFunction<FunctionBitPositionsToArray>();
|
||||
|
||||
@ -65,7 +65,6 @@ namespace ErrorCodes
|
||||
constexpr size_t uuid_bytes_length = 16;
|
||||
constexpr size_t uuid_text_length = 36;
|
||||
|
||||
|
||||
class FunctionIPv6NumToString : public IFunction
|
||||
{
|
||||
public:
|
||||
@ -951,19 +950,22 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class FunctionHex : public IFunction
|
||||
/// Encode number or string to string with binary or hexadecimal representation
|
||||
template <typename Impl>
|
||||
class EncodeToBinaryRepr : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "hex";
|
||||
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionHex>(); }
|
||||
static constexpr auto name = Impl::name;
|
||||
static constexpr size_t word_size = Impl::word_size;
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
return name;
|
||||
}
|
||||
static FunctionPtr create(ContextPtr) { return std::make_shared<EncodeToBinaryRepr>(); }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
bool isInjective(const ColumnsWithTypeAndName &) const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
@ -983,235 +985,6 @@ public:
|
||||
return std::make_shared<DataTypeString>();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void executeOneUInt(T x, char *& out) const
|
||||
{
|
||||
bool was_nonzero = false;
|
||||
for (int offset = (sizeof(T) - 1) * 8; offset >= 0; offset -= 8)
|
||||
{
|
||||
UInt8 byte = x >> offset;
|
||||
|
||||
/// Leading zeros.
|
||||
if (byte == 0 && !was_nonzero && offset) // -V560
|
||||
continue;
|
||||
|
||||
was_nonzero = true;
|
||||
|
||||
writeHexByteUppercase(byte, out);
|
||||
out += 2;
|
||||
}
|
||||
*out = '\0';
|
||||
++out;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool tryExecuteUInt(const IColumn * col, ColumnPtr & col_res) const
|
||||
{
|
||||
const ColumnVector<T> * col_vec = checkAndGetColumn<ColumnVector<T>>(col);
|
||||
|
||||
static constexpr size_t MAX_UINT_HEX_LENGTH = sizeof(T) * 2 + 1; /// Including trailing zero byte.
|
||||
|
||||
if (col_vec)
|
||||
{
|
||||
auto col_str = ColumnString::create();
|
||||
ColumnString::Chars & out_vec = col_str->getChars();
|
||||
ColumnString::Offsets & out_offsets = col_str->getOffsets();
|
||||
|
||||
const typename ColumnVector<T>::Container & in_vec = col_vec->getData();
|
||||
|
||||
size_t size = in_vec.size();
|
||||
out_offsets.resize(size);
|
||||
out_vec.resize(size * 3 + MAX_UINT_HEX_LENGTH); /// 3 is length of one byte in hex plus zero byte.
|
||||
|
||||
size_t pos = 0;
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
/// Manual exponential growth, so as not to rely on the linear amortized work time of `resize` (no one guarantees it).
|
||||
if (pos + MAX_UINT_HEX_LENGTH > out_vec.size())
|
||||
out_vec.resize(out_vec.size() * 2 + MAX_UINT_HEX_LENGTH);
|
||||
|
||||
char * begin = reinterpret_cast<char *>(&out_vec[pos]);
|
||||
char * end = begin;
|
||||
executeOneUInt<T>(in_vec[i], end);
|
||||
|
||||
pos += end - begin;
|
||||
out_offsets[i] = pos;
|
||||
}
|
||||
|
||||
out_vec.resize(pos);
|
||||
|
||||
col_res = std::move(col_str);
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void executeFloatAndDecimal(const T & in_vec, ColumnPtr & col_res, const size_t type_size_in_bytes) const
|
||||
{
|
||||
const size_t hex_length = type_size_in_bytes * 2 + 1; /// Including trailing zero byte.
|
||||
auto col_str = ColumnString::create();
|
||||
|
||||
ColumnString::Chars & out_vec = col_str->getChars();
|
||||
ColumnString::Offsets & out_offsets = col_str->getOffsets();
|
||||
|
||||
size_t size = in_vec.size();
|
||||
out_offsets.resize(size);
|
||||
out_vec.resize(size * hex_length);
|
||||
|
||||
size_t pos = 0;
|
||||
char * out = reinterpret_cast<char *>(&out_vec[0]);
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
const UInt8 * in_pos = reinterpret_cast<const UInt8 *>(&in_vec[i]);
|
||||
executeOneString(in_pos, in_pos + type_size_in_bytes, out);
|
||||
|
||||
pos += hex_length;
|
||||
out_offsets[i] = pos;
|
||||
}
|
||||
col_res = std::move(col_str);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool tryExecuteFloat(const IColumn * col, ColumnPtr & col_res) const
|
||||
{
|
||||
const ColumnVector<T> * col_vec = checkAndGetColumn<ColumnVector<T>>(col);
|
||||
if (col_vec)
|
||||
{
|
||||
const typename ColumnVector<T>::Container & in_vec = col_vec->getData();
|
||||
executeFloatAndDecimal<typename ColumnVector<T>::Container>(in_vec, col_res, sizeof(T));
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool tryExecuteDecimal(const IColumn * col, ColumnPtr & col_res) const
|
||||
{
|
||||
const ColumnDecimal<T> * col_dec = checkAndGetColumn<ColumnDecimal<T>>(col);
|
||||
if (col_dec)
|
||||
{
|
||||
const typename ColumnDecimal<T>::Container & in_vec = col_dec->getData();
|
||||
executeFloatAndDecimal<typename ColumnDecimal<T>::Container>(in_vec, col_res, sizeof(T));
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out)
|
||||
{
|
||||
while (pos < end)
|
||||
{
|
||||
writeHexByteUppercase(*pos, out);
|
||||
++pos;
|
||||
out += 2;
|
||||
}
|
||||
*out = '\0';
|
||||
++out;
|
||||
}
|
||||
|
||||
static bool tryExecuteString(const IColumn * col, ColumnPtr & col_res)
|
||||
{
|
||||
const ColumnString * col_str_in = checkAndGetColumn<ColumnString>(col);
|
||||
|
||||
if (col_str_in)
|
||||
{
|
||||
auto col_str = ColumnString::create();
|
||||
ColumnString::Chars & out_vec = col_str->getChars();
|
||||
ColumnString::Offsets & out_offsets = col_str->getOffsets();
|
||||
|
||||
const ColumnString::Chars & in_vec = col_str_in->getChars();
|
||||
const ColumnString::Offsets & in_offsets = col_str_in->getOffsets();
|
||||
|
||||
size_t size = in_offsets.size();
|
||||
out_offsets.resize(size);
|
||||
out_vec.resize(in_vec.size() * 2 - size);
|
||||
|
||||
char * begin = reinterpret_cast<char *>(out_vec.data());
|
||||
char * pos = begin;
|
||||
size_t prev_offset = 0;
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
size_t new_offset = in_offsets[i];
|
||||
|
||||
executeOneString(&in_vec[prev_offset], &in_vec[new_offset - 1], pos);
|
||||
|
||||
out_offsets[i] = pos - begin;
|
||||
|
||||
prev_offset = new_offset;
|
||||
}
|
||||
|
||||
if (!out_offsets.empty() && out_offsets.back() != out_vec.size())
|
||||
throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
col_res = std::move(col_str);
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static bool tryExecuteFixedString(const IColumn * col, ColumnPtr & col_res)
|
||||
{
|
||||
const ColumnFixedString * col_fstr_in = checkAndGetColumn<ColumnFixedString>(col);
|
||||
|
||||
if (col_fstr_in)
|
||||
{
|
||||
auto col_str = ColumnString::create();
|
||||
ColumnString::Chars & out_vec = col_str->getChars();
|
||||
ColumnString::Offsets & out_offsets = col_str->getOffsets();
|
||||
|
||||
const ColumnString::Chars & in_vec = col_fstr_in->getChars();
|
||||
|
||||
size_t size = col_fstr_in->size();
|
||||
|
||||
out_offsets.resize(size);
|
||||
out_vec.resize(in_vec.size() * 2 + size);
|
||||
|
||||
char * begin = reinterpret_cast<char *>(out_vec.data());
|
||||
char * pos = begin;
|
||||
|
||||
size_t n = col_fstr_in->getN();
|
||||
|
||||
size_t prev_offset = 0;
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
size_t new_offset = prev_offset + n;
|
||||
|
||||
executeOneString(&in_vec[prev_offset], &in_vec[new_offset], pos);
|
||||
|
||||
out_offsets[i] = pos - begin;
|
||||
prev_offset = new_offset;
|
||||
}
|
||||
|
||||
if (!out_offsets.empty() && out_offsets.back() != out_vec.size())
|
||||
throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
col_res = std::move(col_str);
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
|
||||
{
|
||||
const IColumn * column = arguments[0].column.get();
|
||||
@ -1234,19 +1007,185 @@ public:
|
||||
+ " of argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool tryExecuteUInt(const IColumn * col, ColumnPtr & col_res) const
|
||||
{
|
||||
const ColumnVector<T> * col_vec = checkAndGetColumn<ColumnVector<T>>(col);
|
||||
|
||||
static constexpr size_t MAX_LENGTH = sizeof(T) * word_size + 1; /// Including trailing zero byte.
|
||||
|
||||
if (col_vec)
|
||||
{
|
||||
auto col_str = ColumnString::create();
|
||||
ColumnString::Chars & out_vec = col_str->getChars();
|
||||
ColumnString::Offsets & out_offsets = col_str->getOffsets();
|
||||
|
||||
const typename ColumnVector<T>::Container & in_vec = col_vec->getData();
|
||||
|
||||
size_t size = in_vec.size();
|
||||
out_offsets.resize(size);
|
||||
out_vec.resize(size * (word_size+1) + MAX_LENGTH); /// word_size+1 is length of one byte in hex/bin plus zero byte.
|
||||
|
||||
size_t pos = 0;
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
/// Manual exponential growth, so as not to rely on the linear amortized work time of `resize` (no one guarantees it).
|
||||
if (pos + MAX_LENGTH > out_vec.size())
|
||||
out_vec.resize(out_vec.size() * word_size + MAX_LENGTH);
|
||||
|
||||
char * begin = reinterpret_cast<char *>(&out_vec[pos]);
|
||||
char * end = begin;
|
||||
Impl::executeOneUInt(in_vec[i], end);
|
||||
|
||||
pos += end - begin;
|
||||
out_offsets[i] = pos;
|
||||
}
|
||||
out_vec.resize(pos);
|
||||
|
||||
col_res = std::move(col_str);
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool tryExecuteString(const IColumn *col, ColumnPtr &col_res) const
|
||||
{
|
||||
const ColumnString * col_str_in = checkAndGetColumn<ColumnString>(col);
|
||||
|
||||
if (col_str_in)
|
||||
{
|
||||
auto col_str = ColumnString::create();
|
||||
ColumnString::Chars & out_vec = col_str->getChars();
|
||||
ColumnString::Offsets & out_offsets = col_str->getOffsets();
|
||||
|
||||
const ColumnString::Chars & in_vec = col_str_in->getChars();
|
||||
const ColumnString::Offsets & in_offsets = col_str_in->getOffsets();
|
||||
|
||||
size_t size = in_offsets.size();
|
||||
|
||||
out_offsets.resize(size);
|
||||
/// reserve `word_size` bytes for each non trailing zero byte from input + `size` bytes for trailing zeros
|
||||
out_vec.resize((in_vec.size() - size) * word_size + size);
|
||||
|
||||
char * begin = reinterpret_cast<char *>(out_vec.data());
|
||||
char * pos = begin;
|
||||
size_t prev_offset = 0;
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
size_t new_offset = in_offsets[i];
|
||||
|
||||
Impl::executeOneString(&in_vec[prev_offset], &in_vec[new_offset - 1], pos);
|
||||
|
||||
out_offsets[i] = pos - begin;
|
||||
|
||||
prev_offset = new_offset;
|
||||
}
|
||||
if (!out_offsets.empty() && out_offsets.back() != out_vec.size())
|
||||
throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
col_res = std::move(col_str);
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool tryExecuteDecimal(const IColumn * col, ColumnPtr & col_res) const
|
||||
{
|
||||
const ColumnDecimal<T> * col_dec = checkAndGetColumn<ColumnDecimal<T>>(col);
|
||||
if (col_dec)
|
||||
{
|
||||
const typename ColumnDecimal<T>::Container & in_vec = col_dec->getData();
|
||||
Impl::executeFloatAndDecimal(in_vec, col_res, sizeof(T));
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static bool tryExecuteFixedString(const IColumn * col, ColumnPtr & col_res)
|
||||
{
|
||||
const ColumnFixedString * col_fstr_in = checkAndGetColumn<ColumnFixedString>(col);
|
||||
|
||||
if (col_fstr_in)
|
||||
{
|
||||
auto col_str = ColumnString::create();
|
||||
ColumnString::Chars & out_vec = col_str->getChars();
|
||||
ColumnString::Offsets & out_offsets = col_str->getOffsets();
|
||||
|
||||
const ColumnString::Chars & in_vec = col_fstr_in->getChars();
|
||||
|
||||
size_t size = col_fstr_in->size();
|
||||
|
||||
out_offsets.resize(size);
|
||||
out_vec.resize(in_vec.size() * word_size + size);
|
||||
|
||||
char * begin = reinterpret_cast<char *>(out_vec.data());
|
||||
char * pos = begin;
|
||||
|
||||
size_t n = col_fstr_in->getN();
|
||||
|
||||
size_t prev_offset = 0;
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
size_t new_offset = prev_offset + n;
|
||||
|
||||
Impl::executeOneString(&in_vec[prev_offset], &in_vec[new_offset], pos);
|
||||
|
||||
out_offsets[i] = pos - begin;
|
||||
prev_offset = new_offset;
|
||||
}
|
||||
|
||||
if (!out_offsets.empty() && out_offsets.back() != out_vec.size())
|
||||
throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
col_res = std::move(col_str);
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool tryExecuteFloat(const IColumn * col, ColumnPtr & col_res) const
|
||||
{
|
||||
const ColumnVector<T> * col_vec = checkAndGetColumn<ColumnVector<T>>(col);
|
||||
if (col_vec)
|
||||
{
|
||||
const typename ColumnVector<T>::Container & in_vec = col_vec->getData();
|
||||
Impl::executeFloatAndDecimal(in_vec, col_res, sizeof(T));
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class FunctionUnhex : public IFunction
|
||||
/// Decode number or string from string with binary or hexadecimal representation
|
||||
template <typename Impl>
|
||||
class DecodeFromBinaryRepr : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "unhex";
|
||||
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionUnhex>(); }
|
||||
static constexpr auto name = Impl::name;
|
||||
static constexpr size_t word_size = Impl::word_size;
|
||||
static FunctionPtr create(ContextPtr) { return std::make_shared<DecodeFromBinaryRepr>(); }
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
return name;
|
||||
}
|
||||
String getName() const override { return name; }
|
||||
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
bool isInjective(const ColumnsWithTypeAndName &) const override { return true; }
|
||||
@ -1255,29 +1194,11 @@ public:
|
||||
{
|
||||
if (!isString(arguments[0]))
|
||||
throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
||||
return std::make_shared<DataTypeString>();
|
||||
}
|
||||
|
||||
static void unhexOne(const char * pos, const char * end, char *& out)
|
||||
{
|
||||
if ((end - pos) & 1)
|
||||
{
|
||||
*out = unhex(*pos);
|
||||
++out;
|
||||
++pos;
|
||||
}
|
||||
while (pos < end)
|
||||
{
|
||||
*out = unhex2(pos);
|
||||
pos += 2;
|
||||
++out;
|
||||
}
|
||||
*out = '\0';
|
||||
++out;
|
||||
}
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
|
||||
@ -1296,7 +1217,7 @@ public:
|
||||
|
||||
size_t size = in_offsets.size();
|
||||
out_offsets.resize(size);
|
||||
out_vec.resize(in_vec.size() / 2 + size);
|
||||
out_vec.resize(in_vec.size() / word_size + size);
|
||||
|
||||
char * begin = reinterpret_cast<char *>(out_vec.data());
|
||||
char * pos = begin;
|
||||
@ -1306,7 +1227,7 @@ public:
|
||||
{
|
||||
size_t new_offset = in_offsets[i];
|
||||
|
||||
unhexOne(reinterpret_cast<const char *>(&in_vec[prev_offset]), reinterpret_cast<const char *>(&in_vec[new_offset - 1]), pos);
|
||||
Impl::decode(reinterpret_cast<const char *>(&in_vec[prev_offset]), reinterpret_cast<const char *>(&in_vec[new_offset - 1]), pos);
|
||||
|
||||
out_offsets[i] = pos - begin;
|
||||
|
||||
@ -1326,6 +1247,219 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
struct HexImpl
|
||||
{
|
||||
static constexpr auto name = "hex";
|
||||
static constexpr size_t word_size = 2;
|
||||
|
||||
template <typename T>
|
||||
static void executeOneUInt(T x, char *& out)
|
||||
{
|
||||
bool was_nonzero = false;
|
||||
for (int offset = (sizeof(T) - 1) * 8; offset >= 0; offset -= 8)
|
||||
{
|
||||
UInt8 byte = x >> offset;
|
||||
|
||||
/// Skip leading zeros
|
||||
if (byte == 0 && !was_nonzero && offset)
|
||||
continue;
|
||||
|
||||
was_nonzero = true;
|
||||
writeHexByteUppercase(byte, out);
|
||||
out += word_size;
|
||||
}
|
||||
*out = '\0';
|
||||
++out;
|
||||
}
|
||||
|
||||
static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out)
|
||||
{
|
||||
while (pos < end)
|
||||
{
|
||||
writeHexByteUppercase(*pos, out);
|
||||
++pos;
|
||||
out += word_size;
|
||||
}
|
||||
*out = '\0';
|
||||
++out;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void executeFloatAndDecimal(const T & in_vec, ColumnPtr & col_res, const size_t type_size_in_bytes)
|
||||
{
|
||||
const size_t hex_length = type_size_in_bytes * word_size + 1; /// Including trailing zero byte.
|
||||
auto col_str = ColumnString::create();
|
||||
|
||||
ColumnString::Chars & out_vec = col_str->getChars();
|
||||
ColumnString::Offsets & out_offsets = col_str->getOffsets();
|
||||
|
||||
size_t size = in_vec.size();
|
||||
out_offsets.resize(size);
|
||||
out_vec.resize(size * hex_length);
|
||||
|
||||
size_t pos = 0;
|
||||
char * out = reinterpret_cast<char *>(&out_vec[0]);
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
const UInt8 * in_pos = reinterpret_cast<const UInt8 *>(&in_vec[i]);
|
||||
executeOneString(in_pos, in_pos + type_size_in_bytes, out);
|
||||
|
||||
pos += hex_length;
|
||||
out_offsets[i] = pos;
|
||||
}
|
||||
col_res = std::move(col_str);
|
||||
}
|
||||
};
|
||||
|
||||
struct UnhexImpl
|
||||
{
|
||||
static constexpr auto name = "unhex";
|
||||
static constexpr size_t word_size = 2;
|
||||
|
||||
static void decode(const char * pos, const char * end, char *& out)
|
||||
{
|
||||
if ((end - pos) & 1)
|
||||
{
|
||||
*out = unhex(*pos);
|
||||
++out;
|
||||
++pos;
|
||||
}
|
||||
while (pos < end)
|
||||
{
|
||||
*out = unhex2(pos);
|
||||
pos += word_size;
|
||||
++out;
|
||||
}
|
||||
*out = '\0';
|
||||
++out;
|
||||
}
|
||||
};
|
||||
|
||||
struct BinImpl
|
||||
{
|
||||
static constexpr auto name = "bin";
|
||||
static constexpr size_t word_size = 8;
|
||||
|
||||
template <typename T>
|
||||
static void executeOneUInt(T x, char *& out)
|
||||
{
|
||||
bool was_nonzero = false;
|
||||
for (int offset = (sizeof(T) - 1) * 8; offset >= 0; offset -= 8)
|
||||
{
|
||||
UInt8 byte = x >> offset;
|
||||
|
||||
/// Skip leading zeros
|
||||
if (byte == 0 && !was_nonzero && offset)
|
||||
continue;
|
||||
|
||||
was_nonzero = true;
|
||||
writeBinByte(byte, out);
|
||||
out += word_size;
|
||||
}
|
||||
*out = '\0';
|
||||
++out;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void executeFloatAndDecimal(const T & in_vec, ColumnPtr & col_res, const size_t type_size_in_bytes)
|
||||
{
|
||||
const size_t hex_length = type_size_in_bytes * word_size + 1; /// Including trailing zero byte.
|
||||
auto col_str = ColumnString::create();
|
||||
|
||||
ColumnString::Chars & out_vec = col_str->getChars();
|
||||
ColumnString::Offsets & out_offsets = col_str->getOffsets();
|
||||
|
||||
size_t size = in_vec.size();
|
||||
out_offsets.resize(size);
|
||||
out_vec.resize(size * hex_length);
|
||||
|
||||
size_t pos = 0;
|
||||
char * out = reinterpret_cast<char *>(out_vec.data());
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
const UInt8 * in_pos = reinterpret_cast<const UInt8 *>(&in_vec[i]);
|
||||
executeOneString(in_pos, in_pos + type_size_in_bytes, out);
|
||||
|
||||
pos += hex_length;
|
||||
out_offsets[i] = pos;
|
||||
}
|
||||
col_res = std::move(col_str);
|
||||
}
|
||||
|
||||
static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out)
|
||||
{
|
||||
while (pos < end)
|
||||
{
|
||||
writeBinByte(*pos, out);
|
||||
++pos;
|
||||
out += word_size;
|
||||
}
|
||||
*out = '\0';
|
||||
++out;
|
||||
}
|
||||
};
|
||||
|
||||
struct UnbinImpl
|
||||
{
|
||||
static constexpr auto name = "unbin";
|
||||
static constexpr size_t word_size = 8;
|
||||
|
||||
static void decode(const char * pos, const char * end, char *& out)
|
||||
{
|
||||
if (pos == end)
|
||||
{
|
||||
*out = '\0';
|
||||
++out;
|
||||
return;
|
||||
}
|
||||
|
||||
UInt8 left = 0;
|
||||
|
||||
/// end - pos is the length of input.
|
||||
/// (length & 7) to make remain bits length mod 8 is zero to split.
|
||||
/// e.g. the length is 9 and the input is "101000001",
|
||||
/// first left_cnt is 1, left is 0, right shift, pos is 1, left = 1
|
||||
/// then, left_cnt is 0, remain input is '01000001'.
|
||||
for (UInt8 left_cnt = (end - pos) & 7; left_cnt > 0; --left_cnt)
|
||||
{
|
||||
left = left << 1;
|
||||
if (*pos != '0')
|
||||
left += 1;
|
||||
++pos;
|
||||
}
|
||||
|
||||
if (left != 0 || end - pos == 0)
|
||||
{
|
||||
*out = left;
|
||||
++out;
|
||||
}
|
||||
|
||||
assert((end - pos) % 8 == 0);
|
||||
|
||||
while (end - pos != 0)
|
||||
{
|
||||
UInt8 c = 0;
|
||||
for (UInt8 i = 0; i < 8; ++i)
|
||||
{
|
||||
c = c << 1;
|
||||
if (*pos != '0')
|
||||
c += 1;
|
||||
++pos;
|
||||
}
|
||||
*out = c;
|
||||
++out;
|
||||
}
|
||||
|
||||
*out = '\0';
|
||||
++out;
|
||||
}
|
||||
};
|
||||
|
||||
using FunctionHex = EncodeToBinaryRepr<HexImpl>;
|
||||
using FunctionUnhex = DecodeFromBinaryRepr<UnhexImpl>;
|
||||
using FunctionBin = EncodeToBinaryRepr<BinImpl>;
|
||||
using FunctionUnbin = DecodeFromBinaryRepr<UnbinImpl>;
|
||||
|
||||
class FunctionChar : public IFunction
|
||||
{
|
||||
public:
|
||||
|
||||
@ -575,12 +575,12 @@ ColumnPtr FunctionAnyArityLogical<Impl, Name>::getConstantResultForNonConstArgum
|
||||
if constexpr (std::is_same_v<Impl, AndImpl>)
|
||||
{
|
||||
if (has_false_constant)
|
||||
result_type->createColumnConst(0, static_cast<UInt8>(false));
|
||||
result_column = result_type->createColumnConst(0, static_cast<UInt8>(false));
|
||||
}
|
||||
else if constexpr (std::is_same_v<Impl, OrImpl>)
|
||||
{
|
||||
if (has_true_constant)
|
||||
result_type->createColumnConst(0, static_cast<UInt8>(true));
|
||||
result_column = result_type->createColumnConst(0, static_cast<UInt8>(true));
|
||||
}
|
||||
|
||||
return result_column;
|
||||
|
||||
@ -149,7 +149,7 @@ off_t ReadBufferFromFileDescriptor::seek(off_t offset, int whence)
|
||||
off_t res = ::lseek(fd, new_pos, SEEK_SET);
|
||||
if (-1 == res)
|
||||
throwFromErrnoWithPath("Cannot seek through file " + getFileName(), getFileName(),
|
||||
ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
|
||||
ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
|
||||
file_offset_of_buffer_end = new_pos;
|
||||
|
||||
watch.stop();
|
||||
@ -160,6 +160,20 @@ off_t ReadBufferFromFileDescriptor::seek(off_t offset, int whence)
|
||||
}
|
||||
|
||||
|
||||
void ReadBufferFromFileDescriptor::rewind()
|
||||
{
|
||||
ProfileEvents::increment(ProfileEvents::Seek);
|
||||
off_t res = ::lseek(fd, 0, SEEK_SET);
|
||||
if (-1 == res)
|
||||
throwFromErrnoWithPath("Cannot seek through file " + getFileName(), getFileName(),
|
||||
ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
|
||||
|
||||
/// Clearing the buffer with existing data. New data will be read on subsequent call to 'next'.
|
||||
working_buffer.resize(0);
|
||||
pos = working_buffer.begin();
|
||||
}
|
||||
|
||||
|
||||
/// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout.
|
||||
bool ReadBufferFromFileDescriptor::poll(size_t timeout_microseconds)
|
||||
{
|
||||
|
||||
@ -39,6 +39,9 @@ public:
|
||||
/// If 'offset' is small enough to stay in buffer after seek, then true seek in file does not happen.
|
||||
off_t seek(off_t off, int whence) override;
|
||||
|
||||
/// Seek to the beginning, discarding already read data if any. Useful to reread file that changes on every read.
|
||||
void rewind();
|
||||
|
||||
off_t size();
|
||||
|
||||
void setProgressCallback(ContextPtr context);
|
||||
|
||||
@ -18,7 +18,7 @@ NamesAndTypesList AsynchronousMetricLogElement::getNamesAndTypes()
|
||||
{"event_date", std::make_shared<DataTypeDate>()},
|
||||
{"event_time", std::make_shared<DataTypeDateTime>()},
|
||||
{"event_time_microseconds", std::make_shared<DataTypeDateTime64>(6)},
|
||||
{"name", std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())},
|
||||
{"metric", std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())},
|
||||
{"value", std::make_shared<DataTypeFloat64>(),}
|
||||
};
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -3,11 +3,15 @@
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
#include <Common/MemoryStatisticsOS.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
|
||||
#include <condition_variable>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
|
||||
|
||||
@ -15,6 +19,7 @@ namespace DB
|
||||
{
|
||||
|
||||
class ProtocolServerAdapter;
|
||||
class ReadBuffer;
|
||||
|
||||
using AsynchronousMetricValue = double;
|
||||
using AsynchronousMetricValues = std::unordered_map<std::string, AsynchronousMetricValue>;
|
||||
@ -23,10 +28,30 @@ using AsynchronousMetricValues = std::unordered_map<std::string, AsynchronousMet
|
||||
/** Periodically (by default, each minute, starting at 30 seconds offset)
|
||||
* calculates and updates some metrics,
|
||||
* that are not updated automatically (so, need to be asynchronously calculated).
|
||||
*
|
||||
* This includes both ClickHouse-related metrics (like memory usage of ClickHouse process)
|
||||
* and common OS-related metrics (like total memory usage on the server).
|
||||
*/
|
||||
class AsynchronousMetrics : WithContext
|
||||
{
|
||||
public:
|
||||
/// The default value of update_period_seconds is for ClickHouse-over-YT
|
||||
/// in Arcadia -- it uses its own server implementation that also uses these
|
||||
/// metrics.
|
||||
AsynchronousMetrics(
|
||||
ContextPtr global_context_,
|
||||
int update_period_seconds,
|
||||
std::shared_ptr<std::vector<ProtocolServerAdapter>> servers_to_start_before_tables_,
|
||||
std::shared_ptr<std::vector<ProtocolServerAdapter>> servers_);
|
||||
|
||||
~AsynchronousMetrics();
|
||||
|
||||
/// Separate method allows to initialize the `servers` variable beforehand.
|
||||
void start();
|
||||
|
||||
/// Returns copy of all values.
|
||||
AsynchronousMetricValues getValues() const;
|
||||
|
||||
#if defined(ARCADIA_BUILD)
|
||||
/// This constructor needs only to provide backward compatibility with some other projects (hello, Arcadia).
|
||||
/// Never use this in the ClickHouse codebase.
|
||||
@ -39,35 +64,6 @@ public:
|
||||
}
|
||||
#endif
|
||||
|
||||
/// The default value of update_period_seconds is for ClickHouse-over-YT
|
||||
/// in Arcadia -- it uses its own server implementation that also uses these
|
||||
/// metrics.
|
||||
AsynchronousMetrics(
|
||||
ContextPtr global_context_,
|
||||
int update_period_seconds,
|
||||
std::shared_ptr<std::vector<ProtocolServerAdapter>> servers_to_start_before_tables_,
|
||||
std::shared_ptr<std::vector<ProtocolServerAdapter>> servers_)
|
||||
: WithContext(global_context_)
|
||||
, update_period(update_period_seconds)
|
||||
, servers_to_start_before_tables(servers_to_start_before_tables_)
|
||||
, servers(servers_)
|
||||
{
|
||||
}
|
||||
|
||||
~AsynchronousMetrics();
|
||||
|
||||
/// Separate method allows to initialize the `servers` variable beforehand.
|
||||
void start()
|
||||
{
|
||||
/// Update once right now, to make metrics available just after server start
|
||||
/// (without waiting for asynchronous_metrics_update_period_s).
|
||||
update();
|
||||
thread = std::make_unique<ThreadFromGlobalPool>([this] { run(); });
|
||||
}
|
||||
|
||||
/// Returns copy of all values.
|
||||
AsynchronousMetricValues getValues() const;
|
||||
|
||||
private:
|
||||
const std::chrono::seconds update_period;
|
||||
std::shared_ptr<std::vector<ProtocolServerAdapter>> servers_to_start_before_tables{nullptr};
|
||||
@ -78,14 +74,113 @@ private:
|
||||
bool quit {false};
|
||||
AsynchronousMetricValues values;
|
||||
|
||||
/// Some values are incremental and we have to calculate the difference.
|
||||
/// On first run we will only collect the values to subtract later.
|
||||
bool first_run = true;
|
||||
std::chrono::system_clock::time_point previous_update_time;
|
||||
|
||||
#if defined(OS_LINUX)
|
||||
MemoryStatisticsOS memory_stat;
|
||||
|
||||
std::optional<ReadBufferFromFile> meminfo;
|
||||
std::optional<ReadBufferFromFile> loadavg;
|
||||
std::optional<ReadBufferFromFile> proc_stat;
|
||||
std::optional<ReadBufferFromFile> cpuinfo;
|
||||
std::optional<ReadBufferFromFile> file_nr;
|
||||
std::optional<ReadBufferFromFile> uptime;
|
||||
std::optional<ReadBufferFromFile> net_dev;
|
||||
|
||||
std::vector<std::unique_ptr<ReadBufferFromFile>> thermal;
|
||||
|
||||
std::unordered_map<String /* device name */,
|
||||
std::unordered_map<String /* label name */,
|
||||
std::unique_ptr<ReadBufferFromFile>>> hwmon_devices;
|
||||
|
||||
std::vector<std::pair<
|
||||
std::unique_ptr<ReadBufferFromFile> /* correctable errors */,
|
||||
std::unique_ptr<ReadBufferFromFile> /* uncorrectable errors */>> edac;
|
||||
|
||||
std::unordered_map<String /* device name */, std::unique_ptr<ReadBufferFromFile>> block_devs;
|
||||
|
||||
/// TODO: socket statistics.
|
||||
|
||||
struct ProcStatValuesCPU
|
||||
{
|
||||
uint64_t user;
|
||||
uint64_t nice;
|
||||
uint64_t system;
|
||||
uint64_t idle;
|
||||
uint64_t iowait;
|
||||
uint64_t irq;
|
||||
uint64_t softirq;
|
||||
uint64_t steal;
|
||||
uint64_t guest;
|
||||
uint64_t guest_nice;
|
||||
|
||||
void read(ReadBuffer & in);
|
||||
ProcStatValuesCPU operator-(const ProcStatValuesCPU & other) const;
|
||||
};
|
||||
|
||||
struct ProcStatValuesOther
|
||||
{
|
||||
uint64_t interrupts;
|
||||
uint64_t context_switches;
|
||||
uint64_t processes_created;
|
||||
|
||||
ProcStatValuesOther operator-(const ProcStatValuesOther & other) const;
|
||||
};
|
||||
|
||||
ProcStatValuesCPU proc_stat_values_all_cpus{};
|
||||
ProcStatValuesOther proc_stat_values_other{};
|
||||
std::vector<ProcStatValuesCPU> proc_stat_values_per_cpu;
|
||||
|
||||
/// https://www.kernel.org/doc/Documentation/block/stat.txt
|
||||
struct BlockDeviceStatValues
|
||||
{
|
||||
uint64_t read_ios;
|
||||
uint64_t read_merges;
|
||||
uint64_t read_sectors;
|
||||
uint64_t read_ticks;
|
||||
uint64_t write_ios;
|
||||
uint64_t write_merges;
|
||||
uint64_t write_sectors;
|
||||
uint64_t write_ticks;
|
||||
uint64_t in_flight_ios;
|
||||
uint64_t io_ticks;
|
||||
uint64_t time_in_queue;
|
||||
uint64_t discard_ops;
|
||||
uint64_t discard_merges;
|
||||
uint64_t discard_sectors;
|
||||
uint64_t discard_ticks;
|
||||
|
||||
void read(ReadBuffer & in);
|
||||
BlockDeviceStatValues operator-(const BlockDeviceStatValues & other) const;
|
||||
};
|
||||
|
||||
std::unordered_map<String /* device name */, BlockDeviceStatValues> block_device_stats;
|
||||
|
||||
struct NetworkInterfaceStatValues
|
||||
{
|
||||
uint64_t recv_bytes;
|
||||
uint64_t recv_packets;
|
||||
uint64_t recv_errors;
|
||||
uint64_t recv_drop;
|
||||
uint64_t send_bytes;
|
||||
uint64_t send_packets;
|
||||
uint64_t send_errors;
|
||||
uint64_t send_drop;
|
||||
|
||||
NetworkInterfaceStatValues operator-(const NetworkInterfaceStatValues & other) const;
|
||||
};
|
||||
|
||||
std::unordered_map<String /* device name */, NetworkInterfaceStatValues> network_interface_stats;
|
||||
|
||||
#endif
|
||||
|
||||
std::unique_ptr<ThreadFromGlobalPool> thread;
|
||||
|
||||
void run();
|
||||
void update();
|
||||
void update(std::chrono::system_clock::time_point update_time);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -34,6 +34,7 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ACCESS_DENIED;
|
||||
}
|
||||
|
||||
StorageHDFS::StorageHDFS(
|
||||
@ -280,15 +281,7 @@ Pipe StorageHDFS::read(
|
||||
size_t max_block_size,
|
||||
unsigned num_streams)
|
||||
{
|
||||
size_t begin_of_path;
|
||||
/// This uri is checked for correctness in constructor of StorageHDFS and never modified afterwards
|
||||
auto two_slash = uri.find("//");
|
||||
|
||||
if (two_slash == std::string::npos)
|
||||
begin_of_path = uri.find('/');
|
||||
else
|
||||
begin_of_path = uri.find('/', two_slash + 2);
|
||||
|
||||
const size_t begin_of_path = uri.find('/', uri.find("//") + 2);
|
||||
const String path_from_uri = uri.substr(begin_of_path);
|
||||
const String uri_without_path = uri.substr(0, begin_of_path);
|
||||
|
||||
@ -330,6 +323,21 @@ BlockOutputStreamPtr StorageHDFS::write(const ASTPtr & /*query*/, const StorageM
|
||||
chooseCompressionMethod(uri, compression_method));
|
||||
}
|
||||
|
||||
void StorageHDFS::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, ContextPtr context_, TableExclusiveLockHolder &)
|
||||
{
|
||||
const size_t begin_of_path = uri.find('/', uri.find("//") + 2);
|
||||
const String path = uri.substr(begin_of_path);
|
||||
const String url = uri.substr(0, begin_of_path);
|
||||
|
||||
HDFSBuilderWrapper builder = createHDFSBuilder(url + "/", context_->getGlobalContext()->getConfigRef());
|
||||
HDFSFSPtr fs = createHDFSFS(builder.get());
|
||||
|
||||
int ret = hdfsDelete(fs.get(), path.data(), 0);
|
||||
if (ret)
|
||||
throw Exception(ErrorCodes::ACCESS_DENIED, "Unable to truncate hdfs table: {}", std::string(hdfsGetLastError()));
|
||||
}
|
||||
|
||||
|
||||
void registerStorageHDFS(StorageFactory & factory)
|
||||
{
|
||||
factory.registerStorage("HDFS", [](const StorageFactory::Arguments & args)
|
||||
|
||||
@ -34,6 +34,8 @@ public:
|
||||
|
||||
BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context) override;
|
||||
|
||||
void truncate(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context_, TableExclusiveLockHolder &) override;
|
||||
|
||||
NamesAndTypesList getVirtuals() const override;
|
||||
|
||||
protected:
|
||||
|
||||
@ -1818,11 +1818,10 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
|
||||
if (MergeTreeSettings::isPartFormatSetting(setting_name) && !new_value)
|
||||
{
|
||||
/// Use default settings + new and check if doesn't affect part format settings
|
||||
MergeTreeSettings copy = *getSettings();
|
||||
copy.resetToDefault();
|
||||
copy.applyChanges(new_changes);
|
||||
auto copy = getDefaultSettings();
|
||||
copy->applyChanges(new_changes);
|
||||
String reason;
|
||||
if (!canUsePolymorphicParts(copy, &reason) && !reason.empty())
|
||||
if (!canUsePolymorphicParts(*copy, &reason) && !reason.empty())
|
||||
throw Exception("Can't change settings. Reason: " + reason, ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
@ -1984,14 +1983,12 @@ void MergeTreeData::changeSettings(
|
||||
}
|
||||
}
|
||||
|
||||
MergeTreeSettings copy = *getSettings();
|
||||
/// reset to default settings before applying existing
|
||||
copy.resetToDefault();
|
||||
copy.applyChanges(new_changes);
|
||||
/// Reset to default settings before applying existing.
|
||||
auto copy = getDefaultSettings();
|
||||
copy->applyChanges(new_changes);
|
||||
copy->sanityCheck(getContext()->getSettingsRef());
|
||||
|
||||
copy.sanityCheck(getContext()->getSettingsRef());
|
||||
|
||||
storage_settings.set(std::make_unique<const MergeTreeSettings>(copy));
|
||||
storage_settings.set(std::move(copy));
|
||||
StorageInMemoryMetadata new_metadata = getInMemoryMetadata();
|
||||
new_metadata.setSettingsChanges(new_settings);
|
||||
setInMemoryMetadata(new_metadata);
|
||||
|
||||
@ -1087,6 +1087,9 @@ private:
|
||||
|
||||
// Get partition matcher for FREEZE / UNFREEZE queries.
|
||||
MatcherFn getPartitionMatcher(const ASTPtr & partition, ContextPtr context) const;
|
||||
|
||||
/// Returns default settings for storage with possible changes from global config.
|
||||
virtual std::unique_ptr<MergeTreeSettings> getDefaultSettings() const = 0;
|
||||
};
|
||||
|
||||
/// RAII struct to record big parts that are submerging or emerging.
|
||||
|
||||
@ -1604,4 +1604,9 @@ void StorageMergeTree::startBackgroundMovesIfNeeded()
|
||||
background_moves_executor.start();
|
||||
}
|
||||
|
||||
std::unique_ptr<MergeTreeSettings> StorageMergeTree::getDefaultSettings() const
|
||||
{
|
||||
return std::make_unique<MergeTreeSettings>(getContext()->getMergeTreeSettings());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -236,6 +236,8 @@ private:
|
||||
|
||||
void startBackgroundMovesIfNeeded() override;
|
||||
|
||||
std::unique_ptr<MergeTreeSettings> getDefaultSettings() const override;
|
||||
|
||||
friend class MergeTreeProjectionBlockOutputStream;
|
||||
friend class MergeTreeBlockOutputStream;
|
||||
friend class MergeTreeData;
|
||||
|
||||
@ -7174,6 +7174,11 @@ void StorageReplicatedMergeTree::startBackgroundMovesIfNeeded()
|
||||
background_moves_executor.start();
|
||||
}
|
||||
|
||||
std::unique_ptr<MergeTreeSettings> StorageReplicatedMergeTree::getDefaultSettings() const
|
||||
{
|
||||
return std::make_unique<MergeTreeSettings>(getContext()->getReplicatedMergeTreeSettings());
|
||||
}
|
||||
|
||||
|
||||
void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part) const
|
||||
{
|
||||
|
||||
@ -702,6 +702,8 @@ private:
|
||||
|
||||
void startBackgroundMovesIfNeeded() override;
|
||||
|
||||
std::unique_ptr<MergeTreeSettings> getDefaultSettings() const override;
|
||||
|
||||
std::set<String> getPartitionIdsAffectedByCommands(const MutationCommands & commands, ContextPtr query_context) const;
|
||||
PartitionBlockNumbersHolder allocateBlockNumbersInAffectedPartitions(
|
||||
const MutationCommands & commands, ContextPtr query_context, const zkutil::ZooKeeperPtr & zookeeper) const;
|
||||
|
||||
@ -27,6 +27,8 @@
|
||||
#include <aws/core/auth/AWSCredentials.h>
|
||||
#include <aws/s3/S3Client.h>
|
||||
#include <aws/s3/model/ListObjectsV2Request.h>
|
||||
#include <aws/s3/model/CopyObjectRequest.h>
|
||||
#include <aws/s3/model/DeleteObjectsRequest.h>
|
||||
|
||||
#include <Common/parseGlobs.h>
|
||||
#include <Common/quoteString.h>
|
||||
@ -434,6 +436,30 @@ BlockOutputStreamPtr StorageS3::write(const ASTPtr & /*query*/, const StorageMet
|
||||
max_single_part_upload_size);
|
||||
}
|
||||
|
||||
|
||||
void StorageS3::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, ContextPtr local_context, TableExclusiveLockHolder &)
|
||||
{
|
||||
updateClientAndAuthSettings(local_context, client_auth);
|
||||
|
||||
Aws::S3::Model::ObjectIdentifier obj;
|
||||
obj.SetKey(client_auth.uri.key);
|
||||
|
||||
Aws::S3::Model::Delete delkeys;
|
||||
delkeys.AddObjects(std::move(obj));
|
||||
|
||||
Aws::S3::Model::DeleteObjectsRequest request;
|
||||
request.SetBucket(client_auth.uri.bucket);
|
||||
request.SetDelete(delkeys);
|
||||
|
||||
auto response = client_auth.client->DeleteObjects(request);
|
||||
if (!response.IsSuccess())
|
||||
{
|
||||
const auto & err = response.GetError();
|
||||
throw Exception(std::to_string(static_cast<int>(err.GetErrorType())) + ": " + err.GetMessage(), ErrorCodes::S3_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void StorageS3::updateClientAndAuthSettings(ContextPtr ctx, StorageS3::ClientAuthentication & upd)
|
||||
{
|
||||
auto settings = ctx->getStorageS3Settings().getSettings(upd.uri.uri.toString());
|
||||
|
||||
@ -130,6 +130,8 @@ public:
|
||||
|
||||
BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context) override;
|
||||
|
||||
void truncate(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr local_context, TableExclusiveLockHolder &) override;
|
||||
|
||||
NamesAndTypesList getVirtuals() const override;
|
||||
|
||||
private:
|
||||
|
||||
@ -15,7 +15,6 @@ def started_cluster():
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def test_read_write_storage(started_cluster):
|
||||
hdfs_api = started_cluster.hdfs_api
|
||||
|
||||
@ -235,7 +234,7 @@ def test_virtual_columns(started_cluster):
|
||||
expected = "1\tfile1\thdfs://hdfs1:9000//file1\n2\tfile2\thdfs://hdfs1:9000//file2\n3\tfile3\thdfs://hdfs1:9000//file3\n"
|
||||
assert node1.query("select id, _file as file_name, _path as file_path from virtual_cols order by id") == expected
|
||||
|
||||
|
||||
|
||||
def test_read_files_with_spaces(started_cluster):
|
||||
hdfs_api = started_cluster.hdfs_api
|
||||
|
||||
@ -246,6 +245,18 @@ def test_read_files_with_spaces(started_cluster):
|
||||
assert node1.query("select * from test order by id") == "1\n2\n3\n"
|
||||
|
||||
|
||||
def test_truncate_table(started_cluster):
|
||||
hdfs_api = started_cluster.hdfs_api
|
||||
node1.query(
|
||||
"create table test_truncate (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/tr', 'TSV')")
|
||||
node1.query("insert into test_truncate values (1, 'Mark', 72.53)")
|
||||
assert hdfs_api.read_data("/tr") == "1\tMark\t72.53\n"
|
||||
assert node1.query("select * from test_truncate") == "1\tMark\t72.53\n"
|
||||
node1.query("truncate table test_truncate")
|
||||
assert node1.query("select * from test_truncate") == ""
|
||||
node1.query("drop table test_truncate")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cluster.start()
|
||||
input("Cluster created, press any key to destroy...")
|
||||
|
||||
@ -646,3 +646,28 @@ def test_storage_s3_put_gzip(started_cluster, extension, method):
|
||||
f = gzip.GzipFile(fileobj=buf, mode="rb")
|
||||
uncompressed_content = f.read().decode()
|
||||
assert sum([ int(i.split(',')[1]) for i in uncompressed_content.splitlines() ]) == 708
|
||||
|
||||
|
||||
def test_truncate_table(started_cluster):
|
||||
bucket = started_cluster.minio_bucket
|
||||
instance = started_cluster.instances["dummy"] # type: ClickHouseInstance
|
||||
name = "truncate"
|
||||
|
||||
instance.query("CREATE TABLE {} (id UInt32) ENGINE = S3('http://{}:{}/{}/{}', 'CSV')".format(
|
||||
name, started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, name))
|
||||
|
||||
instance.query("INSERT INTO {} SELECT number FROM numbers(10)".format(name))
|
||||
result = instance.query("SELECT * FROM {}".format(name))
|
||||
assert result == instance.query("SELECT number FROM numbers(10)")
|
||||
instance.query("TRUNCATE TABLE {}".format(name))
|
||||
|
||||
minio = started_cluster.minio_client
|
||||
timeout = 30
|
||||
while timeout > 0:
|
||||
if len(list(minio.list_objects(started_cluster.minio_bucket, 'truncate/'))) == 0:
|
||||
return
|
||||
timeout -= 1
|
||||
time.sleep(1)
|
||||
assert(len(list(minio.list_objects(started_cluster.minio_bucket, 'truncate/'))) == 0)
|
||||
assert instance.query("SELECT * FROM {}".format(name)) == ""
|
||||
|
||||
|
||||
35
tests/queries/0_stateless/01926_bin_unbin.reference
Normal file
35
tests/queries/0_stateless/01926_bin_unbin.reference
Normal file
@ -0,0 +1,35 @@
|
||||
|
||||
00000000
|
||||
00000001
|
||||
00001010
|
||||
01111111
|
||||
11111111
|
||||
0000000100000000
|
||||
0000000111111111
|
||||
0000001000000000
|
||||
00110000
|
||||
0011000100110000
|
||||
111001101011010110001011111010001010111110010101
|
||||
11100110101101011000101111101000101011111001010100000000000000000000000000000000
|
||||
10011010100110011001100100111111
|
||||
0011001100110011001100110011001100110011001100111111001100111111
|
||||
00000000000011100010011100000111
|
||||
0000000000000000000011000011110101011101010100111010101000000001
|
||||
0011000100110010001100110011001100110010001101000011001000110100
|
||||
0011000100110010001100110011001100110010001101000011001000110100
|
||||
0011000100110010001100110011001100110010001101000011001000110100
|
||||
0011000100110010001100110011001100110010001101000011001000110100
|
||||
|
||||
1
|
||||
0
|
||||
10
|
||||
测试
|
||||
0
|
||||
0
|
||||
0
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
39
tests/queries/0_stateless/01926_bin_unbin.sql
Normal file
39
tests/queries/0_stateless/01926_bin_unbin.sql
Normal file
@ -0,0 +1,39 @@
|
||||
select bin('');
|
||||
select bin(0);
|
||||
select bin(1);
|
||||
select bin(10);
|
||||
select bin(127);
|
||||
select bin(255);
|
||||
select bin(256);
|
||||
select bin(511);
|
||||
select bin(512);
|
||||
select bin('0');
|
||||
select bin('10');
|
||||
select bin('测试');
|
||||
select bin(toFixedString('测试', 10));
|
||||
select bin(toFloat32(1.2));
|
||||
select bin(toFloat64(1.2));
|
||||
select bin(toDecimal32(1.2, 8));
|
||||
select bin(toDecimal64(1.2, 17));
|
||||
select bin('12332424');
|
||||
select bin(materialize('12332424'));
|
||||
select bin(toNullable(materialize('12332424')));
|
||||
select bin(toLowCardinality(materialize('12332424')));
|
||||
|
||||
select unbin('');
|
||||
select unbin('0') == '\0';
|
||||
select unbin('00110000'); -- 0
|
||||
select unbin('0011000100110000'); -- 10
|
||||
select unbin('111001101011010110001011111010001010111110010101'); -- 测试
|
||||
select unbin(materialize('00110000'));
|
||||
select unbin(toNullable(materialize('00110000')));
|
||||
select unbin(toLowCardinality(materialize('00110000')));
|
||||
|
||||
select unbin(bin('')) == '';
|
||||
select bin(unbin('')) == '';
|
||||
select bin(unbin('0')) == '00000000';
|
||||
|
||||
-- hex and bin consistent for corner cases
|
||||
select hex('') == bin('');
|
||||
select unhex('') == unbin('');
|
||||
select unhex('0') == unbin('0');
|
||||
@ -253,3 +253,4 @@
|
||||
01923_different_expression_name_alias
|
||||
01932_null_valid_identifier
|
||||
00918_json_functions
|
||||
01889_sql_json_functions
|
||||
|
||||
Loading…
Reference in New Issue
Block a user