Merge branch 'master' into support-distinct-on

docker/test/performance-comparison/compare.sh

@@ -1178,11 +1178,11 @@ create view right_async_metric_log as
 -- Use the right log as time reference because it may have higher precision.
 create table metrics engine File(TSV, 'metrics/metrics.tsv') as
     with (select min(event_time) from right_async_metric_log) as min_time
-    select name metric, r.event_time - min_time event_time, l.value as left, r.value as right
+    select metric, r.event_time - min_time event_time, l.value as left, r.value as right
     from right_async_metric_log r
     asof join file('left-async-metric-log.tsv', TSVWithNamesAndTypes,
         '$(cat left-async-metric-log.tsv.columns)') l
-    on l.name = r.name and r.event_time <= l.event_time
+    on l.metric = r.metric and r.event_time <= l.event_time
     order by metric, event_time
     ;
 

docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md

@@ -159,7 +159,7 @@ Configuration fields:
 | Tag                                                  | Description                                                                                                                                                                                                                                                                                                                                     | Required |
 |------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|
 | `name`                                               | Column name.                                                                                                                                                                                                                                                                                                                                    | Yes      |
-| `type`                                               | ClickHouse data type: [UInt8](../../../sql-reference/data-types/int-uint.md), [UInt16](../../../sql-reference/data-types/int-uint.md), [UInt32](../../../sql-reference/data-types/int-uint.md), [UInt64](../../../sql-reference/data-types/int-uint.md), [Int8](../../../sql-reference/data-types/int-uint.md), [Int16](../../../sql-reference/data-types/int-uint.md), [Int32](../../../sql-reference/data-types/int-uint.md), [Int64](../../../sql-reference/data-types/int-uint.md), [Float32](../../../sql-reference/data-types/float.md), [Float64](../../../sql-reference/data-types/float.md), [UUID](../../../sql-reference/data-types/uuid.md), [Decimal32](../../../sql-reference/data-types/decimal.md), [Decimal64](../../../sql-reference/data-types/decimal.md), [Decimal128](../../../sql-reference/data-types/decimal.md), [Decimal256](../../../sql-reference/data-types/decimal.md), [String](../../../sql-reference/data-types/string.md).<br/>ClickHouse tries to cast value from dictionary to the specified data type. For example, for MySQL, the field might be `TEXT`, `VARCHAR`, or `BLOB` in the MySQL source table, but it can be uploaded as `String` in ClickHouse.<br/>[Nullable](../../../sql-reference/data-types/nullable.md) is currently supported for [Flat](external-dicts-dict-layout.md#flat), [Hashed](external-dicts-dict-layout.md#dicts-external_dicts_dict_layout-hashed), [ComplexKeyHashed](external-dicts-dict-layout.md#complex-key-hashed), [Direct](external-dicts-dict-layout.md#direct), [ComplexKeyDirect](external-dicts-dict-layout.md#complex-key-direct), [RangeHashed](external-dicts-dict-layout.md#range-hashed), [Polygon](external-dicts-dict-polygon.md), [Cache](external-dicts-dict-layout.md#cache), [ComplexKeyCache](external-dicts-dict-layout.md#complex-key-cache), [SSDCache](external-dicts-dict-layout.md#ssd-cache), [SSDComplexKeyCache](external-dicts-dict-layout.md#complex-key-ssd-cache) dictionaries. In [IPTrie](external-dicts-dict-layout.md#ip-trie) dictionaries `Nullable` types are not supported.       | Yes      |
+| `type`                                               | ClickHouse data type: [UInt8](../../../sql-reference/data-types/int-uint.md), [UInt16](../../../sql-reference/data-types/int-uint.md), [UInt32](../../../sql-reference/data-types/int-uint.md), [UInt64](../../../sql-reference/data-types/int-uint.md), [Int8](../../../sql-reference/data-types/int-uint.md), [Int16](../../../sql-reference/data-types/int-uint.md), [Int32](../../../sql-reference/data-types/int-uint.md), [Int64](../../../sql-reference/data-types/int-uint.md), [Float32](../../../sql-reference/data-types/float.md), [Float64](../../../sql-reference/data-types/float.md), [UUID](../../../sql-reference/data-types/uuid.md), [Decimal32](../../../sql-reference/data-types/decimal.md), [Decimal64](../../../sql-reference/data-types/decimal.md), [Decimal128](../../../sql-reference/data-types/decimal.md), [Decimal256](../../../sql-reference/data-types/decimal.md), [String](../../../sql-reference/data-types/string.md), [Array](../../../sql-reference/data-types/array.md).<br/>ClickHouse tries to cast value from dictionary to the specified data type. For example, for MySQL, the field might be `TEXT`, `VARCHAR`, or `BLOB` in the MySQL source table, but it can be uploaded as `String` in ClickHouse.<br/>[Nullable](../../../sql-reference/data-types/nullable.md) is currently supported for [Flat](external-dicts-dict-layout.md#flat), [Hashed](external-dicts-dict-layout.md#dicts-external_dicts_dict_layout-hashed), [ComplexKeyHashed](external-dicts-dict-layout.md#complex-key-hashed), [Direct](external-dicts-dict-layout.md#direct), [ComplexKeyDirect](external-dicts-dict-layout.md#complex-key-direct), [RangeHashed](external-dicts-dict-layout.md#range-hashed), [Polygon](external-dicts-dict-polygon.md), [Cache](external-dicts-dict-layout.md#cache), [ComplexKeyCache](external-dicts-dict-layout.md#complex-key-cache), [SSDCache](external-dicts-dict-layout.md#ssd-cache), [SSDComplexKeyCache](external-dicts-dict-layout.md#complex-key-ssd-cache) dictionaries. In [IPTrie](external-dicts-dict-layout.md#ip-trie) dictionaries `Nullable` types are not supported.       | Yes      |
 | `null_value`                                         | Default value for a non-existing element.<br/>In the example, it is an empty string. [NULL](../../syntax.md#null-literal) value can be used only for the `Nullable` types (see the previous line with types description).                                                                                                                                                                                                                       | Yes      |
 | `expression`                                         | [Expression](../../../sql-reference/syntax.md#syntax-expressions) that ClickHouse executes on the value.<br/>The expression can be a column name in the remote SQL database. Thus, you can use it to create an alias for the remote column.<br/><br/>Default value: no expression.                                                              | No       |
 | <a name="hierarchical-dict-attr"></a> `hierarchical` | If `true`, the attribute contains the value of a parent key for the current key. See [Hierarchical Dictionaries](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md).<br/><br/>Default value: `false`.                                                                                               | No       |

docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md

@@ -159,7 +159,7 @@ CREATE DICTIONARY somename (
 | Тег                                                  | Описание                                                                                                                                                                                                                                                                                                                                                      | Обязательный |
 |------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------|
 | `name`                                               | Имя столбца.                                                                                                                                                                                                                                                                                                                                                  | Да           |
-| `type`                                               | Тип данных ClickHouse: [UInt8](../../../sql-reference/data-types/int-uint.md), [UInt16](../../../sql-reference/data-types/int-uint.md), [UInt32](../../../sql-reference/data-types/int-uint.md), [UInt64](../../../sql-reference/data-types/int-uint.md), [Int8](../../../sql-reference/data-types/int-uint.md), [Int16](../../../sql-reference/data-types/int-uint.md), [Int32](../../../sql-reference/data-types/int-uint.md), [Int64](../../../sql-reference/data-types/int-uint.md), [Float32](../../../sql-reference/data-types/float.md), [Float64](../../../sql-reference/data-types/float.md), [UUID](../../../sql-reference/data-types/uuid.md), [Decimal32](../../../sql-reference/data-types/decimal.md), [Decimal64](../../../sql-reference/data-types/decimal.md), [Decimal128](../../../sql-reference/data-types/decimal.md), [Decimal256](../../../sql-reference/data-types/decimal.md), [String](../../../sql-reference/data-types/string.md).<br/>ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`. <br/>[Nullable](../../../sql-reference/data-types/nullable.md) в настоящее время поддерживается для словарей [Flat](external-dicts-dict-layout.md#flat), [Hashed](external-dicts-dict-layout.md#dicts-external_dicts_dict_layout-hashed), [ComplexKeyHashed](external-dicts-dict-layout.md#complex-key-hashed), [Direct](external-dicts-dict-layout.md#direct), [ComplexKeyDirect](external-dicts-dict-layout.md#complex-key-direct), [RangeHashed](external-dicts-dict-layout.md#range-hashed), [Polygon](external-dicts-dict-polygon.md), [Cache](external-dicts-dict-layout.md#cache), [ComplexKeyCache](external-dicts-dict-layout.md#complex-key-cache), [SSDCache](external-dicts-dict-layout.md#ssd-cache), [SSDComplexKeyCache](external-dicts-dict-layout.md#complex-key-ssd-cache). Для словарей [IPTrie](external-dicts-dict-layout.md#ip-trie) `Nullable`-типы не поддерживаются. | Да           |
+| `type`                                               | Тип данных ClickHouse: [UInt8](../../../sql-reference/data-types/int-uint.md), [UInt16](../../../sql-reference/data-types/int-uint.md), [UInt32](../../../sql-reference/data-types/int-uint.md), [UInt64](../../../sql-reference/data-types/int-uint.md), [Int8](../../../sql-reference/data-types/int-uint.md), [Int16](../../../sql-reference/data-types/int-uint.md), [Int32](../../../sql-reference/data-types/int-uint.md), [Int64](../../../sql-reference/data-types/int-uint.md), [Float32](../../../sql-reference/data-types/float.md), [Float64](../../../sql-reference/data-types/float.md), [UUID](../../../sql-reference/data-types/uuid.md), [Decimal32](../../../sql-reference/data-types/decimal.md), [Decimal64](../../../sql-reference/data-types/decimal.md), [Decimal128](../../../sql-reference/data-types/decimal.md), [Decimal256](../../../sql-reference/data-types/decimal.md), [String](../../../sql-reference/data-types/string.md), [Array](../../../sql-reference/data-types/array.md).<br/>ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`. <br/>[Nullable](../../../sql-reference/data-types/nullable.md) в настоящее время поддерживается для словарей [Flat](external-dicts-dict-layout.md#flat), [Hashed](external-dicts-dict-layout.md#dicts-external_dicts_dict_layout-hashed), [ComplexKeyHashed](external-dicts-dict-layout.md#complex-key-hashed), [Direct](external-dicts-dict-layout.md#direct), [ComplexKeyDirect](external-dicts-dict-layout.md#complex-key-direct), [RangeHashed](external-dicts-dict-layout.md#range-hashed), [Polygon](external-dicts-dict-polygon.md), [Cache](external-dicts-dict-layout.md#cache), [ComplexKeyCache](external-dicts-dict-layout.md#complex-key-cache), [SSDCache](external-dicts-dict-layout.md#ssd-cache), [SSDComplexKeyCache](external-dicts-dict-layout.md#complex-key-ssd-cache). Для словарей [IPTrie](external-dicts-dict-layout.md#ip-trie) `Nullable`-типы не поддерживаются. | Да           |
 | `null_value`                                         | Значение по умолчанию для несуществующего элемента.<br/>В примере это пустая строка. Значение [NULL](../../syntax.md#null-literal) можно указывать только для типов `Nullable` (см. предыдущую строку с описанием типов).                                                                                                                                                                                                                                          | Да           |
 | `expression`                                         | [Выражение](../../syntax.md#syntax-expressions), которое ClickHouse выполняет со значением.<br/>Выражением может быть имя столбца в удаленной SQL базе. Таким образом, вы можете использовать его для создания псевдонима удаленного столбца.<br/><br/>Значение по умолчанию: нет выражения.                                                                  | Нет          |
 | <a name="hierarchical-dict-attr"></a> `hierarchical` | Если `true`, то атрибут содержит ключ предка для текущего элемента. Смотрите [Иерархические словари](external-dicts-dict-hierarchical.md).<br/><br/>Значение по умолчанию: `false`.                                                                                                                                                                                   | Нет           |

programs/server/Server.cpp

@@ -1159,7 +1159,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
     {
         /// This object will periodically calculate some metrics.
         AsynchronousMetrics async_metrics(
-            global_context, config().getUInt("asynchronous_metrics_update_period_s", 60), servers_to_start_before_tables, servers);
+            global_context, config().getUInt("asynchronous_metrics_update_period_s", 1), servers_to_start_before_tables, servers);
         attachSystemTablesAsync(*DatabaseCatalog::instance().getSystemDatabase(), async_metrics);
 
         for (const auto & listen_host : listen_hosts)

programs/server/config.xml

@@ -583,7 +583,7 @@
         <port>9019</port>
     </jdbc_bridge>
     -->
-  
+
     <!-- Configuration of clusters that could be used in Distributed tables.
          https://clickhouse.tech/docs/en/operations/table_engines/distributed/
       -->
@@ -917,7 +917,7 @@
             Asynchronous metrics are updated once a minute, so there is
             no need to flush more often.
         -->
-        <flush_interval_milliseconds>60000</flush_interval_milliseconds>
+        <flush_interval_milliseconds>7000</flush_interval_milliseconds>
     </asynchronous_metric_log>
 
     <!--

src/Common/ErrorCodes.cpp

@@ -557,6 +557,7 @@
     M(587, CONCURRENT_ACCESS_NOT_SUPPORTED) \
     M(588, DISTRIBUTED_BROKEN_BATCH_INFO) \
     M(589, DISTRIBUTED_BROKEN_BATCH_FILES) \
+    M(590, CANNOT_SYSCONF) \
     \
     M(998, POSTGRESQL_CONNECTION_FAILURE) \
     M(999, KEEPER_EXCEPTION) \

src/Common/hex.cpp

@@ -56,3 +56,37 @@ const char * const hex_char_to_digit_table =
     "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
     "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
     "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff";
+
+const char * const bin_byte_to_char_table =
+    "0000000000000001000000100000001100000100000001010000011000000111"
+    "0000100000001001000010100000101100001100000011010000111000001111"
+    "0001000000010001000100100001001100010100000101010001011000010111"
+    "0001100000011001000110100001101100011100000111010001111000011111"
+    "0010000000100001001000100010001100100100001001010010011000100111"
+    "0010100000101001001010100010101100101100001011010010111000101111"
+    "0011000000110001001100100011001100110100001101010011011000110111"
+    "0011100000111001001110100011101100111100001111010011111000111111"
+    "0100000001000001010000100100001101000100010001010100011001000111"
+    "0100100001001001010010100100101101001100010011010100111001001111"
+    "0101000001010001010100100101001101010100010101010101011001010111"
+    "0101100001011001010110100101101101011100010111010101111001011111"
+    "0110000001100001011000100110001101100100011001010110011001100111"
+    "0110100001101001011010100110101101101100011011010110111001101111"
+    "0111000001110001011100100111001101110100011101010111011001110111"
+    "0111100001111001011110100111101101111100011111010111111001111111"
+    "1000000010000001100000101000001110000100100001011000011010000111"
+    "1000100010001001100010101000101110001100100011011000111010001111"
+    "1001000010010001100100101001001110010100100101011001011010010111"
+    "1001100010011001100110101001101110011100100111011001111010011111"
+    "1010000010100001101000101010001110100100101001011010011010100111"
+    "1010100010101001101010101010101110101100101011011010111010101111"
+    "1011000010110001101100101011001110110100101101011011011010110111"
+    "1011100010111001101110101011101110111100101111011011111010111111"
+    "1100000011000001110000101100001111000100110001011100011011000111"
+    "1100100011001001110010101100101111001100110011011100111011001111"
+    "1101000011010001110100101101001111010100110101011101011011010111"
+    "1101100011011001110110101101101111011100110111011101111011011111"
+    "1110000011100001111000101110001111100100111001011110011011100111"
+    "1110100011101001111010101110101111101100111011011110111011101111"
+    "1111000011110001111100101111001111110100111101011111011011110111"
+    "1111100011111001111110101111101111111100111111011111111011111111";

src/Common/hex.h

@@ -39,6 +39,12 @@ inline void writeHexByteLowercase(UInt8 byte, void * out)
     memcpy(out, &hex_byte_to_char_lowercase_table[static_cast<size_t>(byte) * 2], 2);
 }
 
+extern const char * const bin_byte_to_char_table;
+
+inline void writeBinByte(UInt8 byte, void * out)
+{
+    memcpy(out, &bin_byte_to_char_table[static_cast<size_t>(byte) * 8], 8);
+}
 
 /// Produces hex representation of an unsigned int with leading zeros (for checksums)
 template <typename TUInt>

src/Functions/FunctionsCoding.cpp

@@ -21,6 +21,8 @@ void registerFunctionsCoding(FunctionFactory & factory)
     factory.registerFunction<FunctionUUIDStringToNum>();
     factory.registerFunction<FunctionHex>(FunctionFactory::CaseInsensitive);
     factory.registerFunction<FunctionUnhex>(FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionBin>(FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionUnbin>(FunctionFactory::CaseInsensitive);
     factory.registerFunction<FunctionChar>(FunctionFactory::CaseInsensitive);
     factory.registerFunction<FunctionBitmaskToArray>();
     factory.registerFunction<FunctionBitPositionsToArray>();

src/Functions/FunctionsCoding.h

@@ -65,7 +65,6 @@ namespace ErrorCodes
 constexpr size_t uuid_bytes_length = 16;
 constexpr size_t uuid_text_length = 36;
 
-
 class FunctionIPv6NumToString : public IFunction
 {
 public:
@@ -951,19 +950,22 @@ public:
     }
 };
 
-
+/// Encode number or string to string with binary or hexadecimal representation
-class FunctionHex : public IFunction
+template <typename Impl>
+class EncodeToBinaryRepr : public IFunction
 {
 public:
-    static constexpr auto name = "hex";
+    static constexpr auto name = Impl::name;
-    static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionHex>(); }
+    static constexpr size_t word_size = Impl::word_size;
 
-    String getName() const override
+    static FunctionPtr create(ContextPtr) { return std::make_shared<EncodeToBinaryRepr>(); }
-    {
+
-        return name;
+    String getName() const override { return name; }
-    }
 
     size_t getNumberOfArguments() const override { return 1; }
+
+    bool useDefaultImplementationForConstants() const override { return true; }
+
     bool isInjective(const ColumnsWithTypeAndName &) const override { return true; }
 
     DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
@@ -983,235 +985,6 @@ public:
         return std::make_shared<DataTypeString>();
     }
 
-    template <typename T>
-    void executeOneUInt(T x, char *& out) const
-    {
-        bool was_nonzero = false;
-        for (int offset = (sizeof(T) - 1) * 8; offset >= 0; offset -= 8)
-        {
-            UInt8 byte = x >> offset;
-
-            /// Leading zeros.
-            if (byte == 0 && !was_nonzero && offset)  // -V560
-                continue;
-
-            was_nonzero = true;
-
-            writeHexByteUppercase(byte, out);
-            out += 2;
-        }
-        *out = '\0';
-        ++out;
-    }
-
-    template <typename T>
-    bool tryExecuteUInt(const IColumn * col, ColumnPtr & col_res) const
-    {
-        const ColumnVector<T> * col_vec = checkAndGetColumn<ColumnVector<T>>(col);
-
-        static constexpr size_t MAX_UINT_HEX_LENGTH = sizeof(T) * 2 + 1;    /// Including trailing zero byte.
-
-        if (col_vec)
-        {
-            auto col_str = ColumnString::create();
-            ColumnString::Chars & out_vec = col_str->getChars();
-            ColumnString::Offsets & out_offsets = col_str->getOffsets();
-
-            const typename ColumnVector<T>::Container & in_vec = col_vec->getData();
-
-            size_t size = in_vec.size();
-            out_offsets.resize(size);
-            out_vec.resize(size * 3 + MAX_UINT_HEX_LENGTH); /// 3 is length of one byte in hex plus zero byte.
-
-            size_t pos = 0;
-            for (size_t i = 0; i < size; ++i)
-            {
-                /// Manual exponential growth, so as not to rely on the linear amortized work time of `resize` (no one guarantees it).
-                if (pos + MAX_UINT_HEX_LENGTH > out_vec.size())
-                    out_vec.resize(out_vec.size() * 2 + MAX_UINT_HEX_LENGTH);
-
-                char * begin = reinterpret_cast<char *>(&out_vec[pos]);
-                char * end = begin;
-                executeOneUInt<T>(in_vec[i], end);
-
-                pos += end - begin;
-                out_offsets[i] = pos;
-            }
-
-            out_vec.resize(pos);
-
-            col_res = std::move(col_str);
-            return true;
-        }
-        else
-        {
-            return false;
-        }
-    }
-
-    template <typename T>
-    void executeFloatAndDecimal(const T & in_vec, ColumnPtr & col_res, const size_t type_size_in_bytes) const
-    {
-        const size_t hex_length = type_size_in_bytes * 2 + 1; /// Including trailing zero byte.
-        auto col_str = ColumnString::create();
-
-        ColumnString::Chars & out_vec = col_str->getChars();
-        ColumnString::Offsets & out_offsets = col_str->getOffsets();
-
-        size_t size = in_vec.size();
-        out_offsets.resize(size);
-        out_vec.resize(size * hex_length);
-
-        size_t pos = 0;
-        char * out = reinterpret_cast<char *>(&out_vec[0]);
-        for (size_t i = 0; i < size; ++i)
-        {
-            const UInt8 * in_pos = reinterpret_cast<const UInt8 *>(&in_vec[i]);
-            executeOneString(in_pos, in_pos + type_size_in_bytes, out);
-
-            pos += hex_length;
-            out_offsets[i] = pos;
-        }
-        col_res = std::move(col_str);
-    }
-
-    template <typename T>
-    bool tryExecuteFloat(const IColumn * col, ColumnPtr & col_res) const
-    {
-        const ColumnVector<T> * col_vec = checkAndGetColumn<ColumnVector<T>>(col);
-        if (col_vec)
-        {
-            const typename ColumnVector<T>::Container & in_vec = col_vec->getData();
-            executeFloatAndDecimal<typename ColumnVector<T>::Container>(in_vec, col_res, sizeof(T));
-            return true;
-        }
-        else
-        {
-            return false;
-        }
-    }
-
-    template <typename T>
-    bool tryExecuteDecimal(const IColumn * col, ColumnPtr & col_res) const
-    {
-        const ColumnDecimal<T> * col_dec = checkAndGetColumn<ColumnDecimal<T>>(col);
-        if (col_dec)
-        {
-            const typename ColumnDecimal<T>::Container & in_vec = col_dec->getData();
-            executeFloatAndDecimal<typename ColumnDecimal<T>::Container>(in_vec, col_res, sizeof(T));
-            return true;
-        }
-        else
-        {
-            return false;
-        }
-    }
-
-
-    static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out)
-    {
-        while (pos < end)
-        {
-            writeHexByteUppercase(*pos, out);
-            ++pos;
-            out += 2;
-        }
-        *out = '\0';
-        ++out;
-    }
-
-    static bool tryExecuteString(const IColumn * col, ColumnPtr & col_res)
-    {
-        const ColumnString * col_str_in = checkAndGetColumn<ColumnString>(col);
-
-        if (col_str_in)
-        {
-            auto col_str = ColumnString::create();
-            ColumnString::Chars & out_vec = col_str->getChars();
-            ColumnString::Offsets & out_offsets = col_str->getOffsets();
-
-            const ColumnString::Chars & in_vec = col_str_in->getChars();
-            const ColumnString::Offsets & in_offsets = col_str_in->getOffsets();
-
-            size_t size = in_offsets.size();
-            out_offsets.resize(size);
-            out_vec.resize(in_vec.size() * 2 - size);
-
-            char * begin = reinterpret_cast<char *>(out_vec.data());
-            char * pos = begin;
-            size_t prev_offset = 0;
-
-            for (size_t i = 0; i < size; ++i)
-            {
-                size_t new_offset = in_offsets[i];
-
-                executeOneString(&in_vec[prev_offset], &in_vec[new_offset - 1], pos);
-
-                out_offsets[i] = pos - begin;
-
-                prev_offset = new_offset;
-            }
-
-            if (!out_offsets.empty() && out_offsets.back() != out_vec.size())
-                throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR);
-
-            col_res = std::move(col_str);
-            return true;
-        }
-        else
-        {
-            return false;
-        }
-    }
-
-    static bool tryExecuteFixedString(const IColumn * col, ColumnPtr & col_res)
-    {
-        const ColumnFixedString * col_fstr_in = checkAndGetColumn<ColumnFixedString>(col);
-
-        if (col_fstr_in)
-        {
-            auto col_str = ColumnString::create();
-            ColumnString::Chars & out_vec = col_str->getChars();
-            ColumnString::Offsets & out_offsets = col_str->getOffsets();
-
-            const ColumnString::Chars & in_vec = col_fstr_in->getChars();
-
-            size_t size = col_fstr_in->size();
-
-            out_offsets.resize(size);
-            out_vec.resize(in_vec.size() * 2 + size);
-
-            char * begin = reinterpret_cast<char *>(out_vec.data());
-            char * pos = begin;
-
-            size_t n = col_fstr_in->getN();
-
-            size_t prev_offset = 0;
-
-            for (size_t i = 0; i < size; ++i)
-            {
-                size_t new_offset = prev_offset + n;
-
-                executeOneString(&in_vec[prev_offset], &in_vec[new_offset], pos);
-
-                out_offsets[i] = pos - begin;
-                prev_offset = new_offset;
-            }
-
-            if (!out_offsets.empty() && out_offsets.back() != out_vec.size())
-                throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR);
-
-            col_res = std::move(col_str);
-            return true;
-        }
-        else
-        {
-            return false;
-        }
-    }
-
-    bool useDefaultImplementationForConstants() const override { return true; }
-
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
     {
         const IColumn * column = arguments[0].column.get();
@@ -1234,19 +1007,185 @@ public:
                         + " of argument of function " + getName(),
                         ErrorCodes::ILLEGAL_COLUMN);
     }
+
+    template <typename T>
+    bool tryExecuteUInt(const IColumn * col, ColumnPtr & col_res) const
+    {
+        const ColumnVector<T> * col_vec = checkAndGetColumn<ColumnVector<T>>(col);
+
+        static constexpr size_t MAX_LENGTH = sizeof(T) * word_size + 1;    /// Including trailing zero byte.
+
+        if (col_vec)
+        {
+            auto col_str = ColumnString::create();
+            ColumnString::Chars & out_vec = col_str->getChars();
+            ColumnString::Offsets & out_offsets = col_str->getOffsets();
+
+            const typename ColumnVector<T>::Container & in_vec = col_vec->getData();
+
+            size_t size = in_vec.size();
+            out_offsets.resize(size);
+            out_vec.resize(size * (word_size+1) + MAX_LENGTH); /// word_size+1 is length of one byte in hex/bin plus zero byte.
+
+            size_t pos = 0;
+            for (size_t i = 0; i < size; ++i)
+            {
+                /// Manual exponential growth, so as not to rely on the linear amortized work time of `resize` (no one guarantees it).
+                if (pos + MAX_LENGTH > out_vec.size())
+                    out_vec.resize(out_vec.size() * word_size + MAX_LENGTH);
+
+                char * begin = reinterpret_cast<char *>(&out_vec[pos]);
+                char * end = begin;
+                Impl::executeOneUInt(in_vec[i], end);
+
+                pos += end - begin;
+                out_offsets[i] = pos;
+            }
+            out_vec.resize(pos);
+
+            col_res = std::move(col_str);
+            return true;
+        }
+        else
+        {
+            return false;
+        }
+    }
+
+    bool tryExecuteString(const IColumn *col, ColumnPtr &col_res) const
+    {
+        const ColumnString * col_str_in = checkAndGetColumn<ColumnString>(col);
+
+        if (col_str_in)
+        {
+            auto col_str = ColumnString::create();
+            ColumnString::Chars & out_vec = col_str->getChars();
+            ColumnString::Offsets & out_offsets = col_str->getOffsets();
+
+            const ColumnString::Chars & in_vec = col_str_in->getChars();
+            const ColumnString::Offsets & in_offsets = col_str_in->getOffsets();
+
+            size_t size = in_offsets.size();
+
+            out_offsets.resize(size);
+            /// reserve `word_size` bytes for each non trailing zero byte from input + `size` bytes for trailing zeros
+            out_vec.resize((in_vec.size() - size) * word_size + size);
+
+            char * begin = reinterpret_cast<char *>(out_vec.data());
+            char * pos = begin;
+            size_t prev_offset = 0;
+
+            for (size_t i = 0; i < size; ++i)
+            {
+                size_t new_offset = in_offsets[i];
+
+                Impl::executeOneString(&in_vec[prev_offset], &in_vec[new_offset - 1], pos);
+
+                out_offsets[i] = pos - begin;
+
+                prev_offset = new_offset;
+            }
+            if (!out_offsets.empty() && out_offsets.back() != out_vec.size())
+                throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR);
+
+            col_res = std::move(col_str);
+            return true;
+        }
+        else
+        {
+            return false;
+        }
+    }
+
+    template <typename T>
+    bool tryExecuteDecimal(const IColumn * col, ColumnPtr & col_res) const
+    {
+        const ColumnDecimal<T> * col_dec = checkAndGetColumn<ColumnDecimal<T>>(col);
+        if (col_dec)
+        {
+            const typename ColumnDecimal<T>::Container & in_vec = col_dec->getData();
+            Impl::executeFloatAndDecimal(in_vec, col_res, sizeof(T));
+            return true;
+        }
+        else
+        {
+            return false;
+        }
+    }
+
+    static bool tryExecuteFixedString(const IColumn * col, ColumnPtr & col_res)
+    {
+         const ColumnFixedString * col_fstr_in = checkAndGetColumn<ColumnFixedString>(col);
+
+         if (col_fstr_in)
+         {
+             auto col_str = ColumnString::create();
+             ColumnString::Chars & out_vec = col_str->getChars();
+             ColumnString::Offsets & out_offsets = col_str->getOffsets();
+
+             const ColumnString::Chars & in_vec = col_fstr_in->getChars();
+
+             size_t size = col_fstr_in->size();
+
+             out_offsets.resize(size);
+             out_vec.resize(in_vec.size() * word_size + size);
+
+             char * begin = reinterpret_cast<char *>(out_vec.data());
+             char * pos = begin;
+
+             size_t n = col_fstr_in->getN();
+
+             size_t prev_offset = 0;
+
+             for (size_t i = 0; i < size; ++i)
+             {
+                 size_t new_offset = prev_offset + n;
+
+                 Impl::executeOneString(&in_vec[prev_offset], &in_vec[new_offset], pos);
+
+                 out_offsets[i] = pos - begin;
+                 prev_offset = new_offset;
+             }
+
+             if (!out_offsets.empty() && out_offsets.back() != out_vec.size())
+                 throw Exception("Column size mismatch (internal logical error)", ErrorCodes::LOGICAL_ERROR);
+
+             col_res = std::move(col_str);
+             return true;
+         }
+         else
+         {
+             return false;
+         }
+     }
+
+    template <typename T>
+    bool tryExecuteFloat(const IColumn * col, ColumnPtr & col_res) const
+    {
+        const ColumnVector<T> * col_vec = checkAndGetColumn<ColumnVector<T>>(col);
+        if (col_vec)
+        {
+            const typename ColumnVector<T>::Container & in_vec = col_vec->getData();
+            Impl::executeFloatAndDecimal(in_vec, col_res, sizeof(T));
+            return true;
+        }
+        else
+        {
+            return false;
+        }
+    }
 };
 
-
+/// Decode number or string from string with binary or hexadecimal representation
-class FunctionUnhex : public IFunction
+template <typename Impl>
+class DecodeFromBinaryRepr : public IFunction
 {
 public:
-    static constexpr auto name = "unhex";
+    static constexpr auto name = Impl::name;
-    static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionUnhex>(); }
+    static constexpr size_t word_size = Impl::word_size;
+    static FunctionPtr create(ContextPtr) { return std::make_shared<DecodeFromBinaryRepr>(); }
 
-    String getName() const override
+    String getName() const override { return name; }
-    {
-        return name;
-    }
 
     size_t getNumberOfArguments() const override { return 1; }
     bool isInjective(const ColumnsWithTypeAndName &) const override { return true; }
@@ -1255,29 +1194,11 @@ public:
     {
         if (!isString(arguments[0]))
             throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
-            ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+                            ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
 
         return std::make_shared<DataTypeString>();
     }
 
-    static void unhexOne(const char * pos, const char * end, char *& out)
-    {
-        if ((end - pos) & 1)
-        {
-            *out = unhex(*pos);
-            ++out;
-            ++pos;
-        }
-        while (pos < end)
-        {
-            *out = unhex2(pos);
-            pos += 2;
-            ++out;
-        }
-        *out = '\0';
-        ++out;
-    }
-
     bool useDefaultImplementationForConstants() const override { return true; }
 
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
@@ -1296,7 +1217,7 @@ public:
 
             size_t size = in_offsets.size();
             out_offsets.resize(size);
-            out_vec.resize(in_vec.size() / 2 + size);
+            out_vec.resize(in_vec.size() / word_size + size);
 
             char * begin = reinterpret_cast<char *>(out_vec.data());
             char * pos = begin;
@@ -1306,7 +1227,7 @@ public:
             {
                 size_t new_offset = in_offsets[i];
 
-                unhexOne(reinterpret_cast<const char *>(&in_vec[prev_offset]), reinterpret_cast<const char *>(&in_vec[new_offset - 1]), pos);
+                Impl::decode(reinterpret_cast<const char *>(&in_vec[prev_offset]), reinterpret_cast<const char *>(&in_vec[new_offset - 1]), pos);
 
                 out_offsets[i] = pos - begin;
 
@@ -1326,6 +1247,219 @@ public:
     }
 };
 
+struct HexImpl
+{
+    static constexpr auto name = "hex";
+    static constexpr size_t word_size = 2;
+
+    template <typename T>
+    static void executeOneUInt(T x, char *& out)
+    {
+        bool was_nonzero = false;
+        for (int offset = (sizeof(T) - 1) * 8; offset >= 0; offset -= 8)
+        {
+            UInt8 byte = x >> offset;
+
+            /// Skip leading zeros
+            if (byte == 0 && !was_nonzero && offset)
+                continue;
+
+            was_nonzero = true;
+            writeHexByteUppercase(byte, out);
+            out += word_size;
+        }
+        *out = '\0';
+        ++out;
+    }
+
+    static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out)
+    {
+        while (pos < end)
+        {
+            writeHexByteUppercase(*pos, out);
+            ++pos;
+            out += word_size;
+        }
+        *out = '\0';
+        ++out;
+    }
+
+    template <typename T>
+    static void executeFloatAndDecimal(const T & in_vec, ColumnPtr & col_res, const size_t type_size_in_bytes)
+    {
+        const size_t hex_length = type_size_in_bytes * word_size + 1; /// Including trailing zero byte.
+        auto col_str = ColumnString::create();
+
+        ColumnString::Chars & out_vec = col_str->getChars();
+        ColumnString::Offsets & out_offsets = col_str->getOffsets();
+
+        size_t size = in_vec.size();
+        out_offsets.resize(size);
+        out_vec.resize(size * hex_length);
+
+        size_t pos = 0;
+        char * out = reinterpret_cast<char *>(&out_vec[0]);
+        for (size_t i = 0; i < size; ++i)
+        {
+            const UInt8 * in_pos = reinterpret_cast<const UInt8 *>(&in_vec[i]);
+            executeOneString(in_pos, in_pos + type_size_in_bytes, out);
+
+            pos += hex_length;
+            out_offsets[i] = pos;
+        }
+        col_res = std::move(col_str);
+    }
+};
+
+struct UnhexImpl
+{
+    static constexpr auto name = "unhex";
+    static constexpr size_t word_size = 2;
+
+    static void decode(const char * pos, const char * end, char *& out)
+    {
+        if ((end - pos) & 1)
+        {
+            *out = unhex(*pos);
+            ++out;
+            ++pos;
+        }
+        while (pos < end)
+        {
+            *out = unhex2(pos);
+            pos += word_size;
+            ++out;
+        }
+        *out = '\0';
+        ++out;
+    }
+};
+
+struct BinImpl
+{
+    static constexpr auto name = "bin";
+    static constexpr size_t word_size = 8;
+
+    template <typename T>
+    static void executeOneUInt(T x, char *& out)
+    {
+        bool was_nonzero = false;
+        for (int offset = (sizeof(T) - 1) * 8; offset >= 0; offset -= 8)
+        {
+            UInt8 byte = x >> offset;
+
+            /// Skip leading zeros
+            if (byte == 0 && !was_nonzero && offset)
+                continue;
+
+            was_nonzero = true;
+            writeBinByte(byte, out);
+            out += word_size;
+        }
+        *out = '\0';
+        ++out;
+    }
+
+    template <typename T>
+    static void executeFloatAndDecimal(const T & in_vec, ColumnPtr & col_res, const size_t type_size_in_bytes)
+    {
+        const size_t hex_length = type_size_in_bytes * word_size + 1; /// Including trailing zero byte.
+        auto col_str = ColumnString::create();
+
+        ColumnString::Chars & out_vec = col_str->getChars();
+        ColumnString::Offsets & out_offsets = col_str->getOffsets();
+
+        size_t size = in_vec.size();
+        out_offsets.resize(size);
+        out_vec.resize(size * hex_length);
+
+        size_t pos = 0;
+        char * out = reinterpret_cast<char *>(out_vec.data());
+        for (size_t i = 0; i < size; ++i)
+        {
+            const UInt8 * in_pos = reinterpret_cast<const UInt8 *>(&in_vec[i]);
+            executeOneString(in_pos, in_pos + type_size_in_bytes, out);
+
+            pos += hex_length;
+            out_offsets[i] = pos;
+        }
+        col_res = std::move(col_str);
+    }
+
+    static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out)
+    {
+        while (pos < end)
+        {
+            writeBinByte(*pos, out);
+            ++pos;
+            out += word_size;
+        }
+        *out = '\0';
+        ++out;
+    }
+};
+
+struct UnbinImpl
+{
+    static constexpr auto name = "unbin";
+    static constexpr size_t word_size = 8;
+
+    static void decode(const char * pos, const char * end, char *& out)
+    {
+        if (pos == end)
+        {
+            *out = '\0';
+            ++out;
+            return;
+        }
+
+        UInt8 left = 0;
+
+        /// end - pos is the length of input.
+        /// (length & 7) to make remain bits length mod 8 is zero to split.
+        /// e.g. the length is 9 and the input is "101000001",
+        /// first left_cnt is 1, left is 0, right shift, pos is 1, left = 1
+        /// then, left_cnt is 0, remain input is '01000001'.
+        for (UInt8 left_cnt = (end - pos) & 7; left_cnt > 0; --left_cnt)
+        {
+            left = left << 1;
+            if (*pos != '0')
+                left += 1;
+            ++pos;
+        }
+
+        if (left != 0 || end - pos == 0)
+        {
+            *out = left;
+            ++out;
+        }
+
+        assert((end - pos) % 8 == 0);
+
+        while (end - pos != 0)
+        {
+            UInt8 c = 0;
+            for (UInt8 i = 0; i < 8; ++i)
+            {
+                c = c << 1;
+                if (*pos != '0')
+                    c += 1;
+                ++pos;
+            }
+            *out = c;
+            ++out;
+        }
+
+        *out = '\0';
+        ++out;
+    }
+};
+
+using FunctionHex = EncodeToBinaryRepr<HexImpl>;
+using FunctionUnhex = DecodeFromBinaryRepr<UnhexImpl>;
+using FunctionBin = EncodeToBinaryRepr<BinImpl>;
+using FunctionUnbin = DecodeFromBinaryRepr<UnbinImpl>;
+
 class FunctionChar : public IFunction
 {
 public:

src/Functions/FunctionsLogical.cpp

@@ -575,12 +575,12 @@ ColumnPtr FunctionAnyArityLogical<Impl, Name>::getConstantResultForNonConstArgum
     if constexpr (std::is_same_v<Impl, AndImpl>)
     {
         if (has_false_constant)
-            result_type->createColumnConst(0, static_cast<UInt8>(false));
+            result_column = result_type->createColumnConst(0, static_cast<UInt8>(false));
     }
     else if constexpr (std::is_same_v<Impl, OrImpl>)
     {
         if (has_true_constant)
-            result_type->createColumnConst(0, static_cast<UInt8>(true));
+            result_column = result_type->createColumnConst(0, static_cast<UInt8>(true));
     }
 
     return result_column;

src/IO/ReadBufferFromFileDescriptor.cpp

@@ -149,7 +149,7 @@ off_t ReadBufferFromFileDescriptor::seek(off_t offset, int whence)
         off_t res = ::lseek(fd, new_pos, SEEK_SET);
         if (-1 == res)
             throwFromErrnoWithPath("Cannot seek through file " + getFileName(), getFileName(),
-                                   ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
+                ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
         file_offset_of_buffer_end = new_pos;
 
         watch.stop();
@@ -160,6 +160,20 @@ off_t ReadBufferFromFileDescriptor::seek(off_t offset, int whence)
 }
 
 
+void ReadBufferFromFileDescriptor::rewind()
+{
+    ProfileEvents::increment(ProfileEvents::Seek);
+    off_t res = ::lseek(fd, 0, SEEK_SET);
+    if (-1 == res)
+        throwFromErrnoWithPath("Cannot seek through file " + getFileName(), getFileName(),
+            ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
+
+    /// Clearing the buffer with existing data. New data will be read on subsequent call to 'next'.
+    working_buffer.resize(0);
+    pos = working_buffer.begin();
+}
+
+
 /// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout.
 bool ReadBufferFromFileDescriptor::poll(size_t timeout_microseconds)
 {

src/IO/ReadBufferFromFileDescriptor.h

@@ -39,6 +39,9 @@ public:
     /// If 'offset' is small enough to stay in buffer after seek, then true seek in file does not happen.
     off_t seek(off_t off, int whence) override;
 
+    /// Seek to the beginning, discarding already read data if any. Useful to reread file that changes on every read.
+    void rewind();
+
     off_t size();
 
     void setProgressCallback(ContextPtr context);

src/Interpreters/AsynchronousMetricLog.cpp

@@ -18,7 +18,7 @@ NamesAndTypesList AsynchronousMetricLogElement::getNamesAndTypes()
         {"event_date", std::make_shared<DataTypeDate>()},
         {"event_time", std::make_shared<DataTypeDateTime>()},
         {"event_time_microseconds", std::make_shared<DataTypeDateTime64>(6)},
-        {"name", std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())},
+        {"metric", std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())},
         {"value", std::make_shared<DataTypeFloat64>(),}
     };
 }

src/Interpreters/AsynchronousMetrics.h

@@ -3,11 +3,15 @@
 #include <Interpreters/Context_fwd.h>
 #include <Common/MemoryStatisticsOS.h>
 #include <Common/ThreadPool.h>
+#include <IO/ReadBufferFromFile.h>
 
 #include <condition_variable>
+#include <map>
 #include <mutex>
 #include <string>
 #include <thread>
+#include <vector>
+#include <optional>
 #include <unordered_map>
 
 
@@ -15,6 +19,7 @@ namespace DB
 {
 
 class ProtocolServerAdapter;
+class ReadBuffer;
 
 using AsynchronousMetricValue = double;
 using AsynchronousMetricValues = std::unordered_map<std::string, AsynchronousMetricValue>;
@@ -23,10 +28,30 @@ using AsynchronousMetricValues = std::unordered_map<std::string, AsynchronousMet
 /** Periodically (by default, each minute, starting at 30 seconds offset)
   *  calculates and updates some metrics,
   *  that are not updated automatically (so, need to be asynchronously calculated).
+  *
+  * This includes both ClickHouse-related metrics (like memory usage of ClickHouse process)
+  *  and common OS-related metrics (like total memory usage on the server).
   */
 class AsynchronousMetrics : WithContext
 {
 public:
+    /// The default value of update_period_seconds is for ClickHouse-over-YT
+    /// in Arcadia -- it uses its own server implementation that also uses these
+    /// metrics.
+    AsynchronousMetrics(
+        ContextPtr global_context_,
+        int update_period_seconds,
+        std::shared_ptr<std::vector<ProtocolServerAdapter>> servers_to_start_before_tables_,
+        std::shared_ptr<std::vector<ProtocolServerAdapter>> servers_);
+
+    ~AsynchronousMetrics();
+
+    /// Separate method allows to initialize the `servers` variable beforehand.
+    void start();
+
+    /// Returns copy of all values.
+    AsynchronousMetricValues getValues() const;
+
 #if defined(ARCADIA_BUILD)
     /// This constructor needs only to provide backward compatibility with some other projects (hello, Arcadia).
     /// Never use this in the ClickHouse codebase.
@@ -39,35 +64,6 @@ public:
     }
 #endif
 
-    /// The default value of update_period_seconds is for ClickHouse-over-YT
-    /// in Arcadia -- it uses its own server implementation that also uses these
-    /// metrics.
-    AsynchronousMetrics(
-        ContextPtr global_context_,
-        int update_period_seconds,
-        std::shared_ptr<std::vector<ProtocolServerAdapter>> servers_to_start_before_tables_,
-        std::shared_ptr<std::vector<ProtocolServerAdapter>> servers_)
-        : WithContext(global_context_)
-        , update_period(update_period_seconds)
-        , servers_to_start_before_tables(servers_to_start_before_tables_)
-        , servers(servers_)
-    {
-    }
-
-    ~AsynchronousMetrics();
-
-    /// Separate method allows to initialize the `servers` variable beforehand.
-    void start()
-    {
-        /// Update once right now, to make metrics available just after server start
-        /// (without waiting for asynchronous_metrics_update_period_s).
-        update();
-        thread = std::make_unique<ThreadFromGlobalPool>([this] { run(); });
-    }
-
-    /// Returns copy of all values.
-    AsynchronousMetricValues getValues() const;
-
 private:
     const std::chrono::seconds update_period;
     std::shared_ptr<std::vector<ProtocolServerAdapter>> servers_to_start_before_tables{nullptr};
@@ -78,14 +74,113 @@ private:
     bool quit {false};
     AsynchronousMetricValues values;
 
+    /// Some values are incremental and we have to calculate the difference.
+    /// On first run we will only collect the values to subtract later.
+    bool first_run = true;
+    std::chrono::system_clock::time_point previous_update_time;
+
 #if defined(OS_LINUX)
     MemoryStatisticsOS memory_stat;
+
+    std::optional<ReadBufferFromFile> meminfo;
+    std::optional<ReadBufferFromFile> loadavg;
+    std::optional<ReadBufferFromFile> proc_stat;
+    std::optional<ReadBufferFromFile> cpuinfo;
+    std::optional<ReadBufferFromFile> file_nr;
+    std::optional<ReadBufferFromFile> uptime;
+    std::optional<ReadBufferFromFile> net_dev;
+
+    std::vector<std::unique_ptr<ReadBufferFromFile>> thermal;
+
+    std::unordered_map<String /* device name */,
+        std::unordered_map<String /* label name */,
+            std::unique_ptr<ReadBufferFromFile>>> hwmon_devices;
+
+    std::vector<std::pair<
+        std::unique_ptr<ReadBufferFromFile> /* correctable errors */,
+        std::unique_ptr<ReadBufferFromFile> /* uncorrectable errors */>> edac;
+
+    std::unordered_map<String /* device name */, std::unique_ptr<ReadBufferFromFile>> block_devs;
+
+    /// TODO: socket statistics.
+
+    struct ProcStatValuesCPU
+    {
+        uint64_t user;
+        uint64_t nice;
+        uint64_t system;
+        uint64_t idle;
+        uint64_t iowait;
+        uint64_t irq;
+        uint64_t softirq;
+        uint64_t steal;
+        uint64_t guest;
+        uint64_t guest_nice;
+
+        void read(ReadBuffer & in);
+        ProcStatValuesCPU operator-(const ProcStatValuesCPU & other) const;
+    };
+
+    struct ProcStatValuesOther
+    {
+        uint64_t interrupts;
+        uint64_t context_switches;
+        uint64_t processes_created;
+
+        ProcStatValuesOther operator-(const ProcStatValuesOther & other) const;
+    };
+
+    ProcStatValuesCPU proc_stat_values_all_cpus{};
+    ProcStatValuesOther proc_stat_values_other{};
+    std::vector<ProcStatValuesCPU> proc_stat_values_per_cpu;
+
+    /// https://www.kernel.org/doc/Documentation/block/stat.txt
+    struct BlockDeviceStatValues
+    {
+        uint64_t read_ios;
+        uint64_t read_merges;
+        uint64_t read_sectors;
+        uint64_t read_ticks;
+        uint64_t write_ios;
+        uint64_t write_merges;
+        uint64_t write_sectors;
+        uint64_t write_ticks;
+        uint64_t in_flight_ios;
+        uint64_t io_ticks;
+        uint64_t time_in_queue;
+        uint64_t discard_ops;
+        uint64_t discard_merges;
+        uint64_t discard_sectors;
+        uint64_t discard_ticks;
+
+        void read(ReadBuffer & in);
+        BlockDeviceStatValues operator-(const BlockDeviceStatValues & other) const;
+    };
+
+    std::unordered_map<String /* device name */, BlockDeviceStatValues> block_device_stats;
+
+    struct NetworkInterfaceStatValues
+    {
+        uint64_t recv_bytes;
+        uint64_t recv_packets;
+        uint64_t recv_errors;
+        uint64_t recv_drop;
+        uint64_t send_bytes;
+        uint64_t send_packets;
+        uint64_t send_errors;
+        uint64_t send_drop;
+
+        NetworkInterfaceStatValues operator-(const NetworkInterfaceStatValues & other) const;
+    };
+
+    std::unordered_map<String /* device name */, NetworkInterfaceStatValues> network_interface_stats;
+
 #endif
 
     std::unique_ptr<ThreadFromGlobalPool> thread;
 
     void run();
-    void update();
+    void update(std::chrono::system_clock::time_point update_time);
 };
 
 }

src/Storages/HDFS/StorageHDFS.cpp

@@ -34,6 +34,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+    extern const int ACCESS_DENIED;
 }
 
 StorageHDFS::StorageHDFS(
@@ -280,15 +281,7 @@ Pipe StorageHDFS::read(
     size_t max_block_size,
     unsigned num_streams)
 {
-    size_t begin_of_path;
+    const size_t begin_of_path = uri.find('/', uri.find("//") + 2);
-    /// This uri is checked for correctness in constructor of StorageHDFS and never modified afterwards
-    auto two_slash = uri.find("//");
-
-    if (two_slash == std::string::npos)
-        begin_of_path = uri.find('/');
-    else
-        begin_of_path = uri.find('/', two_slash + 2);
-
     const String path_from_uri = uri.substr(begin_of_path);
     const String uri_without_path = uri.substr(0, begin_of_path);
 
@@ -330,6 +323,21 @@ BlockOutputStreamPtr StorageHDFS::write(const ASTPtr & /*query*/, const StorageM
         chooseCompressionMethod(uri, compression_method));
 }
 
+void StorageHDFS::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, ContextPtr context_, TableExclusiveLockHolder &)
+{
+    const size_t begin_of_path = uri.find('/', uri.find("//") + 2);
+    const String path = uri.substr(begin_of_path);
+    const String url = uri.substr(0, begin_of_path);
+
+    HDFSBuilderWrapper builder = createHDFSBuilder(url + "/", context_->getGlobalContext()->getConfigRef());
+    HDFSFSPtr fs = createHDFSFS(builder.get());
+
+    int ret = hdfsDelete(fs.get(), path.data(), 0);
+    if (ret)
+        throw Exception(ErrorCodes::ACCESS_DENIED, "Unable to truncate hdfs table: {}", std::string(hdfsGetLastError()));
+}
+
+
 void registerStorageHDFS(StorageFactory & factory)
 {
     factory.registerStorage("HDFS", [](const StorageFactory::Arguments & args)

src/Storages/HDFS/StorageHDFS.h

@@ -34,6 +34,8 @@ public:
 
     BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context) override;
 
+    void truncate(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context_, TableExclusiveLockHolder &) override;
+
     NamesAndTypesList getVirtuals() const override;
 
 protected:

src/Storages/MergeTree/MergeTreeData.cpp

@@ -1818,11 +1818,10 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
             if (MergeTreeSettings::isPartFormatSetting(setting_name) && !new_value)
             {
                 /// Use default settings + new and check if doesn't affect part format settings
-                MergeTreeSettings copy = *getSettings();
+                auto copy = getDefaultSettings();
-                copy.resetToDefault();
+                copy->applyChanges(new_changes);
-                copy.applyChanges(new_changes);
                 String reason;
-                if (!canUsePolymorphicParts(copy, &reason) && !reason.empty())
+                if (!canUsePolymorphicParts(*copy, &reason) && !reason.empty())
                     throw Exception("Can't change settings. Reason: " + reason, ErrorCodes::NOT_IMPLEMENTED);
             }
 
@@ -1984,14 +1983,12 @@ void MergeTreeData::changeSettings(
             }
         }
 
-        MergeTreeSettings copy = *getSettings();
+        /// Reset to default settings before applying existing.
-        /// reset to default settings before applying existing
+        auto copy = getDefaultSettings();
-        copy.resetToDefault();
+        copy->applyChanges(new_changes);
-        copy.applyChanges(new_changes);
+        copy->sanityCheck(getContext()->getSettingsRef());
 
-        copy.sanityCheck(getContext()->getSettingsRef());
+        storage_settings.set(std::move(copy));
-
-        storage_settings.set(std::make_unique<const MergeTreeSettings>(copy));
         StorageInMemoryMetadata new_metadata = getInMemoryMetadata();
         new_metadata.setSettingsChanges(new_settings);
         setInMemoryMetadata(new_metadata);

src/Storages/MergeTree/MergeTreeData.h

@@ -1087,6 +1087,9 @@ private:
 
     // Get partition matcher for FREEZE / UNFREEZE queries.
     MatcherFn getPartitionMatcher(const ASTPtr & partition, ContextPtr context) const;
+
+    /// Returns default settings for storage with possible changes from global config.
+    virtual std::unique_ptr<MergeTreeSettings> getDefaultSettings() const = 0;
 };
 
 /// RAII struct to record big parts that are submerging or emerging.

src/Storages/StorageMergeTree.cpp

@@ -1604,4 +1604,9 @@ void StorageMergeTree::startBackgroundMovesIfNeeded()
         background_moves_executor.start();
 }
 
+std::unique_ptr<MergeTreeSettings> StorageMergeTree::getDefaultSettings() const
+{
+    return std::make_unique<MergeTreeSettings>(getContext()->getMergeTreeSettings());
+}
+
 }

src/Storages/StorageMergeTree.h

@@ -236,6 +236,8 @@ private:
 
     void startBackgroundMovesIfNeeded() override;
 
+    std::unique_ptr<MergeTreeSettings> getDefaultSettings() const override;
+
     friend class MergeTreeProjectionBlockOutputStream;
     friend class MergeTreeBlockOutputStream;
     friend class MergeTreeData;

src/Storages/StorageReplicatedMergeTree.cpp

@@ -7174,6 +7174,11 @@ void StorageReplicatedMergeTree::startBackgroundMovesIfNeeded()
         background_moves_executor.start();
 }
 
+std::unique_ptr<MergeTreeSettings> StorageReplicatedMergeTree::getDefaultSettings() const
+{
+    return std::make_unique<MergeTreeSettings>(getContext()->getReplicatedMergeTreeSettings());
+}
+
 
 void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part) const
 {

src/Storages/StorageReplicatedMergeTree.h

@@ -702,6 +702,8 @@ private:
 
     void startBackgroundMovesIfNeeded() override;
 
+    std::unique_ptr<MergeTreeSettings> getDefaultSettings() const override;
+
     std::set<String> getPartitionIdsAffectedByCommands(const MutationCommands & commands, ContextPtr query_context) const;
     PartitionBlockNumbersHolder allocateBlockNumbersInAffectedPartitions(
         const MutationCommands & commands, ContextPtr query_context, const zkutil::ZooKeeperPtr & zookeeper) const;

src/Storages/StorageS3.cpp

@@ -27,6 +27,8 @@
 #include <aws/core/auth/AWSCredentials.h>
 #include <aws/s3/S3Client.h>
 #include <aws/s3/model/ListObjectsV2Request.h>
+#include <aws/s3/model/CopyObjectRequest.h>
+#include <aws/s3/model/DeleteObjectsRequest.h>
 
 #include <Common/parseGlobs.h>
 #include <Common/quoteString.h>
@@ -434,6 +436,30 @@ BlockOutputStreamPtr StorageS3::write(const ASTPtr & /*query*/, const StorageMet
         max_single_part_upload_size);
 }
 
+
+void StorageS3::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, ContextPtr local_context, TableExclusiveLockHolder &)
+{
+    updateClientAndAuthSettings(local_context, client_auth);
+
+    Aws::S3::Model::ObjectIdentifier obj;
+    obj.SetKey(client_auth.uri.key);
+
+    Aws::S3::Model::Delete delkeys;
+    delkeys.AddObjects(std::move(obj));
+
+    Aws::S3::Model::DeleteObjectsRequest request;
+    request.SetBucket(client_auth.uri.bucket);
+    request.SetDelete(delkeys);
+
+    auto response = client_auth.client->DeleteObjects(request);
+    if (!response.IsSuccess())
+    {
+        const auto & err = response.GetError();
+        throw Exception(std::to_string(static_cast<int>(err.GetErrorType())) + ": " + err.GetMessage(), ErrorCodes::S3_ERROR);
+    }
+}
+
+
 void StorageS3::updateClientAndAuthSettings(ContextPtr ctx, StorageS3::ClientAuthentication & upd)
 {
     auto settings = ctx->getStorageS3Settings().getSettings(upd.uri.uri.toString());

src/Storages/StorageS3.h

@@ -130,6 +130,8 @@ public:
 
     BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context) override;
 
+    void truncate(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr local_context, TableExclusiveLockHolder &) override;
+
     NamesAndTypesList getVirtuals() const override;
 
 private:

tests/integration/test_storage_hdfs/test.py

@@ -15,7 +15,6 @@ def started_cluster():
     finally:
         cluster.shutdown()
 
-
 def test_read_write_storage(started_cluster):
     hdfs_api = started_cluster.hdfs_api
 
@@ -235,7 +234,7 @@ def test_virtual_columns(started_cluster):
     expected = "1\tfile1\thdfs://hdfs1:9000//file1\n2\tfile2\thdfs://hdfs1:9000//file2\n3\tfile3\thdfs://hdfs1:9000//file3\n"
     assert node1.query("select id, _file as file_name, _path as file_path from virtual_cols order by id") == expected
 
-    
+
 def test_read_files_with_spaces(started_cluster):
     hdfs_api = started_cluster.hdfs_api
 
@@ -246,6 +245,18 @@ def test_read_files_with_spaces(started_cluster):
     assert node1.query("select * from test order by id") == "1\n2\n3\n"
 
 
+def test_truncate_table(started_cluster):
+    hdfs_api = started_cluster.hdfs_api
+    node1.query(
+        "create table test_truncate (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/tr', 'TSV')")
+    node1.query("insert into test_truncate values (1, 'Mark', 72.53)")
+    assert hdfs_api.read_data("/tr") == "1\tMark\t72.53\n"
+    assert node1.query("select * from test_truncate") == "1\tMark\t72.53\n"
+    node1.query("truncate table test_truncate")
+    assert node1.query("select * from test_truncate") == ""
+    node1.query("drop table test_truncate")
+
+
 if __name__ == '__main__':
     cluster.start()
     input("Cluster created, press any key to destroy...")

tests/integration/test_storage_s3/test.py

@@ -646,3 +646,28 @@ def test_storage_s3_put_gzip(started_cluster, extension, method):
     f = gzip.GzipFile(fileobj=buf, mode="rb")
     uncompressed_content = f.read().decode()
     assert sum([ int(i.split(',')[1]) for i in uncompressed_content.splitlines() ]) == 708
+
+
+def test_truncate_table(started_cluster):
+    bucket = started_cluster.minio_bucket
+    instance = started_cluster.instances["dummy"]  # type: ClickHouseInstance
+    name = "truncate"
+
+    instance.query("CREATE TABLE {} (id UInt32) ENGINE = S3('http://{}:{}/{}/{}', 'CSV')".format(
+        name, started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, name))
+
+    instance.query("INSERT INTO {} SELECT number FROM numbers(10)".format(name))
+    result = instance.query("SELECT * FROM {}".format(name))
+    assert result == instance.query("SELECT number FROM numbers(10)")
+    instance.query("TRUNCATE TABLE {}".format(name))
+
+    minio = started_cluster.minio_client
+    timeout = 30
+    while timeout > 0:
+        if len(list(minio.list_objects(started_cluster.minio_bucket, 'truncate/'))) == 0:
+            return
+        timeout -= 1
+        time.sleep(1)
+    assert(len(list(minio.list_objects(started_cluster.minio_bucket, 'truncate/'))) == 0)
+    assert instance.query("SELECT * FROM {}".format(name)) == ""
+

tests/queries/0_stateless/01926_bin_unbin.reference

@@ -0,0 +1,35 @@
+
+00000000
+00000001
+00001010
+01111111
+11111111
+0000000100000000
+0000000111111111
+0000001000000000
+00110000
+0011000100110000
+111001101011010110001011111010001010111110010101
+11100110101101011000101111101000101011111001010100000000000000000000000000000000
+10011010100110011001100100111111
+0011001100110011001100110011001100110011001100111111001100111111
+00000000000011100010011100000111
+0000000000000000000011000011110101011101010100111010101000000001
+0011000100110010001100110011001100110010001101000011001000110100
+0011000100110010001100110011001100110010001101000011001000110100
+0011000100110010001100110011001100110010001101000011001000110100
+0011000100110010001100110011001100110010001101000011001000110100
+
+1
+0
+10
+测试
+0
+0
+0
+1
+1
+1
+1
+1
+1

tests/queries/0_stateless/01926_bin_unbin.sql

@@ -0,0 +1,39 @@
+select bin('');
+select bin(0);
+select bin(1);
+select bin(10);
+select bin(127);
+select bin(255);
+select bin(256);
+select bin(511);
+select bin(512);
+select bin('0');
+select bin('10');
+select bin('测试');
+select bin(toFixedString('测试', 10));
+select bin(toFloat32(1.2));
+select bin(toFloat64(1.2));
+select bin(toDecimal32(1.2, 8));
+select bin(toDecimal64(1.2, 17));
+select bin('12332424');
+select bin(materialize('12332424'));
+select bin(toNullable(materialize('12332424')));
+select bin(toLowCardinality(materialize('12332424')));
+
+select unbin('');
+select unbin('0') == '\0';
+select unbin('00110000'); -- 0
+select unbin('0011000100110000'); -- 10
+select unbin('111001101011010110001011111010001010111110010101'); -- 测试
+select unbin(materialize('00110000'));
+select unbin(toNullable(materialize('00110000')));
+select unbin(toLowCardinality(materialize('00110000')));
+
+select unbin(bin('')) == '';
+select bin(unbin('')) == '';
+select bin(unbin('0')) == '00000000';
+
+-- hex and bin consistent for corner cases
+select hex('') == bin('');
+select unhex('') == unbin('');
+select unhex('0') == unbin('0');

tests/queries/0_stateless/arcadia_skip_list.txt

@@ -253,3 +253,4 @@
 01923_different_expression_name_alias
 01932_null_valid_identifier
 00918_json_functions
+01889_sql_json_functions