Merge with master

cmake/autogenerated_versions.txt

@@ -0,0 +1,9 @@
+# This strings autochanged from release_lib.sh:
+SET(VERSION_REVISION 54435)
+SET(VERSION_MAJOR 20)
+SET(VERSION_MINOR 5)
+SET(VERSION_PATCH 1)
+SET(VERSION_GITHASH 91df18a906dcffdbee6816e5389df6c65f86e35f)
+SET(VERSION_DESCRIBE v20.5.1.1-prestable)
+SET(VERSION_STRING 20.5.1.1)
+# end of autochange

cmake/version.cmake

@@ -1,12 +1,4 @@
-# This strings autochanged from release_lib.sh:
+include(${CMAKE_SOURCE_DIR}/cmake/autogenerated_versions.txt)
-set(VERSION_REVISION 54435)
-set(VERSION_MAJOR 20)
-set(VERSION_MINOR 5)
-set(VERSION_PATCH 1)
-set(VERSION_GITHASH 91df18a906dcffdbee6816e5389df6c65f86e35f)
-set(VERSION_DESCRIBE v20.5.1.1-prestable)
-set(VERSION_STRING 20.5.1.1)
-# end of autochange
 
 set(VERSION_EXTRA "" CACHE STRING "")
 set(VERSION_TWEAK "" CACHE STRING "")

cmake/yandex/ya.make.versions.inc

@@ -0,0 +1,25 @@
+INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/autogenerated_versions.txt)
+
+# TODO: not sure if this is customizable per-binary
+SET(VERSION_NAME "ClickHouse")
+
+# TODO: not quite sure how to replace dash with space in ya.make
+SET(VERSION_FULL "${VERSION_NAME}-${VERSION_STRING}")
+
+CFLAGS (GLOBAL -DDBMS_NAME=\"ClickHouse\")
+CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=${VERSION_MAJOR})
+CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=${VERSION_MINOR})
+CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=${VERSION_PATCH})
+CFLAGS (GLOBAL -DVERSION_FULL=\"\\\"${VERSION_FULL}\\\"\")
+CFLAGS (GLOBAL -DVERSION_MAJOR=${VERSION_MAJOR})	
+CFLAGS (GLOBAL -DVERSION_MINOR=${VERSION_MINOR})	
+CFLAGS (GLOBAL -DVERSION_PATCH=${VERSION_PATCH})
+
+# TODO: not supported yet, not sure if ya.make supports arithmetics.
+CFLAGS (GLOBAL -DVERSION_INTEGER=0)
+
+CFLAGS (GLOBAL -DVERSION_NAME=\"\\\"${VERSION_NAME}\\\"\")
+CFLAGS (GLOBAL -DVERSION_OFFICIAL=\"-arcadia\")
+CFLAGS (GLOBAL -DVERSION_REVISION=${VERSION_REVISION})
+CFLAGS (GLOBAL -DVERSION_STRING=\"\\\"${VERSION_STRING}\\\"\")
+

docker/test/test_runner.sh

@@ -12,7 +12,7 @@ readonly CLICKHOUSE_PACKAGES_ARG="${2}"
 CLICKHOUSE_SERVER_IMAGE="${3}"
 
 if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
-	readonly CLICKHOUSE_PACKAGES_DIR="$(realpath ${2})" # or --no-rebuild
+    readonly CLICKHOUSE_PACKAGES_DIR="$(realpath ${2})" # or --no-rebuild
 fi
 
 
@@ -26,19 +26,19 @@ fi
 # TODO: optionally mount most recent clickhouse-test and queries directory from local machine
 
 if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
-	docker build \
+    docker build \
-		-f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \
+        -f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \
-		--target clickhouse-test-runner-base \
+        --target clickhouse-test-runner-base \
-		-t clickhouse-test-runner-base:preinstall \
+        -t clickhouse-test-runner-base:preinstall \
-		"${CLICKHOUSE_DOCKER_DIR}/test/stateless"
+        "${CLICKHOUSE_DOCKER_DIR}/test/stateless"
 
-	docker rm -f clickhouse-test-runner-installing-packages || true
+    docker rm -f clickhouse-test-runner-installing-packages || true
-	docker run \
+    docker run \
-		-v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
+        -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
-		--name clickhouse-test-runner-installing-packages \
+        --name clickhouse-test-runner-installing-packages \
-		clickhouse-test-runner-base:preinstall
+        clickhouse-test-runner-base:preinstall
-	docker commit clickhouse-test-runner-installing-packages clickhouse-statelest-test-runner:local
+    docker commit clickhouse-test-runner-installing-packages clickhouse-statelest-test-runner:local
-	docker rm -f clickhouse-test-runner-installing-packages || true
+    docker rm -f clickhouse-test-runner-installing-packages || true
 fi
 
 # # Create a bind-volume to the clickhouse-test script file
@@ -47,38 +47,38 @@ fi
 
 # Build server image (optional) from local packages
 if [ -z "${CLICKHOUSE_SERVER_IMAGE}" ]; then
-	CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local"
+    CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local"
 
-	if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
+    if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
-		docker build \
+        docker build \
-			-f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
+            -f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
-			--target clickhouse-server-base \
+            --target clickhouse-server-base \
-			-t clickhouse-server-base:preinstall \
+            -t clickhouse-server-base:preinstall \
-			"${CLICKHOUSE_DOCKER_DIR}/server"
+            "${CLICKHOUSE_DOCKER_DIR}/server"
 
-		docker rm -f clickhouse_server_base_installing_server || true
+        docker rm -f clickhouse_server_base_installing_server || true
-		docker run -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
+        docker run -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
-			--name clickhouse_server_base_installing_server \
+            --name clickhouse_server_base_installing_server \
-			clickhouse-server-base:preinstall
+            clickhouse-server-base:preinstall
-		docker commit clickhouse_server_base_installing_server clickhouse-server-base:postinstall
+        docker commit clickhouse_server_base_installing_server clickhouse-server-base:postinstall
 
-		docker build \
+        docker build \
-			-f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
+            -f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
-			--target clickhouse-server \
+            --target clickhouse-server \
-			-t "${CLICKHOUSE_SERVER_IMAGE}" \
+            -t "${CLICKHOUSE_SERVER_IMAGE}" \
-			"${CLICKHOUSE_DOCKER_DIR}/server"
+            "${CLICKHOUSE_DOCKER_DIR}/server"
-	fi
+    fi
 fi
 
 docker rm -f test-runner || true
 docker-compose down
 CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \
-	docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
+    docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
-	create \
+    create \
-	--build --force-recreate
+    --build --force-recreate
 
 CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \
-	docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
+    docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
-	run \
+    run \
-	--name test-runner \
+    --name test-runner \
-	test-runner
+    test-runner

docs/ru/whats-new/extended-roadmap.md

@@ -73,10 +73,10 @@ Upd. Включено для системных таблиц.
 Q1. Закоммичено, но есть технический долг, который исправляется сейчас.
 Готово. Нет, не готово - там всё ещё технический долг.
 
-### 1.9. Использование TTL для прореживания данных {#ispolzovanie-ttl-dlia-prorezhivaniia-dannykh}
+### 1.9. + Использование TTL для прореживания данных {#ispolzovanie-ttl-dlia-prorezhivaniia-dannykh}
 
 Будет делать Сорокин Николай, ВШЭ и Яндекс.
-Upd. Есть pull request.
+Upd. Есть pull request. Upd. Сделано.
 
 Сейчас пользователь может задать в таблице выражение, которое определяет, сколько времени хранятся данные. Обычно это выражение задаётся относительно значения столбца с датой - например: удалять данные через три месяца. https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/\#table_engine-mergetree-ttl
 
@@ -124,7 +124,7 @@ Q2.
 Upd. Олег будет делать только часть про HDFS.
 Upd. Реализация поверх S3 является рабочей на уровне PoC.
 
-### 1.13. Ускорение запросов с FINAL {#uskorenie-zaprosov-s-final}
+### 1.13. + Ускорение запросов с FINAL {#uskorenie-zaprosov-s-final}
 
 Требует 2.1. Делает [Николай Кочетов](https://github.com/KochetovNicolai). Нужно для Яндекс.Метрики. Q2.
 Upd: PR [#10463](https://github.com/ClickHouse/ClickHouse/pull/10463)
@@ -203,10 +203,11 @@ Upd. SharedContext вынесен из Context.
 
 Upd. В очереди. Иван Лежанкин.
 
-### 2.9. Логгировние в format-стиле {#loggirovnie-v-format-stile}
+### 2.9. + Логгировние в format-стиле {#loggirovnie-v-format-stile}
 
-Делает [Иван Лежанкин](https://github.com/abyss7). Низкий приоритет.
+[#6049](https://github.com/ClickHouse/ClickHouse/issues/6049#issuecomment-570836998)
-[\#6049](https://github.com/ClickHouse/ClickHouse/issues/6049#issuecomment-570836998)
+
+Сделано.
 
 ### 2.10. Запрашивать у таблиц не столбцы, а срезы {#zaprashivat-u-tablits-ne-stolbtsy-a-srezy}
 
@@ -282,24 +283,20 @@ Upd. Сейчас обсуждается, как сделать другую з
 
 ### 4.3. Ограничение числа одновременных скачиваний с реплик {#ogranichenie-chisla-odnovremennykh-skachivanii-s-replik}
 
-Дмитрий Григорьев, ВШЭ.
 Изначально делал Олег Алексеенков, но пока решение не готово, хотя там не так уж много доделывать.
 
 ### 4.4. Ограничение сетевой полосы при репликации {#ogranichenie-setevoi-polosy-pri-replikatsii}
 
-Дмитрий Григорьев, ВШЭ. Нужно для Метрики.
+Нужно для Метрики.
 
 ### 4.5. Возможность продолжить передачу куска данных при репликации после сбоя {#vozmozhnost-prodolzhit-peredachu-kuska-dannykh-pri-replikatsii-posle-sboia}
 
-Дмитрий Григорьев, ВШЭ.
-
 ### 4.6. p2p передача для GLOBAL подзапросов {#p2p-peredacha-dlia-global-podzaprosov}
 
 ### 4.7. Ленивая загрузка множеств для IN и JOIN с помощью k/v запросов {#lenivaia-zagruzka-mnozhestv-dlia-in-i-join-s-pomoshchiu-kv-zaprosov}
 
 ### 4.8. Разделить background pool для fetch и merge {#razdelit-background-pool-dlia-fetch-i-merge}
 
-Дмитрий Григорьев, ВШЭ.
 В очереди. Исправить проблему, что восстанавливающаяся реплика перестаёт мержить. Частично компенсируется 4.3.
 
 
@@ -329,6 +326,7 @@ Upd. Сделано. Эффективность работы под вопрос
 Метрика, БК, Маркет, Altinity уже используют более свежие версии чем LTS.
 Upd. Появилась вторая версия LTS - 20.3.
 
+
 ## 6. Инструментирование {#instrumentirovanie}
 
 ### 6.1. + Исправления сэмплирующего профайлера запросов {#ispravleniia-sempliruiushchego-profailera-zaprosov}
@@ -425,11 +423,11 @@ Upd. Рассмотрели все проверки подряд.
 
 UBSan включен в функциональных тестах, но не включен в интеграционных тестах. Требует 7.7.
 
-### 7.11. Включение \*San в unit тестах {#vkliuchenie-san-v-unit-testakh}
+### 7.11. + Включение \*San в unit тестах {#vkliuchenie-san-v-unit-testakh}
 
 У нас мало unit тестов по сравнению с функциональными тестами и их использование не обязательно. Но они всё-равно важны и нет причин не запускать их под всеми видами sanitizers.
 
-Илья Яцишин.
+Илья Яцишин. Сделано.
 
 ### 7.12. Показывать тестовое покрытие нового кода в PR {#pokazyvat-testovoe-pokrytie-novogo-koda-v-pr}
 
@@ -528,6 +526,8 @@ Upd. Есть сборки, [пример](https://clickhouse-builds.s3.yandex.n
 
 Дарья Петрова, УрФУ.
 
+Рабочий прототип: https://pulls-dashboard-demo.herokuapp.com/dashboard/ClickHouse/ClickHouse
+
 Над ClickHouse одновременно работает большое количество разработчиков, которые оформляют свои изменения в виде pull requests. Когда непомерженных pull requests много, то возникает сложность с организацией работы - непонятно, на какой pull request смотреть в первую очередь.
 
 Предлагается реализовать простое одностраничное веб-приложение, в котором отображается список pull requests со следующей информацией:
@@ -627,6 +627,7 @@ Upd. Готово (все директории кроме contrib).
 ### 7.32. Обфускация продакшен запросов {#obfuskatsiia-prodakshen-zaprosov}
 
 Роман Ильговский. Нужно для Яндекс.Метрики.
+Есть pull request, почти готово: https://github.com/ClickHouse/ClickHouse/pull/10973
 
 Имея SQL запрос, требуется вывести структуру таблиц, на которых этот запрос будет выполнен, и заполнить эти таблицы случайными данными, такими, что результат этого запроса зависит от выбора подмножества данных.
 
@@ -1397,11 +1398,11 @@ Constraints позволяют задать выражение, истиннос
 Василий Морозов, Арслан Гумеров, Альберт Кидрачев, ВШЭ.
 В прошлом году задачу начинал делать другой человек, но не добился достаточного прогресса.
 
-1.  Оптимизация top sort.
++ 1.  Оптимизация top sort.
 
 В ClickHouse используется неоптимальный вариант top sort. Суть его в том, что из каждого блока достаётся top N записей, а затем, все блоки мержатся. Но доставание top N записей у каждого следующего блока бессмысленно, если мы знаем, что из них в глобальный top N войдёт меньше. Конечно нужно реализовать вариацию на тему priority queue (heap) с быстрым пропуском целых блоков, если ни одна строка не попадёт в накопленный top.
 
-2.  Рекурсивный вариант сортировки по кортежам.
++ 2.  Рекурсивный вариант сортировки по кортежам.
 
 Для сортировки по кортежам используется обычная сортировка с компаратором, который в цикле по элементам кортежа делает виртуальные вызовы `IColumn::compareAt`. Это неоптимально - как из-за короткого цикла по неизвестному в compile-time количеству элементов, так и из-за виртуальных вызовов. Чтобы обойтись без виртуальных вызовов, есть метод `IColumn::getPermutation`. Он используется в случае сортировки по одному столбцу. Есть вариант, что в случае сортировки по кортежу, что-то похожее тоже можно применить… например, сделать метод `updatePermutation`, принимающий аргументы offset и limit, и допереставляющий перестановку в диапазоне значений, в которых предыдущий столбец имел равные значения.
 

src/Access/Authentication.cpp

@@ -7,8 +7,8 @@ namespace DB
 {
 namespace ErrorCodes
 {
-    extern const int LOGICAL_ERROR;
     extern const int BAD_ARGUMENTS;
+    extern const int NOT_IMPLEMENTED;
 }
 
 
@@ -36,8 +36,11 @@ Authentication::Digest Authentication::getPasswordDoubleSHA1() const
 
         case DOUBLE_SHA1_PASSWORD:
             return password_hash;
+
+        case MAX_TYPE:
+            break;
     }
-    throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
+    throw Exception("getPasswordDoubleSHA1(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED);
 }
 
 
@@ -71,8 +74,11 @@ bool Authentication::isCorrectPassword(const String & password_) const
 
             return encodeSHA1(first_sha1) == password_hash;
         }
+
+        case MAX_TYPE:
+            break;
     }
-    throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
+    throw Exception("Cannot check if the password is correct for authentication type " + toString(type), ErrorCodes::NOT_IMPLEMENTED);
 }
 
 }

src/Access/Authentication.h

@@ -5,6 +5,7 @@
 #include <Common/OpenSSLHelpers.h>
 #include <Poco/SHA1Engine.h>
 #include <boost/algorithm/hex.hpp>
+#include <boost/algorithm/string/case_conv.hpp>
 
 
 namespace DB
@@ -14,6 +15,7 @@ namespace ErrorCodes
     extern const int SUPPORT_IS_DISABLED;
     extern const int BAD_ARGUMENTS;
     extern const int LOGICAL_ERROR;
+    extern const int NOT_IMPLEMENTED;
 }
 
 
@@ -35,6 +37,15 @@ public:
         /// SHA1(SHA1(password)).
         /// This kind of hash is used by the `mysql_native_password` authentication plugin.
         DOUBLE_SHA1_PASSWORD,
+
+        MAX_TYPE,
+    };
+
+    struct TypeInfo
+    {
+        const char * const raw_name;
+        const String name; /// Lowercased with underscores, e.g. "sha256_password".
+        static const TypeInfo & get(Type type_);
     };
 
     using Digest = std::vector<uint8_t>;
@@ -85,6 +96,48 @@ private:
 };
 
 
+inline const Authentication::TypeInfo & Authentication::TypeInfo::get(Type type_)
+{
+    static constexpr auto make_info = [](const char * raw_name_)
+    {
+        String init_name = raw_name_;
+        boost::to_lower(init_name);
+        return TypeInfo{raw_name_, std::move(init_name)};
+    };
+
+    switch (type_)
+    {
+        case NO_PASSWORD:
+        {
+            static const auto info = make_info("NO_PASSWORD");
+            return info;
+        }
+        case PLAINTEXT_PASSWORD:
+        {
+            static const auto info = make_info("PLAINTEXT_PASSWORD");
+            return info;
+        }
+        case SHA256_PASSWORD:
+        {
+            static const auto info = make_info("SHA256_PASSWORD");
+            return info;
+        }
+        case DOUBLE_SHA1_PASSWORD:
+        {
+            static const auto info = make_info("DOUBLE_SHA1_PASSWORD");
+            return info;
+        }
+        case MAX_TYPE: break;
+    }
+    throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type_)), ErrorCodes::LOGICAL_ERROR);
+}
+
+inline String toString(Authentication::Type type_)
+{
+    return Authentication::TypeInfo::get(type_).raw_name;
+}
+
+
 inline Authentication::Digest Authentication::encodeSHA256(const std::string_view & text [[maybe_unused]])
 {
 #if USE_SSL
@@ -122,8 +175,10 @@ inline void Authentication::setPassword(const String & password_)
 
         case DOUBLE_SHA1_PASSWORD:
             return setPasswordHashBinary(encodeDoubleSHA1(password_));
+
+        case MAX_TYPE: break;
     }
-    throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
+    throw Exception("setPassword(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED);
 }
 
 
@@ -186,8 +241,10 @@ inline void Authentication::setPasswordHashBinary(const Digest & hash)
             password_hash = hash;
             return;
         }
+
+        case MAX_TYPE: break;
     }
-    throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
+    throw Exception("setPasswordHashBinary(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED);
 }
 
 }

src/Common/ya.make

@@ -18,20 +18,7 @@ PEERDIR(
     contrib/restricted/ryu
 )
 
-# TODO: stub for config_version.h
+INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/yandex/ya.make.versions.inc)
-CFLAGS (GLOBAL -DDBMS_NAME=\"ClickHouse\")
-CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=0)
-CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=0)
-CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=0)
-CFLAGS (GLOBAL -DVERSION_FULL=\"ClickHouse\")
-CFLAGS (GLOBAL -DVERSION_INTEGER=0)
-CFLAGS (GLOBAL -DVERSION_MAJOR=0)
-CFLAGS (GLOBAL -DVERSION_MINOR=0)
-CFLAGS (GLOBAL -DVERSION_PATCH=0)
-CFLAGS (GLOBAL -DVERSION_NAME=\"ClickHouse\")
-CFLAGS (GLOBAL -DVERSION_OFFICIAL=\"\\\(arcadia\\\)\")
-CFLAGS (GLOBAL -DVERSION_REVISION=0)
-CFLAGS (GLOBAL -DVERSION_STRING=\"Unknown\")
 
 SRCS(
     ActionLock.cpp

src/Formats/ProtobufReader.cpp

@@ -32,11 +32,12 @@ namespace
         BITS32 = 5,
     };
 
-    // The following condition must always be true:
+    // The following conditions must always be true:
-    // any_cursor_position < min(END_OF_VARINT, END_OF_GROUP)
+    // any_cursor_position > END_OF_VARINT
-    // This inequation helps to check conditions in SimpleReader.
+    // any_cursor_position > END_OF_GROUP
-    constexpr UInt64 END_OF_VARINT = static_cast<UInt64>(-1);
+    // Those inequations helps checking conditions in ProtobufReader::SimpleReader.
-    constexpr UInt64 END_OF_GROUP = static_cast<UInt64>(-2);
+    constexpr Int64 END_OF_VARINT = -1;
+    constexpr Int64 END_OF_GROUP = -2;
 
     Int64 decodeZigZag(UInt64 n) { return static_cast<Int64>((n >> 1) ^ (~(n & 1) + 1)); }
 
@@ -77,7 +78,7 @@ void ProtobufReader::SimpleReader::endMessage(bool ignore_errors)
     if (!current_message_level)
         return;
 
-    UInt64 root_message_end = (current_message_level == 1) ? current_message_end : parent_message_ends.front();
+    Int64 root_message_end = (current_message_level == 1) ? current_message_end : parent_message_ends.front();
     if (cursor != root_message_end)
     {
         if (cursor < root_message_end)
@@ -95,6 +96,9 @@ void ProtobufReader::SimpleReader::endMessage(bool ignore_errors)
 void ProtobufReader::SimpleReader::startNestedMessage()
 {
     assert(current_message_level >= 1);
+    if ((cursor > field_end) && (field_end != END_OF_GROUP))
+        throwUnknownFormat();
+
     // Start reading a nested message which is located inside a length-delimited field
     // of another message.
     parent_message_ends.emplace_back(current_message_end);
@@ -146,7 +150,7 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number)
             throwUnknownFormat();
     }
 
-    if (cursor >= current_message_end)
+    if ((cursor >= current_message_end) && (current_message_end != END_OF_GROUP))
         return false;
 
     UInt64 varint = readVarint();
@@ -196,11 +200,17 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number)
 
 bool ProtobufReader::SimpleReader::readUInt(UInt64 & value)
 {
+    if (field_end == END_OF_VARINT)
+    {
+        value = readVarint();
+        field_end = cursor;
+        return true;
+    }
+
     if (unlikely(cursor >= field_end))
         return false;
+
     value = readVarint();
-    if (field_end == END_OF_VARINT)
-        field_end = cursor;
     return true;
 }
 
@@ -227,6 +237,7 @@ bool ProtobufReader::SimpleReader::readFixed(T & value)
 {
     if (unlikely(cursor >= field_end))
         return false;
+
     readBinary(&value, sizeof(T));
     return true;
 }

src/Formats/ProtobufReader.h

@@ -124,12 +124,12 @@ private:
         void ignoreGroup();
 
         ReadBuffer & in;
-        UInt64 cursor;
+        Int64 cursor;
         size_t current_message_level;
-        UInt64 current_message_end;
+        Int64 current_message_end;
-        std::vector<UInt64> parent_message_ends;
+        std::vector<Int64> parent_message_ends;
-        UInt64 field_end;
+        Int64 field_end;
-        UInt64 last_string_pos;
+        Int64 last_string_pos;
     };
 
     class IConverter

src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp

@@ -56,10 +56,10 @@ namespace
                 query->default_roles = user.default_roles.toASTWithNames(*manager);
         }
 
-        if (attach_mode && (user.authentication.getType() != Authentication::NO_PASSWORD))
+        if (user.authentication.getType() != Authentication::NO_PASSWORD)
         {
-            /// We don't show password unless it's an ATTACH statement.
             query->authentication = user.authentication;
+            query->show_password = attach_mode; /// We don't show password unless it's an ATTACH statement.
         }
 
         if (!user.settings.empty())

src/Parsers/ASTCreateUserQuery.cpp

@@ -6,6 +6,12 @@
 
 namespace DB
 {
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
+
 namespace
 {
     void formatRenameTo(const String & new_name, const IAST::FormatSettings & settings)
@@ -15,27 +21,51 @@ namespace
     }
 
 
-    void formatAuthentication(const Authentication & authentication, const IAST::FormatSettings & settings)
+    void formatAuthentication(const Authentication & authentication, bool show_password, const IAST::FormatSettings & settings)
     {
-        settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH " << (settings.hilite ? IAST::hilite_none : "");
+        auto authentication_type = authentication.getType();
-        switch (authentication.getType())
+        if (authentication_type == Authentication::NO_PASSWORD)
         {
-            case Authentication::Type::NO_PASSWORD:
+            settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NOT IDENTIFIED"
-                settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "no_password" << (settings.hilite ? IAST::hilite_none : "");
+                          << (settings.hilite ? IAST::hilite_none : "");
-                break;
+            return;
-            case Authentication::Type::PLAINTEXT_PASSWORD:
-                settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "plaintext_password BY " << (settings.hilite ? IAST::hilite_none : "")
-                              << quoteString(authentication.getPassword());
-                break;
-            case Authentication::Type::SHA256_PASSWORD:
-                settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "sha256_hash BY " << (settings.hilite ? IAST::hilite_none : "")
-                              << quoteString(authentication.getPasswordHashHex());
-                break;
-            case Authentication::Type::DOUBLE_SHA1_PASSWORD:
-                settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "double_sha1_hash BY " << (settings.hilite ? IAST::hilite_none : "")
-                              << quoteString(authentication.getPasswordHashHex());
-                break;
         }
+
+        String authentication_type_name = Authentication::TypeInfo::get(authentication_type).name;
+        std::optional<String> password;
+
+        if (show_password)
+        {
+            switch (authentication_type)
+            {
+                case Authentication::PLAINTEXT_PASSWORD:
+                {
+                    password = authentication.getPassword();
+                    break;
+                }
+                case Authentication::SHA256_PASSWORD:
+                {
+                    authentication_type_name = "sha256_hash";
+                    password = authentication.getPasswordHashHex();
+                    break;
+                }
+                case Authentication::DOUBLE_SHA1_PASSWORD:
+                {
+                    authentication_type_name = "double_sha1_hash";
+                    password = authentication.getPasswordHashHex();
+                    break;
+                }
+
+                case Authentication::NO_PASSWORD: [[fallthrough]];
+                case Authentication::MAX_TYPE:
+                    throw Exception("AST: Unexpected authentication type " + toString(authentication_type), ErrorCodes::LOGICAL_ERROR);
+            }
+        }
+
+        settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH " << authentication_type_name
+                      << (settings.hilite ? IAST::hilite_none : "");
+        if (password)
+            settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " BY " << quoteString(*password);
     }
 
 
@@ -190,7 +220,7 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState &
         formatRenameTo(new_name, format);
 
     if (authentication)
-        formatAuthentication(*authentication, format);
+        formatAuthentication(*authentication, show_password, format);
 
     if (hosts)
         formatHosts(nullptr, *hosts, format);

src/Parsers/ASTCreateUserQuery.h

@@ -12,14 +12,14 @@ class ASTExtendedRoleSet;
 class ASTSettingsProfileElements;
 
 /** CREATE USER [IF NOT EXISTS | OR REPLACE] name
-  *     [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}]
+  *     [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}]
   *     [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
   *     [DEFAULT ROLE role [,...]]
   *     [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
   *
   * ALTER USER [IF EXISTS] name
   *      [RENAME TO new_name]
-  *      [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}]
+  *      [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}]
   *      [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
   *      [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
   *      [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
@@ -38,6 +38,7 @@ public:
     String new_name;
 
     std::optional<Authentication> authentication;
+    bool show_password = true; /// formatImpl() will show the password or hash.
 
     std::optional<AllowedClientHosts> hosts;
     std::optional<AllowedClientHosts> add_hosts;

src/Parsers/ParserCreateUserQuery.cpp

@@ -35,100 +35,74 @@ namespace
     }
 
 
-    bool parseByPassword(IParserBase::Pos & pos, Expected & expected, String & password)
-    {
-        return IParserBase::wrapParseImpl(pos, [&]
-        {
-            if (!ParserKeyword{"BY"}.ignore(pos, expected))
-                return false;
-
-            ASTPtr ast;
-            if (!ParserStringLiteral{}.parse(pos, ast, expected))
-                return false;
-
-            password = ast->as<const ASTLiteral &>().value.safeGet<String>();
-            return true;
-        });
-    }
-
-
     bool parseAuthentication(IParserBase::Pos & pos, Expected & expected, std::optional<Authentication> & authentication)
     {
         return IParserBase::wrapParseImpl(pos, [&]
         {
+            if (ParserKeyword{"NOT IDENTIFIED"}.ignore(pos, expected))
+            {
+                authentication = Authentication{Authentication::NO_PASSWORD};
+                return true;
+            }
+
             if (!ParserKeyword{"IDENTIFIED"}.ignore(pos, expected))
                 return false;
 
-            if (!ParserKeyword{"WITH"}.ignore(pos, expected))
+            std::optional<Authentication::Type> type;
+            bool expect_password = false;
+            bool expect_hash = false;
+
+            if (ParserKeyword{"WITH"}.ignore(pos, expected))
             {
-                String password;
+                for (auto check_type : ext::range(Authentication::MAX_TYPE))
-                if (!parseByPassword(pos, expected, password))
+                {
+                    if (ParserKeyword{Authentication::TypeInfo::get(check_type).raw_name}.ignore(pos, expected))
+                    {
+                        type = check_type;
+                        expect_password = (check_type != Authentication::NO_PASSWORD);
+                        break;
+                    }
+                }
+
+                if (!type)
+                {
+                    if (ParserKeyword{"SHA256_HASH"}.ignore(pos, expected))
+                    {
+                        type = Authentication::SHA256_PASSWORD;
+                        expect_hash = true;
+                    }
+                    else if (ParserKeyword{"DOUBLE_SHA1_HASH"}.ignore(pos, expected))
+                    {
+                        type = Authentication::DOUBLE_SHA1_PASSWORD;
+                        expect_hash = true;
+                    }
+                    else
+                        return false;
+                }
+            }
+
+            if (!type)
+            {
+                type = Authentication::SHA256_PASSWORD;
+                expect_password = true;
+            }
+
+            String password;
+            if (expect_password || expect_hash)
+            {
+                ASTPtr ast;
+                if (!ParserKeyword{"BY"}.ignore(pos, expected) || !ParserStringLiteral{}.parse(pos, ast, expected))
                     return false;
 
-                authentication = Authentication{Authentication::SHA256_PASSWORD};
+                password = ast->as<const ASTLiteral &>().value.safeGet<String>();
+            }
+
+            authentication = Authentication{*type};
+            if (expect_password)
                 authentication->setPassword(password);
-                return true;
+            else if (expect_hash)
-            }
+                authentication->setPasswordHashHex(password);
 
-            if (ParserKeyword{"PLAINTEXT_PASSWORD"}.ignore(pos, expected))
-            {
-                String password;
-                if (!parseByPassword(pos, expected, password))
-                    return false;
-
-                authentication = Authentication{Authentication::PLAINTEXT_PASSWORD};
-                authentication->setPassword(password);
-                return true;
-            }
-
-            if (ParserKeyword{"SHA256_PASSWORD"}.ignore(pos, expected))
-            {
-                String password;
-                if (!parseByPassword(pos, expected, password))
-                    return false;
-
-                authentication = Authentication{Authentication::SHA256_PASSWORD};
-                authentication->setPassword(password);
-                return true;
-            }
-
-            if (ParserKeyword{"SHA256_HASH"}.ignore(pos, expected))
-            {
-                String hash;
-                if (!parseByPassword(pos, expected, hash))
-                    return false;
-
-                authentication = Authentication{Authentication::SHA256_PASSWORD};
-                authentication->setPasswordHashHex(hash);
-                return true;
-            }
-
-            if (ParserKeyword{"DOUBLE_SHA1_PASSWORD"}.ignore(pos, expected))
-            {
-                String password;
-                if (!parseByPassword(pos, expected, password))
-                    return false;
-
-                authentication = Authentication{Authentication::DOUBLE_SHA1_PASSWORD};
-                authentication->setPassword(password);
-                return true;
-            }
-
-            if (ParserKeyword{"DOUBLE_SHA1_HASH"}.ignore(pos, expected))
-            {
-                String hash;
-                if (!parseByPassword(pos, expected, hash))
-                    return false;
-
-                authentication = Authentication{Authentication::DOUBLE_SHA1_PASSWORD};
-                authentication->setPasswordHashHex(hash);
-                return true;
-            }
-
-            if (!ParserKeyword{"NO_PASSWORD"}.ignore(pos, expected))
-                return false;
-
-            authentication = Authentication{Authentication::NO_PASSWORD};
             return true;
         });
     }

src/Parsers/ParserCreateUserQuery.h

@@ -7,13 +7,13 @@ namespace DB
 {
 /** Parses queries like
   * CREATE USER [IF NOT EXISTS | OR REPLACE] name
-  *     [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}]
+  *     [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}]
   *     [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
   *     [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
   *
   * ALTER USER [IF EXISTS] name
   *     [RENAME TO new_name]
-  *     [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}]
+  *     [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}]
   *     [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
   *     [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
   */

src/Storages/System/StorageSystemUsers.cpp

@@ -3,6 +3,7 @@
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeUUID.h>
 #include <DataTypes/DataTypeArray.h>
+#include <DataTypes/DataTypeEnum.h>
 #include <Columns/ColumnArray.h>
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnsNumber.h>
@@ -15,12 +16,26 @@
 
 namespace DB
 {
+namespace
+{
+    DataTypeEnum8::Values getAuthenticationTypeEnumValues()
+    {
+        DataTypeEnum8::Values enum_values;
+        for (auto type : ext::range(Authentication::MAX_TYPE))
+            enum_values.emplace_back(Authentication::TypeInfo::get(type).name, static_cast<Int8>(type));
+        return enum_values;
+    }
+}
+
+
 NamesAndTypesList StorageSystemUsers::getNamesAndTypes()
 {
     NamesAndTypesList names_and_types{
         {"name", std::make_shared<DataTypeString>()},
         {"id", std::make_shared<DataTypeUUID>()},
         {"storage", std::make_shared<DataTypeString>()},
+        {"auth_type", std::make_shared<DataTypeEnum8>(getAuthenticationTypeEnumValues())},
+        {"auth_params", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
         {"host_ip", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
         {"host_names", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
         {"host_names_regexp", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
@@ -43,6 +58,9 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context &
     auto & column_name = assert_cast<ColumnString &>(*res_columns[column_index++]);
     auto & column_id = assert_cast<ColumnUInt128 &>(*res_columns[column_index++]).getData();
     auto & column_storage = assert_cast<ColumnString &>(*res_columns[column_index++]);
+    auto & column_auth_type = assert_cast<ColumnInt8 &>(*res_columns[column_index++]).getData();
+    auto & column_auth_params = assert_cast<ColumnString &>(assert_cast<ColumnArray &>(*res_columns[column_index]).getData());
+    auto & column_auth_params_offsets = assert_cast<ColumnArray &>(*res_columns[column_index++]).getOffsets();
     auto & column_host_ip = assert_cast<ColumnString &>(assert_cast<ColumnArray &>(*res_columns[column_index]).getData());
     auto & column_host_ip_offsets = assert_cast<ColumnArray &>(*res_columns[column_index++]).getOffsets();
     auto & column_host_names = assert_cast<ColumnString &>(assert_cast<ColumnArray &>(*res_columns[column_index]).getData());
@@ -60,12 +78,15 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context &
     auto add_row = [&](const String & name,
                        const UUID & id,
                        const String & storage_name,
+                       const Authentication & authentication,
                        const AllowedClientHosts & allowed_hosts,
                        const ExtendedRoleSet & default_roles)
     {
         column_name.insertData(name.data(), name.length());
         column_id.push_back(id);
         column_storage.insertData(storage_name.data(), storage_name.length());
+        column_auth_type.push_back(static_cast<Int8>(authentication.getType()));
+        column_auth_params_offsets.push_back(column_auth_params.size());
 
         if (allowed_hosts.containsAnyHost())
         {
@@ -128,7 +149,7 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context &
         if (!storage)
             continue;
 
-        add_row(user->getName(), id, storage->getStorageName(), user->allowed_client_hosts, user->default_roles);
+        add_row(user->getName(), id, storage->getStorageName(), user->authentication, user->allowed_client_hosts, user->default_roles);
     }
 }
 

src/Storages/TTLDescription.cpp

@@ -180,7 +180,7 @@ TTLDescription TTLDescription::getTTLFromAST(
                 auto syntax_result = SyntaxAnalyzer(context).analyze(value, columns.getAllPhysical(), {}, true);
                 auto expr_analyzer = ExpressionAnalyzer(value, syntax_result, context);
 
-                result.set_parts.emplace_back(TTLSetPartDescription{
+                result.set_parts.emplace_back(TTLAggregateDescription{
                     name, value->getColumnName(), expr_analyzer.getActions(false)});
 
                 for (const auto & descr : expr_analyzer.getAnalyzedData().aggregate_descriptions)

src/Storages/TTLDescription.h

@@ -10,23 +10,24 @@
 namespace DB
 {
 
-struct TTLSetPartDescription
+/// Assignment expression in TTL with GROUP BY
+struct TTLAggregateDescription
 {
-    /// Name of column in set part of ttl expression
+    /// Name of column in assignment
     /// x = sum(y)
     /// ^
     String column_name;
 
-    /// Name of column on the right hand of the set part of TTL expression
+    /// Name of column on the right hand of the assignment
     /// x = sum(y)
     ///    ^~~~~~^
     String expression_result_column_name;
 
-    /// Expressions to calculate the value of set expression
+    /// Expressions to calculate the value of assignment expression
     ExpressionActionsPtr expression;
 };
 
-using TTLSetPartDescriptions = std::vector<TTLSetPartDescription>;
+using TTLAggregateDescriptions = std::vector<TTLAggregateDescription>;
 
 /// Common struct for TTL record in storage
 struct TTLDescription
@@ -58,7 +59,7 @@ struct TTLDescription
     Names group_by_keys;
 
     /// SET parts of TTL expression
-    TTLSetPartDescriptions set_parts;
+    TTLAggregateDescriptions set_parts;
 
     /// Aggregate descriptions for GROUP BY in TTL
     AggregateDescriptions aggregate_descriptions;

src/Storages/ya.make

@@ -174,7 +174,7 @@ SRCS(
     transformQueryForExternalDatabase.cpp
     VirtualColumnUtils.cpp
     extractKeyExpressionList.cpp
-    TTLDescriptions.cpp
+    TTLDescription.cpp
 )
 
 END()

tests/integration/test_disk_access_storage/test.py

@@ -39,7 +39,7 @@ def test_create():
 
     def check():
         assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n"
-        assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 HOST LOCAL DEFAULT ROLE rx\n"
+        assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE rx\n"
         assert instance.query("SHOW CREATE ROW POLICY p ON mydb.mytable") == "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a < 1000 TO u1, u2\n"
         assert instance.query("SHOW CREATE QUOTA q") == "CREATE QUOTA q KEYED BY \\'none\\' FOR INTERVAL 1 HOUR MAX QUERIES 100 TO ALL EXCEPT rx\n"
         assert instance.query("SHOW GRANTS FOR u1") == ""
@@ -69,7 +69,7 @@ def test_alter():
 
     def check():
         assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n"
-        assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 HOST LOCAL DEFAULT ROLE ry\n"
+        assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE ry\n"
         assert instance.query("SHOW GRANTS FOR u1") == "GRANT SELECT ON mydb.mytable TO u1\n"
         assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx, ry TO u2\n"
         assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s2\n"

tests/integration/test_grant_and_revoke/test.py

@@ -155,9 +155,9 @@ def test_introspection():
     assert instance.query("SHOW ENABLED ROLES", user='A') == TSV([[ "R1", 0, 1, 1 ]])
     assert instance.query("SHOW ENABLED ROLES", user='B') == TSV([[ "R2", 1, 1, 1 ]])
 
-    assert instance.query("SELECT name, storage, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\
+    assert instance.query("SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\
-           TSV([[ "A", "disk", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ],
+           TSV([[ "A", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ],
-                [ "B", "disk", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]])
+                [ "B", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]])
     
     assert instance.query("SELECT name, storage from system.roles WHERE name IN ('R1', 'R2') ORDER BY name") ==\
            TSV([[ "R1", "disk" ],

tests/queries/0_stateless/00825_protobuf_format_input.reference

@@ -8,3 +8,4 @@ a7522158-3d41-4b77-ad69-6c598ee55c49	Ivan	Petrov	male	1980-12-29	png	+7495123456
 0	0
 2	4
 3	9
+ok

tests/queries/0_stateless/00825_protobuf_format_input.sh

@@ -3,7 +3,7 @@
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . $CURDIR/../shell_config.sh
 
-set -e -o pipefail
+set -eo pipefail
 
 # Run the client.
 $CLICKHOUSE_CLIENT --multiquery <<'EOF'
@@ -48,5 +48,12 @@ source $CURDIR/00825_protobuf_format_input.insh
 $CLICKHOUSE_CLIENT --query "SELECT * FROM in_persons_00825 ORDER BY uuid;"
 $CLICKHOUSE_CLIENT --query "SELECT * FROM in_squares_00825 ORDER BY number;"
 
+# Try to input malformed data.
+set +eo pipefail
+echo -ne '\xe0\x80\x3f\x0b' \
+    | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'" 2>&1 \
+    | grep -qF "Protobuf messages are corrupted" && echo "ok" || echo "fail"
+set -eo pipefail
+
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_persons_00825;"
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_squares_00825;"

utils/ci/docker-multiarch/update.sh

@@ -37,27 +37,27 @@ fi
 
 # get the image
 if \
-	wget -q --spider "$baseUrl/current" \
+    wget -q --spider "$baseUrl/current" \
-	&& wget -q --spider "$baseUrl/current/$thisTar" \
+    && wget -q --spider "$baseUrl/current/$thisTar" \
-	; then
+    ; then
-		baseUrl+='/current'
+        baseUrl+='/current'
 fi
 wget -qN "$baseUrl/"{{MD5,SHA{1,256}}SUMS{,.gpg},"$thisTarBase.manifest",'unpacked/build-info.txt'} || true
 wget -N "$baseUrl/$thisTar"
 
 # check checksum
 if [ -f SHA256SUMS ]; then
-	sha256sum="$(sha256sum "$thisTar" | cut -d' ' -f1)"
+    sha256sum="$(sha256sum "$thisTar" | cut -d' ' -f1)"
-	if ! grep -q "$sha256sum" SHA256SUMS; then
+    if ! grep -q "$sha256sum" SHA256SUMS; then
-		echo >&2 "error: '$thisTar' has invalid SHA256"
+        echo >&2 "error: '$thisTar' has invalid SHA256"
-		exit 1
+        exit 1
-	fi
+    fi
 fi
 
 cat > Dockerfile <<-EOF
-	FROM scratch
+    FROM scratch
-	ADD $thisTar /
+    ADD $thisTar /
-	ENV ARCH=${ARCH} UBUNTU_SUITE=${VERSION} DOCKER_REPO=${DOCKER_REPO}
+    ENV ARCH=${ARCH} UBUNTU_SUITE=${VERSION} DOCKER_REPO=${DOCKER_REPO}
 EOF
 
 # add qemu-user-static binary
@@ -70,26 +70,26 @@ EOF
 fi
 
 cat >> Dockerfile <<-EOF
-	# a few minor docker-specific tweaks
+    # a few minor docker-specific tweaks
-	# see https://github.com/docker/docker/blob/master/contrib/mkimage/debootstrap
+    # see https://github.com/docker/docker/blob/master/contrib/mkimage/debootstrap
-	RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d \\
+    RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d \\
-		&& echo 'exit 101' >> /usr/sbin/policy-rc.d \\
+        && echo 'exit 101' >> /usr/sbin/policy-rc.d \\
-		&& chmod +x /usr/sbin/policy-rc.d \\
+        && chmod +x /usr/sbin/policy-rc.d \\
-		&& dpkg-divert --local --rename --add /sbin/initctl \\
+        && dpkg-divert --local --rename --add /sbin/initctl \\
-		&& cp -a /usr/sbin/policy-rc.d /sbin/initctl \\
+        && cp -a /usr/sbin/policy-rc.d /sbin/initctl \\
-		&& sed -i 's/^exit.*/exit 0/' /sbin/initctl \\
+        && sed -i 's/^exit.*/exit 0/' /sbin/initctl \\
-		&& echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \\
+        && echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \\
-		&& echo 'DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' > /etc/apt/apt.conf.d/docker-clean \\
+        && echo 'DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' > /etc/apt/apt.conf.d/docker-clean \\
-		&& echo 'APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' >> /etc/apt/apt.conf.d/docker-clean \\
+        && echo 'APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' >> /etc/apt/apt.conf.d/docker-clean \\
-		&& echo 'Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";' >> /etc/apt/apt.conf.d/docker-clean \\
+        && echo 'Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";' >> /etc/apt/apt.conf.d/docker-clean \\
-		&& echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/docker-no-languages \\
+        && echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/docker-no-languages \\
-		&& echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > /etc/apt/apt.conf.d/docker-gzip-indexes
+        && echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > /etc/apt/apt.conf.d/docker-gzip-indexes
 
-	# enable the universe
+    # enable the universe
-	RUN sed -i 's/^#\s*\(deb.*universe\)$/\1/g' /etc/apt/sources.list
+    RUN sed -i 's/^#\s*\(deb.*universe\)$/\1/g' /etc/apt/sources.list
 
-	# overwrite this with 'CMD []' in a dependent Dockerfile
+    # overwrite this with 'CMD []' in a dependent Dockerfile
-	CMD ["/bin/bash"]
+    CMD ["/bin/bash"]
 EOF
 
 docker build -t "${DOCKER_REPO}:${TAG_ARCH}-${VERSION}" .

utils/release/release_lib.sh

@@ -12,10 +12,10 @@ function gen_version_string {
 function get_version {
     if [ -z "$VERSION_MAJOR" ] && [ -z "$VERSION_MINOR" ] && [ -z "$VERSION_PATCH" ]; then
         BASEDIR=$(dirname "${BASH_SOURCE[0]}")/../../
-        VERSION_REVISION=`grep "set(VERSION_REVISION" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_REVISION \(.*\)$/\1/' | sed 's/[) ].*//'`
+        VERSION_REVISION=`grep "SET(VERSION_REVISION" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_REVISION \(.*\)$/\1/' | sed 's/[) ].*//'`
-        VERSION_MAJOR=`grep "set(VERSION_MAJOR" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_MAJOR \(.*\)/\1/' | sed 's/[) ].*//'`
+        VERSION_MAJOR=`grep "SET(VERSION_MAJOR" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_MAJOR \(.*\)/\1/' | sed 's/[) ].*//'`
-        VERSION_MINOR=`grep "set(VERSION_MINOR" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_MINOR \(.*\)/\1/' | sed 's/[) ].*//'`
+        VERSION_MINOR=`grep "SET(VERSION_MINOR" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_MINOR \(.*\)/\1/' | sed 's/[) ].*//'`
-        VERSION_PATCH=`grep "set(VERSION_PATCH" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_PATCH \(.*\)/\1/' | sed 's/[) ].*//'`
+        VERSION_PATCH=`grep "SET(VERSION_PATCH" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_PATCH \(.*\)/\1/' | sed 's/[) ].*//'`
     fi
     VERSION_PREFIX="${VERSION_PREFIX:-v}"
     VERSION_POSTFIX_TAG="${VERSION_POSTFIX:--testing}"
@@ -90,28 +90,28 @@ function gen_revision_author {
 
             git_describe=`git describe`
             git_hash=`git rev-parse HEAD`
-            sed -i -e "s/set(VERSION_REVISION [^) ]*/set(VERSION_REVISION $VERSION_REVISION/g;" \
+            sed -i -e "s/SET(VERSION_REVISION [^) ]*/SET(VERSION_REVISION $VERSION_REVISION/g;" \
-                -e "s/set(VERSION_DESCRIBE [^) ]*/set(VERSION_DESCRIBE $git_describe/g;" \
+                -e "s/SET(VERSION_DESCRIBE [^) ]*/SET(VERSION_DESCRIBE $git_describe/g;" \
-                -e "s/set(VERSION_GITHASH [^) ]*/set(VERSION_GITHASH $git_hash/g;" \
+                -e "s/SET(VERSION_GITHASH [^) ]*/SET(VERSION_GITHASH $git_hash/g;" \
-                -e "s/set(VERSION_MAJOR [^) ]*/set(VERSION_MAJOR $VERSION_MAJOR/g;" \
+                -e "s/SET(VERSION_MAJOR [^) ]*/SET(VERSION_MAJOR $VERSION_MAJOR/g;" \
-                -e "s/set(VERSION_MINOR [^) ]*/set(VERSION_MINOR $VERSION_MINOR/g;" \
+                -e "s/SET(VERSION_MINOR [^) ]*/SET(VERSION_MINOR $VERSION_MINOR/g;" \
-                -e "s/set(VERSION_PATCH [^) ]*/set(VERSION_PATCH $VERSION_PATCH/g;" \
+                -e "s/SET(VERSION_PATCH [^) ]*/SET(VERSION_PATCH $VERSION_PATCH/g;" \
-                -e "s/set(VERSION_STRING [^) ]*/set(VERSION_STRING $VERSION_STRING/g;" \
+                -e "s/SET(VERSION_STRING [^) ]*/SET(VERSION_STRING $VERSION_STRING/g;" \
-                cmake/version.cmake
+                cmake/autogenerated_versions.txt
 
             gen_changelog "$VERSION_STRING" "" "$AUTHOR" ""
             gen_dockerfiles "$VERSION_STRING"
             src/Storages/System/StorageSystemContributors.sh ||:
             utils/list-versions/list-versions.sh > utils/list-versions/version_date.tsv
 
-            git commit -m "$auto_message [$VERSION_STRING] [$VERSION_REVISION]" cmake/version.cmake debian/changelog docker/*/Dockerfile src/Storages/System/StorageSystemContributors.generated.cpp utils/list-versions/version_date.tsv
+            git commit -m "$auto_message [$VERSION_STRING] [$VERSION_REVISION]" cmake/autogenerated_versions.txt debian/changelog docker/*/Dockerfile src/Storages/System/StorageSystemContributors.generated.cpp utils/list-versions/version_date.tsv
             if [ -z $NO_PUSH ]; then
                 git push
             fi
 
             echo "Generated version: ${VERSION_STRING}, revision: ${VERSION_REVISION}."
 
-            # Second tag for correct version information in version.cmake inside tag
+            # Second tag for correct version information in autogenerated_versions.txt inside tag
             if git tag --force -a "$tag" -m "$tag"
             then
                 if [ -z $NO_PUSH ]; then

website/css/docs.css

@@ -69,7 +69,6 @@ summary {
 #content code {
     color: #111;
     background: #eee;
-    padding: 2px;
 }