thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-29 21:20:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge with master

Browse Source
This commit is contained in:
alesapin 2020-05-29 14:03:11 +03:00
commit 748e8fbfc3
26 changed files with 384 additions and 262 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
cmake/autogenerated_versions.txt Normal file
View File

@ -0,0 +1,9 @@
# This strings autochanged from release_lib.sh:
SET(VERSION_REVISION 54435)
SET(VERSION_MAJOR 20)
SET(VERSION_MINOR 5)
SET(VERSION_PATCH 1)
SET(VERSION_GITHASH 91df18a906dcffdbee6816e5389df6c65f86e35f)
SET(VERSION_DESCRIBE v20.5.1.1-prestable)
SET(VERSION_STRING 20.5.1.1)
# end of autochange

10
cmake/version.cmake
View File

@ -1,12 +1,4 @@
# This strings autochanged from release_lib.sh:
set(VERSION_REVISION 54435)
set(VERSION_MAJOR 20)
set(VERSION_MINOR 5)
set(VERSION_PATCH 1)
set(VERSION_GITHASH 91df18a906dcffdbee6816e5389df6c65f86e35f)
set(VERSION_DESCRIBE v20.5.1.1-prestable)
set(VERSION_STRING 20.5.1.1)
# end of autochange
include(${CMAKE_SOURCE_DIR}/cmake/autogenerated_versions.txt)
set(VERSION_EXTRA "" CACHE STRING "")
set(VERSION_TWEAK "" CACHE STRING "")

25
cmake/yandex/ya.make.versions.inc Normal file
View File

@ -0,0 +1,25 @@
INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/autogenerated_versions.txt)
# TODO: not sure if this is customizable per-binary
SET(VERSION_NAME "ClickHouse")
# TODO: not quite sure how to replace dash with space in ya.make
SET(VERSION_FULL "${VERSION_NAME}-${VERSION_STRING}")
CFLAGS (GLOBAL -DDBMS_NAME=\"ClickHouse\")
CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=${VERSION_MAJOR})
CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=${VERSION_MINOR})
CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=${VERSION_PATCH})
CFLAGS (GLOBAL -DVERSION_FULL=\"\\\"${VERSION_FULL}\\\"\")
CFLAGS (GLOBAL -DVERSION_MAJOR=${VERSION_MAJOR})
CFLAGS (GLOBAL -DVERSION_MINOR=${VERSION_MINOR})
CFLAGS (GLOBAL -DVERSION_PATCH=${VERSION_PATCH})
# TODO: not supported yet, not sure if ya.make supports arithmetics.
CFLAGS (GLOBAL -DVERSION_INTEGER=0)
CFLAGS (GLOBAL -DVERSION_NAME=\"\\\"${VERSION_NAME}\\\"\")
CFLAGS (GLOBAL -DVERSION_OFFICIAL=\"-arcadia\")
CFLAGS (GLOBAL -DVERSION_REVISION=${VERSION_REVISION})
CFLAGS (GLOBAL -DVERSION_STRING=\"\\\"${VERSION_STRING}\\\"\")

76
docker/test/test_runner.sh
View File

@ -12,7 +12,7 @@ readonly CLICKHOUSE_PACKAGES_ARG="${2}"
CLICKHOUSE_SERVER_IMAGE="${3}"
if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
readonly CLICKHOUSE_PACKAGES_DIR="$(realpath ${2})" # or --no-rebuild
readonly CLICKHOUSE_PACKAGES_DIR="$(realpath ${2})" # or --no-rebuild
fi
@ -26,19 +26,19 @@ fi
# TODO: optionally mount most recent clickhouse-test and queries directory from local machine
if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
docker build \
-f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \
--target clickhouse-test-runner-base \
-t clickhouse-test-runner-base:preinstall \
"${CLICKHOUSE_DOCKER_DIR}/test/stateless"
docker build \
-f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \
--target clickhouse-test-runner-base \
-t clickhouse-test-runner-base:preinstall \
"${CLICKHOUSE_DOCKER_DIR}/test/stateless"
docker rm -f clickhouse-test-runner-installing-packages || true
docker run \
-v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
--name clickhouse-test-runner-installing-packages \
clickhouse-test-runner-base:preinstall
docker commit clickhouse-test-runner-installing-packages clickhouse-statelest-test-runner:local
docker rm -f clickhouse-test-runner-installing-packages || true
docker rm -f clickhouse-test-runner-installing-packages || true
docker run \
-v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
--name clickhouse-test-runner-installing-packages \
clickhouse-test-runner-base:preinstall
docker commit clickhouse-test-runner-installing-packages clickhouse-statelest-test-runner:local
docker rm -f clickhouse-test-runner-installing-packages || true
fi
# # Create a bind-volume to the clickhouse-test script file
@ -47,38 +47,38 @@ fi
# Build server image (optional) from local packages
if [ -z "${CLICKHOUSE_SERVER_IMAGE}" ]; then
CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local"
CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local"
if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
docker build \
-f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
--target clickhouse-server-base \
-t clickhouse-server-base:preinstall \
"${CLICKHOUSE_DOCKER_DIR}/server"
if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then
docker build \
-f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
--target clickhouse-server-base \
-t clickhouse-server-base:preinstall \
"${CLICKHOUSE_DOCKER_DIR}/server"
docker rm -f clickhouse_server_base_installing_server || true
docker run -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
--name clickhouse_server_base_installing_server \
clickhouse-server-base:preinstall
docker commit clickhouse_server_base_installing_server clickhouse-server-base:postinstall
docker rm -f clickhouse_server_base_installing_server || true
docker run -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \
--name clickhouse_server_base_installing_server \
clickhouse-server-base:preinstall
docker commit clickhouse_server_base_installing_server clickhouse-server-base:postinstall
docker build \
-f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
--target clickhouse-server \
-t "${CLICKHOUSE_SERVER_IMAGE}" \
"${CLICKHOUSE_DOCKER_DIR}/server"
fi
docker build \
-f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \
--target clickhouse-server \
-t "${CLICKHOUSE_SERVER_IMAGE}" \
"${CLICKHOUSE_DOCKER_DIR}/server"
fi
fi
docker rm -f test-runner || true
docker-compose down
CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \
docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
create \
--build --force-recreate
docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
create \
--build --force-recreate
CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \
docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
run \
--name test-runner \
test-runner
docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \
run \
--name test-runner \
test-runner

31
docs/ru/whats-new/extended-roadmap.md
View File

@ -73,10 +73,10 @@ Upd. Включено для системных таблиц.
Q1. Закоммичено, но есть технический долг, который исправляется сейчас.
Готово. Нет, не готово - там всё ещё технический долг.
### 1.9. Использование TTL для прореживания данных {#ispolzovanie-ttl-dlia-prorezhivaniia-dannykh}
### 1.9. + Использование TTL для прореживания данных {#ispolzovanie-ttl-dlia-prorezhivaniia-dannykh}
Будет делать Сорокин Николай, ВШЭ и Яндекс.
Upd. Есть pull request.
Upd. Есть pull request. Upd. Сделано.
Сейчас пользователь может задать в таблице выражение, которое определяет, сколько времени хранятся данные. Обычно это выражение задаётся относительно значения столбца с датой - например: удалять данные через три месяца. https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/\#table_engine-mergetree-ttl
@ -124,7 +124,7 @@ Q2.
Upd. Олег будет делать только часть про HDFS.
Upd. Реализация поверх S3 является рабочей на уровне PoC.
### 1.13. Ускорение запросов с FINAL {#uskorenie-zaprosov-s-final}
### 1.13. + Ускорение запросов с FINAL {#uskorenie-zaprosov-s-final}
Требует 2.1. Делает [Николай Кочетов](https://github.com/KochetovNicolai). Нужно для Яндекс.Метрики. Q2.
Upd: PR [#10463](https://github.com/ClickHouse/ClickHouse/pull/10463)
@ -203,10 +203,11 @@ Upd. SharedContext вынесен из Context.
Upd. В очереди. Иван Лежанкин.
### 2.9. Логгировние в format-стиле {#loggirovnie-v-format-stile}
### 2.9. + Логгировние в format-стиле {#loggirovnie-v-format-stile}
Делает [Иван Лежанкин](https://github.com/abyss7). Низкий приоритет.
[\#6049](https://github.com/ClickHouse/ClickHouse/issues/6049#issuecomment-570836998)
[#6049](https://github.com/ClickHouse/ClickHouse/issues/6049#issuecomment-570836998)
Сделано.
### 2.10. Запрашивать у таблиц не столбцы, а срезы {#zaprashivat-u-tablits-ne-stolbtsy-a-srezy}
@ -282,24 +283,20 @@ Upd. Сейчас обсуждается, как сделать другую з
### 4.3. Ограничение числа одновременных скачиваний с реплик {#ogranichenie-chisla-odnovremennykh-skachivanii-s-replik}
Дмитрий Григорьев, ВШЭ.
Изначально делал Олег Алексеенков, но пока решение не готово, хотя там не так уж много доделывать.
### 4.4. Ограничение сетевой полосы при репликации {#ogranichenie-setevoi-polosy-pri-replikatsii}
Дмитрий Григорьев, ВШЭ. Нужно для Метрики.
Нужно для Метрики.
### 4.5. Возможность продолжить передачу куска данных при репликации после сбоя {#vozmozhnost-prodolzhit-peredachu-kuska-dannykh-pri-replikatsii-posle-sboia}
Дмитрий Григорьев, ВШЭ.
### 4.6. p2p передача для GLOBAL подзапросов {#p2p-peredacha-dlia-global-podzaprosov}
### 4.7. Ленивая загрузка множеств для IN и JOIN с помощью k/v запросов {#lenivaia-zagruzka-mnozhestv-dlia-in-i-join-s-pomoshchiu-kv-zaprosov}
### 4.8. Разделить background pool для fetch и merge {#razdelit-background-pool-dlia-fetch-i-merge}
Дмитрий Григорьев, ВШЭ.
В очереди. Исправить проблему, что восстанавливающаяся реплика перестаёт мержить. Частично компенсируется 4.3.
@ -329,6 +326,7 @@ Upd. Сделано. Эффективность работы под вопрос
Метрика, БК, Маркет, Altinity уже используют более свежие версии чем LTS.
Upd. Появилась вторая версия LTS - 20.3.
## 6. Инструментирование {#instrumentirovanie}
### 6.1. + Исправления сэмплирующего профайлера запросов {#ispravleniia-sempliruiushchego-profailera-zaprosov}
@ -425,11 +423,11 @@ Upd. Рассмотрели все проверки подряд.
UBSan включен в функциональных тестах, но не включен в интеграционных тестах. Требует 7.7.
### 7.11. Включение \*San в unit тестах {#vkliuchenie-san-v-unit-testakh}
### 7.11. + Включение \*San в unit тестах {#vkliuchenie-san-v-unit-testakh}
У нас мало unit тестов по сравнению с функциональными тестами и их использование не обязательно. Но они всё-равно важны и нет причин не запускать их под всеми видами sanitizers.
Илья Яцишин.
Илья Яцишин. Сделано.
### 7.12. Показывать тестовое покрытие нового кода в PR {#pokazyvat-testovoe-pokrytie-novogo-koda-v-pr}
@ -528,6 +526,8 @@ Upd. Есть сборки, [пример](https://clickhouse-builds.s3.yandex.n
Дарья Петрова, УрФУ.
Рабочий прототип: https://pulls-dashboard-demo.herokuapp.com/dashboard/ClickHouse/ClickHouse
Над ClickHouse одновременно работает большое количество разработчиков, которые оформляют свои изменения в виде pull requests. Когда непомерженных pull requests много, то возникает сложность с организацией работы - непонятно, на какой pull request смотреть в первую очередь.
Предлагается реализовать простое одностраничное веб-приложение, в котором отображается список pull requests со следующей информацией:
@ -627,6 +627,7 @@ Upd. Готово (все директории кроме contrib).
### 7.32. Обфускация продакшен запросов {#obfuskatsiia-prodakshen-zaprosov}
Роман Ильговский. Нужно для Яндекс.Метрики.
Есть pull request, почти готово: https://github.com/ClickHouse/ClickHouse/pull/10973
Имея SQL запрос, требуется вывести структуру таблиц, на которых этот запрос будет выполнен, и заполнить эти таблицы случайными данными, такими, что результат этого запроса зависит от выбора подмножества данных.
@ -1397,11 +1398,11 @@ Constraints позволяют задать выражение, истиннос
Василий Морозов, Арслан Гумеров, Альберт Кидрачев, ВШЭ.
В прошлом году задачу начинал делать другой человек, но не добился достаточного прогресса.
1. Оптимизация top sort.
+ 1. Оптимизация top sort.
В ClickHouse используется неоптимальный вариант top sort. Суть его в том, что из каждого блока достаётся top N записей, а затем, все блоки мержатся. Но доставание top N записей у каждого следующего блока бессмысленно, если мы знаем, что из них в глобальный top N войдёт меньше. Конечно нужно реализовать вариацию на тему priority queue (heap) с быстрым пропуском целых блоков, если ни одна строка не попадёт в накопленный top.
2. Рекурсивный вариант сортировки по кортежам.
+ 2. Рекурсивный вариант сортировки по кортежам.
Для сортировки по кортежам используется обычная сортировка с компаратором, который в цикле по элементам кортежа делает виртуальные вызовы `IColumn::compareAt`. Это неоптимально - как из-за короткого цикла по неизвестному в compile-time количеству элементов, так и из-за виртуальных вызовов. Чтобы обойтись без виртуальных вызовов, есть метод `IColumn::getPermutation`. Он используется в случае сортировки по одному столбцу. Есть вариант, что в случае сортировки по кортежу, что-то похожее тоже можно применить… например, сделать метод `updatePermutation`, принимающий аргументы offset и limit, и допереставляющий перестановку в диапазоне значений, в которых предыдущий столбец имел равные значения.

12
src/Access/Authentication.cpp
View File

@ -7,8 +7,8 @@ namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int BAD_ARGUMENTS;
extern const int NOT_IMPLEMENTED;
}
@ -36,8 +36,11 @@ Authentication::Digest Authentication::getPasswordDoubleSHA1() const
case DOUBLE_SHA1_PASSWORD:
return password_hash;
case MAX_TYPE:
break;
}
throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
throw Exception("getPasswordDoubleSHA1(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED);
}
@ -71,8 +74,11 @@ bool Authentication::isCorrectPassword(const String & password_) const
return encodeSHA1(first_sha1) == password_hash;
}
case MAX_TYPE:
break;
}
throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
throw Exception("Cannot check if the password is correct for authentication type " + toString(type), ErrorCodes::NOT_IMPLEMENTED);
}
}

61
src/Access/Authentication.h
View File

@ -5,6 +5,7 @@
#include <Common/OpenSSLHelpers.h>
#include <Poco/SHA1Engine.h>
#include <boost/algorithm/hex.hpp>
#include <boost/algorithm/string/case_conv.hpp>
namespace DB
@ -14,6 +15,7 @@ namespace ErrorCodes
extern const int SUPPORT_IS_DISABLED;
extern const int BAD_ARGUMENTS;
extern const int LOGICAL_ERROR;
extern const int NOT_IMPLEMENTED;
}
@ -35,6 +37,15 @@ public:
/// SHA1(SHA1(password)).
/// This kind of hash is used by the `mysql_native_password` authentication plugin.
DOUBLE_SHA1_PASSWORD,
MAX_TYPE,
};
struct TypeInfo
{
const char * const raw_name;
const String name; /// Lowercased with underscores, e.g. "sha256_password".
static const TypeInfo & get(Type type_);
};
using Digest = std::vector<uint8_t>;
@ -85,6 +96,48 @@ private:
};
inline const Authentication::TypeInfo & Authentication::TypeInfo::get(Type type_)
{
static constexpr auto make_info = [](const char * raw_name_)
{
String init_name = raw_name_;
boost::to_lower(init_name);
return TypeInfo{raw_name_, std::move(init_name)};
};
switch (type_)
{
case NO_PASSWORD:
{
static const auto info = make_info("NO_PASSWORD");
return info;
}
case PLAINTEXT_PASSWORD:
{
static const auto info = make_info("PLAINTEXT_PASSWORD");
return info;
}
case SHA256_PASSWORD:
{
static const auto info = make_info("SHA256_PASSWORD");
return info;
}
case DOUBLE_SHA1_PASSWORD:
{
static const auto info = make_info("DOUBLE_SHA1_PASSWORD");
return info;
}
case MAX_TYPE: break;
}
throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type_)), ErrorCodes::LOGICAL_ERROR);
}
inline String toString(Authentication::Type type_)
{
return Authentication::TypeInfo::get(type_).raw_name;
}
inline Authentication::Digest Authentication::encodeSHA256(const std::string_view & text [[maybe_unused]])
{
#if USE_SSL
@ -122,8 +175,10 @@ inline void Authentication::setPassword(const String & password_)
case DOUBLE_SHA1_PASSWORD:
return setPasswordHashBinary(encodeDoubleSHA1(password_));
case MAX_TYPE: break;
}
throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
throw Exception("setPassword(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED);
}
@ -186,8 +241,10 @@ inline void Authentication::setPasswordHashBinary(const Digest & hash)
password_hash = hash;
return;
}
case MAX_TYPE: break;
}
throw Exception("Unknown authentication type: " + std::to_string(static_cast<int>(type)), ErrorCodes::LOGICAL_ERROR);
throw Exception("setPasswordHashBinary(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED);
}
}

15
src/Common/ya.make
View File

@ -18,20 +18,7 @@ PEERDIR(
contrib/restricted/ryu
)
# TODO: stub for config_version.h
CFLAGS (GLOBAL -DDBMS_NAME=\"ClickHouse\")
CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=0)
CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=0)
CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=0)
CFLAGS (GLOBAL -DVERSION_FULL=\"ClickHouse\")
CFLAGS (GLOBAL -DVERSION_INTEGER=0)
CFLAGS (GLOBAL -DVERSION_MAJOR=0)
CFLAGS (GLOBAL -DVERSION_MINOR=0)
CFLAGS (GLOBAL -DVERSION_PATCH=0)
CFLAGS (GLOBAL -DVERSION_NAME=\"ClickHouse\")
CFLAGS (GLOBAL -DVERSION_OFFICIAL=\"\\\(arcadia\\\)\")
CFLAGS (GLOBAL -DVERSION_REVISION=0)
CFLAGS (GLOBAL -DVERSION_STRING=\"Unknown\")
INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/yandex/ya.make.versions.inc)
SRCS(
ActionLock.cpp

29
src/Formats/ProtobufReader.cpp
View File

@ -32,11 +32,12 @@ namespace
BITS32 = 5,
};
// The following condition must always be true:
// any_cursor_position < min(END_OF_VARINT, END_OF_GROUP)
// This inequation helps to check conditions in SimpleReader.
constexpr UInt64 END_OF_VARINT = static_cast<UInt64>(-1);
constexpr UInt64 END_OF_GROUP = static_cast<UInt64>(-2);
// The following conditions must always be true:
// any_cursor_position > END_OF_VARINT
// any_cursor_position > END_OF_GROUP
// Those inequations helps checking conditions in ProtobufReader::SimpleReader.
constexpr Int64 END_OF_VARINT = -1;
constexpr Int64 END_OF_GROUP = -2;
Int64 decodeZigZag(UInt64 n) { return static_cast<Int64>((n >> 1) ^ (~(n & 1) + 1)); }
@ -77,7 +78,7 @@ void ProtobufReader::SimpleReader::endMessage(bool ignore_errors)
if (!current_message_level)
return;
UInt64 root_message_end = (current_message_level == 1) ? current_message_end : parent_message_ends.front();
Int64 root_message_end = (current_message_level == 1) ? current_message_end : parent_message_ends.front();
if (cursor != root_message_end)
{
if (cursor < root_message_end)
@ -95,6 +96,9 @@ void ProtobufReader::SimpleReader::endMessage(bool ignore_errors)
void ProtobufReader::SimpleReader::startNestedMessage()
{
assert(current_message_level >= 1);
if ((cursor > field_end) && (field_end != END_OF_GROUP))
throwUnknownFormat();
// Start reading a nested message which is located inside a length-delimited field
// of another message.
parent_message_ends.emplace_back(current_message_end);
@ -146,7 +150,7 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number)
throwUnknownFormat();
}
if (cursor >= current_message_end)
if ((cursor >= current_message_end) && (current_message_end != END_OF_GROUP))
return false;
UInt64 varint = readVarint();
@ -196,11 +200,17 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number)
bool ProtobufReader::SimpleReader::readUInt(UInt64 & value)
{
if (field_end == END_OF_VARINT)
{
value = readVarint();
field_end = cursor;
return true;
}
if (unlikely(cursor >= field_end))
return false;
value = readVarint();
if (field_end == END_OF_VARINT)
field_end = cursor;
return true;
}
@ -227,6 +237,7 @@ bool ProtobufReader::SimpleReader::readFixed(T & value)
{
if (unlikely(cursor >= field_end))
return false;
readBinary(&value, sizeof(T));
return true;
}

10
src/Formats/ProtobufReader.h
View File

@ -124,12 +124,12 @@ private:
void ignoreGroup();
ReadBuffer & in;
UInt64 cursor;
Int64 cursor;
size_t current_message_level;
UInt64 current_message_end;
std::vector<UInt64> parent_message_ends;
UInt64 field_end;
UInt64 last_string_pos;
Int64 current_message_end;
std::vector<Int64> parent_message_ends;
Int64 field_end;
Int64 last_string_pos;
};
class IConverter

4
src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp
View File

@ -56,10 +56,10 @@ namespace
query->default_roles = user.default_roles.toASTWithNames(*manager);
}
if (attach_mode && (user.authentication.getType() != Authentication::NO_PASSWORD))
if (user.authentication.getType() != Authentication::NO_PASSWORD)
{
/// We don't show password unless it's an ATTACH statement.
query->authentication = user.authentication;
query->show_password = attach_mode; /// We don't show password unless it's an ATTACH statement.
}
if (!user.settings.empty())

68
src/Parsers/ASTCreateUserQuery.cpp
View File

@ -6,6 +6,12 @@
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
namespace
{
void formatRenameTo(const String & new_name, const IAST::FormatSettings & settings)
@ -15,27 +21,51 @@ namespace
}
void formatAuthentication(const Authentication & authentication, const IAST::FormatSettings & settings)
void formatAuthentication(const Authentication & authentication, bool show_password, const IAST::FormatSettings & settings)
{
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH " << (settings.hilite ? IAST::hilite_none : "");
switch (authentication.getType())
auto authentication_type = authentication.getType();
if (authentication_type == Authentication::NO_PASSWORD)
{
case Authentication::Type::NO_PASSWORD:
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "no_password" << (settings.hilite ? IAST::hilite_none : "");
break;
case Authentication::Type::PLAINTEXT_PASSWORD:
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "plaintext_password BY " << (settings.hilite ? IAST::hilite_none : "")
<< quoteString(authentication.getPassword());
break;
case Authentication::Type::SHA256_PASSWORD:
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "sha256_hash BY " << (settings.hilite ? IAST::hilite_none : "")
<< quoteString(authentication.getPasswordHashHex());
break;
case Authentication::Type::DOUBLE_SHA1_PASSWORD:
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "double_sha1_hash BY " << (settings.hilite ? IAST::hilite_none : "")
<< quoteString(authentication.getPasswordHashHex());
break;
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NOT IDENTIFIED"
<< (settings.hilite ? IAST::hilite_none : "");
return;
}
String authentication_type_name = Authentication::TypeInfo::get(authentication_type).name;
std::optional<String> password;
if (show_password)
{
switch (authentication_type)
{
case Authentication::PLAINTEXT_PASSWORD:
{
password = authentication.getPassword();
break;
}
case Authentication::SHA256_PASSWORD:
{
authentication_type_name = "sha256_hash";
password = authentication.getPasswordHashHex();
break;
}
case Authentication::DOUBLE_SHA1_PASSWORD:
{
authentication_type_name = "double_sha1_hash";
password = authentication.getPasswordHashHex();
break;
}
case Authentication::NO_PASSWORD: [[fallthrough]];
case Authentication::MAX_TYPE:
throw Exception("AST: Unexpected authentication type " + toString(authentication_type), ErrorCodes::LOGICAL_ERROR);
}
}
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH " << authentication_type_name
<< (settings.hilite ? IAST::hilite_none : "");
if (password)
settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " BY " << quoteString(*password);
}
@ -190,7 +220,7 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState &
formatRenameTo(new_name, format);
if (authentication)
formatAuthentication(*authentication, format);
formatAuthentication(*authentication, show_password, format);
if (hosts)
formatHosts(nullptr, *hosts, format);

5
src/Parsers/ASTCreateUserQuery.h
View File

@ -12,14 +12,14 @@ class ASTExtendedRoleSet;
class ASTSettingsProfileElements;
/** CREATE USER [IF NOT EXISTS | OR REPLACE] name
* [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}]
* [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}]
* [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
* [DEFAULT ROLE role [,...]]
* [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
*
* ALTER USER [IF EXISTS] name
* [RENAME TO new_name]
* [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}]
* [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}]
* [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
* [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
* [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
@ -38,6 +38,7 @@ public:
String new_name;
std::optional<Authentication> authentication;
bool show_password = true; /// formatImpl() will show the password or hash.
std::optional<AllowedClientHosts> hosts;
std::optional<AllowedClientHosts> add_hosts;

138
src/Parsers/ParserCreateUserQuery.cpp
View File

@ -35,100 +35,74 @@ namespace
}
bool parseByPassword(IParserBase::Pos & pos, Expected & expected, String & password)
{
return IParserBase::wrapParseImpl(pos, [&]
{
if (!ParserKeyword{"BY"}.ignore(pos, expected))
return false;
ASTPtr ast;
if (!ParserStringLiteral{}.parse(pos, ast, expected))
return false;
password = ast->as<const ASTLiteral &>().value.safeGet<String>();
return true;
});
}
bool parseAuthentication(IParserBase::Pos & pos, Expected & expected, std::optional<Authentication> & authentication)
{
return IParserBase::wrapParseImpl(pos, [&]
{
if (ParserKeyword{"NOT IDENTIFIED"}.ignore(pos, expected))
{
authentication = Authentication{Authentication::NO_PASSWORD};
return true;
}
if (!ParserKeyword{"IDENTIFIED"}.ignore(pos, expected))
return false;
if (!ParserKeyword{"WITH"}.ignore(pos, expected))
std::optional<Authentication::Type> type;
bool expect_password = false;
bool expect_hash = false;
if (ParserKeyword{"WITH"}.ignore(pos, expected))
{
String password;
if (!parseByPassword(pos, expected, password))
for (auto check_type : ext::range(Authentication::MAX_TYPE))
{
if (ParserKeyword{Authentication::TypeInfo::get(check_type).raw_name}.ignore(pos, expected))
{
type = check_type;
expect_password = (check_type != Authentication::NO_PASSWORD);
break;
}
}
if (!type)
{
if (ParserKeyword{"SHA256_HASH"}.ignore(pos, expected))
{
type = Authentication::SHA256_PASSWORD;
expect_hash = true;
}
else if (ParserKeyword{"DOUBLE_SHA1_HASH"}.ignore(pos, expected))
{
type = Authentication::DOUBLE_SHA1_PASSWORD;
expect_hash = true;
}
else
return false;
}
}
if (!type)
{
type = Authentication::SHA256_PASSWORD;
expect_password = true;
}
String password;
if (expect_password || expect_hash)
{
ASTPtr ast;
if (!ParserKeyword{"BY"}.ignore(pos, expected) || !ParserStringLiteral{}.parse(pos, ast, expected))
return false;
authentication = Authentication{Authentication::SHA256_PASSWORD};
password = ast->as<const ASTLiteral &>().value.safeGet<String>();
}
authentication = Authentication{*type};
if (expect_password)
authentication->setPassword(password);
return true;
}
else if (expect_hash)
authentication->setPasswordHashHex(password);
if (ParserKeyword{"PLAINTEXT_PASSWORD"}.ignore(pos, expected))
{
String password;
if (!parseByPassword(pos, expected, password))
return false;
authentication = Authentication{Authentication::PLAINTEXT_PASSWORD};
authentication->setPassword(password);
return true;
}
if (ParserKeyword{"SHA256_PASSWORD"}.ignore(pos, expected))
{
String password;
if (!parseByPassword(pos, expected, password))
return false;
authentication = Authentication{Authentication::SHA256_PASSWORD};
authentication->setPassword(password);
return true;
}
if (ParserKeyword{"SHA256_HASH"}.ignore(pos, expected))
{
String hash;
if (!parseByPassword(pos, expected, hash))
return false;
authentication = Authentication{Authentication::SHA256_PASSWORD};
authentication->setPasswordHashHex(hash);
return true;
}
if (ParserKeyword{"DOUBLE_SHA1_PASSWORD"}.ignore(pos, expected))
{
String password;
if (!parseByPassword(pos, expected, password))
return false;
authentication = Authentication{Authentication::DOUBLE_SHA1_PASSWORD};
authentication->setPassword(password);
return true;
}
if (ParserKeyword{"DOUBLE_SHA1_HASH"}.ignore(pos, expected))
{
String hash;
if (!parseByPassword(pos, expected, hash))
return false;
authentication = Authentication{Authentication::DOUBLE_SHA1_PASSWORD};
authentication->setPasswordHashHex(hash);
return true;
}
if (!ParserKeyword{"NO_PASSWORD"}.ignore(pos, expected))
return false;
authentication = Authentication{Authentication::NO_PASSWORD};
return true;
});
}

4
src/Parsers/ParserCreateUserQuery.h
View File

@ -7,13 +7,13 @@ namespace DB
{
/** Parses queries like
* CREATE USER [IF NOT EXISTS | OR REPLACE] name
* [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}]
* [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}]
* [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
* [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
*
* ALTER USER [IF EXISTS] name
* [RENAME TO new_name]
* [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}]
* [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}]
* [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
* [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
*/

23
src/Storages/System/StorageSystemUsers.cpp
View File

@ -3,6 +3,7 @@
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeUUID.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeEnum.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnsNumber.h>
@ -15,12 +16,26 @@
namespace DB
{
namespace
{
DataTypeEnum8::Values getAuthenticationTypeEnumValues()
{
DataTypeEnum8::Values enum_values;
for (auto type : ext::range(Authentication::MAX_TYPE))
enum_values.emplace_back(Authentication::TypeInfo::get(type).name, static_cast<Int8>(type));
return enum_values;
}
}
NamesAndTypesList StorageSystemUsers::getNamesAndTypes()
{
NamesAndTypesList names_and_types{
{"name", std::make_shared<DataTypeString>()},
{"id", std::make_shared<DataTypeUUID>()},
{"storage", std::make_shared<DataTypeString>()},
{"auth_type", std::make_shared<DataTypeEnum8>(getAuthenticationTypeEnumValues())},
{"auth_params", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
{"host_ip", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
{"host_names", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
{"host_names_regexp", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
@ -43,6 +58,9 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context &
auto & column_name = assert_cast<ColumnString &>(*res_columns[column_index++]);
auto & column_id = assert_cast<ColumnUInt128 &>(*res_columns[column_index++]).getData();
auto & column_storage = assert_cast<ColumnString &>(*res_columns[column_index++]);
auto & column_auth_type = assert_cast<ColumnInt8 &>(*res_columns[column_index++]).getData();
auto & column_auth_params = assert_cast<ColumnString &>(assert_cast<ColumnArray &>(*res_columns[column_index]).getData());
auto & column_auth_params_offsets = assert_cast<ColumnArray &>(*res_columns[column_index++]).getOffsets();
auto & column_host_ip = assert_cast<ColumnString &>(assert_cast<ColumnArray &>(*res_columns[column_index]).getData());
auto & column_host_ip_offsets = assert_cast<ColumnArray &>(*res_columns[column_index++]).getOffsets();
auto & column_host_names = assert_cast<ColumnString &>(assert_cast<ColumnArray &>(*res_columns[column_index]).getData());
@ -60,12 +78,15 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context &
auto add_row = [&](const String & name,
const UUID & id,
const String & storage_name,
const Authentication & authentication,
const AllowedClientHosts & allowed_hosts,
const ExtendedRoleSet & default_roles)
{
column_name.insertData(name.data(), name.length());
column_id.push_back(id);
column_storage.insertData(storage_name.data(), storage_name.length());
column_auth_type.push_back(static_cast<Int8>(authentication.getType()));
column_auth_params_offsets.push_back(column_auth_params.size());
if (allowed_hosts.containsAnyHost())
{
@ -128,7 +149,7 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context &
if (!storage)
continue;
add_row(user->getName(), id, storage->getStorageName(), user->allowed_client_hosts, user->default_roles);
add_row(user->getName(), id, storage->getStorageName(), user->authentication, user->allowed_client_hosts, user->default_roles);
}
}

2
src/Storages/TTLDescription.cpp
View File

@ -180,7 +180,7 @@ TTLDescription TTLDescription::getTTLFromAST(
auto syntax_result = SyntaxAnalyzer(context).analyze(value, columns.getAllPhysical(), {}, true);
auto expr_analyzer = ExpressionAnalyzer(value, syntax_result, context);
result.set_parts.emplace_back(TTLSetPartDescription{
result.set_parts.emplace_back(TTLAggregateDescription{
name, value->getColumnName(), expr_analyzer.getActions(false)});
for (const auto & descr : expr_analyzer.getAnalyzedData().aggregate_descriptions)

13
src/Storages/TTLDescription.h
View File

@ -10,23 +10,24 @@
namespace DB
{
struct TTLSetPartDescription
/// Assignment expression in TTL with GROUP BY
struct TTLAggregateDescription
{
/// Name of column in set part of ttl expression
/// Name of column in assignment
/// x = sum(y)
/// ^
String column_name;
/// Name of column on the right hand of the set part of TTL expression
/// Name of column on the right hand of the assignment
/// x = sum(y)
/// ^~~~~~^
String expression_result_column_name;
/// Expressions to calculate the value of set expression
/// Expressions to calculate the value of assignment expression
ExpressionActionsPtr expression;
};
using TTLSetPartDescriptions = std::vector<TTLSetPartDescription>;
using TTLAggregateDescriptions = std::vector<TTLAggregateDescription>;
/// Common struct for TTL record in storage
struct TTLDescription
@ -58,7 +59,7 @@ struct TTLDescription
Names group_by_keys;
/// SET parts of TTL expression
TTLSetPartDescriptions set_parts;
TTLAggregateDescriptions set_parts;
/// Aggregate descriptions for GROUP BY in TTL
AggregateDescriptions aggregate_descriptions;

2
src/Storages/ya.make
View File

@ -174,7 +174,7 @@ SRCS(
transformQueryForExternalDatabase.cpp
VirtualColumnUtils.cpp
extractKeyExpressionList.cpp
TTLDescriptions.cpp
TTLDescription.cpp
)
END()

4
tests/integration/test_disk_access_storage/test.py
View File

@ -39,7 +39,7 @@ def test_create():
def check():
assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n"
assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 HOST LOCAL DEFAULT ROLE rx\n"
assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE rx\n"
assert instance.query("SHOW CREATE ROW POLICY p ON mydb.mytable") == "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a < 1000 TO u1, u2\n"
assert instance.query("SHOW CREATE QUOTA q") == "CREATE QUOTA q KEYED BY \\'none\\' FOR INTERVAL 1 HOUR MAX QUERIES 100 TO ALL EXCEPT rx\n"
assert instance.query("SHOW GRANTS FOR u1") == ""
@ -69,7 +69,7 @@ def test_alter():
def check():
assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n"
assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 HOST LOCAL DEFAULT ROLE ry\n"
assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE ry\n"
assert instance.query("SHOW GRANTS FOR u1") == "GRANT SELECT ON mydb.mytable TO u1\n"
assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx, ry TO u2\n"
assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s2\n"

6
tests/integration/test_grant_and_revoke/test.py
View File

@ -155,9 +155,9 @@ def test_introspection():
assert instance.query("SHOW ENABLED ROLES", user='A') == TSV([[ "R1", 0, 1, 1 ]])
assert instance.query("SHOW ENABLED ROLES", user='B') == TSV([[ "R2", 1, 1, 1 ]])
assert instance.query("SELECT name, storage, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\
TSV([[ "A", "disk", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ],
[ "B", "disk", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]])
assert instance.query("SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\
TSV([[ "A", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ],
[ "B", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]])
assert instance.query("SELECT name, storage from system.roles WHERE name IN ('R1', 'R2') ORDER BY name") ==\
TSV([[ "R1", "disk" ],

1
tests/queries/0_stateless/00825_protobuf_format_input.reference
View File

@ -8,3 +8,4 @@ a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +7495123456
0 0
2 4
3 9
ok

9
tests/queries/0_stateless/00825_protobuf_format_input.sh
View File

@ -3,7 +3,7 @@
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. $CURDIR/../shell_config.sh
set -e -o pipefail
set -eo pipefail
# Run the client.
$CLICKHOUSE_CLIENT --multiquery <<'EOF'
@ -48,5 +48,12 @@ source $CURDIR/00825_protobuf_format_input.insh
$CLICKHOUSE_CLIENT --query "SELECT * FROM in_persons_00825 ORDER BY uuid;"
$CLICKHOUSE_CLIENT --query "SELECT * FROM in_squares_00825 ORDER BY number;"
# Try to input malformed data.
set +eo pipefail
echo -ne '\xe0\x80\x3f\x0b' \
| $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'" 2>&1 \
| grep -qF "Protobuf messages are corrupted" && echo "ok" || echo "fail"
set -eo pipefail
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_persons_00825;"
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_squares_00825;"

60
utils/ci/docker-multiarch/update.sh
View File

@ -37,27 +37,27 @@ fi
# get the image
if \
wget -q --spider "$baseUrl/current" \
&& wget -q --spider "$baseUrl/current/$thisTar" \
; then
baseUrl+='/current'
wget -q --spider "$baseUrl/current" \
&& wget -q --spider "$baseUrl/current/$thisTar" \
; then
baseUrl+='/current'
fi
wget -qN "$baseUrl/"{{MD5,SHA{1,256}}SUMS{,.gpg},"$thisTarBase.manifest",'unpacked/build-info.txt'} || true
wget -N "$baseUrl/$thisTar"
# check checksum
if [ -f SHA256SUMS ]; then
sha256sum="$(sha256sum "$thisTar" | cut -d' ' -f1)"
if ! grep -q "$sha256sum" SHA256SUMS; then
echo >&2 "error: '$thisTar' has invalid SHA256"
exit 1
fi
sha256sum="$(sha256sum "$thisTar" | cut -d' ' -f1)"
if ! grep -q "$sha256sum" SHA256SUMS; then
echo >&2 "error: '$thisTar' has invalid SHA256"
exit 1
fi
fi
cat > Dockerfile <<-EOF
FROM scratch
ADD $thisTar /
ENV ARCH=${ARCH} UBUNTU_SUITE=${VERSION} DOCKER_REPO=${DOCKER_REPO}
FROM scratch
ADD $thisTar /
ENV ARCH=${ARCH} UBUNTU_SUITE=${VERSION} DOCKER_REPO=${DOCKER_REPO}
EOF
# add qemu-user-static binary
@ -70,26 +70,26 @@ EOF
fi
cat >> Dockerfile <<-EOF
# a few minor docker-specific tweaks
# see https://github.com/docker/docker/blob/master/contrib/mkimage/debootstrap
RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d \\
&& echo 'exit 101' >> /usr/sbin/policy-rc.d \\
&& chmod +x /usr/sbin/policy-rc.d \\
&& dpkg-divert --local --rename --add /sbin/initctl \\
&& cp -a /usr/sbin/policy-rc.d /sbin/initctl \\
&& sed -i 's/^exit.*/exit 0/' /sbin/initctl \\
&& echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \\
&& echo 'DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' > /etc/apt/apt.conf.d/docker-clean \\
&& echo 'APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' >> /etc/apt/apt.conf.d/docker-clean \\
&& echo 'Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";' >> /etc/apt/apt.conf.d/docker-clean \\
&& echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/docker-no-languages \\
&& echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > /etc/apt/apt.conf.d/docker-gzip-indexes
# a few minor docker-specific tweaks
# see https://github.com/docker/docker/blob/master/contrib/mkimage/debootstrap
RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d \\
&& echo 'exit 101' >> /usr/sbin/policy-rc.d \\
&& chmod +x /usr/sbin/policy-rc.d \\
&& dpkg-divert --local --rename --add /sbin/initctl \\
&& cp -a /usr/sbin/policy-rc.d /sbin/initctl \\
&& sed -i 's/^exit.*/exit 0/' /sbin/initctl \\
&& echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \\
&& echo 'DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' > /etc/apt/apt.conf.d/docker-clean \\
&& echo 'APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' >> /etc/apt/apt.conf.d/docker-clean \\
&& echo 'Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";' >> /etc/apt/apt.conf.d/docker-clean \\
&& echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/docker-no-languages \\
&& echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > /etc/apt/apt.conf.d/docker-gzip-indexes
# enable the universe
RUN sed -i 's/^#\s*\(deb.*universe\)$/\1/g' /etc/apt/sources.list
# enable the universe
RUN sed -i 's/^#\s*\(deb.*universe\)$/\1/g' /etc/apt/sources.list
# overwrite this with 'CMD []' in a dependent Dockerfile
CMD ["/bin/bash"]
# overwrite this with 'CMD []' in a dependent Dockerfile
CMD ["/bin/bash"]
EOF
docker build -t "${DOCKER_REPO}:${TAG_ARCH}-${VERSION}" .

28
utils/release/release_lib.sh
View File

@ -12,10 +12,10 @@ function gen_version_string {
function get_version {
if [ -z "$VERSION_MAJOR" ] && [ -z "$VERSION_MINOR" ] && [ -z "$VERSION_PATCH" ]; then
BASEDIR=$(dirname "${BASH_SOURCE[0]}")/../../
VERSION_REVISION=`grep "set(VERSION_REVISION" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_REVISION \(.*\)$/\1/' | sed 's/[) ].*//'`
VERSION_MAJOR=`grep "set(VERSION_MAJOR" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_MAJOR \(.*\)/\1/' | sed 's/[) ].*//'`
VERSION_MINOR=`grep "set(VERSION_MINOR" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_MINOR \(.*\)/\1/' | sed 's/[) ].*//'`
VERSION_PATCH=`grep "set(VERSION_PATCH" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_PATCH \(.*\)/\1/' | sed 's/[) ].*//'`
VERSION_REVISION=`grep "SET(VERSION_REVISION" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_REVISION \(.*\)$/\1/' | sed 's/[) ].*//'`
VERSION_MAJOR=`grep "SET(VERSION_MAJOR" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_MAJOR \(.*\)/\1/' | sed 's/[) ].*//'`
VERSION_MINOR=`grep "SET(VERSION_MINOR" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_MINOR \(.*\)/\1/' | sed 's/[) ].*//'`
VERSION_PATCH=`grep "SET(VERSION_PATCH" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_PATCH \(.*\)/\1/' | sed 's/[) ].*//'`
fi
VERSION_PREFIX="${VERSION_PREFIX:-v}"
VERSION_POSTFIX_TAG="${VERSION_POSTFIX:--testing}"
@ -90,28 +90,28 @@ function gen_revision_author {
git_describe=`git describe`
git_hash=`git rev-parse HEAD`
sed -i -e "s/set(VERSION_REVISION [^) ]*/set(VERSION_REVISION $VERSION_REVISION/g;" \
-e "s/set(VERSION_DESCRIBE [^) ]*/set(VERSION_DESCRIBE $git_describe/g;" \
-e "s/set(VERSION_GITHASH [^) ]*/set(VERSION_GITHASH $git_hash/g;" \
-e "s/set(VERSION_MAJOR [^) ]*/set(VERSION_MAJOR $VERSION_MAJOR/g;" \
-e "s/set(VERSION_MINOR [^) ]*/set(VERSION_MINOR $VERSION_MINOR/g;" \
-e "s/set(VERSION_PATCH [^) ]*/set(VERSION_PATCH $VERSION_PATCH/g;" \
-e "s/set(VERSION_STRING [^) ]*/set(VERSION_STRING $VERSION_STRING/g;" \
cmake/version.cmake
sed -i -e "s/SET(VERSION_REVISION [^) ]*/SET(VERSION_REVISION $VERSION_REVISION/g;" \
-e "s/SET(VERSION_DESCRIBE [^) ]*/SET(VERSION_DESCRIBE $git_describe/g;" \
-e "s/SET(VERSION_GITHASH [^) ]*/SET(VERSION_GITHASH $git_hash/g;" \
-e "s/SET(VERSION_MAJOR [^) ]*/SET(VERSION_MAJOR $VERSION_MAJOR/g;" \
-e "s/SET(VERSION_MINOR [^) ]*/SET(VERSION_MINOR $VERSION_MINOR/g;" \
-e "s/SET(VERSION_PATCH [^) ]*/SET(VERSION_PATCH $VERSION_PATCH/g;" \
-e "s/SET(VERSION_STRING [^) ]*/SET(VERSION_STRING $VERSION_STRING/g;" \
cmake/autogenerated_versions.txt
gen_changelog "$VERSION_STRING" "" "$AUTHOR" ""
gen_dockerfiles "$VERSION_STRING"
src/Storages/System/StorageSystemContributors.sh ||:
utils/list-versions/list-versions.sh > utils/list-versions/version_date.tsv
git commit -m "$auto_message [$VERSION_STRING] [$VERSION_REVISION]" cmake/version.cmake debian/changelog docker/*/Dockerfile src/Storages/System/StorageSystemContributors.generated.cpp utils/list-versions/version_date.tsv
git commit -m "$auto_message [$VERSION_STRING] [$VERSION_REVISION]" cmake/autogenerated_versions.txt debian/changelog docker/*/Dockerfile src/Storages/System/StorageSystemContributors.generated.cpp utils/list-versions/version_date.tsv
if [ -z $NO_PUSH ]; then
git push
fi
echo "Generated version: ${VERSION_STRING}, revision: ${VERSION_REVISION}."
# Second tag for correct version information in version.cmake inside tag
# Second tag for correct version information in autogenerated_versions.txt inside tag
if git tag --force -a "$tag" -m "$tag"
then
if [ -z $NO_PUSH ]; then

1
website/css/docs.css
View File

@ -69,7 +69,6 @@ summary {
#content code {
color: #111;
background: #eee;
padding: 2px;
}