mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-22 23:52:03 +00:00
Merge branch 'master' into fix_bad_exception
This commit is contained in:
alesapin
GitHub
commit
61d014f6ec
@ -54,8 +54,6 @@ Checks: '*,
|
||||
-cppcoreguidelines-slicing,
|
||||
-cppcoreguidelines-special-member-functions,
|
||||
|
||||
-concurrency-mt-unsafe,
|
||||
|
||||
-darwin-*,
|
||||
|
||||
-fuchsia-*,
|
||||
|
||||
@ -27,7 +27,7 @@ void trim(String & s)
|
||||
|
||||
std::string getEditor()
|
||||
{
|
||||
const char * editor = std::getenv("EDITOR");
|
||||
const char * editor = std::getenv("EDITOR"); // NOLINT(concurrency-mt-unsafe)
|
||||
|
||||
if (!editor || !*editor)
|
||||
editor = "vim";
|
||||
@ -76,7 +76,7 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
|
||||
if (!in)
|
||||
{
|
||||
rx.print("Cannot open %s reading (for conversion): %s\n",
|
||||
path.c_str(), errnoToString(errno).c_str());
|
||||
path.c_str(), errnoToString().c_str());
|
||||
return;
|
||||
}
|
||||
|
||||
@ -84,7 +84,7 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
|
||||
if (getline(in, line).bad())
|
||||
{
|
||||
rx.print("Cannot read from %s (for conversion): %s\n",
|
||||
path.c_str(), errnoToString(errno).c_str());
|
||||
path.c_str(), errnoToString().c_str());
|
||||
return;
|
||||
}
|
||||
|
||||
@ -113,7 +113,7 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
|
||||
if (!out)
|
||||
{
|
||||
rx.print("Cannot open %s for writing (for conversion): %s\n",
|
||||
path.c_str(), errnoToString(errno).c_str());
|
||||
path.c_str(), errnoToString().c_str());
|
||||
return;
|
||||
}
|
||||
|
||||
@ -151,7 +151,7 @@ ReplxxLineReader::ReplxxLineReader(
|
||||
history_file_fd = open(history_file_path.c_str(), O_RDWR);
|
||||
if (history_file_fd < 0)
|
||||
{
|
||||
rx.print("Open of history file failed: %s\n", errnoToString(errno).c_str());
|
||||
rx.print("Open of history file failed: %s\n", errnoToString().c_str());
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -159,18 +159,18 @@ ReplxxLineReader::ReplxxLineReader(
|
||||
|
||||
if (flock(history_file_fd, LOCK_SH))
|
||||
{
|
||||
rx.print("Shared lock of history file failed: %s\n", errnoToString(errno).c_str());
|
||||
rx.print("Shared lock of history file failed: %s\n", errnoToString().c_str());
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!rx.history_load(history_file_path))
|
||||
{
|
||||
rx.print("Loading history failed: %s\n", errnoToString(errno).c_str());
|
||||
rx.print("Loading history failed: %s\n", errnoToString().c_str());
|
||||
}
|
||||
|
||||
if (flock(history_file_fd, LOCK_UN))
|
||||
{
|
||||
rx.print("Unlock of history file failed: %s\n", errnoToString(errno).c_str());
|
||||
rx.print("Unlock of history file failed: %s\n", errnoToString().c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -225,7 +225,7 @@ ReplxxLineReader::ReplxxLineReader(
|
||||
ReplxxLineReader::~ReplxxLineReader()
|
||||
{
|
||||
if (close(history_file_fd))
|
||||
rx.print("Close of history file failed: %s\n", errnoToString(errno).c_str());
|
||||
rx.print("Close of history file failed: %s\n", errnoToString().c_str());
|
||||
}
|
||||
|
||||
LineReader::InputStatus ReplxxLineReader::readOneLine(const String & prompt)
|
||||
@ -250,7 +250,7 @@ void ReplxxLineReader::addToHistory(const String & line)
|
||||
// and that is why flock() is added here.
|
||||
bool locked = false;
|
||||
if (flock(history_file_fd, LOCK_EX))
|
||||
rx.print("Lock of history file failed: %s\n", errnoToString(errno).c_str());
|
||||
rx.print("Lock of history file failed: %s\n", errnoToString().c_str());
|
||||
else
|
||||
locked = true;
|
||||
|
||||
@ -258,10 +258,10 @@ void ReplxxLineReader::addToHistory(const String & line)
|
||||
|
||||
// flush changes to the disk
|
||||
if (!rx.history_save(history_file_path))
|
||||
rx.print("Saving history failed: %s\n", errnoToString(errno).c_str());
|
||||
rx.print("Saving history failed: %s\n", errnoToString().c_str());
|
||||
|
||||
if (locked && 0 != flock(history_file_fd, LOCK_UN))
|
||||
rx.print("Unlock of history file failed: %s\n", errnoToString(errno).c_str());
|
||||
rx.print("Unlock of history file failed: %s\n", errnoToString().c_str());
|
||||
}
|
||||
|
||||
/// See comments in ShellCommand::executeImpl()
|
||||
@ -275,7 +275,7 @@ int ReplxxLineReader::executeEditor(const std::string & path)
|
||||
static void * real_vfork = dlsym(RTLD_DEFAULT, "vfork");
|
||||
if (!real_vfork)
|
||||
{
|
||||
rx.print("Cannot find symbol vfork in myself: %s\n", errnoToString(errno).c_str());
|
||||
rx.print("Cannot find symbol vfork in myself: %s\n", errnoToString().c_str());
|
||||
return -1;
|
||||
}
|
||||
|
||||
@ -283,7 +283,7 @@ int ReplxxLineReader::executeEditor(const std::string & path)
|
||||
|
||||
if (-1 == pid)
|
||||
{
|
||||
rx.print("Cannot vfork: %s\n", errnoToString(errno).c_str());
|
||||
rx.print("Cannot vfork: %s\n", errnoToString().c_str());
|
||||
return -1;
|
||||
}
|
||||
|
||||
@ -292,11 +292,11 @@ int ReplxxLineReader::executeEditor(const std::string & path)
|
||||
{
|
||||
sigset_t mask;
|
||||
sigemptyset(&mask);
|
||||
sigprocmask(0, nullptr, &mask);
|
||||
sigprocmask(SIG_UNBLOCK, &mask, nullptr);
|
||||
sigprocmask(0, nullptr, &mask); // NOLINT(concurrency-mt-unsafe) // ok in newly created process
|
||||
sigprocmask(SIG_UNBLOCK, &mask, nullptr); // NOLINT(concurrency-mt-unsafe) // ok in newly created process
|
||||
|
||||
execvp(editor.c_str(), argv);
|
||||
rx.print("Cannot execute %s: %s\n", editor.c_str(), errnoToString(errno).c_str());
|
||||
rx.print("Cannot execute %s: %s\n", editor.c_str(), errnoToString().c_str());
|
||||
_exit(-1);
|
||||
}
|
||||
|
||||
@ -309,7 +309,7 @@ int ReplxxLineReader::executeEditor(const std::string & path)
|
||||
if (errno == EINTR)
|
||||
continue;
|
||||
|
||||
rx.print("Cannot waitpid: %s\n", errnoToString(errno).c_str());
|
||||
rx.print("Cannot waitpid: %s\n", errnoToString().c_str());
|
||||
return -1;
|
||||
}
|
||||
else
|
||||
@ -324,7 +324,7 @@ void ReplxxLineReader::openEditor()
|
||||
int fd = ::mkstemps(filename, 4);
|
||||
if (-1 == fd)
|
||||
{
|
||||
rx.print("Cannot create temporary file to edit query: %s\n", errnoToString(errno).c_str());
|
||||
rx.print("Cannot create temporary file to edit query: %s\n", errnoToString().c_str());
|
||||
return;
|
||||
}
|
||||
|
||||
@ -338,7 +338,7 @@ void ReplxxLineReader::openEditor()
|
||||
ssize_t res = ::write(fd, begin + bytes_written, offset - bytes_written);
|
||||
if ((-1 == res || 0 == res) && errno != EINTR)
|
||||
{
|
||||
rx.print("Cannot write to temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
|
||||
rx.print("Cannot write to temporary query file %s: %s\n", filename, errnoToString().c_str());
|
||||
break;
|
||||
}
|
||||
bytes_written += res;
|
||||
@ -346,7 +346,7 @@ void ReplxxLineReader::openEditor()
|
||||
|
||||
if (0 != ::close(fd))
|
||||
{
|
||||
rx.print("Cannot close temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
|
||||
rx.print("Cannot close temporary query file %s: %s\n", filename, errnoToString().c_str());
|
||||
return;
|
||||
}
|
||||
|
||||
@ -364,7 +364,7 @@ void ReplxxLineReader::openEditor()
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
rx.print("Cannot read from temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
|
||||
rx.print("Cannot read from temporary query file %s: %s\n", filename, errnoToString().c_str());
|
||||
return;
|
||||
}
|
||||
}
|
||||
@ -373,7 +373,7 @@ void ReplxxLineReader::openEditor()
|
||||
enableBracketedPaste();
|
||||
|
||||
if (0 != ::unlink(filename))
|
||||
rx.print("Cannot remove temporary query file %s: %s\n", filename, errnoToString(errno).c_str());
|
||||
rx.print("Cannot remove temporary query file %s: %s\n", filename, errnoToString().c_str());
|
||||
}
|
||||
|
||||
void ReplxxLineReader::enableBracketedPaste()
|
||||
|
||||
@ -3,10 +3,11 @@
|
||||
#include <fmt/format.h>
|
||||
|
||||
|
||||
std::string errnoToString(int code, int the_errno)
|
||||
std::string errnoToString(int the_errno)
|
||||
{
|
||||
const size_t buf_size = 128;
|
||||
char buf[buf_size];
|
||||
|
||||
#ifndef _GNU_SOURCE
|
||||
int rc = strerror_r(the_errno, buf, buf_size);
|
||||
#ifdef OS_DARWIN
|
||||
@ -15,7 +16,7 @@ std::string errnoToString(int code, int the_errno)
|
||||
if (rc != 0)
|
||||
#endif
|
||||
{
|
||||
std::string tmp = std::to_string(code);
|
||||
std::string tmp = std::to_string(the_errno);
|
||||
const char * code_str = tmp.c_str();
|
||||
const char * unknown_message = "Unknown error ";
|
||||
strcpy(buf, unknown_message);
|
||||
@ -23,7 +24,6 @@ std::string errnoToString(int code, int the_errno)
|
||||
}
|
||||
return fmt::format("errno: {}, strerror: {}", the_errno, buf);
|
||||
#else
|
||||
(void)code;
|
||||
return fmt::format("errno: {}, strerror: {}", the_errno, strerror_r(the_errno, buf, sizeof(buf)));
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -3,4 +3,4 @@
|
||||
#include <cerrno>
|
||||
#include <string>
|
||||
|
||||
std::string errnoToString(int code, int the_errno = errno);
|
||||
std::string errnoToString(int the_errno = errno);
|
||||
|
||||
@ -16,7 +16,7 @@ void setTerminalEcho(bool enable)
|
||||
struct termios tty{};
|
||||
|
||||
if (0 != tcgetattr(STDIN_FILENO, &tty))
|
||||
throw std::runtime_error(std::string("setTerminalEcho failed get: ") + errnoToString(errno));
|
||||
throw std::runtime_error(std::string("setTerminalEcho failed get: ") + errnoToString());
|
||||
|
||||
if (enable)
|
||||
tty.c_lflag |= ECHO;
|
||||
@ -24,5 +24,5 @@ void setTerminalEcho(bool enable)
|
||||
tty.c_lflag &= ~ECHO;
|
||||
|
||||
if (0 != tcsetattr(STDIN_FILENO, TCSANOW, &tty))
|
||||
throw std::runtime_error(std::string("setTerminalEcho failed set: ") + errnoToString(errno));
|
||||
throw std::runtime_error(std::string("setTerminalEcho failed set: ") + errnoToString());
|
||||
}
|
||||
|
||||
@ -17,9 +17,10 @@ empty([x])
|
||||
|
||||
Массив считается пустым, если он не содержит ни одного элемента.
|
||||
|
||||
:::note "Примечание"
|
||||
:::note "Примечание"
|
||||
Функцию можно оптимизировать, если включить настройку [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns). При `optimize_functions_to_subcolumns = 1` функция читает только подстолбец [size0](../../sql-reference/data-types/array.md#array-size) вместо чтения и обработки всего столбца массива. Запрос `SELECT empty(arr) FROM TABLE` преобразуется к запросу `SELECT arr.size0 = 0 FROM TABLE`.
|
||||
:::
|
||||
:::
|
||||
|
||||
Функция также поддерживает работу с типами [String](string-functions.md#empty) и [UUID](uuid-functions.md#empty).
|
||||
|
||||
**Параметры**
|
||||
@ -60,9 +61,10 @@ notEmpty([x])
|
||||
|
||||
Массив считается непустым, если он содержит хотя бы один элемент.
|
||||
|
||||
:::note "Примечание"
|
||||
:::note "Примечание"
|
||||
Функцию можно оптимизировать, если включить настройку [optimize_functions_to_subcolumns](../../operations/settings/settings.md#optimize-functions-to-subcolumns). При `optimize_functions_to_subcolumns = 1` функция читает только подстолбец [size0](../../sql-reference/data-types/array.md#array-size) вместо чтения и обработки всего столбца массива. Запрос `SELECT notEmpty(arr) FROM table` преобразуется к запросу `SELECT arr.size0 != 0 FROM TABLE`.
|
||||
:::
|
||||
:::
|
||||
|
||||
Функция также поддерживает работу с типами [String](string-functions.md#notempty) и [UUID](uuid-functions.md#notempty).
|
||||
|
||||
**Параметры**
|
||||
@ -689,9 +691,10 @@ SELECT arraySort((x, y) -> -y, [0, 1, 2], [1, 2, 3]) as res;
|
||||
└─────────┘
|
||||
```
|
||||
|
||||
:::note "Примечание"
|
||||
:::note "Примечание"
|
||||
Для улучшения эффективности сортировки применяется [преобразование Шварца](https://ru.wikipedia.org/wiki/%D0%9F%D1%80%D0%B5%D0%BE%D0%B1%D1%80%D0%B0%D0%B7%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5_%D0%A8%D0%B2%D0%B0%D1%80%D1%86%D0%B0).
|
||||
:::
|
||||
:::
|
||||
|
||||
## arrayReverseSort(\[func,\] arr, …) {#array_functions-reverse-sort}
|
||||
|
||||
Возвращает массив `arr`, отсортированный в нисходящем порядке. Если указана функция `func`, то массив `arr` сначала сортируется в порядке, который определяется функцией `func`, а затем отсортированный массив переворачивается. Если функция `func` принимает несколько аргументов, то в функцию `arrayReverseSort` необходимо передавать несколько массивов, которые будут соответствовать аргументам функции `func`. Подробные примеры рассмотрены в конце описания функции `arrayReverseSort`.
|
||||
|
||||
@ -266,9 +266,10 @@ SELECT toUnixTimestamp('2017-11-05 08:07:47', 'Asia/Tokyo') AS unix_timestamp;
|
||||
└────────────────┘
|
||||
```
|
||||
|
||||
:::note "Attention"
|
||||
:::note "Attention"
|
||||
`Date` или `DateTime` это возвращаемый тип функций `toStartOf*`, который описан ниже. Несмотря на то, что эти функции могут принимать `DateTime64` в качестве аргумента, если переданное значение типа `DateTime64` выходит за пределы нормального диапазона (с 1900 по 2299 год), то это даст неверный результат.
|
||||
:::
|
||||
:::
|
||||
|
||||
## toStartOfYear {#tostartofyear}
|
||||
|
||||
Округляет дату или дату-с-временем вниз до первого дня года.
|
||||
@ -302,9 +303,10 @@ SELECT toStartOfISOYear(toDate('2017-01-01')) AS ISOYear20170101;
|
||||
Округляет дату или дату-с-временем вниз до первого дня месяца.
|
||||
Возвращается дата.
|
||||
|
||||
:::note "Attention"
|
||||
:::note "Attention"
|
||||
Возвращаемое значение для некорректных дат зависит от реализации. ClickHouse может вернуть нулевую дату, выбросить исключение, или выполнить «естественное» перетекание дат между месяцами.
|
||||
:::
|
||||
:::
|
||||
|
||||
## toMonday {#tomonday}
|
||||
|
||||
Округляет дату или дату-с-временем вниз до ближайшего понедельника.
|
||||
|
||||
@ -153,9 +153,10 @@ SELECT hex(toFloat64(number)) AS hex_presentation FROM numbers(15, 2);
|
||||
|
||||
Если вы хотите преобразовать результат в число, вы можете использовать функции [reverse](../../sql-reference/functions/string-functions.md#reverse) и [reinterpretAs<Type>](../../sql-reference/functions/type-conversion-functions.md#type-conversion-functions).
|
||||
|
||||
:::note "Примечание"
|
||||
Если `unhex` вызывается из `clickhouse-client`, двоичные строки отображаются с использованием UTF-8.
|
||||
:::
|
||||
:::note "Примечание"
|
||||
Если `unhex` вызывается из `clickhouse-client`, двоичные строки отображаются с использованием UTF-8.
|
||||
:::
|
||||
|
||||
Синоним: `UNHEX`.
|
||||
|
||||
**Синтаксис**
|
||||
@ -294,9 +295,10 @@ unbin(arg)
|
||||
|
||||
Для числового аргумента `unbin()` не возвращает значение, обратное результату `bin()`. Чтобы преобразовать результат в число, используйте функции [reverse](../../sql-reference/functions/string-functions.md#reverse) и [reinterpretAs<Type>](../../sql-reference/functions/type-conversion-functions.md#reinterpretasuint8163264).
|
||||
|
||||
:::note "Примечание"
|
||||
Если `unbin` вызывается из клиента `clickhouse-client`, бинарная строка возвращается в кодировке UTF-8.
|
||||
:::
|
||||
:::note "Примечание"
|
||||
Если `unbin` вызывается из клиента `clickhouse-client`, бинарная строка возвращается в кодировке UTF-8.
|
||||
:::
|
||||
|
||||
Поддерживает двоичные цифры `0` и `1`. Количество двоичных цифр не обязательно должно быть кратно восьми. Если строка аргумента содержит что-либо, кроме двоичных цифр, возвращается некоторый результат, определенный реализацией (ошибки не возникает).
|
||||
|
||||
**Аргументы**
|
||||
|
||||
@ -3,9 +3,10 @@ sidebar_position: 58
|
||||
sidebar_label: "Функции для работы с внешними словарями"
|
||||
---
|
||||
|
||||
:::note "Внимание"
|
||||
:::note "Внимание"
|
||||
Для словарей, созданных с помощью [DDL-запросов](../../sql-reference/statements/create/dictionary.md), в параметре `dict_name` указывается полное имя словаря вместе с базой данных, например: `<database>.<dict_name>`. Если база данных не указана, используется текущая.
|
||||
:::
|
||||
:::
|
||||
|
||||
# Функции для работы с внешними словарями {#ext_dict_functions}
|
||||
|
||||
Информацию о подключении и настройке внешних словарей смотрите в разделе [Внешние словари](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md).
|
||||
|
||||
@ -9,6 +9,7 @@ sidebar_label: "Функции интроспекции"
|
||||
|
||||
:::danger "Предупреждение"
|
||||
Эти функции выполняются медленно и могут приводить к нежелательным последствиям в плане безопасности.
|
||||
:::
|
||||
|
||||
Для правильной работы функций интроспекции:
|
||||
|
||||
|
||||
@ -360,9 +360,10 @@ SELECT JSON_EXISTS('{"hello":["world"]}', '$.hello[*]');
|
||||
SELECT JSON_EXISTS('{"hello":["world"]}', '$.hello[0]');
|
||||
```
|
||||
|
||||
:::note "Примечание"
|
||||
:::note "Примечание"
|
||||
до версии 21.11 порядок аргументов функции был обратный, т.е. JSON_EXISTS(path, json)
|
||||
:::
|
||||
:::
|
||||
|
||||
## JSON_QUERY(json, path) {#json-query}
|
||||
|
||||
Парсит JSON и извлекает значение как JSON массив или JSON объект.
|
||||
@ -386,9 +387,10 @@ SELECT toTypeName(JSON_QUERY('{"hello":2}', '$.hello'));
|
||||
[2]
|
||||
String
|
||||
```
|
||||
:::note "Примечание"
|
||||
:::note "Примечание"
|
||||
до версии 21.11 порядок аргументов функции был обратный, т.е. JSON_QUERY(path, json)
|
||||
:::
|
||||
:::
|
||||
|
||||
## JSON_VALUE(json, path) {#json-value}
|
||||
|
||||
Парсит JSON и извлекает значение как JSON скаляр.
|
||||
@ -413,9 +415,10 @@ world
|
||||
String
|
||||
```
|
||||
|
||||
:::note "Примечание"
|
||||
:::note "Примечание"
|
||||
до версии 21.11 порядок аргументов функции был обратный, т.е. JSON_VALUE(path, json)
|
||||
:::
|
||||
:::
|
||||
|
||||
## toJSONString {#tojsonstring}
|
||||
|
||||
Сериализует значение в JSON представление. Поддерживаются различные типы данных и вложенные структуры.
|
||||
|
||||
@ -7,6 +7,7 @@ sidebar_label: NLP
|
||||
|
||||
:::danger "Предупреждение"
|
||||
Сейчас использование функций для работы с естественным языком является экспериментальной возможностью. Чтобы использовать данные функции, включите настройку `allow_experimental_nlp_functions = 1`.
|
||||
:::
|
||||
|
||||
## stem {#stem}
|
||||
|
||||
@ -129,4 +130,4 @@ SELECT synonyms('list', 'important');
|
||||
<path>en/</path>
|
||||
</extension>
|
||||
</synonyms_extensions>
|
||||
```
|
||||
```
|
||||
|
||||
@ -2020,9 +2020,10 @@ countDigits(x)
|
||||
|
||||
Тип: [UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges).
|
||||
|
||||
:::note "Примечание"
|
||||
:::note "Примечание"
|
||||
Для `Decimal` значений учитывается их масштаб: вычисляется результат по базовому целочисленному типу, полученному как `(value * scale)`. Например: `countDigits(42) = 2`, `countDigits(42.000) = 5`, `countDigits(0.04200) = 4`. То есть вы можете проверить десятичное переполнение для `Decimal64` с помощью `countDecimal(x) > 18`. Это медленный вариант [isDecimalOverflow](#is-decimal-overflow).
|
||||
:::
|
||||
:::
|
||||
|
||||
**Пример**
|
||||
|
||||
Запрос:
|
||||
|
||||
@ -27,9 +27,10 @@ position(needle IN haystack)
|
||||
|
||||
Алиас: `locate(haystack, needle[, start_pos])`.
|
||||
|
||||
:::note "Примечание"
|
||||
:::note "Примечание"
|
||||
Синтаксис `position(needle IN haystack)` обеспечивает совместимость с SQL, функция работает так же, как `position(haystack, needle)`.
|
||||
:::
|
||||
:::
|
||||
|
||||
**Аргументы**
|
||||
|
||||
- `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal).
|
||||
@ -327,9 +328,10 @@ Result:
|
||||
|
||||
Для поиска без учета регистра и/или в кодировке UTF-8 используйте функции `multiSearchAnyCaseInsensitive, multiSearchAnyUTF8, multiSearchAnyCaseInsensitiveUTF8`.
|
||||
|
||||
:::note "Примечание"
|
||||
:::note "Примечание"
|
||||
Во всех функциях `multiSearch*` количество needles должно быть меньше 2<sup>8</sup> из-за особенностей реализации.
|
||||
:::
|
||||
:::
|
||||
|
||||
## match(haystack, pattern) {#matchhaystack-pattern}
|
||||
|
||||
Проверка строки на соответствие регулярному выражению pattern. Регулярное выражение **re2**. Синтаксис регулярных выражений **re2** является более ограниченным по сравнению с регулярными выражениями **Perl** ([подробнее](https://github.com/google/re2/wiki/Syntax)).
|
||||
@ -344,9 +346,9 @@ Result:
|
||||
|
||||
То же, что и `match`, но возвращает ноль, если ни одно регулярное выражение не подошло и один, если хотя бы одно. Используется библиотека [hyperscan](https://github.com/intel/hyperscan) для соответствия регулярных выражений. Для шаблонов на поиск многих подстрок в строке, лучше используйте `multiSearchAny`, так как она работает существенно быстрее.
|
||||
|
||||
:::note "Примечание"
|
||||
:::note "Примечание"
|
||||
Длина любой строки из `haystack` должна быть меньше 2<sup>32</sup> байт, иначе бросается исключение. Это ограничение связано с ограничением hyperscan API.
|
||||
:::
|
||||
:::
|
||||
## multiMatchAnyIndex(haystack, \[pattern<sub>1</sub>, pattern<sub>2</sub>, …, pattern<sub>n</sub>\]) {#multimatchanyindexhaystack-pattern1-pattern2-patternn}
|
||||
|
||||
То же, что и `multiMatchAny`, только возвращает любой индекс подходящего регулярного выражения.
|
||||
@ -367,12 +369,13 @@ Result:
|
||||
|
||||
То же, что и `multiFuzzyMatchAny`, только возвращает массив всех индексов всех подходящих регулярных выражений в любом порядке в пределах константного редакционного расстояния.
|
||||
|
||||
:::note "Примечание"
|
||||
:::note "Примечание"
|
||||
`multiFuzzyMatch*` функции не поддерживают UTF-8 закодированные регулярные выражения, и такие выражения рассматриваются как байтовые из-за ограничения hyperscan.
|
||||
:::
|
||||
:::note "Примечание"
|
||||
:::
|
||||
|
||||
:::note "Примечание"
|
||||
Чтобы выключить все функции, использующие hyperscan, используйте настройку `SET allow_hyperscan = 0;`.
|
||||
:::
|
||||
:::
|
||||
## extract(haystack, pattern) {#extracthaystack-pattern}
|
||||
|
||||
Извлечение фрагмента строки по регулярному выражению. Если haystack не соответствует регулярному выражению pattern, то возвращается пустая строка. Если регулярное выражение не содержит subpattern-ов, то вынимается фрагмент, который подпадает под всё регулярное выражение. Иначе вынимается фрагмент, который подпадает под первый subpattern.
|
||||
@ -385,9 +388,10 @@ Result:
|
||||
|
||||
Разбирает строку `haystack` на фрагменты, соответствующие группам регулярного выражения `pattern`. Возвращает массив массивов, где первый массив содержит все фрагменты, соответствующие первой группе регулярного выражения, второй массив - соответствующие второй группе, и т.д.
|
||||
|
||||
:::note "Замечание"
|
||||
:::note "Замечание"
|
||||
Функция `extractAllGroupsHorizontal` работает медленнее, чем функция [extractAllGroupsVertical](#extractallgroups-vertical).
|
||||
:::
|
||||
:::
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
@ -556,9 +560,10 @@ SELECT * FROM Months WHERE ilike(name, '%j%');
|
||||
|
||||
Для поиска без учета регистра и/или в формате UTF-8 используйте функции `ngramSearchCaseInsensitive, ngramSearchUTF8, ngramSearchCaseInsensitiveUTF8`.
|
||||
|
||||
:::note "Примечание"
|
||||
:::note "Примечание"
|
||||
Для случая UTF-8 мы используем триграммное расстояние. Вычисление n-граммного расстояния не совсем честное. Мы используем 2-х байтные хэши для хэширования n-грамм, а затем вычисляем (не)симметрическую разность между хэш таблицами – могут возникнуть коллизии. В формате UTF-8 без учета регистра мы не используем честную функцию `tolower` – мы обнуляем 5-й бит (нумерация с нуля) каждого байта кодовой точки, а также первый бит нулевого байта, если байтов больше 1 – это работает для латиницы и почти для всех кириллических букв.
|
||||
:::
|
||||
:::
|
||||
|
||||
## countMatches(haystack, pattern) {#countmatcheshaystack-pattern}
|
||||
|
||||
Возвращает количество совпадений, найденных в строке `haystack`, для регулярного выражения `pattern`.
|
||||
|
||||
@ -684,9 +684,10 @@ x::t
|
||||
|
||||
- Преобразованное значение.
|
||||
|
||||
:::note "Примечание"
|
||||
Если входное значение выходит за границы нового типа, то результат переполняется. Например, `CAST(-1, 'UInt8')` возвращает `255`.
|
||||
:::
|
||||
:::note "Примечание"
|
||||
Если входное значение выходит за границы нового типа, то результат переполняется. Например, `CAST(-1, 'UInt8')` возвращает `255`.
|
||||
:::
|
||||
|
||||
**Примеры**
|
||||
|
||||
Запрос:
|
||||
|
||||
@ -120,7 +120,7 @@ public:
|
||||
void initialize(Poco::Util::Application & self [[maybe_unused]]) override
|
||||
{
|
||||
std::string home_path;
|
||||
const char * home_path_cstr = getenv("HOME");
|
||||
const char * home_path_cstr = getenv("HOME"); // NOLINT(concurrency-mt-unsafe)
|
||||
if (home_path_cstr)
|
||||
home_path = home_path_cstr;
|
||||
|
||||
@ -613,15 +613,15 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
|
||||
std::optional<std::string> env_password_str;
|
||||
std::optional<std::string> env_quota_key_str;
|
||||
|
||||
const char * env_user = getenv("CLICKHOUSE_USER");
|
||||
const char * env_user = getenv("CLICKHOUSE_USER"); // NOLINT(concurrency-mt-unsafe)
|
||||
if (env_user != nullptr)
|
||||
env_user_str.emplace(std::string(env_user));
|
||||
|
||||
const char * env_password = getenv("CLICKHOUSE_PASSWORD");
|
||||
const char * env_password = getenv("CLICKHOUSE_PASSWORD"); // NOLINT(concurrency-mt-unsafe)
|
||||
if (env_password != nullptr)
|
||||
env_password_str.emplace(std::string(env_password));
|
||||
|
||||
const char * env_quota_key = getenv("CLICKHOUSE_QUOTA_KEY");
|
||||
const char * env_quota_key = getenv("CLICKHOUSE_QUOTA_KEY"); // NOLINT(concurrency-mt-unsafe)
|
||||
if (env_quota_key != nullptr)
|
||||
env_quota_key_str.emplace(std::string(env_quota_key));
|
||||
|
||||
|
||||
@ -183,7 +183,7 @@ void Client::initialize(Poco::Util::Application & self)
|
||||
{
|
||||
Poco::Util::Application::initialize(self);
|
||||
|
||||
const char * home_path_cstr = getenv("HOME");
|
||||
const char * home_path_cstr = getenv("HOME"); // NOLINT(concurrency-mt-unsafe)
|
||||
if (home_path_cstr)
|
||||
home_path = home_path_cstr;
|
||||
|
||||
@ -202,11 +202,11 @@ void Client::initialize(Poco::Util::Application & self)
|
||||
* may be statically allocated, and can be modified by a subsequent call to getenv(), putenv(3), setenv(3), or unsetenv(3).
|
||||
*/
|
||||
|
||||
const char * env_user = getenv("CLICKHOUSE_USER");
|
||||
const char * env_user = getenv("CLICKHOUSE_USER"); // NOLINT(concurrency-mt-unsafe)
|
||||
if (env_user)
|
||||
config().setString("user", env_user);
|
||||
|
||||
const char * env_password = getenv("CLICKHOUSE_PASSWORD");
|
||||
const char * env_password = getenv("CLICKHOUSE_PASSWORD"); // NOLINT(concurrency-mt-unsafe)
|
||||
if (env_password)
|
||||
config().setString("password", env_password);
|
||||
|
||||
@ -620,7 +620,7 @@ bool Client::processWithFuzzing(const String & full_query)
|
||||
stderr,
|
||||
"Found error: IAST::clone() is broken for some AST node. This is a bug. The original AST ('dump before fuzz') and its cloned copy ('dump of cloned AST') refer to the same nodes, which must never happen. This means that their parent node doesn't implement clone() correctly.");
|
||||
|
||||
exit(1);
|
||||
_exit(1);
|
||||
}
|
||||
|
||||
auto fuzzed_text = ast_to_process->formatForErrorMessage();
|
||||
@ -770,7 +770,7 @@ bool Client::processWithFuzzing(const String & full_query)
|
||||
fmt::print(stderr, "Text-3 (AST-3 formatted):\n'{}'\n", text_3);
|
||||
fmt::print(stderr, "Text-3 must be equal to Text-2, but it is not.\n");
|
||||
|
||||
exit(1);
|
||||
_exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -909,7 +909,7 @@ void Client::processOptions(const OptionsDescription & options_description,
|
||||
auto exit_code = e.code() % 256;
|
||||
if (exit_code == 0)
|
||||
exit_code = 255;
|
||||
exit(exit_code);
|
||||
_exit(exit_code);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -110,7 +110,7 @@ void DisksApp::init(std::vector<String> & common_arguments)
|
||||
if (options.count("help"))
|
||||
{
|
||||
printHelpMessage(options_description);
|
||||
exit(0);
|
||||
exit(0); // NOLINT(concurrency-mt-unsafe)
|
||||
}
|
||||
|
||||
if (!supported_commands.contains(command_name))
|
||||
|
||||
@ -708,7 +708,7 @@ int mainEntryClickHouseInstall(int argc, char ** argv)
|
||||
|
||||
/// dpkg or apt installers can ask for non-interactive work explicitly.
|
||||
|
||||
const char * debian_frontend_var = getenv("DEBIAN_FRONTEND");
|
||||
const char * debian_frontend_var = getenv("DEBIAN_FRONTEND"); // NOLINT(concurrency-mt-unsafe)
|
||||
bool noninteractive = debian_frontend_var && debian_frontend_var == std::string_view("noninteractive");
|
||||
|
||||
bool is_interactive = !noninteractive && stdin_is_a_tty && stdout_is_a_tty;
|
||||
|
||||
@ -366,10 +366,10 @@ void checkHarmfulEnvironmentVariables(char ** argv)
|
||||
bool require_reexec = false;
|
||||
for (const auto * var : harmful_env_variables)
|
||||
{
|
||||
if (const char * value = getenv(var); value && value[0])
|
||||
if (const char * value = getenv(var); value && value[0]) // NOLINT(concurrency-mt-unsafe)
|
||||
{
|
||||
/// NOTE: setenv() is used over unsetenv() since unsetenv() marked as harmful
|
||||
if (setenv(var, "", true))
|
||||
if (setenv(var, "", true)) // NOLINT(concurrency-mt-unsafe) // this is safe if not called concurrently
|
||||
{
|
||||
fmt::print(stderr, "Cannot override {} environment variable", var);
|
||||
_exit(1);
|
||||
|
||||
@ -4,18 +4,14 @@
|
||||
#include <sys/resource.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/wait.h>
|
||||
#include <cerrno>
|
||||
#include <pwd.h>
|
||||
#include <unistd.h>
|
||||
#include <Poco/Version.h>
|
||||
#include <Poco/DirectoryIterator.h>
|
||||
#include <Poco/Net/HTTPServer.h>
|
||||
#include <Poco/Net/NetException.h>
|
||||
#include <Poco/Util/HelpFormatter.h>
|
||||
#include <Poco/Environment.h>
|
||||
#include <Common/scope_guard_safe.h>
|
||||
#include <base/defines.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <base/phdr_cache.h>
|
||||
#include <Common/ErrorHandlers.h>
|
||||
@ -45,7 +41,6 @@
|
||||
#include <Common/remapExecutable.h>
|
||||
#include <Common/TLDListsHolder.h>
|
||||
#include <Core/ServerUUID.h>
|
||||
#include <IO/HTTPCommon.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
#include <IO/IOThreadPool.h>
|
||||
@ -84,7 +79,6 @@
|
||||
#include <Common/ThreadFuzzer.h>
|
||||
#include <Common/getHashOfLoadedBinary.h>
|
||||
#include <Common/filesystemHelpers.h>
|
||||
#include <Common/Elf.h>
|
||||
#include <Compression/CompressionCodecEncrypted.h>
|
||||
#include <Server/MySQLHandlerFactory.h>
|
||||
#include <Server/PostgreSQLHandlerFactory.h>
|
||||
@ -170,7 +164,7 @@ int mainEntryClickHouseServer(int argc, char ** argv)
|
||||
/// Can be overridden by environment variable (cannot use server config at this moment).
|
||||
if (argc > 0)
|
||||
{
|
||||
const char * env_watchdog = getenv("CLICKHOUSE_WATCHDOG_ENABLE");
|
||||
const char * env_watchdog = getenv("CLICKHOUSE_WATCHDOG_ENABLE"); // NOLINT(concurrency-mt-unsafe)
|
||||
if (env_watchdog)
|
||||
{
|
||||
if (0 == strcmp(env_watchdog, "1"))
|
||||
@ -268,7 +262,6 @@ namespace ErrorCodes
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
|
||||
extern const int INVALID_CONFIG_PARAMETER;
|
||||
extern const int SYSTEM_ERROR;
|
||||
extern const int FAILED_TO_GETPWUID;
|
||||
extern const int MISMATCHING_USERS_FOR_PROCESS_AND_DATA;
|
||||
extern const int NETWORK_ERROR;
|
||||
@ -658,6 +651,24 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
|
||||
StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true));
|
||||
|
||||
#if USE_HDFS
|
||||
/// This will point libhdfs3 to the right location for its config.
|
||||
/// Note: this has to be done once at server initialization, because 'setenv' is not thread-safe.
|
||||
|
||||
String libhdfs3_conf = config().getString("hdfs.libhdfs3_conf", "");
|
||||
if (!libhdfs3_conf.empty())
|
||||
{
|
||||
if (std::filesystem::path{libhdfs3_conf}.is_relative() && !std::filesystem::exists(libhdfs3_conf))
|
||||
{
|
||||
const String config_path = config().getString("config-file", "config.xml");
|
||||
const auto config_dir = std::filesystem::path{config_path}.remove_filename();
|
||||
if (std::filesystem::exists(config_dir / libhdfs3_conf))
|
||||
libhdfs3_conf = std::filesystem::absolute(config_dir / libhdfs3_conf);
|
||||
}
|
||||
setenv("LIBHDFS3_CONF", libhdfs3_conf.c_str(), true /* overwrite */); // NOLINT
|
||||
}
|
||||
#endif
|
||||
|
||||
registerFunctions();
|
||||
registerAggregateFunctions();
|
||||
registerTableFunctions();
|
||||
@ -698,8 +709,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
GlobalThreadPool::initialize(
|
||||
config().getUInt("max_thread_pool_size", 10000),
|
||||
config().getUInt("max_thread_pool_free_size", 1000),
|
||||
config().getUInt("thread_pool_queue_size", 10000)
|
||||
);
|
||||
config().getUInt("thread_pool_queue_size", 10000));
|
||||
|
||||
IOThreadPool::initialize(
|
||||
config().getUInt("max_io_thread_pool_size", 100),
|
||||
@ -840,7 +850,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
|
||||
LOG_TRACE(log, "Will do mlock to prevent executable memory from being paged out. It may take a few seconds.");
|
||||
if (0 != mlock(addr, len))
|
||||
LOG_WARNING(log, "Failed mlock: {}", errnoToString(ErrorCodes::SYSTEM_ERROR));
|
||||
LOG_WARNING(log, "Failed mlock: {}", errnoToString());
|
||||
else
|
||||
LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed, total {}", ReadableSize(len));
|
||||
}
|
||||
@ -908,7 +918,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
rlim.rlim_cur = config().getUInt("max_open_files", rlim.rlim_max);
|
||||
int rc = setrlimit(RLIMIT_NOFILE, &rlim);
|
||||
if (rc != 0)
|
||||
LOG_WARNING(log, "Cannot set max number of file descriptors to {}. Try to specify max_open_files according to your system limits. error: {}", rlim.rlim_cur, strerror(errno));
|
||||
LOG_WARNING(log, "Cannot set max number of file descriptors to {}. Try to specify max_open_files according to your system limits. error: {}", rlim.rlim_cur, errnoToString());
|
||||
else
|
||||
LOG_DEBUG(log, "Set max number of file descriptors to {} (was {}).", rlim.rlim_cur, old);
|
||||
}
|
||||
@ -931,7 +941,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
int rc = setrlimit(RLIMIT_NPROC, &rlim);
|
||||
if (rc != 0)
|
||||
{
|
||||
LOG_WARNING(log, "Cannot set max number of threads to {}. error: {}", rlim.rlim_cur, strerror(errno));
|
||||
LOG_WARNING(log, "Cannot set max number of threads to {}. error: {}", rlim.rlim_cur, errnoToString());
|
||||
rlim.rlim_cur = old;
|
||||
}
|
||||
else
|
||||
|
||||
@ -160,7 +160,7 @@ try
|
||||
if (options.empty() || options.count("help"))
|
||||
{
|
||||
std::cout << description << std::endl;
|
||||
exit(0);
|
||||
exit(0); // NOLINT(concurrency-mt-unsafe)
|
||||
}
|
||||
|
||||
String metadata_path;
|
||||
|
||||
@ -108,7 +108,7 @@ try
|
||||
if (argc < 3)
|
||||
{
|
||||
std::cout << "Usage: ./clickhouse su user:group ..." << std::endl;
|
||||
exit(0);
|
||||
exit(0); // NOLINT(concurrency-mt-unsafe)
|
||||
}
|
||||
|
||||
std::string_view user_and_group = argv[1];
|
||||
|
||||
@ -179,7 +179,7 @@ void IBridge::initialize(Application & self)
|
||||
limit.rlim_max = limit.rlim_cur = gb;
|
||||
if (setrlimit(RLIMIT_RSS, &limit))
|
||||
LOG_WARNING(log, "Unable to set maximum RSS to 1GB: {} (current rlim_cur={}, rlim_max={})",
|
||||
errnoToString(errno), limit.rlim_cur, limit.rlim_max);
|
||||
errnoToString(), limit.rlim_cur, limit.rlim_max);
|
||||
|
||||
if (!getrlimit(RLIMIT_RSS, &limit))
|
||||
LOG_INFO(log, "RSS limit: cur={}, max={}", limit.rlim_cur, limit.rlim_max);
|
||||
|
||||
@ -1952,7 +1952,7 @@ void ClientBase::runInteractive()
|
||||
|
||||
if (home_path.empty())
|
||||
{
|
||||
const char * home_path_cstr = getenv("HOME");
|
||||
const char * home_path_cstr = getenv("HOME"); // NOLINT(concurrency-mt-unsafe)
|
||||
if (home_path_cstr)
|
||||
home_path = home_path_cstr;
|
||||
}
|
||||
@ -1962,7 +1962,7 @@ void ClientBase::runInteractive()
|
||||
history_file = config().getString("history_file");
|
||||
else
|
||||
{
|
||||
auto * history_file_from_env = getenv("CLICKHOUSE_HISTORY_FILE");
|
||||
auto * history_file_from_env = getenv("CLICKHOUSE_HISTORY_FILE"); // NOLINT(concurrency-mt-unsafe)
|
||||
if (history_file_from_env)
|
||||
history_file = history_file_from_env;
|
||||
else if (!home_path.empty())
|
||||
@ -2260,13 +2260,13 @@ void ClientBase::init(int argc, char ** argv)
|
||||
if (options.count("version") || options.count("V"))
|
||||
{
|
||||
showClientVersion();
|
||||
exit(0);
|
||||
exit(0); // NOLINT(concurrency-mt-unsafe)
|
||||
}
|
||||
|
||||
if (options.count("version-clean"))
|
||||
{
|
||||
std::cout << VERSION_STRING;
|
||||
exit(0);
|
||||
exit(0); // NOLINT(concurrency-mt-unsafe)
|
||||
}
|
||||
|
||||
/// Output of help message.
|
||||
@ -2274,7 +2274,7 @@ void ClientBase::init(int argc, char ** argv)
|
||||
|| (options.count("host") && options["host"].as<std::string>() == "elp")) /// If user writes -help instead of --help.
|
||||
{
|
||||
printHelpMessage(options_description);
|
||||
exit(0);
|
||||
exit(0); // NOLINT(concurrency-mt-unsafe)
|
||||
}
|
||||
|
||||
/// Common options for clickhouse-client and clickhouse-local.
|
||||
|
||||
@ -419,7 +419,7 @@ void ConfigProcessor::doIncludesRecursive(
|
||||
XMLDocumentPtr env_document;
|
||||
auto get_env_node = [&](const std::string & name) -> const Node *
|
||||
{
|
||||
const char * env_val = std::getenv(name.c_str());
|
||||
const char * env_val = std::getenv(name.c_str()); // NOLINT(concurrency-mt-unsafe) // this is safe on Linux glibc/Musl, but potentially not safe on other platforms
|
||||
if (env_val == nullptr)
|
||||
return nullptr;
|
||||
|
||||
|
||||
@ -30,12 +30,12 @@ std::string determineDefaultTimeZone()
|
||||
{
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
const char * tzdir_env_var = std::getenv("TZDIR");
|
||||
const char * tzdir_env_var = std::getenv("TZDIR"); // NOLINT(concurrency-mt-unsafe) // ok, because it does not run concurrently with other getenv calls
|
||||
fs::path tz_database_path = tzdir_env_var ? tzdir_env_var : "/usr/share/zoneinfo/";
|
||||
|
||||
fs::path tz_file_path;
|
||||
std::string error_prefix;
|
||||
const char * tz_env_var = std::getenv("TZ");
|
||||
const char * tz_env_var = std::getenv("TZ"); // NOLINT(concurrency-mt-unsafe) // ok, because it does not run concurrently with other getenv calls
|
||||
|
||||
/// In recent tzdata packages some files now are symlinks and canonical path resolution
|
||||
/// may give wrong timezone names - store the name as it is, if possible.
|
||||
|
||||
@ -152,12 +152,12 @@ Exception::FramePointers Exception::getStackFramePointers() const
|
||||
|
||||
void throwFromErrno(const std::string & s, int code, int the_errno)
|
||||
{
|
||||
throw ErrnoException(s + ", " + errnoToString(code, the_errno), code, the_errno);
|
||||
throw ErrnoException(s + ", " + errnoToString(the_errno), code, the_errno);
|
||||
}
|
||||
|
||||
void throwFromErrnoWithPath(const std::string & s, const std::string & path, int code, int the_errno)
|
||||
{
|
||||
throw ErrnoException(s + ", " + errnoToString(code, the_errno), code, the_errno, path);
|
||||
throw ErrnoException(s + ", " + errnoToString(the_errno), code, the_errno, path);
|
||||
}
|
||||
|
||||
static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string & start_of_message)
|
||||
|
||||
@ -106,7 +106,7 @@ void LazyPipeFDs::tryIncreaseSize(int desired_size)
|
||||
{
|
||||
if (errno == EINVAL)
|
||||
{
|
||||
LOG_INFO(log, "Cannot get pipe capacity, {}. Very old Linux kernels have no support for this fcntl.", errnoToString(ErrorCodes::CANNOT_FCNTL));
|
||||
LOG_INFO(log, "Cannot get pipe capacity, {}. Very old Linux kernels have no support for this fcntl.", errnoToString());
|
||||
/// It will work nevertheless.
|
||||
}
|
||||
else
|
||||
|
||||
@ -81,7 +81,6 @@ namespace ErrorCodes
|
||||
extern const int CANNOT_SET_SIGNAL_HANDLER;
|
||||
extern const int CANNOT_CREATE_TIMER;
|
||||
extern const int CANNOT_SET_TIMER_PERIOD;
|
||||
extern const int CANNOT_DELETE_TIMER;
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
@ -188,7 +187,7 @@ void QueryProfilerBase<ProfilerImpl>::tryCleanup()
|
||||
if (timer_id.has_value())
|
||||
{
|
||||
if (timer_delete(*timer_id))
|
||||
LOG_ERROR(log, "Failed to delete query profiler timer {}", errnoToString(ErrorCodes::CANNOT_DELETE_TIMER));
|
||||
LOG_ERROR(log, "Failed to delete query profiler timer {}", errnoToString());
|
||||
timer_id.reset();
|
||||
}
|
||||
|
||||
|
||||
@ -17,7 +17,7 @@ SharedLibrary::SharedLibrary(std::string_view path, int flags)
|
||||
{
|
||||
handle = dlopen(path.data(), flags);
|
||||
if (!handle)
|
||||
throw Exception(ErrorCodes::CANNOT_DLOPEN, "Cannot dlopen: ({})", dlerror());
|
||||
throw Exception(ErrorCodes::CANNOT_DLOPEN, "Cannot dlopen: ({})", dlerror()); // NOLINT(concurrency-mt-unsafe) // MT-Safe on Linux, see man dlerror
|
||||
|
||||
updatePHDRCache();
|
||||
|
||||
@ -33,11 +33,11 @@ SharedLibrary::~SharedLibrary()
|
||||
|
||||
void * SharedLibrary::getImpl(std::string_view name, bool no_throw)
|
||||
{
|
||||
dlerror();
|
||||
dlerror(); // NOLINT(concurrency-mt-unsafe) // MT-Safe on Linux, see man dlerror
|
||||
|
||||
auto * res = dlsym(handle, name.data());
|
||||
|
||||
if (char * error = dlerror())
|
||||
if (char * error = dlerror()) // NOLINT(concurrency-mt-unsafe) // MT-Safe on Linux, see man dlerror
|
||||
{
|
||||
if (no_throw)
|
||||
return nullptr;
|
||||
|
||||
@ -76,7 +76,7 @@ ShellCommand::~ShellCommand()
|
||||
|
||||
int retcode = kill(pid, SIGTERM);
|
||||
if (retcode != 0)
|
||||
LOG_WARNING(getLogger(), "Cannot kill shell command pid {} errno '{}'", pid, errnoToString(retcode));
|
||||
LOG_WARNING(getLogger(), "Cannot kill shell command pid {} errno '{}'", pid, errnoToString());
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -201,8 +201,8 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(
|
||||
// by the child process, which might not expect this.
|
||||
sigset_t mask;
|
||||
sigemptyset(&mask);
|
||||
sigprocmask(0, nullptr, &mask);
|
||||
sigprocmask(SIG_UNBLOCK, &mask, nullptr);
|
||||
sigprocmask(0, nullptr, &mask); // NOLINT(concurrency-mt-unsafe)
|
||||
sigprocmask(SIG_UNBLOCK, &mask, nullptr); // NOLINT(concurrency-mt-unsafe)
|
||||
|
||||
execv(filename, argv);
|
||||
/// If the process is running, then `execv` does not return here.
|
||||
|
||||
@ -23,7 +23,6 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_OPEN_FILE;
|
||||
extern const int CANNOT_CLOSE_FILE;
|
||||
extern const int CANNOT_TRUNCATE_FILE;
|
||||
extern const int CANNOT_SEEK_THROUGH_FILE;
|
||||
}
|
||||
@ -98,10 +97,10 @@ StatusFile::StatusFile(std::string path_, FillFunction fill_)
|
||||
StatusFile::~StatusFile()
|
||||
{
|
||||
if (0 != close(fd))
|
||||
LOG_ERROR(&Poco::Logger::get("StatusFile"), "Cannot close file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
|
||||
LOG_ERROR(&Poco::Logger::get("StatusFile"), "Cannot close file {}, {}", path, errnoToString());
|
||||
|
||||
if (0 != unlink(path.c_str()))
|
||||
LOG_ERROR(&Poco::Logger::get("StatusFile"), "Cannot unlink file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE));
|
||||
LOG_ERROR(&Poco::Logger::get("StatusFile"), "Cannot unlink file {}, {}", path, errnoToString());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -82,7 +82,7 @@ ThreadFuzzer::ThreadFuzzer()
|
||||
template <typename T>
|
||||
static void initFromEnv(T & what, const char * name)
|
||||
{
|
||||
const char * env = getenv(name);
|
||||
const char * env = getenv(name); // NOLINT(concurrency-mt-unsafe)
|
||||
if (!env)
|
||||
return;
|
||||
what = parse<T>(env);
|
||||
@ -91,7 +91,7 @@ static void initFromEnv(T & what, const char * name)
|
||||
template <typename T>
|
||||
static void initFromEnv(std::atomic<T> & what, const char * name)
|
||||
{
|
||||
const char * env = getenv(name);
|
||||
const char * env = getenv(name); // NOLINT(concurrency-mt-unsafe)
|
||||
if (!env)
|
||||
return;
|
||||
what.store(parse<T>(env), std::memory_order_relaxed);
|
||||
|
||||
@ -301,7 +301,7 @@ static void enablePerfEvent(int event_fd)
|
||||
{
|
||||
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
|
||||
"Can't enable perf event with file descriptor {}: '{}' ({})",
|
||||
event_fd, errnoToString(errno), errno);
|
||||
event_fd, errnoToString(), errno);
|
||||
}
|
||||
}
|
||||
|
||||
@ -311,7 +311,7 @@ static void disablePerfEvent(int event_fd)
|
||||
{
|
||||
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
|
||||
"Can't disable perf event with file descriptor {}: '{}' ({})",
|
||||
event_fd, errnoToString(errno), errno);
|
||||
event_fd, errnoToString(), errno);
|
||||
}
|
||||
}
|
||||
|
||||
@ -321,7 +321,7 @@ static void releasePerfEvent(int event_fd)
|
||||
{
|
||||
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
|
||||
"Can't close perf event file descriptor {}: {} ({})",
|
||||
event_fd, errnoToString(errno), errno);
|
||||
event_fd, errnoToString(), errno);
|
||||
}
|
||||
}
|
||||
|
||||
@ -339,7 +339,7 @@ static bool validatePerfEventDescriptor(int & fd)
|
||||
{
|
||||
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
|
||||
"Error while checking availability of event descriptor {}: {} ({})",
|
||||
fd, errnoToString(errno), errno);
|
||||
fd, errnoToString(), errno);
|
||||
|
||||
disablePerfEvent(fd);
|
||||
releasePerfEvent(fd);
|
||||
@ -446,7 +446,7 @@ bool PerfEventsCounters::processThreadLocalChanges(const std::string & needed_ev
|
||||
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
|
||||
"Failed to open perf event {} (event_type={}, event_config={}): "
|
||||
"'{}' ({})", event_info.settings_name, event_info.event_type,
|
||||
event_info.event_config, errnoToString(errno), errno);
|
||||
event_info.event_config, errnoToString(), errno);
|
||||
}
|
||||
}
|
||||
|
||||
@ -532,7 +532,7 @@ void PerfEventsCounters::finalizeProfileEvents(ProfileEvents::Counters & profile
|
||||
{
|
||||
LOG_WARNING(&Poco::Logger::get("PerfEvents"),
|
||||
"Can't read event value from file descriptor {}: '{}' ({})",
|
||||
fd, errnoToString(errno), errno);
|
||||
fd, errnoToString(), errno);
|
||||
current_values[i] = {};
|
||||
}
|
||||
}
|
||||
|
||||
@ -120,7 +120,7 @@ ThreadStatus::ThreadStatus()
|
||||
|
||||
if (0 != sigaltstack(&altstack_description, nullptr))
|
||||
{
|
||||
LOG_WARNING(log, "Cannot set alternative signal stack for thread, {}", errnoToString(errno));
|
||||
LOG_WARNING(log, "Cannot set alternative signal stack for thread, {}", errnoToString());
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -128,7 +128,7 @@ ThreadStatus::ThreadStatus()
|
||||
struct sigaction action{};
|
||||
if (0 != sigaction(SIGSEGV, nullptr, &action))
|
||||
{
|
||||
LOG_WARNING(log, "Cannot obtain previous signal action to set alternative signal stack for thread, {}", errnoToString(errno));
|
||||
LOG_WARNING(log, "Cannot obtain previous signal action to set alternative signal stack for thread, {}", errnoToString());
|
||||
}
|
||||
else if (!(action.sa_flags & SA_ONSTACK))
|
||||
{
|
||||
@ -136,7 +136,7 @@ ThreadStatus::ThreadStatus()
|
||||
|
||||
if (0 != sigaction(SIGSEGV, &action, nullptr))
|
||||
{
|
||||
LOG_WARNING(log, "Cannot set action with alternative signal stack for thread, {}", errnoToString(errno));
|
||||
LOG_WARNING(log, "Cannot set action with alternative signal stack for thread, {}", errnoToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -6,6 +6,3 @@ target_link_libraries(zkutil_test_commands_new_lib PRIVATE clickhouse_common_zoo
|
||||
|
||||
clickhouse_add_executable(zkutil_test_async zkutil_test_async.cpp)
|
||||
target_link_libraries(zkutil_test_async PRIVATE clickhouse_common_zookeeper_no_log)
|
||||
|
||||
clickhouse_add_executable (zookeeper_impl zookeeper_impl.cpp)
|
||||
target_link_libraries (zookeeper_impl PRIVATE clickhouse_common_zookeeper_no_log)
|
||||
|
||||
@ -1,26 +0,0 @@
|
||||
#include <Common/ZooKeeper/ZooKeeperImpl.h>
|
||||
#include <iostream>
|
||||
|
||||
|
||||
int main()
|
||||
try
|
||||
{
|
||||
Coordination::ZooKeeper zookeeper({Coordination::ZooKeeper::Node{Poco::Net::SocketAddress{"localhost:2181"}, false}}, "", "", "", {30, 0}, {0, 50000}, {0, 50000}, nullptr);
|
||||
|
||||
zookeeper.create("/test", "hello", false, false, {}, [](const Coordination::CreateResponse & response)
|
||||
{
|
||||
if (response.error != Coordination::Error::ZOK)
|
||||
std::cerr << "Error: " << Coordination::errorMessage(response.error) << "\n";
|
||||
else
|
||||
std::cerr << "Path created: " << response.path_created << "\n";
|
||||
});
|
||||
|
||||
sleep(100);
|
||||
|
||||
return 0;
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
DB::tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
return 1;
|
||||
}
|
||||
@ -13,6 +13,8 @@
|
||||
#include <array>
|
||||
#include <sys/resource.h>
|
||||
#include <base/bit_cast.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <pcg_random.hpp>
|
||||
|
||||
#include <base/StringRef.h>
|
||||
#include <base/arraySize.h>
|
||||
@ -218,6 +220,7 @@ int main(int argc, char ** argv)
|
||||
}
|
||||
|
||||
std::cerr << std::fixed << std::setprecision(2);
|
||||
pcg64 rng(randomSeed());
|
||||
|
||||
size_t n = parse<size_t>(argv[1]);
|
||||
std::vector<std::string> data;
|
||||
@ -281,8 +284,8 @@ int main(int argc, char ** argv)
|
||||
size_t bytes = 0;
|
||||
for (size_t i = 0, size = data.size(); i < size; ++i)
|
||||
{
|
||||
size_t index_from = lrand48() % size;
|
||||
size_t index_to = lrand48() % size;
|
||||
size_t index_from = rng() % size;
|
||||
size_t index_to = rng() % size;
|
||||
|
||||
arena.free(const_cast<char *>(refs[index_to].data), refs[index_to].size);
|
||||
const auto & s = data[index_from];
|
||||
@ -318,8 +321,8 @@ int main(int argc, char ** argv)
|
||||
size_t bytes = 0;
|
||||
for (size_t i = 0, size = data.size(); i < size; ++i)
|
||||
{
|
||||
size_t index_from = lrand48() % size;
|
||||
size_t index_to = lrand48() % cache_size;
|
||||
size_t index_from = rng() % size;
|
||||
size_t index_to = rng() % cache_size;
|
||||
|
||||
dictionary.setAttributeValue(attr, index_to, data[index_from]);
|
||||
|
||||
|
||||
@ -8,9 +8,11 @@
|
||||
|
||||
#include <iostream>
|
||||
#include <iomanip>
|
||||
#include <pcg_random.hpp>
|
||||
#include <Poco/Exception.h>
|
||||
#include <Common/HashTable/Hash.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <Core/Defines.h>
|
||||
|
||||
|
||||
@ -266,9 +268,9 @@ int main(int argc, char ** argv)
|
||||
{
|
||||
Stopwatch watch;
|
||||
|
||||
srand48(rdtsc());
|
||||
pcg64 rng(randomSeed());
|
||||
for (size_t i = 0; i < BUF_SIZE; ++i)
|
||||
data[i] = lrand48();
|
||||
data[i] = rng();
|
||||
|
||||
watch.stop();
|
||||
double elapsed = watch.elapsedSeconds();
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
#include <cerrno>
|
||||
#include <base/errnoToString.h>
|
||||
#include <future>
|
||||
#include <Coordination/KeeperSnapshotManager.h>
|
||||
#include <Coordination/KeeperStateMachine.h>
|
||||
@ -11,6 +12,7 @@
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include "Coordination/KeeperStorage.h"
|
||||
|
||||
|
||||
namespace ProfileEvents
|
||||
{
|
||||
extern const Event KeeperCommits;
|
||||
@ -446,7 +448,7 @@ static int bufferFromFile(Poco::Logger * log, const std::string & path, nuraft::
|
||||
LOG_INFO(log, "Opening file {} for read_logical_snp_obj", path);
|
||||
if (fd < 0)
|
||||
{
|
||||
LOG_WARNING(log, "Error opening {}, error: {}, errno: {}", path, std::strerror(errno), errno);
|
||||
LOG_WARNING(log, "Error opening {}, error: {}, errno: {}", path, errnoToString(), errno);
|
||||
return errno;
|
||||
}
|
||||
auto file_size = ::lseek(fd, 0, SEEK_END);
|
||||
@ -454,7 +456,7 @@ static int bufferFromFile(Poco::Logger * log, const std::string & path, nuraft::
|
||||
auto * chunk = reinterpret_cast<nuraft::byte *>(::mmap(nullptr, file_size, PROT_READ, MAP_FILE | MAP_SHARED, fd, 0));
|
||||
if (chunk == MAP_FAILED)
|
||||
{
|
||||
LOG_WARNING(log, "Error mmapping {}, error: {}, errno: {}", path, std::strerror(errno), errno);
|
||||
LOG_WARNING(log, "Error mmapping {}, error: {}, errno: {}", path, errnoToString(), errno);
|
||||
::close(fd);
|
||||
return errno;
|
||||
}
|
||||
|
||||
@ -137,7 +137,7 @@ struct SimpliestRaftServer
|
||||
if (!raft_instance)
|
||||
{
|
||||
std::cerr << "Failed to initialize launcher" << std::endl;
|
||||
exit(-1);
|
||||
_exit(1);
|
||||
}
|
||||
|
||||
std::cout << "init Raft instance " << server_id;
|
||||
|
||||
@ -4,12 +4,14 @@
|
||||
|
||||
#include <Daemon/BaseDaemon.h>
|
||||
#include <Daemon/SentryWriter.h>
|
||||
#include <base/errnoToString.h>
|
||||
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/wait.h>
|
||||
#include <sys/resource.h>
|
||||
|
||||
#if defined(OS_LINUX)
|
||||
#include <sys/prctl.h>
|
||||
#endif
|
||||
@ -315,13 +317,13 @@ private:
|
||||
{
|
||||
LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (no query) Received signal {} ({})",
|
||||
VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info,
|
||||
thread_num, strsignal(sig), sig);
|
||||
thread_num, strsignal(sig), sig); // NOLINT(concurrency-mt-unsafe) // it is not thread-safe but ok in this context
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (query_id: {}) (query: {}) Received signal {} ({})",
|
||||
VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info,
|
||||
thread_num, query_id, query, strsignal(sig), sig);
|
||||
thread_num, query_id, query, strsignal(sig), sig); // NOLINT(concurrency-mt-unsafe) // it is not thread-safe but ok in this context)
|
||||
}
|
||||
|
||||
String error_message;
|
||||
@ -665,7 +667,7 @@ void BaseDaemon::initialize(Application & self)
|
||||
if (config().has("timezone"))
|
||||
{
|
||||
const std::string config_timezone = config().getString("timezone");
|
||||
if (0 != setenv("TZ", config_timezone.data(), 1))
|
||||
if (0 != setenv("TZ", config_timezone.data(), 1)) // NOLINT(concurrency-mt-unsafe) // ok if not called concurrently with other setenv/getenv
|
||||
throw Poco::Exception("Cannot setenv TZ variable");
|
||||
|
||||
tzset();
|
||||
@ -940,13 +942,13 @@ void BaseDaemon::handleSignal(int signal_id)
|
||||
onInterruptSignals(signal_id);
|
||||
}
|
||||
else
|
||||
throw DB::Exception(std::string("Unsupported signal: ") + strsignal(signal_id), 0);
|
||||
throw DB::Exception(std::string("Unsupported signal: ") + strsignal(signal_id), 0); // NOLINT(concurrency-mt-unsafe) // it is not thread-safe but ok in this context
|
||||
}
|
||||
|
||||
void BaseDaemon::onInterruptSignals(int signal_id)
|
||||
{
|
||||
is_cancelled = true;
|
||||
LOG_INFO(&logger(), "Received termination signal ({})", strsignal(signal_id));
|
||||
LOG_INFO(&logger(), "Received termination signal ({})", strsignal(signal_id)); // NOLINT(concurrency-mt-unsafe) // it is not thread-safe but ok in this context
|
||||
|
||||
if (sigint_signals_counter >= 2)
|
||||
{
|
||||
@ -1064,7 +1066,7 @@ void BaseDaemon::setupWatchdog()
|
||||
break;
|
||||
}
|
||||
else if (errno != EINTR)
|
||||
throw Poco::Exception("Cannot waitpid, errno: " + std::string(strerror(errno)));
|
||||
throw Poco::Exception("Cannot waitpid, errno: " + errnoToString());
|
||||
} while (true);
|
||||
|
||||
if (errno == ECHILD)
|
||||
|
||||
@ -146,7 +146,7 @@ void SentryWriter::onFault(int sig, const std::string & error_message, const Sta
|
||||
if (initialized)
|
||||
{
|
||||
sentry_value_t event = sentry_value_new_message_event(SENTRY_LEVEL_FATAL, "fault", error_message.c_str());
|
||||
sentry_set_tag("signal", strsignal(sig));
|
||||
sentry_set_tag("signal", strsignal(sig)); // NOLINT(concurrency-mt-unsafe) // not thread-safe but ok in this context
|
||||
sentry_set_extra("signal_number", sentry_value_new_int32(sig));
|
||||
|
||||
#if defined(__ELF__) && !defined(OS_FREEBSD)
|
||||
|
||||
@ -165,8 +165,7 @@ std::future<IAsynchronousReader::Result> ThreadPoolReader::submit(Request reques
|
||||
{
|
||||
ProfileEvents::increment(ProfileEvents::ReadBufferFromFileDescriptorReadFailed);
|
||||
promise.set_exception(std::make_exception_ptr(ErrnoException(
|
||||
fmt::format("Cannot read from file {}, {}", fd,
|
||||
errnoToString(ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR, errno)),
|
||||
fmt::format("Cannot read from file {}, {}", fd, errnoToString()),
|
||||
ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR, errno)));
|
||||
return future;
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
#include <IO/Archives/ZipArchiveWriter.h>
|
||||
#include <IO/ReadBufferFromFileBase.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include <base/errnoToString.h>
|
||||
#include <unzip.h>
|
||||
|
||||
|
||||
@ -553,11 +554,11 @@ void ZipArchiveReader::checkResult(int code) const
|
||||
if (code >= UNZ_OK)
|
||||
return;
|
||||
|
||||
String message = "Code= ";
|
||||
String message = "Code = ";
|
||||
switch (code)
|
||||
{
|
||||
case UNZ_OK: return;
|
||||
case UNZ_ERRNO: message += "ERRNO, errno= " + String{strerror(errno)}; break;
|
||||
case UNZ_ERRNO: message += "ERRNO, errno = " + errnoToString(); break;
|
||||
case UNZ_PARAMERROR: message += "PARAMERROR"; break;
|
||||
case UNZ_BADZIPFILE: message += "BADZIPFILE"; break;
|
||||
case UNZ_INTERNALERROR: message += "INTERNALERROR"; break;
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#if USE_MINIZIP
|
||||
#include <IO/WriteBufferFromFileBase.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include <base/errnoToString.h>
|
||||
#include <zip.h>
|
||||
#include <boost/algorithm/string/predicate.hpp>
|
||||
|
||||
@ -380,10 +381,10 @@ void ZipArchiveWriter::checkResult(int code) const
|
||||
if (code >= ZIP_OK)
|
||||
return;
|
||||
|
||||
String message = "Code= ";
|
||||
String message = "Code = ";
|
||||
switch (code)
|
||||
{
|
||||
case ZIP_ERRNO: message += "ERRNO, errno= " + String{strerror(errno)}; break;
|
||||
case ZIP_ERRNO: message += "ERRNO, errno = " + errnoToString(); break;
|
||||
case ZIP_PARAMERROR: message += "PARAMERROR"; break;
|
||||
case ZIP_BADZIPFILE: message += "BADZIPFILE"; break;
|
||||
case ZIP_INTERNALERROR: message += "INTERNALERROR"; break;
|
||||
|
||||
@ -37,12 +37,6 @@ target_link_libraries (read_write_int PRIVATE clickhouse_common_io)
|
||||
clickhouse_add_executable (o_direct_and_dirty_pages o_direct_and_dirty_pages.cpp)
|
||||
target_link_libraries (o_direct_and_dirty_pages PRIVATE clickhouse_common_io)
|
||||
|
||||
clickhouse_add_executable (hashing_write_buffer hashing_write_buffer.cpp)
|
||||
target_link_libraries (hashing_write_buffer PRIVATE clickhouse_common_io)
|
||||
|
||||
clickhouse_add_executable (hashing_read_buffer hashing_read_buffer.cpp)
|
||||
target_link_libraries (hashing_read_buffer PRIVATE clickhouse_common_io)
|
||||
|
||||
clickhouse_add_executable (io_operators io_operators.cpp)
|
||||
target_link_libraries (io_operators PRIVATE clickhouse_common_io)
|
||||
|
||||
|
||||
@ -1,21 +0,0 @@
|
||||
#pragma once
|
||||
#include <IO/HashingWriteBuffer.h>
|
||||
#include <IO/WriteBufferFromFile.h>
|
||||
|
||||
#define FAIL(msg) do { std::cout << msg; exit(1); } while (false)
|
||||
|
||||
|
||||
static CityHash_v1_0_2::uint128 referenceHash(const char * data, size_t len)
|
||||
{
|
||||
const size_t block_size = DBMS_DEFAULT_HASHING_BLOCK_SIZE;
|
||||
CityHash_v1_0_2::uint128 state(0, 0);
|
||||
size_t pos;
|
||||
|
||||
for (pos = 0; pos + block_size <= len; pos += block_size)
|
||||
state = CityHash_v1_0_2::CityHash128WithSeed(data + pos, block_size, state);
|
||||
|
||||
if (pos < len)
|
||||
state = CityHash_v1_0_2::CityHash128WithSeed(data + pos, len - pos, state);
|
||||
|
||||
return state;
|
||||
}
|
||||
@ -1,69 +0,0 @@
|
||||
#include <IO/ReadBufferFromIStream.h>
|
||||
#include <IO/HashingReadBuffer.h>
|
||||
#include <IO/WriteBufferFromOStream.h>
|
||||
#include "hashing_buffer.h"
|
||||
#include <iostream>
|
||||
#include <pcg_random.hpp>
|
||||
|
||||
|
||||
static void test(size_t data_size)
|
||||
{
|
||||
pcg64 rng;
|
||||
|
||||
std::vector<char> vec(data_size);
|
||||
char * data = vec.data();
|
||||
|
||||
for (size_t i = 0; i < data_size; ++i)
|
||||
data[i] = rng() & 255;
|
||||
|
||||
CityHash_v1_0_2::uint128 reference = referenceHash(data, data_size);
|
||||
|
||||
std::vector<size_t> block_sizes = {56, 128, 513, 2048, 3055, 4097, 4096};
|
||||
for (size_t read_buffer_block_size : block_sizes)
|
||||
{
|
||||
std::cout << "block size " << read_buffer_block_size << std::endl;
|
||||
std::stringstream io; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
|
||||
io.exceptions(std::ios::failbit);
|
||||
DB::WriteBufferFromOStream out_impl(io);
|
||||
DB::HashingWriteBuffer out(out_impl);
|
||||
out.write(data, data_size);
|
||||
out.next();
|
||||
|
||||
DB::ReadBufferFromIStream source(io, read_buffer_block_size);
|
||||
DB::HashingReadBuffer buf(source);
|
||||
|
||||
std::vector<char> read_buf(data_size);
|
||||
buf.read(read_buf.data(), data_size);
|
||||
|
||||
bool failed_to_read = false;
|
||||
for (size_t i = 0; i < data_size; ++i)
|
||||
if (read_buf[i] != vec[i])
|
||||
failed_to_read = true;
|
||||
|
||||
if (failed_to_read)
|
||||
{
|
||||
std::cout.write(data, data_size);
|
||||
std::cout << std::endl;
|
||||
std::cout.write(read_buf.data(), data_size);
|
||||
std::cout << std::endl;
|
||||
FAIL("Fail to read data");
|
||||
}
|
||||
|
||||
if (buf.getHash() != reference)
|
||||
FAIL("failed on data size " << data_size << " reading by blocks of size " << read_buffer_block_size);
|
||||
if (buf.getHash() != out.getHash())
|
||||
FAIL("Hash of HashingReadBuffer doesn't match with hash of HashingWriteBuffer on data size " << data_size << " reading by blocks of size " << read_buffer_block_size);
|
||||
}
|
||||
}
|
||||
|
||||
int main()
|
||||
{
|
||||
test(5);
|
||||
test(100);
|
||||
test(2048);
|
||||
test(2049);
|
||||
test(100000);
|
||||
test(1 << 17);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -1,86 +0,0 @@
|
||||
#include <IO/HashingWriteBuffer.h>
|
||||
#include <IO/WriteBufferFromFile.h>
|
||||
#include <pcg_random.hpp>
|
||||
|
||||
#include "hashing_buffer.h"
|
||||
|
||||
static void test(size_t data_size)
|
||||
{
|
||||
pcg64 rng;
|
||||
|
||||
std::vector<char> vec(data_size);
|
||||
char * data = vec.data();
|
||||
|
||||
for (size_t i = 0; i < data_size; ++i)
|
||||
data[i] = rng() & 255;
|
||||
|
||||
CityHash_v1_0_2::uint128 reference = referenceHash(data, data_size);
|
||||
|
||||
DB::WriteBufferFromFile sink("/dev/null", 1 << 16);
|
||||
|
||||
{
|
||||
DB::HashingWriteBuffer buf(sink);
|
||||
|
||||
for (size_t pos = 0; pos < data_size;)
|
||||
{
|
||||
size_t len = std::min(static_cast<size_t>(rng() % 10000 + 1), data_size - pos);
|
||||
buf.write(data + pos, len);
|
||||
buf.next();
|
||||
pos += len;
|
||||
}
|
||||
|
||||
if (buf.getHash() != reference)
|
||||
FAIL("failed on data size " << data_size << " writing rngom chunks of up to 10000 bytes");
|
||||
}
|
||||
|
||||
{
|
||||
DB::HashingWriteBuffer buf(sink);
|
||||
|
||||
for (size_t pos = 0; pos < data_size;)
|
||||
{
|
||||
size_t len = std::min(static_cast<size_t>(rng() % 5 + 1), data_size - pos);
|
||||
buf.write(data + pos, len);
|
||||
buf.next();
|
||||
pos += len;
|
||||
}
|
||||
|
||||
if (buf.getHash() != reference)
|
||||
FAIL("failed on data size " << data_size << " writing rngom chunks of up to 5 bytes");
|
||||
}
|
||||
|
||||
{
|
||||
DB::HashingWriteBuffer buf(sink);
|
||||
|
||||
for (size_t pos = 0; pos < data_size;)
|
||||
{
|
||||
size_t len = std::min(static_cast<size_t>(2048 + rng() % 3 - 1), data_size - pos);
|
||||
buf.write(data + pos, len);
|
||||
buf.next();
|
||||
pos += len;
|
||||
}
|
||||
|
||||
if (buf.getHash() != reference)
|
||||
FAIL("failed on data size " << data_size << " writing rngom chunks of 2048 +-1 bytes");
|
||||
}
|
||||
|
||||
{
|
||||
DB::HashingWriteBuffer buf(sink);
|
||||
|
||||
buf.write(data, data_size);
|
||||
|
||||
if (buf.getHash() != reference)
|
||||
FAIL("failed on data size " << data_size << " writing all at once");
|
||||
}
|
||||
}
|
||||
|
||||
int main()
|
||||
{
|
||||
test(5);
|
||||
test(100);
|
||||
test(2048);
|
||||
test(2049);
|
||||
test(100000);
|
||||
test(1 << 17);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -1,5 +1,6 @@
|
||||
#include <iostream>
|
||||
#include <iomanip>
|
||||
#include <pcg_random.hpp>
|
||||
|
||||
#include <base/types.h>
|
||||
|
||||
@ -7,7 +8,6 @@
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/WriteIntText.h>
|
||||
#include <IO/WriteBufferFromVector.h>
|
||||
#include <Compression/CompressedReadBuffer.h>
|
||||
|
||||
#include <Common/Stopwatch.h>
|
||||
|
||||
@ -27,6 +27,8 @@ static UInt64 rdtsc()
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
{
|
||||
pcg64 rng;
|
||||
|
||||
try
|
||||
{
|
||||
if (argc < 2)
|
||||
@ -47,7 +49,7 @@ int main(int argc, char ** argv)
|
||||
Stopwatch watch;
|
||||
|
||||
for (size_t i = 0; i < n; ++i)
|
||||
data[i] = lrand48();// / lrand48();// ^ (lrand48() << 24) ^ (lrand48() << 48);
|
||||
data[i] = rng();
|
||||
|
||||
watch.stop();
|
||||
std::cerr << std::fixed << std::setprecision(2)
|
||||
|
||||
@ -8,32 +8,8 @@
|
||||
#include <Poco/HexBinaryEncoder.h>
|
||||
|
||||
|
||||
static void parse_trash_string_as_uint_must_fail(const std::string & str)
|
||||
{
|
||||
using namespace DB;
|
||||
|
||||
unsigned x = 0xFF;
|
||||
|
||||
try
|
||||
{
|
||||
x = parse<unsigned>(str);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
/// Ok
|
||||
return;
|
||||
}
|
||||
|
||||
std::cerr << "Parsing must fail, but finished successfully x=" << x;
|
||||
exit(-1);
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
{
|
||||
parse_trash_string_as_uint_must_fail("trash");
|
||||
parse_trash_string_as_uint_must_fail("-1");
|
||||
|
||||
if (argc != 2)
|
||||
{
|
||||
std::cerr << "Usage: " << std::endl
|
||||
|
||||
@ -416,7 +416,7 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits)
|
||||
LOG_TRACE(log, "Resetting nice");
|
||||
|
||||
if (0 != setpriority(PRIO_PROCESS, thread_id, 0))
|
||||
LOG_ERROR(log, "Cannot 'setpriority' back to zero: {}", errnoToString(ErrorCodes::CANNOT_SET_THREAD_PRIORITY, errno));
|
||||
LOG_ERROR(log, "Cannot 'setpriority' back to zero: {}", errnoToString());
|
||||
|
||||
os_thread_priority = 0;
|
||||
}
|
||||
|
||||
@ -12,6 +12,8 @@
|
||||
#include <AggregateFunctions/registerAggregateFunctions.h>
|
||||
#include <Processors/Merges/Algorithms/Graphite.h>
|
||||
#include <Common/Config/ConfigProcessor.h>
|
||||
#include <base/errnoToString.h>
|
||||
|
||||
|
||||
using namespace DB;
|
||||
|
||||
@ -43,7 +45,7 @@ static ConfigProcessor::LoadedConfig loadConfigurationFromString(std::string & s
|
||||
int fd = mkstemp(tmp_file);
|
||||
if (fd == -1)
|
||||
{
|
||||
throw std::runtime_error(strerror(errno));
|
||||
throw std::runtime_error(errnoToString());
|
||||
}
|
||||
try
|
||||
{
|
||||
@ -61,7 +63,7 @@ static ConfigProcessor::LoadedConfig loadConfigurationFromString(std::string & s
|
||||
{
|
||||
int err = errno;
|
||||
(void)remove(tmp_file);
|
||||
throw std::runtime_error(strerror(err));
|
||||
throw std::runtime_error(errnoToString(err));
|
||||
}
|
||||
ConfigProcessor::LoadedConfig config = loadConfiguration(config_path);
|
||||
(void)remove(tmp_file);
|
||||
|
||||
@ -23,12 +23,6 @@ int callSetCertificate(SSL * ssl, [[maybe_unused]] void * arg)
|
||||
}
|
||||
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_STAT;
|
||||
}
|
||||
|
||||
|
||||
/// This is callback for OpenSSL. It will be called on every connection to obtain a certificate and private key.
|
||||
int CertificateReloader::setCertificate(SSL * ssl)
|
||||
{
|
||||
@ -118,7 +112,7 @@ bool CertificateReloader::File::changeIfModified(std::string new_path, Poco::Log
|
||||
if (ec)
|
||||
{
|
||||
LOG_ERROR(logger, "Cannot obtain modification time for {} file {}, skipping update. {}",
|
||||
description, new_path, errnoToString(ErrorCodes::CANNOT_STAT, ec.value()));
|
||||
description, new_path, errnoToString(ec.value()));
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@ -7,15 +7,17 @@
|
||||
#if USE_HDFS
|
||||
#include <Common/ShellCommand.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
#include <IO/Operators.h>
|
||||
#include <Common/logger_useful.h>
|
||||
|
||||
#if USE_KRB5
|
||||
#include <Access/KerberosInit.h>
|
||||
#endif // USE_KRB5
|
||||
#include <Access/KerberosInit.h>
|
||||
#endif
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int BAD_ARGUMENTS;
|
||||
@ -26,10 +28,15 @@ namespace ErrorCodes
|
||||
#endif // USE_KRB5
|
||||
}
|
||||
|
||||
const String HDFSBuilderWrapper::CONFIG_PREFIX = "hdfs";
|
||||
const String HDFS_URL_REGEXP = "^hdfs://[^/]*/.*";
|
||||
static constexpr std::string_view CONFIG_PREFIX = "hdfs";
|
||||
static constexpr std::string_view HDFS_URL_REGEXP = "^hdfs://[^/]*/.*";
|
||||
|
||||
|
||||
HDFSFileInfo::~HDFSFileInfo()
|
||||
{
|
||||
hdfsFreeFileInfo(file_info, length);
|
||||
}
|
||||
|
||||
std::once_flag init_libhdfs3_conf_flag;
|
||||
|
||||
void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration & config,
|
||||
const String & prefix, bool isUser)
|
||||
@ -111,23 +118,6 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A
|
||||
if (host.empty())
|
||||
throw Exception("Illegal HDFS URI: " + uri.toString(), ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
// Shall set env LIBHDFS3_CONF *before* HDFSBuilderWrapper construction.
|
||||
std::call_once(init_libhdfs3_conf_flag, [&config]()
|
||||
{
|
||||
String libhdfs3_conf = config.getString(HDFSBuilderWrapper::CONFIG_PREFIX + ".libhdfs3_conf", "");
|
||||
if (!libhdfs3_conf.empty())
|
||||
{
|
||||
if (std::filesystem::path{libhdfs3_conf}.is_relative() && !std::filesystem::exists(libhdfs3_conf))
|
||||
{
|
||||
const String config_path = config.getString("config-file", "config.xml");
|
||||
const auto config_dir = std::filesystem::path{config_path}.remove_filename();
|
||||
if (std::filesystem::exists(config_dir / libhdfs3_conf))
|
||||
libhdfs3_conf = std::filesystem::absolute(config_dir / libhdfs3_conf);
|
||||
}
|
||||
setenv("LIBHDFS3_CONF", libhdfs3_conf.c_str(), 1);
|
||||
}
|
||||
});
|
||||
|
||||
HDFSBuilderWrapper builder;
|
||||
if (builder.get() == nullptr)
|
||||
throw Exception("Unable to create builder to connect to HDFS: " +
|
||||
@ -157,14 +147,14 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A
|
||||
hdfsBuilderSetNameNodePort(builder.get(), port);
|
||||
}
|
||||
|
||||
if (config.has(HDFSBuilderWrapper::CONFIG_PREFIX))
|
||||
if (config.has(std::string(CONFIG_PREFIX)))
|
||||
{
|
||||
builder.loadFromConfig(config, HDFSBuilderWrapper::CONFIG_PREFIX);
|
||||
builder.loadFromConfig(config, std::string(CONFIG_PREFIX));
|
||||
}
|
||||
|
||||
if (!user.empty())
|
||||
{
|
||||
String user_config_prefix = HDFSBuilderWrapper::CONFIG_PREFIX + "_" + user;
|
||||
String user_config_prefix = std::string(CONFIG_PREFIX) + "_" + user;
|
||||
if (config.has(user_config_prefix))
|
||||
{
|
||||
builder.loadFromConfig(config, user_config_prefix, true);
|
||||
@ -208,7 +198,7 @@ String getNameNodeCluster(const String &hdfs_url)
|
||||
|
||||
void checkHDFSURL(const String & url)
|
||||
{
|
||||
if (!re2::RE2::FullMatch(url, HDFS_URL_REGEXP))
|
||||
if (!re2::RE2::FullMatch(url, std::string(HDFS_URL_REGEXP)))
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs url: {}. It should have structure 'hdfs://<host_name>:<port>/<path>'", url);
|
||||
}
|
||||
|
||||
|
||||
@ -40,11 +40,7 @@ struct HDFSFileInfo
|
||||
HDFSFileInfo(HDFSFileInfo && other) = default;
|
||||
HDFSFileInfo & operator=(const HDFSFileInfo & other) = delete;
|
||||
HDFSFileInfo & operator=(HDFSFileInfo && other) = default;
|
||||
|
||||
~HDFSFileInfo()
|
||||
{
|
||||
hdfsFreeFileInfo(file_info, length);
|
||||
}
|
||||
~HDFSFileInfo();
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -18,7 +18,7 @@ NATSConnectionManager::NATSConnectionManager(const NATSConfiguration & configura
|
||||
, log(log_)
|
||||
, event_handler(loop.getLoop(), log)
|
||||
{
|
||||
const char * val = std::getenv("CLICKHOUSE_NATS_TLS_SECURE");
|
||||
const char * val = std::getenv("CLICKHOUSE_NATS_TLS_SECURE"); // NOLINT(concurrency-mt-unsafe) // this is safe on Linux glibc/Musl, but potentially not safe on other platforms
|
||||
std::string tls_secure = val == nullptr ? std::string("1") : std::string(val);
|
||||
if (tls_secure == "0")
|
||||
skip_verification = true;
|
||||
|
||||
77
tests/queries/0_stateless/02293_part_log_has_merge_reason.sh
Executable file
77
tests/queries/0_stateless/02293_part_log_has_merge_reason.sh
Executable file
@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q 'DROP TABLE IF EXISTS t_part_log_has_merge_type_table'
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q '
|
||||
CREATE TABLE t_part_log_has_merge_type_table
|
||||
(
|
||||
event_time DateTime,
|
||||
UserID UInt64,
|
||||
Comment String
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY tuple()
|
||||
TTL event_time + INTERVAL 3 MONTH
|
||||
SETTINGS min_bytes_for_wide_part = 0, materialize_ttl_recalculate_only = true, max_number_of_merges_with_ttl_in_pool = 100
|
||||
'
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "INSERT INTO t_part_log_has_merge_type_table VALUES (now(), 1, 'username1');"
|
||||
${CLICKHOUSE_CLIENT} -q "INSERT INTO t_part_log_has_merge_type_table VALUES (now() - INTERVAL 4 MONTH, 2, 'username2');"
|
||||
|
||||
function get_parts_count() {
|
||||
table_name=$1
|
||||
${CLICKHOUSE_CLIENT} -q '
|
||||
SELECT
|
||||
count(*)
|
||||
FROM
|
||||
system.parts
|
||||
WHERE
|
||||
table = '"'${table_name}'"'
|
||||
AND
|
||||
active = 1
|
||||
AND
|
||||
database = '"'${CLICKHOUSE_DATABASE}'"'
|
||||
'
|
||||
}
|
||||
|
||||
function wait_table_parts_are_merged_into_one_part() {
|
||||
table_name=$1
|
||||
|
||||
while true
|
||||
do
|
||||
count=$(get_parts_count $table_name)
|
||||
if [[ count -gt 1 ]]
|
||||
then
|
||||
sleep 1
|
||||
else
|
||||
break
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
export -f get_parts_count
|
||||
export -f wait_table_parts_are_merged_into_one_part
|
||||
|
||||
timeout 30 bash -c 'wait_table_parts_are_merged_into_one_part t_part_log_has_merge_type_table'
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q 'SYSTEM FLUSH LOGS'
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q '
|
||||
SELECT
|
||||
event_type,
|
||||
merge_reason
|
||||
FROM
|
||||
system.part_log
|
||||
WHERE
|
||||
event_type = '"'MergeParts'"'
|
||||
AND
|
||||
table = '"'t_part_log_has_merge_type_table'"'
|
||||
AND
|
||||
database = '"'${CLICKHOUSE_DATABASE}'"'
|
||||
ORDER BY event_type, merge_reason
|
||||
'
|
||||
@ -1,35 +0,0 @@
|
||||
-- Tags: disabled
|
||||
|
||||
DROP TABLE IF EXISTS t_part_log_has_merge_type_table;
|
||||
|
||||
CREATE TABLE t_part_log_has_merge_type_table
|
||||
(
|
||||
event_time DateTime,
|
||||
UserID UInt64,
|
||||
Comment String
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY tuple()
|
||||
TTL event_time + INTERVAL 3 MONTH
|
||||
SETTINGS min_bytes_for_wide_part = 0, materialize_ttl_recalculate_only = true, max_number_of_merges_with_ttl_in_pool = 100;
|
||||
|
||||
INSERT INTO t_part_log_has_merge_type_table VALUES (now(), 1, 'username1');
|
||||
INSERT INTO t_part_log_has_merge_type_table VALUES (now() - INTERVAL 4 MONTH, 2, 'username2');
|
||||
|
||||
OPTIMIZE TABLE t_part_log_has_merge_type_table FINAL;
|
||||
|
||||
SYSTEM FLUSH LOGS;
|
||||
|
||||
SELECT
|
||||
event_type,
|
||||
merge_reason
|
||||
FROM
|
||||
system.part_log
|
||||
WHERE
|
||||
table = 't_part_log_has_merge_type_table'
|
||||
AND
|
||||
merge_reason = 'TTLDeleteMerge'
|
||||
AND
|
||||
database = currentDatabase();
|
||||
|
||||
DROP TABLE t_part_log_has_merge_type_table;
|
||||
@ -23,7 +23,6 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS)
|
||||
add_subdirectory (iotest)
|
||||
add_subdirectory (corrector_utf8)
|
||||
add_subdirectory (zookeeper-cli)
|
||||
add_subdirectory (zookeeper-test)
|
||||
add_subdirectory (zookeeper-dump-tree)
|
||||
add_subdirectory (zookeeper-remove-by-list)
|
||||
add_subdirectory (zookeeper-create-entry-to-download-part)
|
||||
@ -35,7 +34,6 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS)
|
||||
add_subdirectory (wal-dump)
|
||||
add_subdirectory (check-mysql-binlog)
|
||||
add_subdirectory (keeper-bench)
|
||||
add_subdirectory (graphite-rollup)
|
||||
|
||||
if (TARGET ch_contrib::nuraft)
|
||||
add_subdirectory (keeper-data-dumper)
|
||||
|
||||
@ -1,23 +0,0 @@
|
||||
clickhouse_add_executable(graphite-rollup-bench graphite-rollup-bench.cpp)
|
||||
target_link_libraries(
|
||||
graphite-rollup-bench
|
||||
PRIVATE
|
||||
clickhouse_storages_system
|
||||
clickhouse_aggregate_functions
|
||||
clickhouse_common_config
|
||||
dbms
|
||||
)
|
||||
target_include_directories(
|
||||
graphite-rollup-bench
|
||||
SYSTEM PRIVATE
|
||||
${ClickHouse_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src
|
||||
${ClickHouse_SOURCE_DIR}/base ${ClickHouse_SOURCE_DIR}/base/pcg-random
|
||||
${CMAKE_BINARY_DIR}/src/Core/include
|
||||
${POCO_INCLUDE_DIR}
|
||||
${ClickHouse_SOURCE_DIR}/contrib/double-conversion ${ClickHouse_SOURCE_DIR}/contrib/dragonbox/include
|
||||
${ClickHouse_SOURCE_DIR}/contrib/fmtlib/include
|
||||
${ClickHouse_SOURCE_DIR}/contrib/cityhash102/include
|
||||
${RE2_INCLUDE_DIR} ${CMAKE_BINARY_DIR}/contrib/re2-cmake
|
||||
)
|
||||
|
||||
target_compile_definitions(graphite-rollup-bench PRIVATE RULES_DIR="${CMAKE_CURRENT_SOURCE_DIR}")
|
||||
@ -1,130 +0,0 @@
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
|
||||
#include <boost/program_options.hpp>
|
||||
|
||||
#include <AggregateFunctions/registerAggregateFunctions.h>
|
||||
#include <Processors/Merges/Algorithms/Graphite.h>
|
||||
#include <Processors/Merges/Algorithms/GraphiteRollupSortedAlgorithm.h>
|
||||
#include <Storages/System/StorageSystemGraphite.h>
|
||||
#include <Common/Config/ConfigProcessor.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
|
||||
using namespace DB;
|
||||
|
||||
static SharedContextHolder shared_context = Context::createShared();
|
||||
|
||||
auto loadMetrics(const std::string & metrics_file)
|
||||
{
|
||||
std::vector<std::string> metrics;
|
||||
ReadBufferFromFile in(metrics_file);
|
||||
String line;
|
||||
|
||||
while (!in.eof())
|
||||
{
|
||||
readEscapedStringUntilEOL(line, in);
|
||||
if (!in.eof())
|
||||
{
|
||||
++in.position();
|
||||
}
|
||||
if (!line.empty() && line.back() == '\n')
|
||||
{
|
||||
line.pop_back();
|
||||
}
|
||||
if (!line.empty())
|
||||
{
|
||||
metrics.emplace_back(line);
|
||||
}
|
||||
}
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
ConfigProcessor::LoadedConfig loadConfiguration(const std::string & config_path)
|
||||
{
|
||||
ConfigProcessor config_processor(config_path, true, true);
|
||||
ConfigProcessor::LoadedConfig config = config_processor.loadConfig(false);
|
||||
return config;
|
||||
}
|
||||
|
||||
void bench(const std::string & config_path, const std::string & metrics_file, size_t n, bool verbose)
|
||||
{
|
||||
auto config = loadConfiguration(config_path);
|
||||
|
||||
auto context = Context::createGlobal(shared_context.get());
|
||||
context->setConfig(config.configuration.get());
|
||||
|
||||
Graphite::Params params;
|
||||
setGraphitePatternsFromConfig(context, "graphite_rollup", params);
|
||||
|
||||
auto metrics = loadMetrics(metrics_file);
|
||||
|
||||
std::vector<double> durations(metrics.size());
|
||||
size_t j, i;
|
||||
for (j = 0; j < n; j++)
|
||||
{
|
||||
for (i = 0; i < metrics.size(); i++)
|
||||
{
|
||||
auto start = std::chrono::high_resolution_clock::now();
|
||||
|
||||
auto rule = DB::Graphite::selectPatternForPath(params, metrics[i]);
|
||||
(void)rule;
|
||||
|
||||
auto end = std::chrono::high_resolution_clock::now();
|
||||
double duration = (duration_cast<std::chrono::duration<double>>(end - start)).count() * 1E9;
|
||||
durations[i] += duration;
|
||||
|
||||
if (j == 0 && verbose)
|
||||
{
|
||||
std::cout << metrics[i] << ": rule with regexp '" << rule.second->regexp_str << "' found\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (i = 0; i < metrics.size(); i++)
|
||||
{
|
||||
std::cout << metrics[i] << " " << durations[i] / n << " ns\n";
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
{
|
||||
registerAggregateFunctions();
|
||||
|
||||
std::string config_file, metrics_file;
|
||||
|
||||
using namespace std::literals;
|
||||
|
||||
std::string config_default = RULES_DIR + "/rollup.xml"s;
|
||||
std::string metrics_default = RULES_DIR + "/metrics.txt"s;
|
||||
|
||||
namespace po = boost::program_options;
|
||||
po::variables_map vm;
|
||||
|
||||
po::options_description desc;
|
||||
desc.add_options()("help,h", "produce help")(
|
||||
"config,c", po::value<std::string>()->default_value(config_default), "XML config with rollup rules")(
|
||||
"metrics,m", po::value<std::string>()->default_value(metrics_default), "metrcis files (one metric per line) for run benchmark")(
|
||||
"verbose,V", po::bool_switch()->default_value(false), "verbose output (print found rule)");
|
||||
|
||||
po::parsed_options parsed = po::command_line_parser(argc, argv).options(desc).run();
|
||||
po::store(parsed, vm);
|
||||
po::notify(vm);
|
||||
|
||||
if (vm.count("help"))
|
||||
{
|
||||
std::cout << desc << '\n';
|
||||
exit(1);
|
||||
}
|
||||
|
||||
bench(vm["config"].as<std::string>(), vm["metrics"].as<std::string>(), 10000, vm["verbose"].as<bool>());
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
test.sum
|
||||
sum?env=test&tag=Fake3
|
||||
test.max
|
||||
max?env=test&tag=Fake4
|
||||
test.min
|
||||
min?env=test&tag=Fake5
|
||||
fake5?env=test&tag=Fake5
|
||||
test.p95
|
||||
p95?env=test&tag=FakeNo
|
||||
default
|
||||
default?env=test&tag=FakeNo
|
||||
@ -1,167 +0,0 @@
|
||||
<clickhouse>
|
||||
<graphite_rollup>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.sum$</regexp>
|
||||
<function>sum</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tagged</rule_type>
|
||||
<regexp>^((.*)|.)sum\?</regexp>
|
||||
<function>sum</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.max$</regexp>
|
||||
<function>max</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tagged</rule_type>
|
||||
<regexp>^((.*)|.)max\?</regexp>
|
||||
<function>max</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.min$</regexp>
|
||||
<function>min</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tagged</rule_type>
|
||||
<regexp>^((.*)|.)min\?</regexp>
|
||||
<function>min</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.fake1\..*\.Fake1\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tag_list</rule_type>
|
||||
<regexp>fake1;tag=Fake1</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.fake2\..*\.Fake2\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tag_list</rule_type>
|
||||
<regexp>fake2;tag=Fake2</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.fake3\..*\.Fake3\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tag_list</rule_type>
|
||||
<regexp>fake3;tag=Fake3</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.fake4\..*\.Fake4\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tag_list</rule_type>
|
||||
<regexp>fake4;tag=Fake4</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.fake5\..*\.Fake5\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tag_list</rule_type>
|
||||
<regexp>fake5;tag=Fake5</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.fake6\..*\.Fake6\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tag_list</rule_type>
|
||||
<regexp>fake6;tag=Fake6</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.fake7\..*\.Fake7\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tag_list</rule_type>
|
||||
<regexp>fake7;tag=Fake7</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<default>
|
||||
<function>avg</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>3600</age>
|
||||
<precision>300</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</default>
|
||||
</graphite_rollup>
|
||||
</clickhouse>
|
||||
@ -1,167 +0,0 @@
|
||||
<clickhouse>
|
||||
<graphite_rollup>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.sum$</regexp>
|
||||
<function>sum</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tagged</rule_type>
|
||||
<regexp>^((.*)|.)sum\?</regexp>
|
||||
<function>sum</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.max$</regexp>
|
||||
<function>max</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tagged</rule_type>
|
||||
<regexp>^((.*)|.)max\?</regexp>
|
||||
<function>max</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.min$</regexp>
|
||||
<function>min</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tagged</rule_type>
|
||||
<regexp>^((.*)|.)min\?</regexp>
|
||||
<function>min</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.fake1\..*\.Fake1\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tagged</rule_type>
|
||||
<regexp><![CDATA[^fake1\?(.*&)*tag=Fake1(&|$)]]></regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.fake2\..*\.Fake2\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tagged</rule_type>
|
||||
<regexp><![CDATA[^fake2\?(.*&)*tag=Fake2(&|$)]]></regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.fake3\..*\.Fake3\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tagged</rule_type>
|
||||
<regexp><![CDATA[^fake3\?(.*&)*tag=Fake3(&|$)]]></regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.fake4\..*\.Fake4\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tagged</rule_type>
|
||||
<regexp><![CDATA[^fake4\?(.*&)*tag=Fake4(&|$)]]></regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.fake5\..*\.Fake5\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tagged</rule_type>
|
||||
<regexp><![CDATA[^fake5\?(.*&)*tag=Fake5(&|$)]]></regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.fake6\..*\.Fake6\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tagged</rule_type>
|
||||
<regexp><![CDATA[^fake6\?(.*&)*tag=Fake6(&|$)]]></regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>plain</rule_type>
|
||||
<regexp>\.fake7\..*\.Fake7\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<rule_type>tagged</rule_type>
|
||||
<regexp><![CDATA[^fake7\?(.*&)*tag=Fake7(&|$)]]></regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<default>
|
||||
<function>avg</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>3600</age>
|
||||
<precision>300</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</default>
|
||||
</graphite_rollup>
|
||||
</clickhouse>
|
||||
@ -1,147 +0,0 @@
|
||||
<clickhouse>
|
||||
<graphite_rollup>
|
||||
<pattern>
|
||||
<regexp>\.sum$</regexp>
|
||||
<function>sum</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp>^((.*)|.)sum\?</regexp>
|
||||
<function>sum</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp>\.max$</regexp>
|
||||
<function>max</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp>^((.*)|.)max\?</regexp>
|
||||
<function>max</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp>\.min$</regexp>
|
||||
<function>min</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp>^((.*)|.)min\?</regexp>
|
||||
<function>min</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp>\.fake1\..*\.Fake1\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp><![CDATA[^fake1\?(.*&)*tag=Fake1(&|$)]]></regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp>\.fake2\..*\.Fake2\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp><![CDATA[^fake2\?(.*&)*tag=Fake2(&|$)]]></regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp>\.fake3\..*\.Fake3\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp><![CDATA[^fake3\?(.*&)*tag=Fake3(&|$)]]></regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp>\.fake4\..*\.Fake4\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp><![CDATA[^fake4\?(.*&)*tag=Fake4(&|$)]]></regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp>\.fake5\..*\.Fake5\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp><![CDATA[^fake5\?(.*&)*tag=Fake5(&|$)]]></regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp>\.fake6\..*\.Fake6\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp><![CDATA[^fake6\?(.*&)*tag=Fake6(&|$)]]></regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp>\.fake7\..*\.Fake7\.</regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<pattern>
|
||||
<regexp><![CDATA[^fake7\?(.*&)*tag=Fake7(&|$)]]></regexp>
|
||||
<function>sum</function>
|
||||
</pattern>
|
||||
<default>
|
||||
<function>avg</function>
|
||||
<retention>
|
||||
<age>0</age>
|
||||
<precision>60</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>3600</age>
|
||||
<precision>300</precision>
|
||||
</retention>
|
||||
<retention>
|
||||
<age>86400</age>
|
||||
<precision>3600</precision>
|
||||
</retention>
|
||||
</default>
|
||||
</graphite_rollup>
|
||||
</clickhouse>
|
||||
@ -101,8 +101,8 @@ int decompress(char * input, char * output, off_t start, off_t end, size_t max_n
|
||||
{
|
||||
/// Decompress data in child process.
|
||||
if (0 != doDecompress(input, output, in_pointer, out_pointer, size, decompressed_size, dctx))
|
||||
exit(1);
|
||||
exit(0);
|
||||
_exit(1);
|
||||
_exit(0);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@ -1,2 +0,0 @@
|
||||
clickhouse_add_executable(zk-test main.cpp)
|
||||
target_link_libraries(zk-test PRIVATE clickhouse_common_zookeeper_no_log)
|
||||
@ -1,364 +0,0 @@
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <Poco/ConsoleChannel.h>
|
||||
#include <Common/ZooKeeper/KeeperException.h>
|
||||
#include <Common/ZooKeeper/ZooKeeper.h>
|
||||
#include <base/LineReader.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <fmt/format.h>
|
||||
#include <random>
|
||||
#include <iterator>
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <Common/Stopwatch.h>
|
||||
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <exception>
|
||||
#include <future>
|
||||
|
||||
using namespace std;
|
||||
|
||||
/// TODO: Remove ME
|
||||
|
||||
void checkEq(zkutil::ZooKeeper & zk, const std::string & path, const std::string & expected)
|
||||
{
|
||||
auto result = zk.get(path);
|
||||
if (result != expected)
|
||||
throw std::runtime_error(fmt::format("Data on path '{}' = '{}' doesn't match expected '{}'",
|
||||
path, result, expected));
|
||||
}
|
||||
|
||||
void checkExists(zkutil::ZooKeeper & zk, const std::string & path)
|
||||
{
|
||||
if (!zk.exists(path))
|
||||
throw std::runtime_error(fmt::format("Path '{}' doesn't exists", path));
|
||||
}
|
||||
|
||||
void testCreateGetExistsNode(zkutil::ZooKeeper & zk)
|
||||
{
|
||||
zk.create("/data", "test_string", zkutil::CreateMode::Persistent);
|
||||
zk.create("/data/seq-", "another_string", zkutil::CreateMode::PersistentSequential);
|
||||
checkEq(zk, "/data", "test_string");
|
||||
checkExists(zk, "/data/seq-0000000000");
|
||||
checkEq(zk, "/data/seq-0000000000", "another_string");
|
||||
}
|
||||
|
||||
void testCreateSetNode(zkutil::ZooKeeper & zk)
|
||||
{
|
||||
zk.create("/data/set", "sssss", zkutil::CreateMode::Persistent);
|
||||
checkEq(zk, "/data/set", "sssss");
|
||||
zk.set("/data/set", "qqqqq");
|
||||
checkEq(zk, "/data/set", "qqqqq");
|
||||
}
|
||||
|
||||
void testCreateList(zkutil::ZooKeeper & zk)
|
||||
{
|
||||
zk.create("/data/lst", "", zkutil::CreateMode::Persistent);
|
||||
zk.create("/data/lst/d1", "", zkutil::CreateMode::Persistent);
|
||||
zk.create("/data/lst/d2", "", zkutil::CreateMode::Persistent);
|
||||
zk.create("/data/lst/d3", "", zkutil::CreateMode::Persistent);
|
||||
auto children = zk.getChildren("/data/lst");
|
||||
if (children.size() != 3)
|
||||
throw std::runtime_error("Children of /data/lst doesn't equal to three");
|
||||
for (size_t i = 0; i < children.size(); ++i)
|
||||
{
|
||||
std::cerr << "children:" << children[i] << std::endl;
|
||||
std::cerr << "children size:" << children[i].size() << std::endl;
|
||||
if (children[i] != "d" + std::to_string(i + 1))
|
||||
throw std::runtime_error(fmt::format("Incorrect children #{} got {}, expected {}", i, children[i], "d" + std::to_string(i + 1)));
|
||||
}
|
||||
}
|
||||
|
||||
void testCreateSetVersionRequest(zkutil::ZooKeeper & zk)
|
||||
{
|
||||
zk.create("/data/check_data", "d", zkutil::CreateMode::Persistent);
|
||||
Coordination::Stat stat{};
|
||||
try
|
||||
{
|
||||
zk.set("/data/check_data", "e", stat.version + 2);
|
||||
std::terminate();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
std::cerr << "Got exception on incorrect version (it's ok)\n";
|
||||
}
|
||||
|
||||
checkEq(zk, "/data/check_data", "d");
|
||||
zk.set("/data/check_data", "e", stat.version);
|
||||
|
||||
checkEq(zk, "/data/check_data", "e");
|
||||
}
|
||||
|
||||
void testCreateSetWatchEvent(zkutil::ZooKeeper & zk)
|
||||
{
|
||||
|
||||
std::shared_ptr<Poco::Event> event = std::make_shared<Poco::Event>();
|
||||
zk.create("/data/nodeforwatch", "", zkutil::CreateMode::Persistent);
|
||||
Coordination::Stat stat;
|
||||
zk.get("/data/nodeforwatch", &stat, event);
|
||||
|
||||
if (event->tryWait(300))
|
||||
throw std::runtime_error(fmt::format("Event for path {} was set without any actions", "/data/nodeforwatch"));
|
||||
|
||||
zk.set("/data/nodeforwatch", "x");
|
||||
if (!event->tryWait(300))
|
||||
throw std::runtime_error(fmt::format("Event for path {} was not set after set", "/data/nodeforwatch"));
|
||||
else
|
||||
std::cerr << "Event was set well\n";
|
||||
}
|
||||
|
||||
void testCreateListWatchEvent(zkutil::ZooKeeper & zk)
|
||||
{
|
||||
std::shared_ptr<Poco::Event> event = std::make_shared<Poco::Event>();
|
||||
std::string path = "/data/pathforwatch";
|
||||
zk.create(path, "", zkutil::CreateMode::Persistent);
|
||||
zk.create(path + "/n1", "", zkutil::CreateMode::Persistent);
|
||||
zk.create(path + "/n2", "", zkutil::CreateMode::Persistent);
|
||||
zk.getChildren(path, nullptr, event);
|
||||
|
||||
if (event->tryWait(300))
|
||||
throw std::runtime_error(fmt::format("ListEvent for path {} was set without any actions", path));
|
||||
|
||||
zk.create(path + "/n3", "", zkutil::CreateMode::Persistent);
|
||||
if (!event->tryWait(300))
|
||||
throw std::runtime_error(fmt::format("ListEvent for path {} was not set after create", path));
|
||||
else
|
||||
std::cerr << "ListEvent was set well\n";
|
||||
}
|
||||
|
||||
void testMultiRequest(zkutil::ZooKeeper & zk)
|
||||
{
|
||||
std::cerr << "Testing multi request\n";
|
||||
Coordination::Requests requests;
|
||||
requests.push_back(zkutil::makeCreateRequest("/data/multirequest", "aaa", zkutil::CreateMode::Persistent));
|
||||
requests.push_back(zkutil::makeSetRequest("/data/multirequest", "bbb", -1));
|
||||
zk.multi(requests);
|
||||
std::cerr << "Multi executed\n";
|
||||
|
||||
try
|
||||
{
|
||||
requests.clear();
|
||||
std::cerr << "Testing bad multi\n";
|
||||
requests.push_back(zkutil::makeCreateRequest("/data/multirequest", "qweqwe", zkutil::CreateMode::Persistent));
|
||||
requests.push_back(zkutil::makeSetRequest("/data/multirequest", "bbb", -1));
|
||||
requests.push_back(zkutil::makeSetRequest("/data/multirequest", "ccc", -1));
|
||||
zk.multi(requests);
|
||||
std::cerr << "Bad multi executed\n";
|
||||
std::terminate();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
std::cerr << "Got exception on multy request (it's ok)\n";
|
||||
}
|
||||
|
||||
checkEq(zk, "/data/multirequest", "bbb");
|
||||
std::cerr << "Multi request finished\n";
|
||||
}
|
||||
|
||||
std::mutex elements_mutex;
|
||||
std::vector<int> current_elements;
|
||||
std::atomic<int> watches_triggered = 0;
|
||||
|
||||
void triggerWatch(const Coordination::WatchResponse &)
|
||||
{
|
||||
watches_triggered++;
|
||||
}
|
||||
|
||||
template<typename Iter, typename RandomGenerator>
|
||||
Iter select_randomly(Iter start, Iter end, RandomGenerator& g)
|
||||
{
|
||||
std::uniform_int_distribution<> dis(0, std::distance(start, end) - 1);
|
||||
std::advance(start, dis(g));
|
||||
return start;
|
||||
}
|
||||
|
||||
template<typename Iter>
|
||||
Iter select_randomly(Iter start, Iter end)
|
||||
{
|
||||
static std::random_device rd;
|
||||
static std::mt19937 gen(rd());
|
||||
return select_randomly(start, end, gen);
|
||||
}
|
||||
|
||||
std::atomic<int> element_counter = 0;
|
||||
std::atomic<int> failed_setup_counter = 0;
|
||||
|
||||
void createPathAndSetWatch(zkutil::ZooKeeper & zk, const String & path_prefix, size_t total)
|
||||
{
|
||||
for (size_t i = 0; i < total; ++i)
|
||||
{
|
||||
int element = element_counter++;
|
||||
zk.createIfNotExists(path_prefix + "/" + std::to_string(element), "");
|
||||
|
||||
std::string result;
|
||||
if (!zk.tryGetWatch(path_prefix + "/" + std::to_string(element), result, nullptr, triggerWatch))
|
||||
failed_setup_counter++;
|
||||
|
||||
{
|
||||
std::lock_guard lock(elements_mutex);
|
||||
current_elements.push_back(element);
|
||||
}
|
||||
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(200));
|
||||
|
||||
{
|
||||
std::lock_guard lock(elements_mutex);
|
||||
if (current_elements.empty())
|
||||
continue;
|
||||
element = *select_randomly(current_elements.begin(), current_elements.end());
|
||||
current_elements.erase(std::remove(current_elements.begin(), current_elements.end(), element), current_elements.end());
|
||||
}
|
||||
zk.tryRemove(path_prefix + "/" + std::to_string(element));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
std::string random_string(size_t length)
|
||||
{
|
||||
auto randchar = []() -> char
|
||||
{
|
||||
const char charset[] =
|
||||
"0123456789"
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
"abcdefghijklmnopqrstuvwxyz";
|
||||
const size_t max_index = (sizeof(charset) - 1);
|
||||
return charset[rand() % max_index]; /// NOLINT
|
||||
};
|
||||
std::string str(length, 0);
|
||||
std::generate_n(str.begin(), length, randchar);
|
||||
return str;
|
||||
}
|
||||
|
||||
std::string currentDateTime()
|
||||
{
|
||||
time_t now = time(nullptr);
|
||||
tm tstruct;
|
||||
char buf[80];
|
||||
tstruct = *localtime(&now);
|
||||
// Visit http://en.cppreference.com/w/cpp/chrono/c/strftime
|
||||
// for more information about date/time format
|
||||
size_t size = strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", &tstruct);
|
||||
|
||||
return std::string(buf, size);
|
||||
}
|
||||
|
||||
|
||||
void createOnPrefix(const std::string & zkhost, const String & path_prefix, size_t datasize, size_t total)
|
||||
{
|
||||
zkutil::ZooKeeper zk(zkhost);
|
||||
std::vector<std::future<Coordination::CreateResponse>> holder_futures;
|
||||
using namespace std::chrono;
|
||||
try
|
||||
{
|
||||
for (size_t i = 0; i < total; ++i)
|
||||
{
|
||||
std::cerr << currentDateTime() << "] Request:" << i << std::endl;
|
||||
std::string path = path_prefix + "/element" + std::to_string(i);
|
||||
holder_futures.push_back(zk.asyncCreate(path, random_string(datasize), zkutil::CreateMode::Persistent));
|
||||
}
|
||||
|
||||
for (auto & future : holder_futures)
|
||||
future.get();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
::exit(-1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void createConcurrent(zkutil::ZooKeeper & testzk, const std::string & zkhost, size_t threads, size_t requests, size_t blobsize)
|
||||
{
|
||||
std::vector<std::future<void>> asyncs;
|
||||
for (size_t i = 0; i < threads; ++i)
|
||||
{
|
||||
std::string path_prefix = "/data/create_test" + std::to_string(i);
|
||||
testzk.createIfNotExists(path_prefix, "");
|
||||
auto callback = [&zkhost, path_prefix, requests, blobsize] ()
|
||||
{
|
||||
createOnPrefix(zkhost, path_prefix, blobsize, requests);
|
||||
};
|
||||
asyncs.push_back(std::async(std::launch::async, callback));
|
||||
}
|
||||
|
||||
for (auto & async : asyncs)
|
||||
{
|
||||
async.wait();
|
||||
}
|
||||
}
|
||||
|
||||
void tryConcurrentWatches(zkutil::ZooKeeper & zk)
|
||||
{
|
||||
std::string path_prefix = "/concurrent_watches";
|
||||
std::vector<std::future<void>> asyncs;
|
||||
zk.createIfNotExists(path_prefix, "");
|
||||
for (size_t i = 0; i < 100; ++i)
|
||||
{
|
||||
auto callback = [&zk, path_prefix] ()
|
||||
{
|
||||
createPathAndSetWatch(zk, path_prefix, 100);
|
||||
};
|
||||
asyncs.push_back(std::async(std::launch::async, callback));
|
||||
}
|
||||
|
||||
for (auto & async : asyncs)
|
||||
{
|
||||
async.wait();
|
||||
}
|
||||
|
||||
size_t counter = 0;
|
||||
while (watches_triggered != 100 * 100)
|
||||
{
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
if (counter++ > 20)
|
||||
break;
|
||||
}
|
||||
|
||||
std::cerr << "Failed setup counter:" << failed_setup_counter << std::endl;
|
||||
std::cerr << "Current elements size:" << current_elements.size() << std::endl;
|
||||
std::cerr << "WatchesTriggered:" << watches_triggered << std::endl;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
if (argc != 2)
|
||||
{
|
||||
std::cerr << "usage: " << argv[0] << " hosts" << std::endl;
|
||||
return 2;
|
||||
}
|
||||
Poco::AutoPtr<Poco::ConsoleChannel> channel = new Poco::ConsoleChannel(std::cerr);
|
||||
Poco::Logger::root().setChannel(channel);
|
||||
Poco::Logger::root().setLevel("trace");
|
||||
|
||||
zkutil::ZooKeeper zk(argv[1]);
|
||||
|
||||
try
|
||||
{
|
||||
std::cerr << "Removing\n";
|
||||
zk.tryRemoveRecursive("/data");
|
||||
std::cerr << "Creating\n";
|
||||
zk.createIfNotExists("/data", "");
|
||||
std::cerr << "Created\n";
|
||||
|
||||
Stopwatch watch;
|
||||
createConcurrent(zk, argv[1], 1, 1005000, 10);
|
||||
std::cerr << "Finished in: " << watch.elapsedMilliseconds() << "ms" << std::endl;
|
||||
|
||||
//testCreateGetExistsNode(zk);
|
||||
//testCreateSetNode(zk);
|
||||
//testCreateList(zk);
|
||||
//testCreateSetVersionRequest(zk);
|
||||
//testMultiRequest(zk);
|
||||
//testCreateSetWatchEvent(zk);
|
||||
//testCreateListWatchEvent(zk);
|
||||
//tryConcurrentWatches(zk);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
zk.tryRemoveRecursive("/data");
|
||||
throw;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user