thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into replace-yandex-to-noto

Browse Source
This commit is contained in:
Alexey Milovidov 2022-08-09 22:43:52 +03:00 committed by GitHub
commit f7d995681b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
85 changed files with 1673 additions and 1163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docker/packager/binary/build.sh
View File

@ -104,7 +104,6 @@ if [ -n "$MAKE_DEB" ]; then
fi
mv ./programs/clickhouse* /output
[ -x ./programs/self-extracting/clickhouse ] && mv ./programs/self-extracting/clickhouse /output
mv ./src/unit_tests_dbms /output ||: # may not exist for some binary builds
find . -name '*.so' -print -exec mv '{}' /output \;
find . -name '*.so.*' -print -exec mv '{}' /output \;

2
docker/test/fuzzer/run-fuzzer.sh
View File

@ -69,8 +69,6 @@ function download
wget_with_retry "$BINARY_URL_TO_DOWNLOAD"
chmod +x clickhouse
# clickhouse may be compressed - run once to decompress
./clickhouse ||:
ln -s ./clickhouse ./clickhouse-server
ln -s ./clickhouse ./clickhouse-client

6
docker/test/integration/runner/compose/docker_compose_nats.yml
View File

@ -4,4 +4,8 @@ services:
image: nats
ports:
- "${NATS_EXTERNAL_PORT}:${NATS_INTERNAL_PORT}"
command: "-p 4444 --user click --pass house"
command: "-p 4444 --user click --pass house --tls --tlscert=/etc/certs/server-cert.pem --tlskey=/etc/certs/server-key.pem"
volumes:
- type: bind
source: "${NATS_CERT_DIR}/nats"
target: /etc/certs

2
docker/test/performance-comparison/compare.sh
View File

@ -1338,8 +1338,6 @@ EOF
set -x
}
# clickhouse may be compressed - run once to decompress
/workspace/right/clickhouse ||:
# Check that local and client are in PATH
clickhouse-local --version > /dev/null
clickhouse-client --version > /dev/null

30
docs/changelogs/v22.6.5.22-stable.md Normal file
View File

@ -0,0 +1,30 @@
---
sidebar_position: 1
sidebar_label: 2022
---
# 2022 Changelog
### ClickHouse release v22.6.5.22-stable (47ca5f14a34) FIXME as compared to v22.6.4.35-stable (b9202cae6f4)
#### Bug Fix
* Backported in [#39749](https://github.com/ClickHouse/ClickHouse/issues/39749): Fix seeking while reading from encrypted disk. This PR fixes [#38381](https://github.com/ClickHouse/ClickHouse/issues/38381). [#39687](https://github.com/ClickHouse/ClickHouse/pull/39687) ([Vitaly Baranov](https://github.com/vitlibar)).
#### Build/Testing/Packaging Improvement
* Backported in [#39883](https://github.com/ClickHouse/ClickHouse/issues/39883): Former packages used to install systemd.service file to `/etc`. The files there are marked as `conf` and are not cleaned out, and not updated automatically. This PR cleans them out. [#39323](https://github.com/ClickHouse/ClickHouse/pull/39323) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
#### Bug Fix (user-visible misbehavior in official stable or prestable release)
* Backported in [#39207](https://github.com/ClickHouse/ClickHouse/issues/39207): Fix reading of sparse columns from `MergeTree` tables that store their data in S3. [#37978](https://github.com/ClickHouse/ClickHouse/pull/37978) ([Anton Popov](https://github.com/CurtizJ)).
* Backported in [#38932](https://github.com/ClickHouse/ClickHouse/issues/38932): Fix `parallel_view_processing=1` with `optimize_trivial_insert_select=1`. Fix `max_insert_threads` while pushing to views. [#38731](https://github.com/ClickHouse/ClickHouse/pull/38731) ([Azat Khuzhin](https://github.com/azat)).
* Backported in [#39590](https://github.com/ClickHouse/ClickHouse/issues/39590): Fix data race and possible heap-buffer-overflow in Avro format. Closes [#39094](https://github.com/ClickHouse/ClickHouse/issues/39094) Closes [#33652](https://github.com/ClickHouse/ClickHouse/issues/33652). [#39498](https://github.com/ClickHouse/ClickHouse/pull/39498) ([Kruglov Pavel](https://github.com/Avogar)).
* Backported in [#39612](https://github.com/ClickHouse/ClickHouse/issues/39612): Fix bug with maxsplit argument for splitByChar, which was not working correctly. [#39552](https://github.com/ClickHouse/ClickHouse/pull/39552) ([filimonov](https://github.com/filimonov)).
* Backported in [#39791](https://github.com/ClickHouse/ClickHouse/issues/39791): Fix wrong index analysis with tuples and operator `IN`, which could lead to wrong query result. [#39752](https://github.com/ClickHouse/ClickHouse/pull/39752) ([Anton Popov](https://github.com/CurtizJ)).
* Backported in [#39836](https://github.com/ClickHouse/ClickHouse/issues/39836): Fix `CANNOT_READ_ALL_DATA` exception with `local_filesystem_read_method=pread_threadpool`. This bug affected only Linux kernel version 5.9 and 5.10 according to [man](https://manpages.debian.org/testing/manpages-dev/preadv2.2.en.html#BUGS). [#39800](https://github.com/ClickHouse/ClickHouse/pull/39800) ([Anton Popov](https://github.com/CurtizJ)).
#### NOT FOR CHANGELOG / INSIGNIFICANT
* Fix reading from s3 in some corner cases [#38239](https://github.com/ClickHouse/ClickHouse/pull/38239) ([Anton Popov](https://github.com/CurtizJ)).
* Replace MemoryTrackerBlockerInThread to LockMemoryExceptionInThread [#39619](https://github.com/ClickHouse/ClickHouse/pull/39619) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Change mysql-odbc url [#39702](https://github.com/ClickHouse/ClickHouse/pull/39702) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).

5
docs/en/development/developer-instruction.md
View File

@ -285,4 +285,9 @@ If you are not interested in functionality provided by third-party libraries, yo
-DENABLE_LIBRARIES=0 -DENABLE_EMBEDDED_COMPILER=0
```
Compressing the binary at the end of the build may take a while, disable the self-extraction feature via
```
-DENABLE_CLICKHOUSE_SELF_EXTRACTING=0
```
In case of problems with any of the development options, you are on your own!

62
docs/en/interfaces/formats.md
View File

@ -49,10 +49,17 @@ The supported formats are:
| [JSONCompactStringsEachRowWithNamesAndTypes](#jsoncompactstringseachrowwithnamesandtypes) | ✔ | ✔ |
| [TSKV](#tskv) | ✔ | ✔ |
| [Pretty](#pretty) | ✗ | ✔ |
| [PrettyCompact](#prettycompact) | ✗ | ✔ |
| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ |
| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ |
| [PrettyMonoBlock](#prettymonoblock) | ✗ | ✔ |
| [PrettyNoEscapesMonoBlock](#prettynoescapesmonoblock) | ✗ | ✔ |
| [PrettyCompact](#prettycompact) | ✗ | ✔ |
| [PrettyCompactNoEscapes](#prettycompactnoescapes) | ✗ | ✔ |
| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ |
| [PrettyCompactNoEscapesMonoBlock](#prettycompactnoescapesmonoblock) | ✗ | ✔ |
| [PrettySpace](#prettyspace) | ✗ | ✔ |
| [PrettySpaceNoEscapes](#prettyspacenoescapes) | ✗ | ✔ |
| [PrettySpaceMonoBlock](#prettyspacemonoblock) | ✗ | ✔ |
| [PrettySpaceNoEscapesMonoBlock](#prettyspacenoescapesmonoblock) | ✗ | ✔ |
| [Prometheus](#prometheus) | ✗ | ✔ |
| [Protobuf](#protobuf) | ✔ | ✔ |
| [ProtobufSingle](#protobufsingle) | ✔ | ✔ |
@ -1198,18 +1205,9 @@ Extremes:
└────────────┴─────────┘
```
## PrettyCompact {#prettycompact}
Differs from [Pretty](#pretty) in that the grid is drawn between rows and the result is more compact.
This format is used by default in the command-line client in interactive mode.
## PrettyCompactMonoBlock {#prettycompactmonoblock}
Differs from [PrettyCompact](#prettycompact) in that up to 10,000 rows are buffered, then output as a single table, not by blocks.
## PrettyNoEscapes {#prettynoescapes}
Differs from Pretty in that ANSI-escape sequences aren’t used. This is necessary for displaying this format in a browser, as well as for using the ‘watch’ command-line utility.
Differs from [Pretty](#pretty) in that ANSI-escape sequences aren’t used. This is necessary for displaying this format in a browser, as well as for using the ‘watch’ command-line utility.
Example:
@ -1219,19 +1217,49 @@ $ watch -n1 "clickhouse-client --query='SELECT event, value FROM system.events F
You can use the HTTP interface for displaying in the browser.
### PrettyCompactNoEscapes {#prettycompactnoescapes}
## PrettyMonoBlock {#prettymonoblock}
The same as the previous setting.
Differs from [Pretty](#pretty) in that up to 10,000 rows are buffered, then output as a single table, not by blocks.
### PrettySpaceNoEscapes {#prettyspacenoescapes}
## PrettyNoEscapesMonoBlock {#prettynoescapesmonoblock}
The same as the previous setting.
Differs from [PrettyNoEscapes](#prettynoescapes) in that up to 10,000 rows are buffered, then output as a single table, not by blocks.
## PrettyCompact {#prettycompact}
Differs from [Pretty](#pretty) in that the grid is drawn between rows and the result is more compact.
This format is used by default in the command-line client in interactive mode.
## PrettyCompactNoEscapes {#prettynoescapes}
Differs from [PrettyCompact](#prettycompact) in that ANSI-escape sequences aren’t used. This is necessary for displaying this format in a browser, as well as for using the ‘watch’ command-line utility.
## PrettyCompactMonoBlock {#prettycompactmonoblock}
Differs from [PrettyCompact](#prettycompact) in that up to 10,000 rows are buffered, then output as a single table, not by blocks.
## PrettyCompactNoEscapesMonoBlock {#prettycompactnoescapesmonoblock}
Differs from [PrettyCompactNoEscapes](#prettycompactnoescapes) in that up to 10,000 rows are buffered, then output as a single table, not by blocks.
## PrettySpace {#prettyspace}
Differs from [PrettyCompact](#prettycompact) in that whitespace (space characters) is used instead of the grid.
### Pretty formats settings {#pretty-formats-settings}
## PrettySpaceNoEscapes {#prettyspacenoescapes}
Differs from [PrettySpace](#prettyspace) in that ANSI-escape sequences aren’t used. This is necessary for displaying this format in a browser, as well as for using the ‘watch’ command-line utility.
## PrettySpaceMonoBlock {#prettyspacemonoblock}
Differs from [PrettySpace](#prettyspace) in that up to 10,000 rows are buffered, then output as a single table, not by blocks.
## PrettySpaceNoEscapesMonoBlock {#prettyspacenoescapesmonoblock}
Differs from [PrettySpaceNoEscapes](#prettyspacenoescapes) in that up to 10,000 rows are buffered, then output as a single table, not by blocks.
## Pretty formats settings {#pretty-formats-settings}
- [output_format_pretty_max_rows](../operations/settings/settings.md#output_format_pretty_max_rows) - rows limit for Pretty formats. Default value - `10000`.
- [output_format_pretty_max_column_pad_width](../operations/settings/settings.md#output_format_pretty_max_column_pad_width) - maximum width to pad all values in a column in Pretty formats. Default value - `250`.

6
programs/CMakeLists.txt
View File

@ -49,15 +49,13 @@ option (ENABLE_CLICKHOUSE_OBFUSCATOR "Table data obfuscator (convert real data t
# https://clickhouse.com/docs/en/operations/utilities/odbc-bridge/
# TODO Also needs NANODBC.
if (ENABLE_ODBC AND NOT USE_MUSL)
option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "HTTP-server working like a proxy to ODBC driver"
${ENABLE_CLICKHOUSE_ALL})
option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "HTTP-server working like a proxy to ODBC driver" ${ENABLE_CLICKHOUSE_ALL})
else ()
option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "HTTP-server working like a proxy to ODBC driver" OFF)
endif ()
if (NOT USE_MUSL)
option (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE "HTTP-server working like a proxy to Library dictionary source"
${ENABLE_CLICKHOUSE_ALL})
option (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE "HTTP-server working like a proxy to Library dictionary source" ${ENABLE_CLICKHOUSE_ALL})
endif ()
# https://presentations.clickhouse.com/matemarketing_2020/

12
programs/library-bridge/CMakeLists.txt
View File

@ -1,13 +1,13 @@
include(${ClickHouse_SOURCE_DIR}/cmake/split_debug_symbols.cmake)
set (CLICKHOUSE_LIBRARY_BRIDGE_SOURCES
library-bridge.cpp
LibraryInterface.cpp
ExternalDictionaryLibraryAPI.cpp
ExternalDictionaryLibraryHandler.cpp
ExternalDictionaryLibraryHandlerFactory.cpp
LibraryBridge.cpp
Handlers.cpp
HandlerFactory.cpp
SharedLibraryHandler.cpp
SharedLibraryHandlerFactory.cpp
LibraryBridgeHandlerFactory.cpp
LibraryBridgeHandlers.cpp
library-bridge.cpp
)
if (OS_LINUX)

23
programs/library-bridge/LibraryInterface.cpp → programs/library-bridge/ExternalDictionaryLibraryAPI.cpp
View File

@ -1,4 +1,4 @@
#include "LibraryInterface.h"
#include "ExternalDictionaryLibraryAPI.h"
#include <Common/logger_useful.h>
@ -7,24 +7,7 @@ namespace
const char DICT_LOGGER_NAME[] = "LibraryDictionarySourceExternal";
}
namespace ClickHouseLibrary
{
std::string_view LIBRARY_CREATE_NEW_FUNC_NAME = "ClickHouseDictionary_v3_libNew";
std::string_view LIBRARY_CLONE_FUNC_NAME = "ClickHouseDictionary_v3_libClone";
std::string_view LIBRARY_DELETE_FUNC_NAME = "ClickHouseDictionary_v3_libDelete";
std::string_view LIBRARY_DATA_NEW_FUNC_NAME = "ClickHouseDictionary_v3_dataNew";
std::string_view LIBRARY_DATA_DELETE_FUNC_NAME = "ClickHouseDictionary_v3_dataDelete";
std::string_view LIBRARY_LOAD_ALL_FUNC_NAME = "ClickHouseDictionary_v3_loadAll";
std::string_view LIBRARY_LOAD_IDS_FUNC_NAME = "ClickHouseDictionary_v3_loadIds";
std::string_view LIBRARY_LOAD_KEYS_FUNC_NAME = "ClickHouseDictionary_v3_loadKeys";
std::string_view LIBRARY_IS_MODIFIED_FUNC_NAME = "ClickHouseDictionary_v3_isModified";
std::string_view LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME = "ClickHouseDictionary_v3_supportsSelectiveLoad";
void log(LogLevel level, CString msg)
void ExternalDictionaryLibraryAPI::log(LogLevel level, CString msg)
{
auto & logger = Poco::Logger::get(DICT_LOGGER_NAME);
switch (level)
@ -63,5 +46,3 @@ void log(LogLevel level, CString msg)
break;
}
}
}

106
programs/library-bridge/ExternalDictionaryLibraryAPI.h Normal file
View File

@ -0,0 +1,106 @@
#pragma once
#include <cstdint>
#include <string>
#define CLICKHOUSE_DICTIONARY_LIBRARY_API 1
struct ExternalDictionaryLibraryAPI
{
using CString = const char *;
using ColumnName = CString;
using ColumnNames = ColumnName[];
struct CStrings
{
CString * data = nullptr;
uint64_t size = 0;
};
struct VectorUInt64
{
const uint64_t * data = nullptr;
uint64_t size = 0;
};
struct ColumnsUInt64
{
VectorUInt64 * data = nullptr;
uint64_t size = 0;
};
struct Field
{
const void * data = nullptr;
uint64_t size = 0;
};
struct Row
{
const Field * data = nullptr;
uint64_t size = 0;
};
struct Table
{
const Row * data = nullptr;
uint64_t size = 0;
uint64_t error_code = 0; // 0 = ok; !0 = error, with message in error_string
const char * error_string = nullptr;
};
enum LogLevel
{
FATAL = 1,
CRITICAL,
ERROR,
WARNING,
NOTICE,
INFORMATION,
DEBUG,
TRACE,
};
static void log(LogLevel level, CString msg);
using LibraryContext = void *;
using LibraryLoggerFunc = void (*)(LogLevel, CString /* message */);
using LibrarySettings = CStrings *;
using LibraryData = void *;
using RawClickHouseLibraryTable = void *;
/// Can be safely casted into const Table * with static_cast<const ClickHouseLibrary::Table *>
using RequestedColumnsNames = CStrings *;
using RequestedIds = const VectorUInt64 *;
using RequestedKeys = Table *;
using LibraryNewFunc = LibraryContext (*)(LibrarySettings, LibraryLoggerFunc);
static constexpr const char * LIBRARY_CREATE_NEW_FUNC_NAME = "ClickHouseDictionary_v3_libNew";
using LibraryCloneFunc = LibraryContext (*)(LibraryContext);
static constexpr const char * LIBRARY_CLONE_FUNC_NAME = "ClickHouseDictionary_v3_libClone";
using LibraryDeleteFunc = void (*)(LibraryContext);
static constexpr const char * LIBRARY_DELETE_FUNC_NAME = "ClickHouseDictionary_v3_libDelete";
using LibraryDataNewFunc = LibraryData (*)(LibraryContext);
static constexpr const char * LIBRARY_DATA_NEW_FUNC_NAME = "ClickHouseDictionary_v3_dataNew";
using LibraryDataDeleteFunc = void (*)(LibraryContext, LibraryData);
static constexpr const char * LIBRARY_DATA_DELETE_FUNC_NAME = "ClickHouseDictionary_v3_dataDelete";
using LibraryLoadAllFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedColumnsNames);
static constexpr const char * LIBRARY_LOAD_ALL_FUNC_NAME = "ClickHouseDictionary_v3_loadAll";
using LibraryLoadIdsFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedColumnsNames, RequestedIds);
static constexpr const char * LIBRARY_LOAD_IDS_FUNC_NAME = "ClickHouseDictionary_v3_loadIds";
/// There are no requested column names for load keys func
using LibraryLoadKeysFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedKeys);
static constexpr const char * LIBRARY_LOAD_KEYS_FUNC_NAME = "ClickHouseDictionary_v3_loadKeys";
using LibraryIsModifiedFunc = bool (*)(LibraryContext, LibrarySettings);
static constexpr const char * LIBRARY_IS_MODIFIED_FUNC_NAME = "ClickHouseDictionary_v3_isModified";
using LibrarySupportsSelectiveLoadFunc = bool (*)(LibraryContext, LibrarySettings);
static constexpr const char * LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME = "ClickHouseDictionary_v3_supportsSelectiveLoad";
};

214
programs/library-bridge/ExternalDictionaryLibraryHandler.cpp Normal file
View File

@ -0,0 +1,214 @@
#include "ExternalDictionaryLibraryHandler.h"
#include <base/scope_guard.h>
#include <base/bit_cast.h>
#include <base/find_symbols.h>
#include <IO/ReadHelpers.h>
namespace DB
{
namespace ErrorCodes
{
extern const int EXTERNAL_LIBRARY_ERROR;
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
}
ExternalDictionaryLibraryHandler::ExternalDictionaryLibraryHandler(
const std::string & library_path_,
const std::vector<std::string> & library_settings,
const Block & sample_block_,
const std::vector<std::string> & attributes_names_)
: library_path(library_path_)
, sample_block(sample_block_)
, attributes_names(attributes_names_)
{
library = std::make_shared<SharedLibrary>(library_path);
settings_holder = std::make_shared<CStringsHolder>(CStringsHolder(library_settings));
auto lib_new = library->tryGet<ExternalDictionaryLibraryAPI::LibraryNewFunc>(ExternalDictionaryLibraryAPI::LIBRARY_CREATE_NEW_FUNC_NAME);
if (lib_new)
lib_data = lib_new(&settings_holder->strings, ExternalDictionaryLibraryAPI::log);
else
throw Exception("Method extDict_libNew failed", ErrorCodes::EXTERNAL_LIBRARY_ERROR);
}
ExternalDictionaryLibraryHandler::ExternalDictionaryLibraryHandler(const ExternalDictionaryLibraryHandler & other)
: library_path{other.library_path}
, sample_block{other.sample_block}
, attributes_names{other.attributes_names}
, library{other.library}
, settings_holder{other.settings_holder}
{
auto lib_clone = library->tryGet<ExternalDictionaryLibraryAPI::LibraryCloneFunc>(ExternalDictionaryLibraryAPI::LIBRARY_CLONE_FUNC_NAME);
if (lib_clone)
{
lib_data = lib_clone(other.lib_data);
}
else
{
auto lib_new = library->tryGet<ExternalDictionaryLibraryAPI::LibraryNewFunc>(ExternalDictionaryLibraryAPI::LIBRARY_CREATE_NEW_FUNC_NAME);
if (lib_new)
lib_data = lib_new(&settings_holder->strings, ExternalDictionaryLibraryAPI::log);
}
}
ExternalDictionaryLibraryHandler::~ExternalDictionaryLibraryHandler()
{
auto lib_delete = library->tryGet<ExternalDictionaryLibraryAPI::LibraryDeleteFunc>(ExternalDictionaryLibraryAPI::LIBRARY_DELETE_FUNC_NAME);
if (lib_delete)
lib_delete(lib_data);
}
bool ExternalDictionaryLibraryHandler::isModified()
{
auto func_is_modified = library->tryGet<ExternalDictionaryLibraryAPI::LibraryIsModifiedFunc>(ExternalDictionaryLibraryAPI::LIBRARY_IS_MODIFIED_FUNC_NAME);
if (func_is_modified)
return func_is_modified(lib_data, &settings_holder->strings);
return true;
}
bool ExternalDictionaryLibraryHandler::supportsSelectiveLoad()
{
auto func_supports_selective_load = library->tryGet<ExternalDictionaryLibraryAPI::LibrarySupportsSelectiveLoadFunc>(ExternalDictionaryLibraryAPI::LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME);
if (func_supports_selective_load)
return func_supports_selective_load(lib_data, &settings_holder->strings);
return true;
}
Block ExternalDictionaryLibraryHandler::loadAll()
{
auto columns_holder = std::make_unique<ExternalDictionaryLibraryAPI::CString[]>(attributes_names.size());
ExternalDictionaryLibraryAPI::CStrings columns{static_cast<decltype(ExternalDictionaryLibraryAPI::CStrings::data)>(columns_holder.get()), attributes_names.size()};
for (size_t i = 0; i < attributes_names.size(); ++i)
columns.data[i] = attributes_names[i].c_str();
auto load_all_func = library->get<ExternalDictionaryLibraryAPI::LibraryLoadAllFunc>(ExternalDictionaryLibraryAPI::LIBRARY_LOAD_ALL_FUNC_NAME);
auto data_new_func = library->get<ExternalDictionaryLibraryAPI::LibraryDataNewFunc>(ExternalDictionaryLibraryAPI::LIBRARY_DATA_NEW_FUNC_NAME);
auto data_delete_func = library->get<ExternalDictionaryLibraryAPI::LibraryDataDeleteFunc>(ExternalDictionaryLibraryAPI::LIBRARY_DATA_DELETE_FUNC_NAME);
ExternalDictionaryLibraryAPI::LibraryData data_ptr = data_new_func(lib_data);
SCOPE_EXIT(data_delete_func(lib_data, data_ptr));
ExternalDictionaryLibraryAPI::RawClickHouseLibraryTable data = load_all_func(data_ptr, &settings_holder->strings, &columns);
return dataToBlock(data);
}
Block ExternalDictionaryLibraryHandler::loadIds(const std::vector<uint64_t> & ids)
{
const ExternalDictionaryLibraryAPI::VectorUInt64 ids_data{bit_cast<decltype(ExternalDictionaryLibraryAPI::VectorUInt64::data)>(ids.data()), ids.size()};
auto columns_holder = std::make_unique<ExternalDictionaryLibraryAPI::CString[]>(attributes_names.size());
ExternalDictionaryLibraryAPI::CStrings columns_pass{static_cast<decltype(ExternalDictionaryLibraryAPI::CStrings::data)>(columns_holder.get()), attributes_names.size()};
auto load_ids_func = library->get<ExternalDictionaryLibraryAPI::LibraryLoadIdsFunc>(ExternalDictionaryLibraryAPI::LIBRARY_LOAD_IDS_FUNC_NAME);
auto data_new_func = library->get<ExternalDictionaryLibraryAPI::LibraryDataNewFunc>(ExternalDictionaryLibraryAPI::LIBRARY_DATA_NEW_FUNC_NAME);
auto data_delete_func = library->get<ExternalDictionaryLibraryAPI::LibraryDataDeleteFunc>(ExternalDictionaryLibraryAPI::LIBRARY_DATA_DELETE_FUNC_NAME);
ExternalDictionaryLibraryAPI::LibraryData data_ptr = data_new_func(lib_data);
SCOPE_EXIT(data_delete_func(lib_data, data_ptr));
ExternalDictionaryLibraryAPI::RawClickHouseLibraryTable data = load_ids_func(data_ptr, &settings_holder->strings, &columns_pass, &ids_data);
return dataToBlock(data);
}
Block ExternalDictionaryLibraryHandler::loadKeys(const Columns & key_columns)
{
auto holder = std::make_unique<ExternalDictionaryLibraryAPI::Row[]>(key_columns.size());
std::vector<std::unique_ptr<ExternalDictionaryLibraryAPI::Field[]>> column_data_holders;
for (size_t i = 0; i < key_columns.size(); ++i)
{
auto cell_holder = std::make_unique<ExternalDictionaryLibraryAPI::Field[]>(key_columns[i]->size());
for (size_t j = 0; j < key_columns[i]->size(); ++j)
{
auto data_ref = key_columns[i]->getDataAt(j);
cell_holder[j] = ExternalDictionaryLibraryAPI::Field{
.data = static_cast<const void *>(data_ref.data),
.size = data_ref.size};
}
holder[i] = ExternalDictionaryLibraryAPI::Row{
.data = static_cast<ExternalDictionaryLibraryAPI::Field *>(cell_holder.get()),
.size = key_columns[i]->size()};
column_data_holders.push_back(std::move(cell_holder));
}
ExternalDictionaryLibraryAPI::Table request_cols{
.data = static_cast<ExternalDictionaryLibraryAPI::Row *>(holder.get()),
.size = key_columns.size()};
auto load_keys_func = library->get<ExternalDictionaryLibraryAPI::LibraryLoadKeysFunc>(ExternalDictionaryLibraryAPI::LIBRARY_LOAD_KEYS_FUNC_NAME);
auto data_new_func = library->get<ExternalDictionaryLibraryAPI::LibraryDataNewFunc>(ExternalDictionaryLibraryAPI::LIBRARY_DATA_NEW_FUNC_NAME);
auto data_delete_func = library->get<ExternalDictionaryLibraryAPI::LibraryDataDeleteFunc>(ExternalDictionaryLibraryAPI::LIBRARY_DATA_DELETE_FUNC_NAME);
ExternalDictionaryLibraryAPI::LibraryData data_ptr = data_new_func(lib_data);
SCOPE_EXIT(data_delete_func(lib_data, data_ptr));
ExternalDictionaryLibraryAPI::RawClickHouseLibraryTable data = load_keys_func(data_ptr, &settings_holder->strings, &request_cols);
return dataToBlock(data);
}
Block ExternalDictionaryLibraryHandler::dataToBlock(ExternalDictionaryLibraryAPI::RawClickHouseLibraryTable data)
{
if (!data)
throw Exception("LibraryDictionarySource: No data returned", ErrorCodes::EXTERNAL_LIBRARY_ERROR);
const auto * columns_received = static_cast<const ExternalDictionaryLibraryAPI::Table *>(data);
if (columns_received->error_code)
throw Exception(
"LibraryDictionarySource: Returned error: " + std::to_string(columns_received->error_code) + " " + (columns_received->error_string ? columns_received->error_string : ""),
ErrorCodes::EXTERNAL_LIBRARY_ERROR);
MutableColumns columns = sample_block.cloneEmptyColumns();
for (size_t col_n = 0; col_n < columns_received->size; ++col_n)
{
if (columns.size() != columns_received->data[col_n].size)
throw Exception(
"LibraryDictionarySource: Returned unexpected number of columns: " + std::to_string(columns_received->data[col_n].size) + ", must be " + std::to_string(columns.size()),
ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
for (size_t row_n = 0; row_n < columns_received->data[col_n].size; ++row_n)
{
const auto & field = columns_received->data[col_n].data[row_n];
if (!field.data)
{
/// sample_block contains null_value (from config) inside corresponding column
const auto & col = sample_block.getByPosition(row_n);
columns[row_n]->insertFrom(*(col.column), 0);
}
else
{
const auto & size = field.size;
columns[row_n]->insertData(static_cast<const char *>(field.data), size);
}
}
}
return sample_block.cloneWithColumns(std::move(columns));
}
}

16
programs/library-bridge/SharedLibraryHandler.h → programs/library-bridge/ExternalDictionaryLibraryHandler.h
View File

@ -2,7 +2,7 @@
#include <Common/SharedLibrary.h>
#include <Common/logger_useful.h>
#include "LibraryUtils.h"
#include "ExternalDictionaryLibraryUtils.h"
namespace DB
@ -10,21 +10,21 @@ namespace DB
/// A class that manages all operations with library dictionary.
/// Every library dictionary source has its own object of this class, accessed by UUID.
class SharedLibraryHandler
class ExternalDictionaryLibraryHandler
{
public:
SharedLibraryHandler(
ExternalDictionaryLibraryHandler(
const std::string & library_path_,
const std::vector<std::string> & library_settings,
const Block & sample_block_,
const std::vector<std::string> & attributes_names_);
SharedLibraryHandler(const SharedLibraryHandler & other);
ExternalDictionaryLibraryHandler(const ExternalDictionaryLibraryHandler & other);
SharedLibraryHandler & operator=(const SharedLibraryHandler & other) = delete;
ExternalDictionaryLibraryHandler & operator=(const ExternalDictionaryLibraryHandler & other) = delete;
~SharedLibraryHandler();
~ExternalDictionaryLibraryHandler();
Block loadAll();
@ -39,7 +39,7 @@ public:
const Block & getSampleBlock() { return sample_block; }
private:
Block dataToBlock(const ClickHouseLibrary::RawClickHouseLibraryTable data);
Block dataToBlock(ExternalDictionaryLibraryAPI::RawClickHouseLibraryTable data);
std::string library_path;
const Block sample_block;
@ -50,6 +50,6 @@ private:
void * lib_data;
};
using SharedLibraryHandlerPtr = std::shared_ptr<SharedLibraryHandler>;
using SharedLibraryHandlerPtr = std::shared_ptr<ExternalDictionaryLibraryHandler>;
}

62
programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.cpp Normal file
View File

@ -0,0 +1,62 @@
#include "ExternalDictionaryLibraryHandlerFactory.h"
namespace DB
{
SharedLibraryHandlerPtr ExternalDictionaryLibraryHandlerFactory::get(const std::string & dictionary_id)
{
std::lock_guard lock(mutex);
auto library_handler = library_handlers.find(dictionary_id);
if (library_handler != library_handlers.end())
return library_handler->second;
return nullptr;
}
void ExternalDictionaryLibraryHandlerFactory::create(
const std::string & dictionary_id,
const std::string & library_path,
const std::vector<std::string> & library_settings,
const Block & sample_block,
const std::vector<std::string> & attributes_names)
{
std::lock_guard lock(mutex);
if (!library_handlers.contains(dictionary_id))
library_handlers.emplace(std::make_pair(dictionary_id, std::make_shared<ExternalDictionaryLibraryHandler>(library_path, library_settings, sample_block, attributes_names)));
else
LOG_WARNING(&Poco::Logger::get("ExternalDictionaryLibraryHandlerFactory"), "Library handler with dictionary id {} already exists", dictionary_id);
}
bool ExternalDictionaryLibraryHandlerFactory::clone(const std::string & from_dictionary_id, const std::string & to_dictionary_id)
{
std::lock_guard lock(mutex);
auto from_library_handler = library_handlers.find(from_dictionary_id);
if (from_library_handler == library_handlers.end())
return false;
/// extDict_libClone method will be called in copy constructor
library_handlers[to_dictionary_id] = std::make_shared<ExternalDictionaryLibraryHandler>(*from_library_handler->second);
return true;
}
bool ExternalDictionaryLibraryHandlerFactory::remove(const std::string & dictionary_id)
{
std::lock_guard lock(mutex);
/// extDict_libDelete is called in destructor.
return library_handlers.erase(dictionary_id);
}
ExternalDictionaryLibraryHandlerFactory & ExternalDictionaryLibraryHandlerFactory::instance()
{
static ExternalDictionaryLibraryHandlerFactory instance;
return instance;
}
}

8
programs/library-bridge/SharedLibraryHandlerFactory.h → programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.h
View File

@ -1,6 +1,6 @@
#pragma once
#include "SharedLibraryHandler.h"
#include "ExternalDictionaryLibraryHandler.h"
#include <base/defines.h>
#include <unordered_map>
@ -11,11 +11,11 @@ namespace DB
{
/// Each library dictionary source has unique UUID. When clone() method is called, a new UUID is generated.
/// There is a unique mapping from diciotnary UUID to sharedLibraryHandler.
class SharedLibraryHandlerFactory final : private boost::noncopyable
/// There is a unique mapping from dictionary UUID to sharedLibraryHandler.
class ExternalDictionaryLibraryHandlerFactory final : private boost::noncopyable
{
public:
static SharedLibraryHandlerFactory & instance();
static ExternalDictionaryLibraryHandlerFactory & instance();
SharedLibraryHandlerPtr get(const std::string & dictionary_id);

8
programs/library-bridge/LibraryUtils.h → programs/library-bridge/ExternalDictionaryLibraryUtils.h
View File

@ -5,7 +5,7 @@
#include <base/bit_cast.h>
#include <base/range.h>
#include "LibraryInterface.h"
#include "ExternalDictionaryLibraryAPI.h"
namespace DB
@ -22,7 +22,7 @@ public:
strings_holder = strings_pass;
strings.size = strings_holder.size();
ptr_holder = std::make_unique<ClickHouseLibrary::CString[]>(strings.size);
ptr_holder = std::make_unique<ExternalDictionaryLibraryAPI::CString[]>(strings.size);
strings.data = ptr_holder.get();
size_t i = 0;
@ -33,10 +33,10 @@ public:
}
}
ClickHouseLibrary::CStrings strings; // will pass pointer to lib
ExternalDictionaryLibraryAPI::CStrings strings; // will pass pointer to lib
private:
std::unique_ptr<ClickHouseLibrary::CString[]> ptr_holder = nullptr;
std::unique_ptr<ExternalDictionaryLibraryAPI::CString[]> ptr_holder = nullptr;
Container strings_holder;
};

23
programs/library-bridge/HandlerFactory.cpp
View File

@ -1,23 +0,0 @@
#include "HandlerFactory.h"
#include <Poco/Net/HTTPServerRequest.h>
#include <Server/HTTP/HTMLForm.h>
#include "Handlers.h"
namespace DB
{
std::unique_ptr<HTTPRequestHandler> LibraryBridgeHandlerFactory::createRequestHandler(const HTTPServerRequest & request)
{
Poco::URI uri{request.getURI()};
LOG_DEBUG(log, "Request URI: {}", uri.toString());
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET)
return std::make_unique<LibraryExistsHandler>(keep_alive_timeout, getContext());
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST)
return std::make_unique<LibraryRequestHandler>(keep_alive_timeout, getContext());
return nullptr;
}
}

37
programs/library-bridge/HandlerFactory.h
View File

@ -1,37 +0,0 @@
#pragma once
#include <Interpreters/Context.h>
#include <Server/HTTP/HTTPRequestHandlerFactory.h>
#include <Common/logger_useful.h>
namespace DB
{
class SharedLibraryHandler;
using SharedLibraryHandlerPtr = std::shared_ptr<SharedLibraryHandler>;
/// Factory for '/ping', '/' handlers.
class LibraryBridgeHandlerFactory : public HTTPRequestHandlerFactory, WithContext
{
public:
LibraryBridgeHandlerFactory(
const std::string & name_,
size_t keep_alive_timeout_,
ContextPtr context_)
: WithContext(context_)
, log(&Poco::Logger::get(name_))
, name(name_)
, keep_alive_timeout(keep_alive_timeout_)
{
}
std::unique_ptr<HTTPRequestHandler> createRequestHandler(const HTTPServerRequest & request) override;
private:
Poco::Logger * log;
std::string name;
size_t keep_alive_timeout;
};
}

16
programs/library-bridge/LibraryBridge.cpp
View File

@ -1,6 +1,5 @@
#include "LibraryBridge.h"
#pragma GCC diagnostic ignored "-Wmissing-declarations"
int mainEntryClickHouseLibraryBridge(int argc, char ** argv)
{
DB::LibraryBridge app;
@ -15,3 +14,18 @@ int mainEntryClickHouseLibraryBridge(int argc, char ** argv)
return code ? code : 1;
}
}
namespace DB
{
std::string LibraryBridge::bridgeName() const
{
return "LibraryBridge";
}
LibraryBridge::HandlerFactoryPtr LibraryBridge::getHandlerFactoryPtr(ContextPtr context) const
{
return std::make_shared<LibraryBridgeHandlerFactory>("LibraryRequestHandlerFactory", keep_alive_timeout, context);
}
}

13
programs/library-bridge/LibraryBridge.h
View File

@ -2,7 +2,7 @@
#include <Interpreters/Context.h>
#include <Bridge/IBridge.h>
#include "HandlerFactory.h"
#include "LibraryBridgeHandlerFactory.h"
namespace DB
@ -12,15 +12,8 @@ class LibraryBridge : public IBridge
{
protected:
std::string bridgeName() const override
{
return "LibraryBridge";
}
HandlerFactoryPtr getHandlerFactoryPtr(ContextPtr context) const override
{
return std::make_shared<LibraryBridgeHandlerFactory>("LibraryRequestHandlerFactory-factory", keep_alive_timeout, context);
}
std::string bridgeName() const override;
HandlerFactoryPtr getHandlerFactoryPtr(ContextPtr context) const override;
};
}

40
programs/library-bridge/LibraryBridgeHandlerFactory.cpp Normal file
View File

@ -0,0 +1,40 @@
#include "LibraryBridgeHandlerFactory.h"
#include <Poco/Net/HTTPServerRequest.h>
#include <Server/HTTP/HTMLForm.h>
#include "LibraryBridgeHandlers.h"
namespace DB
{
LibraryBridgeHandlerFactory::LibraryBridgeHandlerFactory(
const std::string & name_,
size_t keep_alive_timeout_,
ContextPtr context_)
: WithContext(context_)
, log(&Poco::Logger::get(name_))
, name(name_)
, keep_alive_timeout(keep_alive_timeout_)
{
}
std::unique_ptr<HTTPRequestHandler> LibraryBridgeHandlerFactory::createRequestHandler(const HTTPServerRequest & request)
{
Poco::URI uri{request.getURI()};
LOG_DEBUG(log, "Request URI: {}", uri.toString());
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET)
{
if (uri.getPath() == "/extdict_ping")
return std::make_unique<LibraryBridgeExistsHandler>(keep_alive_timeout, getContext());
}
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST)
{
if (uri.getPath() == "/extdict_request")
return std::make_unique<LibraryBridgeRequestHandler>(keep_alive_timeout, getContext());
}
return nullptr;
}
}

27
programs/library-bridge/LibraryBridgeHandlerFactory.h Normal file
View File

@ -0,0 +1,27 @@
#pragma once
#include <Interpreters/Context.h>
#include <Server/HTTP/HTTPRequestHandlerFactory.h>
#include <Common/logger_useful.h>
namespace DB
{
class LibraryBridgeHandlerFactory : public HTTPRequestHandlerFactory, WithContext
{
public:
LibraryBridgeHandlerFactory(
const std::string & name_,
size_t keep_alive_timeout_,
ContextPtr context_);
std::unique_ptr<HTTPRequestHandler> createRequestHandler(const HTTPServerRequest & request) override;
private:
Poco::Logger * log;
const std::string name;
const size_t keep_alive_timeout;
};
}

78
programs/library-bridge/Handlers.cpp → programs/library-bridge/LibraryBridgeHandlers.cpp
View File

@ -1,5 +1,5 @@
#include "Handlers.h"
#include "SharedLibraryHandlerFactory.h"
#include "LibraryBridgeHandlers.h"
#include "ExternalDictionaryLibraryHandlerFactory.h"
#include <Formats/FormatFactory.h>
#include <Server/HTTP/WriteBufferFromHTTPServerResponse.h>
@ -78,8 +78,14 @@ static void writeData(Block data, OutputFormatPtr format)
executor.execute();
}
LibraryBridgeRequestHandler::LibraryBridgeRequestHandler(size_t keep_alive_timeout_, ContextPtr context_)
: WithContext(context_)
, log(&Poco::Logger::get("LibraryBridgeRequestHandler"))
, keep_alive_timeout(keep_alive_timeout_)
{
}
void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response)
void LibraryBridgeRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response)
{
LOG_TRACE(log, "Request URI: {}", request.getURI());
HTMLForm params(getContext()->getSettingsRef(), request);
@ -104,8 +110,8 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe
try
{
bool lib_new = (method == "libNew");
if (method == "libClone")
bool lib_new = (method == "extDict_libNew");
if (method == "extDict_libClone")
{
if (!params.has("from_dictionary_id"))
{
@ -115,7 +121,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe
std::string from_dictionary_id = params.get("from_dictionary_id");
bool cloned = false;
cloned = SharedLibraryHandlerFactory::instance().clone(from_dictionary_id, dictionary_id);
cloned = ExternalDictionaryLibraryHandlerFactory::instance().clone(from_dictionary_id, dictionary_id);
if (cloned)
{
@ -123,7 +129,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe
}
else
{
LOG_TRACE(log, "Cannot clone from dictionary with id: {}, will call libNew instead", from_dictionary_id);
LOG_TRACE(log, "Cannot clone from dictionary with id: {}, will call extDict_libNew instead", from_dictionary_id);
lib_new = true;
}
}
@ -138,13 +144,14 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe
return;
}
std::string library_path = params.get("library_path");
if (!params.has("library_settings"))
{
processError(response, "No 'library_settings' in request URL");
return;
}
std::string library_path = params.get("library_path");
const auto & settings_string = params.get("library_settings");
LOG_DEBUG(log, "Parsing library settings from binary string");
@ -197,12 +204,12 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe
LOG_DEBUG(log, "Dictionary sample block with null values: {}", sample_block_with_nulls.dumpStructure());
SharedLibraryHandlerFactory::instance().create(dictionary_id, library_path, library_settings, sample_block_with_nulls, attributes_names);
ExternalDictionaryLibraryHandlerFactory::instance().create(dictionary_id, library_path, library_settings, sample_block_with_nulls, attributes_names);
writeStringBinary("1", out);
}
else if (method == "libDelete")
else if (method == "extDict_libDelete")
{
auto deleted = SharedLibraryHandlerFactory::instance().remove(dictionary_id);
bool deleted = ExternalDictionaryLibraryHandlerFactory::instance().remove(dictionary_id);
/// Do not throw, a warning is ok.
if (!deleted)
@ -210,57 +217,57 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe
writeStringBinary("1", out);
}
else if (method == "isModified")
else if (method == "extDict_isModified")
{
auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id);
auto library_handler = ExternalDictionaryLibraryHandlerFactory::instance().get(dictionary_id);
if (!library_handler)
throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Not found dictionary with id: {}", dictionary_id);
bool res = library_handler->isModified();
writeStringBinary(std::to_string(res), out);
}
else if (method == "supportsSelectiveLoad")
else if (method == "extDict_supportsSelectiveLoad")
{
auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id);
auto library_handler = ExternalDictionaryLibraryHandlerFactory::instance().get(dictionary_id);
if (!library_handler)
throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Not found dictionary with id: {}", dictionary_id);
bool res = library_handler->supportsSelectiveLoad();
writeStringBinary(std::to_string(res), out);
}
else if (method == "loadAll")
else if (method == "extDict_loadAll")
{
auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id);
auto library_handler = ExternalDictionaryLibraryHandlerFactory::instance().get(dictionary_id);
if (!library_handler)
throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Not found dictionary with id: {}", dictionary_id);
const auto & sample_block = library_handler->getSampleBlock();
LOG_DEBUG(log, "Calling loadAll() for dictionary id: {}", dictionary_id);
LOG_DEBUG(log, "Calling extDict_loadAll() for dictionary id: {}", dictionary_id);
auto input = library_handler->loadAll();
LOG_DEBUG(log, "Started sending result data for dictionary id: {}", dictionary_id);
auto output = FormatFactory::instance().getOutputFormat(FORMAT, out, sample_block, getContext());
writeData(std::move(input), std::move(output));
}
else if (method == "loadIds")
else if (method == "extDict_loadIds")
{
LOG_DEBUG(log, "Getting diciontary ids for dictionary with id: {}", dictionary_id);
String ids_string;
std::vector<uint64_t> ids = parseIdsFromBinary(request.getStream());
auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id);
auto library_handler = ExternalDictionaryLibraryHandlerFactory::instance().get(dictionary_id);
if (!library_handler)
throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Not found dictionary with id: {}", dictionary_id);
const auto & sample_block = library_handler->getSampleBlock();
LOG_DEBUG(log, "Calling loadIds() for dictionary id: {}", dictionary_id);
LOG_DEBUG(log, "Calling extDict_loadIds() for dictionary id: {}", dictionary_id);
auto input = library_handler->loadIds(ids);
LOG_DEBUG(log, "Started sending result data for dictionary id: {}", dictionary_id);
auto output = FormatFactory::instance().getOutputFormat(FORMAT, out, sample_block, getContext());
writeData(std::move(input), std::move(output));
}
else if (method == "loadKeys")
else if (method == "extDict_loadKeys")
{
if (!params.has("requested_block_sample"))
{
@ -289,18 +296,22 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe
Block block;
executor.pull(block);
auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id);
auto library_handler = ExternalDictionaryLibraryHandlerFactory::instance().get(dictionary_id);
if (!library_handler)
throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Not found dictionary with id: {}", dictionary_id);
const auto & sample_block = library_handler->getSampleBlock();
LOG_DEBUG(log, "Calling loadKeys() for dictionary id: {}", dictionary_id);
LOG_DEBUG(log, "Calling extDict_loadKeys() for dictionary id: {}", dictionary_id);
auto input = library_handler->loadKeys(block.getColumns());
LOG_DEBUG(log, "Started sending result data for dictionary id: {}", dictionary_id);
auto output = FormatFactory::instance().getOutputFormat(FORMAT, out, sample_block, getContext());
writeData(std::move(input), std::move(output));
}
else
{
LOG_WARNING(log, "Unknown library method: '{}'", method);
}
}
catch (...)
{
@ -329,8 +340,14 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe
}
}
LibraryBridgeExistsHandler::LibraryBridgeExistsHandler(size_t keep_alive_timeout_, ContextPtr context_)
: WithContext(context_)
, keep_alive_timeout(keep_alive_timeout_)
, log(&Poco::Logger::get("LibraryBridgeExistsHandler"))
{
}
void LibraryExistsHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response)
void LibraryBridgeExistsHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response)
{
try
{
@ -344,15 +361,12 @@ void LibraryExistsHandler::handleRequest(HTTPServerRequest & request, HTTPServer
}
std::string dictionary_id = params.get("dictionary_id");
auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id);
String res;
if (library_handler)
res = "1";
else
res = "0";
auto library_handler = ExternalDictionaryLibraryHandlerFactory::instance().get(dictionary_id);
String res = library_handler ? "1" : "0";
setResponseDefaultHeaders(response, keep_alive_timeout);
LOG_TRACE(log, "Senging ping response: {} (dictionary id: {})", res, dictionary_id);
LOG_TRACE(log, "Sending ping response: {} (dictionary id: {})", res, dictionary_id);
response.sendBuffer(res.data(), res.size());
}
catch (...)

27
programs/library-bridge/Handlers.h → programs/library-bridge/LibraryBridgeHandlers.h
View File

@ -3,7 +3,7 @@
#include <Interpreters/Context.h>
#include <Server/HTTP/HTTPRequestHandler.h>
#include <Common/logger_useful.h>
#include "SharedLibraryHandler.h"
#include "ExternalDictionaryLibraryHandler.h"
namespace DB
@ -11,23 +11,16 @@ namespace DB
/// Handler for requests to Library Dictionary Source, returns response in RowBinary format.
/// When a library dictionary source is created, it sends libNew request to library bridge (which is started on first
/// When a library dictionary source is created, it sends 'extDict_libNew' request to library bridge (which is started on first
/// request to it, if it was not yet started). On this request a new sharedLibrayHandler is added to a
/// sharedLibraryHandlerFactory by a dictionary uuid. With libNew request come: library_path, library_settings,
/// sharedLibraryHandlerFactory by a dictionary uuid. With 'extDict_libNew' request come: library_path, library_settings,
/// names of dictionary attributes, sample block to parse block of null values, block of null values. Everything is
/// passed in binary format and is urlencoded. When dictionary is cloned, a new handler is created.
/// Each handler is unique to dictionary.
class LibraryRequestHandler : public HTTPRequestHandler, WithContext
class LibraryBridgeRequestHandler : public HTTPRequestHandler, WithContext
{
public:
LibraryRequestHandler(
size_t keep_alive_timeout_, ContextPtr context_)
: WithContext(context_)
, log(&Poco::Logger::get("LibraryRequestHandler"))
, keep_alive_timeout(keep_alive_timeout_)
{
}
LibraryBridgeRequestHandler(size_t keep_alive_timeout_, ContextPtr context_);
void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override;
@ -39,22 +32,16 @@ private:
};
class LibraryExistsHandler : public HTTPRequestHandler, WithContext
class LibraryBridgeExistsHandler : public HTTPRequestHandler, WithContext
{
public:
explicit LibraryExistsHandler(size_t keep_alive_timeout_, ContextPtr context_)
: WithContext(context_)
, keep_alive_timeout(keep_alive_timeout_)
, log(&Poco::Logger::get("LibraryRequestHandler"))
{
}
LibraryBridgeExistsHandler(size_t keep_alive_timeout_, ContextPtr context_);
void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override;
private:
const size_t keep_alive_timeout;
Poco::Logger * log;
};
}

110
programs/library-bridge/LibraryInterface.h
View File

@ -1,110 +0,0 @@
#pragma once
#include <cstdint>
#include <string>
#define CLICKHOUSE_DICTIONARY_LIBRARY_API 1
namespace ClickHouseLibrary
{
using CString = const char *;
using ColumnName = CString;
using ColumnNames = ColumnName[];
struct CStrings
{
CString * data = nullptr;
uint64_t size = 0;
};
struct VectorUInt64
{
const uint64_t * data = nullptr;
uint64_t size = 0;
};
struct ColumnsUInt64
{
VectorUInt64 * data = nullptr;
uint64_t size = 0;
};
struct Field
{
const void * data = nullptr;
uint64_t size = 0;
};
struct Row
{
const Field * data = nullptr;
uint64_t size = 0;
};
struct Table
{
const Row * data = nullptr;
uint64_t size = 0;
uint64_t error_code = 0; // 0 = ok; !0 = error, with message in error_string
const char * error_string = nullptr;
};
enum LogLevel
{
FATAL = 1,
CRITICAL,
ERROR,
WARNING,
NOTICE,
INFORMATION,
DEBUG,
TRACE,
};
void log(LogLevel level, CString msg);
extern std::string_view LIBRARY_CREATE_NEW_FUNC_NAME;
extern std::string_view LIBRARY_CLONE_FUNC_NAME;
extern std::string_view LIBRARY_DELETE_FUNC_NAME;
extern std::string_view LIBRARY_DATA_NEW_FUNC_NAME;
extern std::string_view LIBRARY_DATA_DELETE_FUNC_NAME;
extern std::string_view LIBRARY_LOAD_ALL_FUNC_NAME;
extern std::string_view LIBRARY_LOAD_IDS_FUNC_NAME;
extern std::string_view LIBRARY_LOAD_KEYS_FUNC_NAME;
extern std::string_view LIBRARY_IS_MODIFIED_FUNC_NAME;
extern std::string_view LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME;
using LibraryContext = void *;
using LibraryLoggerFunc = void (*)(LogLevel, CString /* message */);
using LibrarySettings = CStrings *;
using LibraryNewFunc = LibraryContext (*)(LibrarySettings, LibraryLoggerFunc);
using LibraryCloneFunc = LibraryContext (*)(LibraryContext);
using LibraryDeleteFunc = void (*)(LibraryContext);
using LibraryData = void *;
using LibraryDataNewFunc = LibraryData (*)(LibraryContext);
using LibraryDataDeleteFunc = void (*)(LibraryContext, LibraryData);
/// Can be safely casted into const Table * with static_cast<const ClickHouseLibrary::Table *>
using RawClickHouseLibraryTable = void *;
using RequestedColumnsNames = CStrings *;
using LibraryLoadAllFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedColumnsNames);
using RequestedIds = const VectorUInt64 *;
using LibraryLoadIdsFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedColumnsNames, RequestedIds);
using RequestedKeys = Table *;
/// There are no requested column names for load keys func
using LibraryLoadKeysFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedKeys);
using LibraryIsModifiedFunc = bool (*)(LibraryContext, LibrarySettings);
using LibrarySupportsSelectiveLoadFunc = bool (*)(LibraryContext, LibrarySettings);
}

214
programs/library-bridge/SharedLibraryHandler.cpp
View File

@ -1,214 +0,0 @@
#include "SharedLibraryHandler.h"
#include <base/scope_guard.h>
#include <base/bit_cast.h>
#include <base/find_symbols.h>
#include <IO/ReadHelpers.h>
namespace DB
{
namespace ErrorCodes
{
extern const int EXTERNAL_LIBRARY_ERROR;
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
}
SharedLibraryHandler::SharedLibraryHandler(
const std::string & library_path_,
const std::vector<std::string> & library_settings,
const Block & sample_block_,
const std::vector<std::string> & attributes_names_)
: library_path(library_path_)
, sample_block(sample_block_)
, attributes_names(attributes_names_)
{
library = std::make_shared<SharedLibrary>(library_path, RTLD_LAZY);
settings_holder = std::make_shared<CStringsHolder>(CStringsHolder(library_settings));
auto lib_new = library->tryGet<ClickHouseLibrary::LibraryNewFunc>(ClickHouseLibrary::LIBRARY_CREATE_NEW_FUNC_NAME);
if (lib_new)
lib_data = lib_new(&settings_holder->strings, ClickHouseLibrary::log);
else
throw Exception("Method libNew failed", ErrorCodes::EXTERNAL_LIBRARY_ERROR);
}
SharedLibraryHandler::SharedLibraryHandler(const SharedLibraryHandler & other)
: library_path{other.library_path}
, sample_block{other.sample_block}
, attributes_names{other.attributes_names}
, library{other.library}
, settings_holder{other.settings_holder}
{
auto lib_clone = library->tryGet<ClickHouseLibrary::LibraryCloneFunc>(ClickHouseLibrary::LIBRARY_CLONE_FUNC_NAME);
if (lib_clone)
{
lib_data = lib_clone(other.lib_data);
}
else
{
auto lib_new = library->tryGet<ClickHouseLibrary::LibraryNewFunc>(ClickHouseLibrary::LIBRARY_CREATE_NEW_FUNC_NAME);
if (lib_new)
lib_data = lib_new(&settings_holder->strings, ClickHouseLibrary::log);
}
}
SharedLibraryHandler::~SharedLibraryHandler()
{
auto lib_delete = library->tryGet<ClickHouseLibrary::LibraryDeleteFunc>(ClickHouseLibrary::LIBRARY_DELETE_FUNC_NAME);
if (lib_delete)
lib_delete(lib_data);
}
bool SharedLibraryHandler::isModified()
{
auto func_is_modified = library->tryGet<ClickHouseLibrary::LibraryIsModifiedFunc>(ClickHouseLibrary::LIBRARY_IS_MODIFIED_FUNC_NAME);
if (func_is_modified)
return func_is_modified(lib_data, &settings_holder->strings);
return true;
}
bool SharedLibraryHandler::supportsSelectiveLoad()
{
auto func_supports_selective_load = library->tryGet<ClickHouseLibrary::LibrarySupportsSelectiveLoadFunc>(ClickHouseLibrary::LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME);
if (func_supports_selective_load)
return func_supports_selective_load(lib_data, &settings_holder->strings);
return true;
}
Block SharedLibraryHandler::loadAll()
{
auto columns_holder = std::make_unique<ClickHouseLibrary::CString[]>(attributes_names.size());
ClickHouseLibrary::CStrings columns{static_cast<decltype(ClickHouseLibrary::CStrings::data)>(columns_holder.get()), attributes_names.size()};
for (size_t i = 0; i < attributes_names.size(); ++i)
columns.data[i] = attributes_names[i].c_str();
auto load_all_func = library->get<ClickHouseLibrary::LibraryLoadAllFunc>(ClickHouseLibrary::LIBRARY_LOAD_ALL_FUNC_NAME);
auto data_new_func = library->get<ClickHouseLibrary::LibraryDataNewFunc>(ClickHouseLibrary::LIBRARY_DATA_NEW_FUNC_NAME);
auto data_delete_func = library->get<ClickHouseLibrary::LibraryDataDeleteFunc>(ClickHouseLibrary::LIBRARY_DATA_DELETE_FUNC_NAME);
ClickHouseLibrary::LibraryData data_ptr = data_new_func(lib_data);
SCOPE_EXIT(data_delete_func(lib_data, data_ptr));
ClickHouseLibrary::RawClickHouseLibraryTable data = load_all_func(data_ptr, &settings_holder->strings, &columns);
return dataToBlock(data);
}
Block SharedLibraryHandler::loadIds(const std::vector<uint64_t> & ids)
{
const ClickHouseLibrary::VectorUInt64 ids_data{bit_cast<decltype(ClickHouseLibrary::VectorUInt64::data)>(ids.data()), ids.size()};
auto columns_holder = std::make_unique<ClickHouseLibrary::CString[]>(attributes_names.size());
ClickHouseLibrary::CStrings columns_pass{static_cast<decltype(ClickHouseLibrary::CStrings::data)>(columns_holder.get()), attributes_names.size()};
auto load_ids_func = library->get<ClickHouseLibrary::LibraryLoadIdsFunc>(ClickHouseLibrary::LIBRARY_LOAD_IDS_FUNC_NAME);
auto data_new_func = library->get<ClickHouseLibrary::LibraryDataNewFunc>(ClickHouseLibrary::LIBRARY_DATA_NEW_FUNC_NAME);
auto data_delete_func = library->get<ClickHouseLibrary::LibraryDataDeleteFunc>(ClickHouseLibrary::LIBRARY_DATA_DELETE_FUNC_NAME);
ClickHouseLibrary::LibraryData data_ptr = data_new_func(lib_data);
SCOPE_EXIT(data_delete_func(lib_data, data_ptr));
ClickHouseLibrary::RawClickHouseLibraryTable data = load_ids_func(data_ptr, &settings_holder->strings, &columns_pass, &ids_data);
return dataToBlock(data);
}
Block SharedLibraryHandler::loadKeys(const Columns & key_columns)
{
auto holder = std::make_unique<ClickHouseLibrary::Row[]>(key_columns.size());
std::vector<std::unique_ptr<ClickHouseLibrary::Field[]>> column_data_holders;
for (size_t i = 0; i < key_columns.size(); ++i)
{
auto cell_holder = std::make_unique<ClickHouseLibrary::Field[]>(key_columns[i]->size());
for (size_t j = 0; j < key_columns[i]->size(); ++j)
{
auto data_ref = key_columns[i]->getDataAt(j);
cell_holder[j] = ClickHouseLibrary::Field{
.data = static_cast<const void *>(data_ref.data),
.size = data_ref.size};
}
holder[i] = ClickHouseLibrary::Row{
.data = static_cast<ClickHouseLibrary::Field *>(cell_holder.get()),
.size = key_columns[i]->size()};
column_data_holders.push_back(std::move(cell_holder));
}
ClickHouseLibrary::Table request_cols{
.data = static_cast<ClickHouseLibrary::Row *>(holder.get()),
.size = key_columns.size()};
auto load_keys_func = library->get<ClickHouseLibrary::LibraryLoadKeysFunc>(ClickHouseLibrary::LIBRARY_LOAD_KEYS_FUNC_NAME);
auto data_new_func = library->get<ClickHouseLibrary::LibraryDataNewFunc>(ClickHouseLibrary::LIBRARY_DATA_NEW_FUNC_NAME);
auto data_delete_func = library->get<ClickHouseLibrary::LibraryDataDeleteFunc>(ClickHouseLibrary::LIBRARY_DATA_DELETE_FUNC_NAME);
ClickHouseLibrary::LibraryData data_ptr = data_new_func(lib_data);
SCOPE_EXIT(data_delete_func(lib_data, data_ptr));
ClickHouseLibrary::RawClickHouseLibraryTable data = load_keys_func(data_ptr, &settings_holder->strings, &request_cols);
return dataToBlock(data);
}
Block SharedLibraryHandler::dataToBlock(const ClickHouseLibrary::RawClickHouseLibraryTable data)
{
if (!data)
throw Exception("LibraryDictionarySource: No data returned", ErrorCodes::EXTERNAL_LIBRARY_ERROR);
const auto * columns_received = static_cast<const ClickHouseLibrary::Table *>(data);
if (columns_received->error_code)
throw Exception(
"LibraryDictionarySource: Returned error: " + std::to_string(columns_received->error_code) + " " + (columns_received->error_string ? columns_received->error_string : ""),
ErrorCodes::EXTERNAL_LIBRARY_ERROR);
MutableColumns columns = sample_block.cloneEmptyColumns();
for (size_t col_n = 0; col_n < columns_received->size; ++col_n)
{
if (columns.size() != columns_received->data[col_n].size)
throw Exception(
"LibraryDictionarySource: Returned unexpected number of columns: " + std::to_string(columns_received->data[col_n].size) + ", must be " + std::to_string(columns.size()),
ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
for (size_t row_n = 0; row_n < columns_received->data[col_n].size; ++row_n)
{
const auto & field = columns_received->data[col_n].data[row_n];
if (!field.data)
{
/// sample_block contains null_value (from config) inside corresponding column
const auto & col = sample_block.getByPosition(row_n);
columns[row_n]->insertFrom(*(col.column), 0);
}
else
{
const auto & size = field.size;
columns[row_n]->insertData(static_cast<const char *>(field.data), size);
}
}
}
return sample_block.cloneWithColumns(std::move(columns));
}
}

62
programs/library-bridge/SharedLibraryHandlerFactory.cpp
View File

@ -1,62 +0,0 @@
#include "SharedLibraryHandlerFactory.h"
namespace DB
{
SharedLibraryHandlerPtr SharedLibraryHandlerFactory::get(const std::string & dictionary_id)
{
std::lock_guard lock(mutex);
auto library_handler = library_handlers.find(dictionary_id);
if (library_handler != library_handlers.end())
return library_handler->second;
return nullptr;
}
void SharedLibraryHandlerFactory::create(
const std::string & dictionary_id,
const std::string & library_path,
const std::vector<std::string> & library_settings,
const Block & sample_block,
const std::vector<std::string> & attributes_names)
{
std::lock_guard lock(mutex);
if (!library_handlers.contains(dictionary_id))
library_handlers.emplace(std::make_pair(dictionary_id, std::make_shared<SharedLibraryHandler>(library_path, library_settings, sample_block, attributes_names)));
else
LOG_WARNING(&Poco::Logger::get("SharedLibraryHandlerFactory"), "Library handler with dictionary id {} already exists", dictionary_id);
}
bool SharedLibraryHandlerFactory::clone(const std::string & from_dictionary_id, const std::string & to_dictionary_id)
{
std::lock_guard lock(mutex);
auto from_library_handler = library_handlers.find(from_dictionary_id);
if (from_library_handler == library_handlers.end())
return false;
/// libClone method will be called in copy constructor
library_handlers[to_dictionary_id] = std::make_shared<SharedLibraryHandler>(*from_library_handler->second);
return true;
}
bool SharedLibraryHandlerFactory::remove(const std::string & dictionary_id)
{
std::lock_guard lock(mutex);
/// libDelete is called in destructor.
return library_handlers.erase(dictionary_id);
}
SharedLibraryHandlerFactory & SharedLibraryHandlerFactory::instance()
{
static SharedLibraryHandlerFactory ret;
return ret;
}
}

1
programs/library-bridge/library-bridge.cpp
View File

@ -1,3 +1,2 @@
int mainEntryClickHouseLibraryBridge(int argc, char ** argv);
int main(int argc_, char ** argv_) { return mainEntryClickHouseLibraryBridge(argc_, argv_); }

6
programs/odbc-bridge/CMakeLists.txt
View File

@ -2,17 +2,17 @@ include(${ClickHouse_SOURCE_DIR}/cmake/split_debug_symbols.cmake)
set (CLICKHOUSE_ODBC_BRIDGE_SOURCES
ColumnInfoHandler.cpp
getIdentifierQuote.cpp
HandlerFactory.cpp
IdentifierQuoteHandler.cpp
MainHandler.cpp
ODBCBlockInputStream.cpp
ODBCBlockOutputStream.cpp
ODBCBridge.cpp
ODBCHandlerFactory.cpp
PingHandler.cpp
SchemaAllowedHandler.cpp
validateODBCConnectionString.cpp
getIdentifierQuote.cpp
odbc-bridge.cpp
validateODBCConnectionString.cpp
)
if (OS_LINUX)

16
programs/odbc-bridge/ODBCBridge.cpp
View File

@ -1,6 +1,5 @@
#include "ODBCBridge.h"
#pragma GCC diagnostic ignored "-Wmissing-declarations"
int mainEntryClickHouseODBCBridge(int argc, char ** argv)
{
DB::ODBCBridge app;
@ -15,3 +14,18 @@ int mainEntryClickHouseODBCBridge(int argc, char ** argv)
return code ? code : 1;
}
}
namespace DB
{
std::string ODBCBridge::bridgeName() const
{
return "ODBCBridge";
}
ODBCBridge::HandlerFactoryPtr ODBCBridge::getHandlerFactoryPtr(ContextPtr context) const
{
return std::make_shared<ODBCBridgeHandlerFactory>("ODBCRequestHandlerFactory-factory", keep_alive_timeout, context);
}
}

13
programs/odbc-bridge/ODBCBridge.h
View File

@ -3,7 +3,7 @@
#include <Interpreters/Context.h>
#include <Poco/Logger.h>
#include <Bridge/IBridge.h>
#include "HandlerFactory.h"
#include "ODBCHandlerFactory.h"
namespace DB
@ -13,14 +13,7 @@ class ODBCBridge : public IBridge
{
protected:
std::string bridgeName() const override
{
return "ODBCBridge";
}
HandlerFactoryPtr getHandlerFactoryPtr(ContextPtr context) const override
{
return std::make_shared<ODBCBridgeHandlerFactory>("ODBCRequestHandlerFactory-factory", keep_alive_timeout, context);
}
std::string bridgeName() const override;
HandlerFactoryPtr getHandlerFactoryPtr(ContextPtr context) const override;
};
}

10
programs/odbc-bridge/HandlerFactory.cpp → programs/odbc-bridge/ODBCHandlerFactory.cpp
View File

@ -1,4 +1,4 @@
#include "HandlerFactory.h"
#include "ODBCHandlerFactory.h"
#include "PingHandler.h"
#include "ColumnInfoHandler.h"
#include <Common/config.h>
@ -9,6 +9,14 @@
namespace DB
{
ODBCBridgeHandlerFactory::ODBCBridgeHandlerFactory(const std::string & name_, size_t keep_alive_timeout_, ContextPtr context_)
: WithContext(context_)
, log(&Poco::Logger::get(name_))
, name(name_)
, keep_alive_timeout(keep_alive_timeout_)
{
}
std::unique_ptr<HTTPRequestHandler> ODBCBridgeHandlerFactory::createRequestHandler(const HTTPServerRequest & request)
{
Poco::URI uri{request.getURI()};

8
programs/odbc-bridge/HandlerFactory.h → programs/odbc-bridge/ODBCHandlerFactory.h
View File

@ -17,13 +17,7 @@ namespace DB
class ODBCBridgeHandlerFactory : public HTTPRequestHandlerFactory, WithContext
{
public:
ODBCBridgeHandlerFactory(const std::string & name_, size_t keep_alive_timeout_, ContextPtr context_)
: WithContext(context_)
, log(&Poco::Logger::get(name_))
, name(name_)
, keep_alive_timeout(keep_alive_timeout_)
{
}
ODBCBridgeHandlerFactory(const std::string & name_, size_t keep_alive_timeout_, ContextPtr context_);
std::unique_ptr<HTTPRequestHandler> createRequestHandler(const HTTPServerRequest & request) override;

89
src/BridgeHelper/LibraryBridgeHelper.cpp → src/BridgeHelper/ExternalDictionaryLibraryBridgeHelper.cpp
View File

@ -1,4 +1,4 @@
#include "LibraryBridgeHelper.h"
#include "ExternalDictionaryLibraryBridgeHelper.h"
#include <Formats/formatBlock.h>
#include <Dictionaries/DictionarySourceHelpers.h>
@ -26,26 +26,43 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
}
LibraryBridgeHelper::LibraryBridgeHelper(
ExternalDictionaryLibraryBridgeHelper::ExternalDictionaryLibraryBridgeHelper(
ContextPtr context_,
const Block & sample_block_,
const Field & dictionary_id_,
const LibraryInitData & library_data_)
: IBridgeHelper(context_->getGlobalContext())
, log(&Poco::Logger::get("LibraryBridgeHelper"))
, log(&Poco::Logger::get("ExternalDictionaryLibraryBridgeHelper"))
, sample_block(sample_block_)
, config(context_->getConfigRef())
, http_timeout(context_->getGlobalContext()->getSettingsRef().http_receive_timeout.value)
, library_data(library_data_)
, dictionary_id(dictionary_id_)
, bridge_host(config.getString("library_bridge.host", DEFAULT_HOST))
, bridge_port(config.getUInt("library_bridge.port", DEFAULT_PORT))
, http_timeouts(ConnectionTimeouts::getHTTPTimeouts(context_))
{
bridge_port = config.getUInt("library_bridge.port", DEFAULT_PORT);
bridge_host = config.getString("library_bridge.host", DEFAULT_HOST);
}
Poco::URI LibraryBridgeHelper::createRequestURI(const String & method) const
Poco::URI ExternalDictionaryLibraryBridgeHelper::getPingURI() const
{
auto uri = createBaseURI();
uri.setPath(PING_HANDLER);
uri.addQueryParameter("dictionary_id", toString(dictionary_id));
return uri;
}
Poco::URI ExternalDictionaryLibraryBridgeHelper::getMainURI() const
{
auto uri = createBaseURI();
uri.setPath(MAIN_HANDLER);
return uri;
}
Poco::URI ExternalDictionaryLibraryBridgeHelper::createRequestURI(const String & method) const
{
auto uri = getMainURI();
uri.addQueryParameter("dictionary_id", toString(dictionary_id));
@ -54,7 +71,7 @@ Poco::URI LibraryBridgeHelper::createRequestURI(const String & method) const
}
Poco::URI LibraryBridgeHelper::createBaseURI() const
Poco::URI ExternalDictionaryLibraryBridgeHelper::createBaseURI() const
{
Poco::URI uri;
uri.setHost(bridge_host);
@ -64,18 +81,18 @@ Poco::URI LibraryBridgeHelper::createBaseURI() const
}
void LibraryBridgeHelper::startBridge(std::unique_ptr<ShellCommand> cmd) const
void ExternalDictionaryLibraryBridgeHelper::startBridge(std::unique_ptr<ShellCommand> cmd) const
{
getContext()->addBridgeCommand(std::move(cmd));
}
bool LibraryBridgeHelper::bridgeHandShake()
bool ExternalDictionaryLibraryBridgeHelper::bridgeHandShake()
{
String result;
try
{
ReadWriteBufferFromHTTP buf(createRequestURI(PING), Poco::Net::HTTPRequest::HTTP_GET, {}, http_timeouts, credentials);
ReadWriteBufferFromHTTP buf(getPingURI(), Poco::Net::HTTPRequest::HTTP_GET, {}, http_timeouts, credentials);
readString(result, buf);
}
catch (...)
@ -91,12 +108,12 @@ bool LibraryBridgeHelper::bridgeHandShake()
* 2. Bridge crashed or restarted for some reason while server did not.
**/
if (result.size() != 1)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {}. Check bridge and server have the same version.", result);
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {}. Check that bridge and server have the same version.", result);
UInt8 dictionary_id_exists;
auto parsed = tryParse<UInt8>(dictionary_id_exists, result);
if (!parsed || (dictionary_id_exists != 0 && dictionary_id_exists != 1))
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {} ({}). Check bridge and server have the same version.",
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {} ({}). Check that bridge and server have the same version.",
result, parsed ? toString(dictionary_id_exists) : "failed to parse");
LOG_TRACE(log, "dictionary_id: {}, dictionary_id_exists on bridge side: {}, library confirmed to be initialized on server side: {}",
@ -113,7 +130,7 @@ bool LibraryBridgeHelper::bridgeHandShake()
bool reinitialized = false;
try
{
auto uri = createRequestURI(LIB_NEW_METHOD);
auto uri = createRequestURI(EXT_DICT_LIB_NEW_METHOD);
reinitialized = executeRequest(uri, getInitLibraryCallback());
}
catch (...)
@ -131,11 +148,11 @@ bool LibraryBridgeHelper::bridgeHandShake()
}
ReadWriteBufferFromHTTP::OutStreamCallback LibraryBridgeHelper::getInitLibraryCallback() const
ReadWriteBufferFromHTTP::OutStreamCallback ExternalDictionaryLibraryBridgeHelper::getInitLibraryCallback() const
{
/// Sample block must contain null values
WriteBufferFromOwnString out;
auto output_format = getContext()->getOutputFormat(LibraryBridgeHelper::DEFAULT_FORMAT, out, sample_block);
auto output_format = getContext()->getOutputFormat(ExternalDictionaryLibraryBridgeHelper::DEFAULT_FORMAT, out, sample_block);
formatBlock(output_format, sample_block);
auto block_string = out.str();
@ -150,19 +167,19 @@ ReadWriteBufferFromHTTP::OutStreamCallback LibraryBridgeHelper::getInitLibraryCa
}
bool LibraryBridgeHelper::initLibrary()
bool ExternalDictionaryLibraryBridgeHelper::initLibrary()
{
startBridgeSync();
auto uri = createRequestURI(LIB_NEW_METHOD);
auto uri = createRequestURI(EXT_DICT_LIB_NEW_METHOD);
library_initialized = executeRequest(uri, getInitLibraryCallback());
return library_initialized;
}
bool LibraryBridgeHelper::cloneLibrary(const Field & other_dictionary_id)
bool ExternalDictionaryLibraryBridgeHelper::cloneLibrary(const Field & other_dictionary_id)
{
startBridgeSync();
auto uri = createRequestURI(LIB_CLONE_METHOD);
auto uri = createRequestURI(EXT_DICT_LIB_CLONE_METHOD);
uri.addQueryParameter("from_dictionary_id", toString(other_dictionary_id));
/// We also pass initialization settings in order to create a library handler
/// in case from_dictionary_id does not exist in bridge side (possible in case of bridge crash).
@ -171,70 +188,70 @@ bool LibraryBridgeHelper::cloneLibrary(const Field & other_dictionary_id)
}
bool LibraryBridgeHelper::removeLibrary()
bool ExternalDictionaryLibraryBridgeHelper::removeLibrary()
{
/// Do not force bridge restart if it is not running in case of removeLibrary
/// because in this case after restart it will not have this dictionaty id in memory anyway.
if (bridgeHandShake())
{
auto uri = createRequestURI(LIB_DELETE_METHOD);
auto uri = createRequestURI(EXT_DICT_LIB_DELETE_METHOD);
return executeRequest(uri);
}
return true;
}
bool LibraryBridgeHelper::isModified()
bool ExternalDictionaryLibraryBridgeHelper::isModified()
{
startBridgeSync();
auto uri = createRequestURI(IS_MODIFIED_METHOD);
auto uri = createRequestURI(EXT_DICT_IS_MODIFIED_METHOD);
return executeRequest(uri);
}
bool LibraryBridgeHelper::supportsSelectiveLoad()
bool ExternalDictionaryLibraryBridgeHelper::supportsSelectiveLoad()
{
startBridgeSync();
auto uri = createRequestURI(SUPPORTS_SELECTIVE_LOAD_METHOD);
auto uri = createRequestURI(EXT_DICT_SUPPORTS_SELECTIVE_LOAD_METHOD);
return executeRequest(uri);
}
QueryPipeline LibraryBridgeHelper::loadAll()
QueryPipeline ExternalDictionaryLibraryBridgeHelper::loadAll()
{
startBridgeSync();
auto uri = createRequestURI(LOAD_ALL_METHOD);
auto uri = createRequestURI(EXT_DICT_LOAD_ALL_METHOD);
return QueryPipeline(loadBase(uri));
}
QueryPipeline LibraryBridgeHelper::loadIds(const std::vector<uint64_t> & ids)
QueryPipeline ExternalDictionaryLibraryBridgeHelper::loadIds(const std::vector<uint64_t> & ids)
{
startBridgeSync();
auto uri = createRequestURI(LOAD_IDS_METHOD);
auto uri = createRequestURI(EXT_DICT_LOAD_IDS_METHOD);
uri.addQueryParameter("ids_num", toString(ids.size())); /// Not used parameter, but helpful
auto ids_string = getDictIdsString(ids);
return QueryPipeline(loadBase(uri, [ids_string](std::ostream & os) { os << ids_string; }));
}
QueryPipeline LibraryBridgeHelper::loadKeys(const Block & requested_block)
QueryPipeline ExternalDictionaryLibraryBridgeHelper::loadKeys(const Block & requested_block)
{
startBridgeSync();
auto uri = createRequestURI(LOAD_KEYS_METHOD);
auto uri = createRequestURI(EXT_DICT_LOAD_KEYS_METHOD);
/// Sample block to parse block from callback
uri.addQueryParameter("requested_block_sample", requested_block.getNamesAndTypesList().toString());
ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = [requested_block, this](std::ostream & os)
{
WriteBufferFromOStream out_buffer(os);
auto output_format = getContext()->getOutputFormat(LibraryBridgeHelper::DEFAULT_FORMAT, out_buffer, requested_block.cloneEmpty());
auto output_format = getContext()->getOutputFormat(ExternalDictionaryLibraryBridgeHelper::DEFAULT_FORMAT, out_buffer, requested_block.cloneEmpty());
formatBlock(output_format, requested_block);
};
return QueryPipeline(loadBase(uri, out_stream_callback));
}
bool LibraryBridgeHelper::executeRequest(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback) const
bool ExternalDictionaryLibraryBridgeHelper::executeRequest(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback) const
{
ReadWriteBufferFromHTTP buf(
uri,
@ -248,7 +265,7 @@ bool LibraryBridgeHelper::executeRequest(const Poco::URI & uri, ReadWriteBufferF
}
QueryPipeline LibraryBridgeHelper::loadBase(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback)
QueryPipeline ExternalDictionaryLibraryBridgeHelper::loadBase(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback)
{
auto read_buf_ptr = std::make_unique<ReadWriteBufferFromHTTP>(
uri,
@ -261,13 +278,13 @@ QueryPipeline LibraryBridgeHelper::loadBase(const Poco::URI & uri, ReadWriteBuff
getContext()->getReadSettings(),
ReadWriteBufferFromHTTP::HTTPHeaderEntries{});
auto source = FormatFactory::instance().getInput(LibraryBridgeHelper::DEFAULT_FORMAT, *read_buf_ptr, sample_block, getContext(), DEFAULT_BLOCK_SIZE);
auto source = FormatFactory::instance().getInput(ExternalDictionaryLibraryBridgeHelper::DEFAULT_FORMAT, *read_buf_ptr, sample_block, getContext(), DEFAULT_BLOCK_SIZE);
source->addBuffer(std::move(read_buf_ptr));
return QueryPipeline(std::move(source));
}
String LibraryBridgeHelper::getDictIdsString(const std::vector<UInt64> & ids)
String ExternalDictionaryLibraryBridgeHelper::getDictIdsString(const std::vector<UInt64> & ids)
{
WriteBufferFromOwnString out;
writeVectorBinary(ids, out);

35
src/BridgeHelper/LibraryBridgeHelper.h → src/BridgeHelper/ExternalDictionaryLibraryBridgeHelper.h
View File

@ -14,7 +14,7 @@ namespace DB
class Pipe;
class LibraryBridgeHelper : public IBridgeHelper
class ExternalDictionaryLibraryBridgeHelper : public IBridgeHelper
{
public:
@ -26,8 +26,10 @@ public:
};
static constexpr inline size_t DEFAULT_PORT = 9012;
static constexpr inline auto PING_HANDLER = "/extdict_ping";
static constexpr inline auto MAIN_HANDLER = "/extdict_request";
LibraryBridgeHelper(ContextPtr context_, const Block & sample_block, const Field & dictionary_id_, const LibraryInitData & library_data_);
ExternalDictionaryLibraryBridgeHelper(ContextPtr context_, const Block & sample_block, const Field & dictionary_id_, const LibraryInitData & library_data_);
bool initLibrary();
@ -45,13 +47,13 @@ public:
QueryPipeline loadKeys(const Block & requested_block);
QueryPipeline loadBase(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = {});
bool executeRequest(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = {}) const;
LibraryInitData getLibraryData() const { return library_data; }
protected:
Poco::URI getPingURI() const override;
Poco::URI getMainURI() const override;
bool bridgeHandShake() override;
void startBridge(std::unique_ptr<ShellCommand> cmd) const override;
@ -74,18 +76,21 @@ protected:
Poco::URI createBaseURI() const override;
QueryPipeline loadBase(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = {});
bool executeRequest(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = {}) const;
ReadWriteBufferFromHTTP::OutStreamCallback getInitLibraryCallback() const;
private:
static constexpr inline auto LIB_NEW_METHOD = "libNew";
static constexpr inline auto LIB_CLONE_METHOD = "libClone";
static constexpr inline auto LIB_DELETE_METHOD = "libDelete";
static constexpr inline auto LOAD_ALL_METHOD = "loadAll";
static constexpr inline auto LOAD_IDS_METHOD = "loadIds";
static constexpr inline auto LOAD_KEYS_METHOD = "loadKeys";
static constexpr inline auto IS_MODIFIED_METHOD = "isModified";
static constexpr inline auto PING = "ping";
static constexpr inline auto SUPPORTS_SELECTIVE_LOAD_METHOD = "supportsSelectiveLoad";
static constexpr inline auto EXT_DICT_LIB_NEW_METHOD = "extDict_libNew";
static constexpr inline auto EXT_DICT_LIB_CLONE_METHOD = "extDict_libClone";
static constexpr inline auto EXT_DICT_LIB_DELETE_METHOD = "extDict_libDelete";
static constexpr inline auto EXT_DICT_LOAD_ALL_METHOD = "extDict_loadAll";
static constexpr inline auto EXT_DICT_LOAD_IDS_METHOD = "extDict_loadIds";
static constexpr inline auto EXT_DICT_LOAD_KEYS_METHOD = "extDict_loadKeys";
static constexpr inline auto EXT_DICT_IS_MODIFIED_METHOD = "extDict_isModified";
static constexpr inline auto EXT_DICT_SUPPORTS_SELECTIVE_LOAD_METHOD = "extDict_supportsSelectiveLoad";
Poco::URI createRequestURI(const String & method) const;

16
src/BridgeHelper/IBridgeHelper.cpp
View File

@ -18,22 +18,6 @@ namespace ErrorCodes
}
Poco::URI IBridgeHelper::getMainURI() const
{
auto uri = createBaseURI();
uri.setPath(MAIN_HANDLER);
return uri;
}
Poco::URI IBridgeHelper::getPingURI() const
{
auto uri = createBaseURI();
uri.setPath(PING_HANDLER);
return uri;
}
void IBridgeHelper::startBridgeSync()
{
if (!bridgeHandShake())

7
src/BridgeHelper/IBridgeHelper.h
View File

@ -19,8 +19,6 @@ class IBridgeHelper: protected WithContext
public:
static constexpr inline auto DEFAULT_HOST = "127.0.0.1";
static constexpr inline auto PING_HANDLER = "/ping";
static constexpr inline auto MAIN_HANDLER = "/";
static constexpr inline auto DEFAULT_FORMAT = "RowBinary";
static constexpr inline auto PING_OK_ANSWER = "Ok.";
@ -31,9 +29,9 @@ public:
virtual ~IBridgeHelper() = default;
Poco::URI getMainURI() const;
virtual Poco::URI getMainURI() const = 0;
Poco::URI getPingURI() const;
virtual Poco::URI getPingURI() const = 0;
void startBridgeSync();
@ -41,7 +39,6 @@ protected:
/// Check bridge is running. Can also check something else in the mean time.
virtual bool bridgeHandShake() = 0;
/// clickhouse-odbc-bridge, clickhouse-library-bridge
virtual String serviceAlias() const = 0;
virtual String serviceFileName() const = 0;

18
src/BridgeHelper/XDBCBridgeHelper.h
View File

@ -53,6 +53,8 @@ class XDBCBridgeHelper : public IXDBCBridgeHelper
public:
static constexpr inline auto DEFAULT_PORT = BridgeHelperMixin::DEFAULT_PORT;
static constexpr inline auto PING_HANDLER = "/ping";
static constexpr inline auto MAIN_HANDLER = "/";
static constexpr inline auto COL_INFO_HANDLER = "/columns_info";
static constexpr inline auto IDENTIFIER_QUOTE_HANDLER = "/identifier_quote";
static constexpr inline auto SCHEMA_ALLOWED_HANDLER = "/schema_allowed";
@ -72,6 +74,22 @@ public:
}
protected:
Poco::URI getPingURI() const override
{
auto uri = createBaseURI();
uri.setPath(PING_HANDLER);
return uri;
}
Poco::URI getMainURI() const override
{
auto uri = createBaseURI();
uri.setPath(MAIN_HANDLER);
return uri;
}
bool bridgeHandShake() override
{
try

146
src/Common/waitForPid.cpp
View File

@ -17,6 +17,17 @@
eintr_wrapper_result; \
})
namespace DB
{
enum PollPidResult
{
RESTART,
FAILED
};
}
#if defined(OS_LINUX)
#include <poll.h>
@ -43,16 +54,9 @@ namespace DB
static int syscall_pidfd_open(pid_t pid)
{
// pidfd_open cannot be interrupted, no EINTR handling
return syscall(SYS_pidfd_open, pid, 0);
}
static int dir_pidfd_open(pid_t pid)
{
std::string path = "/proc/" + std::to_string(pid);
return HANDLE_EINTR(open(path.c_str(), O_DIRECTORY));
}
static bool supportsPidFdOpen()
{
VersionNumber pidfd_open_minimal_version(5, 3, 0);
@ -60,36 +64,52 @@ static bool supportsPidFdOpen()
return linux_version >= pidfd_open_minimal_version;
}
static int pidFdOpen(pid_t pid)
static PollPidResult pollPid(pid_t pid, int timeout_in_ms)
{
// use pidfd_open or just plain old /proc/[pid] open for Linux
int pid_fd = 0;
if (supportsPidFdOpen())
{
return syscall_pidfd_open(pid);
// pidfd_open cannot be interrupted, no EINTR handling
pid_fd = syscall_pidfd_open(pid);
if (pid_fd < 0)
{
if (errno == ESRCH)
return PollPidResult::RESTART;
return PollPidResult::FAILED;
}
}
else
{
return dir_pidfd_open(pid);
}
}
std::string path = "/proc/" + std::to_string(pid);
pid_fd = HANDLE_EINTR(open(path.c_str(), O_DIRECTORY));
if (pid_fd < 0)
{
if (errno == ENOENT)
return PollPidResult::RESTART;
return PollPidResult::FAILED;
}
}
static int pollPid(pid_t pid, int timeout_in_ms)
{
struct pollfd pollfd;
int pid_fd = pidFdOpen(pid);
if (pid_fd == -1)
{
return false;
}
pollfd.fd = pid_fd;
pollfd.events = POLLIN;
int ready = poll(&pollfd, 1, timeout_in_ms);
int save_errno = errno;
int ready = HANDLE_EINTR(poll(&pollfd, 1, timeout_in_ms));
if (ready <= 0)
return PollPidResult::FAILED;
close(pid_fd);
errno = save_errno;
return ready;
return PollPidResult::RESTART;
}
#elif defined(OS_DARWIN) || defined(OS_FREEBSD)
#pragma clang diagnostic ignored "-Wreserved-identifier"
@ -100,38 +120,32 @@ static int pollPid(pid_t pid, int timeout_in_ms)
namespace DB
{
static int pollPid(pid_t pid, int timeout_in_ms)
static PollPidResult pollPid(pid_t pid, int timeout_in_ms)
{
int status = 0;
int kq = HANDLE_EINTR(kqueue());
int kq = kqueue();
if (kq == -1)
{
return false;
}
return PollPidResult::FAILED;
struct kevent change = {.ident = NULL};
EV_SET(&change, pid, EVFILT_PROC, EV_ADD, NOTE_EXIT, 0, NULL);
int result = HANDLE_EINTR(kevent(kq, &change, 1, NULL, 0, NULL));
if (result == -1)
int event_add_result = HANDLE_EINTR(kevent(kq, &change, 1, NULL, 0, NULL));
if (event_add_result == -1)
{
if (errno != ESRCH)
{
return false;
}
// check if pid already died while we called kevent()
if (waitpid(pid, &status, WNOHANG) == pid)
{
return true;
}
return false;
if (errno == ESRCH)
return PollPidResult::RESTART;
return PollPidResult::FAILED;
}
struct kevent event = {.ident = NULL};
struct timespec remaining_timespec = {.tv_sec = timeout_in_ms / 1000, .tv_nsec = (timeout_in_ms % 1000) * 1000000};
int ready = kevent(kq, nullptr, 0, &event, 1, &remaining_timespec);
int save_errno = errno;
int ready = HANDLE_EINTR(kevent(kq, nullptr, 0, &event, 1, &remaining_timespec));
PollPidResult result = ready < 0 ? PollPidResult::FAILED : PollPidResult::RESTART;
close(kq);
errno = save_errno;
return ready;
return result;
}
#else
#error "Unsupported OS type"
@ -148,7 +162,7 @@ bool waitForPid(pid_t pid, size_t timeout_in_seconds)
/// If there is no timeout before signal try to waitpid 1 time without block so we can avoid sending
/// signal if process is already normally terminated.
int waitpid_res = waitpid(pid, &status, WNOHANG);
int waitpid_res = HANDLE_EINTR(waitpid(pid, &status, WNOHANG));
bool process_terminated_normally = (waitpid_res == pid);
return process_terminated_normally;
}
@ -159,34 +173,24 @@ bool waitForPid(pid_t pid, size_t timeout_in_seconds)
int timeout_in_ms = timeout_in_seconds * 1000;
while (timeout_in_ms > 0)
{
int waitpid_res = waitpid(pid, &status, WNOHANG);
int waitpid_res = HANDLE_EINTR(waitpid(pid, &status, WNOHANG));
bool process_terminated_normally = (waitpid_res == pid);
if (process_terminated_normally)
{
return true;
}
else if (waitpid_res == 0)
{
watch.restart();
int ready = pollPid(pid, timeout_in_ms);
if (ready <= 0)
{
if (errno == EINTR || errno == EAGAIN)
{
timeout_in_ms -= watch.elapsedMilliseconds();
}
else
{
return false;
}
}
continue;
}
else if (waitpid_res == -1 && errno != EINTR)
{
if (waitpid_res != 0)
return false;
}
watch.restart();
PollPidResult result = pollPid(pid, timeout_in_ms);
if (result == PollPidResult::FAILED)
return false;
timeout_in_ms -= watch.elapsedMilliseconds();
}
return false;
}

4
src/Common/waitForPid.h
View File

@ -4,8 +4,8 @@
namespace DB
{
/*
* Waits for a specific pid with timeout, using modern Linux and OSX facilities
* Returns `true` if process terminated successfully or `false` otherwise
* Waits for a specific pid with timeout
* Returns `true` if process terminated successfully in specified timeout or `false` otherwise
*/
bool waitForPid(pid_t pid, size_t timeout_in_seconds);

7
src/Coordination/KeeperSnapshotManager.cpp
View File

@ -635,12 +635,13 @@ nuraft::ptr<nuraft::buffer> KeeperSnapshotManager::serializeSnapshotToBuffer(con
bool KeeperSnapshotManager::isZstdCompressed(nuraft::ptr<nuraft::buffer> buffer)
{
static constexpr uint32_t ZSTD_COMPRESSED_MAGIC = 0xFD2FB528;
static constexpr unsigned char ZSTD_COMPRESSED_MAGIC[4] = {0x28, 0xB5, 0x2F, 0xFD};
ReadBufferFromNuraftBuffer reader(buffer);
uint32_t magic_from_buffer;
unsigned char magic_from_buffer[4]{};
reader.readStrict(reinterpret_cast<char *>(&magic_from_buffer), sizeof(magic_from_buffer));
buffer->pos(0);
return magic_from_buffer == ZSTD_COMPRESSED_MAGIC;
return memcmp(magic_from_buffer, ZSTD_COMPRESSED_MAGIC, 4) == 0;
}
SnapshotDeserializationResult KeeperSnapshotManager::deserializeSnapshotFromBuffer(nuraft::ptr<nuraft::buffer> buffer) const

10
src/Dictionaries/LibraryDictionarySource.cpp
View File

@ -12,8 +12,6 @@
#include <Dictionaries/DictionaryStructure.h>
#include <Dictionaries/registerDictionaries.h>
namespace fs = std::filesystem;
namespace DB
{
@ -44,19 +42,21 @@ LibraryDictionarySource::LibraryDictionarySource(
if (created_from_ddl && !fileOrSymlinkPathStartsWith(path, dictionaries_lib_path))
throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", path, dictionaries_lib_path);
namespace fs = std::filesystem;
if (!fs::exists(path))
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "LibraryDictionarySource: Can't load library {}: file doesn't exist", path);
description.init(sample_block);
LibraryBridgeHelper::LibraryInitData library_data
ExternalDictionaryLibraryBridgeHelper::LibraryInitData library_data
{
.library_path = path,
.library_settings = getLibrarySettingsString(config, config_prefix + ".settings"),
.dict_attributes = getDictAttributesString()
};
bridge_helper = std::make_shared<LibraryBridgeHelper>(context, description.sample_block, dictionary_id, library_data);
bridge_helper = std::make_shared<ExternalDictionaryLibraryBridgeHelper>(context, description.sample_block, dictionary_id, library_data);
if (!bridge_helper->initLibrary())
throw Exception(ErrorCodes::EXTERNAL_LIBRARY_ERROR, "Failed to create shared library from path: {}", path);
@ -87,7 +87,7 @@ LibraryDictionarySource::LibraryDictionarySource(const LibraryDictionarySource &
, context(other.context)
, description{other.description}
{
bridge_helper = std::make_shared<LibraryBridgeHelper>(context, description.sample_block, dictionary_id, other.bridge_helper->getLibraryData());
bridge_helper = std::make_shared<ExternalDictionaryLibraryBridgeHelper>(context, description.sample_block, dictionary_id, other.bridge_helper->getLibraryData());
if (!bridge_helper->cloneLibrary(other.dictionary_id))
throw Exception(ErrorCodes::EXTERNAL_LIBRARY_ERROR, "Failed to clone library");
}

6
src/Dictionaries/LibraryDictionarySource.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <BridgeHelper/LibraryBridgeHelper.h>
#include <BridgeHelper/ExternalDictionaryLibraryBridgeHelper.h>
#include <Common/LocalDateTime.h>
#include <Core/UUID.h>
#include "DictionaryStructure.h"
@ -28,7 +28,7 @@ namespace ErrorCodes
}
class CStringsHolder;
using LibraryBridgeHelperPtr = std::shared_ptr<LibraryBridgeHelper>;
using ExternalDictionaryLibraryBridgeHelperPtr = std::shared_ptr<ExternalDictionaryLibraryBridgeHelper>;
class LibraryDictionarySource final : public IDictionarySource
{
@ -85,7 +85,7 @@ private:
Block sample_block;
ContextPtr context;
LibraryBridgeHelperPtr bridge_helper;
ExternalDictionaryLibraryBridgeHelperPtr bridge_helper;
ExternalResultDescription description;
};

2
src/Functions/dateDiff.cpp
View File

@ -114,7 +114,7 @@ public:
dispatchForColumns<ToRelativeWeekNumImpl>(x, y, timezone_x, timezone_y, res->getData());
else if (unit == "day" || unit == "dd" || unit == "d")
dispatchForColumns<ToRelativeDayNumImpl>(x, y, timezone_x, timezone_y, res->getData());
else if (unit == "hour" || unit == "hh")
else if (unit == "hour" || unit == "hh" || unit == "h")
dispatchForColumns<ToRelativeHourNumImpl>(x, y, timezone_x, timezone_y, res->getData());
else if (unit == "minute" || unit == "mi" || unit == "n")
dispatchForColumns<ToRelativeMinuteNumImpl>(x, y, timezone_x, timezone_y, res->getData());

10
src/Interpreters/InterpreterDeleteQuery.cpp
View File

@ -12,9 +12,8 @@
#include <Storages/AlterCommands.h>
#include <Storages/IStorage.h>
#include <Storages/MutationCommands.h>
#include <Storages/StorageMergeTree.h>
#include <Storages/StorageReplicatedMergeTree.h>
#include <Storages/LightweightDeleteDescription.h>
#include <Storages/MergeTree/MergeTreeData.h>
namespace DB
@ -50,10 +49,9 @@ BlockIO InterpreterDeleteQuery::execute()
/// First check table storage for validations.
StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext());
auto storage_merge_tree = std::dynamic_pointer_cast<StorageMergeTree>(table);
auto storage_replicated_merge_tree = std::dynamic_pointer_cast<StorageReplicatedMergeTree>(table);
if (!storage_merge_tree && !storage_replicated_merge_tree)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Only MergeTree or ReplicatedMergeTree tables are supported");
auto merge_tree = std::dynamic_pointer_cast<MergeTreeData>(table);
if (!merge_tree)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Only MergeTree family tables are supported");
checkStorageSupportsTransactionsIfNeeded(table, getContext());
if (table->isStaticStorage())

77
src/Parsers/IAST.cpp
View File

@ -28,34 +28,65 @@ const char * IAST::hilite_none = "\033[0m";
IAST::~IAST()
{
/// If deleter was set, move our children to it.
/// Will avoid recursive destruction.
if (deleter)
{
deleter->push_back(std::move(children));
return;
}
/** Create intrusive linked list of children to delete.
* Each ASTPtr child contains pointer to next child to delete.
*/
ASTPtr delete_list_head_holder = nullptr;
const bool delete_directly = next_to_delete_list_head == nullptr;
ASTPtr & delete_list_head_reference = next_to_delete_list_head ? *next_to_delete_list_head : delete_list_head_holder;
std::list<ASTs> queue;
queue.push_back(std::move(children));
while (!queue.empty())
/// Move children into intrusive list
for (auto & child : children)
{
for (auto & node : queue.front())
/** If two threads remove ASTPtr concurrently,
* it is possible that neither thead will see use_count == 1.
* It is ok. Will need one more extra stack frame in this case.
*/
if (child.use_count() != 1)
continue;
ASTPtr child_to_delete;
child_to_delete.swap(child);
if (!delete_list_head_reference)
{
/// If two threads remove ASTPtr concurrently,
/// it is possible that neither thead will see use_count == 1.
/// It is ok. Will need one more extra stack frame in this case.
if (node.use_count() == 1)
{
/// Deleter is only set when current thread is the single owner.
/// No extra synchronisation is needed.
ASTPtr to_delete;
node.swap(to_delete);
to_delete->deleter = &queue;
}
/// Initialize list first time
delete_list_head_reference = std::move(child_to_delete);
continue;
}
queue.pop_front();
ASTPtr previous_head = std::move(delete_list_head_reference);
delete_list_head_reference = std::move(child_to_delete);
delete_list_head_reference->next_to_delete = std::move(previous_head);
}
if (!delete_directly)
return;
while (delete_list_head_reference)
{
/** Extract child to delete from current list head.
* Child will be destroyed at the end of scope.
*/
ASTPtr child_to_delete;
child_to_delete.swap(delete_list_head_reference);
/// Update list head
delete_list_head_reference = std::move(child_to_delete->next_to_delete);
/** Pass list head into child before destruction.
* It is important to properly handle cases where subclass has member same as one of its children.
*
* class ASTSubclass : IAST
* {
* ASTPtr first_child; /// Same as first child
* }
*
* In such case we must move children into list only in IAST destructor.
* If we try to move child to delete children into list before subclasses desruction,
* first child use count will be 2.
*/
child_to_delete->next_to_delete_list_head = &delete_list_head_reference;
}
}

7
src/Parsers/IAST.h
View File

@ -275,8 +275,11 @@ public:
private:
size_t checkDepthImpl(size_t max_depth, size_t level) const;
/// This deleter is used in ~IAST to avoid possible stack overflow in destructor.
std::list<ASTs> * deleter = nullptr;
/** Forward linked list of ASTPtr to delete.
* Used in IAST destructor to avoid possible stack overflow.
*/
ASTPtr next_to_delete = nullptr;
ASTPtr * next_to_delete_list_head = nullptr;
};
template <typename AstArray>

2
src/Parsers/parseIntervalKind.cpp
View File

@ -43,7 +43,7 @@ bool parseIntervalKind(IParser::Pos & pos, Expected & expected, IntervalKind & r
}
if (ParserKeyword("HOUR").ignore(pos, expected) || ParserKeyword("SQL_TSI_HOUR").ignore(pos, expected)
|| ParserKeyword("HH").ignore(pos, expected))
|| ParserKeyword("HH").ignore(pos, expected) || ParserKeyword("H").ignore(pos, expected))
{
result = IntervalKind::Hour;
return true;

67
src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp
View File

@ -9,7 +9,6 @@
#include <IO/WriteHelpers.h>
#include <IO/WriteBufferFromString.h>
#include <IO/Operators.h>
#include <Common/PODArray.h>
#include <Common/UTF8Helpers.h>
namespace DB
@ -21,8 +20,8 @@ namespace ErrorCodes
PrettyBlockOutputFormat::PrettyBlockOutputFormat(
WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_)
: IOutputFormat(header_, out_), format_settings(format_settings_), serializations(header_.getSerializations())
WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_, bool mono_block_)
: IOutputFormat(header_, out_), format_settings(format_settings_), serializations(header_.getSerializations()), mono_block(mono_block_)
{
struct winsize w;
if (0 == ioctl(STDOUT_FILENO, TIOCGWINSZ, &w))
@ -142,17 +141,33 @@ GridSymbols ascii_grid_symbols {
}
void PrettyBlockOutputFormat::write(Chunk chunk, PortKind port_kind)
{
UInt64 max_rows = format_settings.pretty.max_rows;
if (total_rows >= max_rows)
if (total_rows >= format_settings.pretty.max_rows)
{
total_rows += chunk.getNumRows();
return;
}
if (mono_block)
{
if (port_kind == PortKind::Main)
{
if (mono_chunk)
mono_chunk.append(chunk);
else
mono_chunk = std::move(chunk);
return;
}
/// Should be written from writeSuffix()
assert(!mono_chunk);
}
writeChunk(chunk, port_kind);
}
void PrettyBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port_kind)
{
auto num_rows = chunk.getNumRows();
auto num_columns = chunk.getNumColumns();
const auto & columns = chunk.getColumns();
@ -265,7 +280,7 @@ void PrettyBlockOutputFormat::write(Chunk chunk, PortKind port_kind)
}
writeString(middle_names_separator_s, out);
for (size_t i = 0; i < num_rows && total_rows + i < max_rows; ++i)
for (size_t i = 0; i < num_rows && total_rows + i < format_settings.pretty.max_rows; ++i)
{
if (i != 0)
{
@ -385,8 +400,19 @@ void PrettyBlockOutputFormat::consumeExtremes(Chunk chunk)
}
void PrettyBlockOutputFormat::writeMonoChunkIfNeeded()
{
if (mono_chunk)
{
writeChunk(mono_chunk, PortKind::Main);
mono_chunk.clear();
}
}
void PrettyBlockOutputFormat::writeSuffix()
{
writeMonoChunkIfNeeded();
if (total_rows >= format_settings.pretty.max_rows)
{
writeCString(" Showed first ", out);
@ -395,32 +421,9 @@ void PrettyBlockOutputFormat::writeSuffix()
}
}
void registerOutputFormatPretty(FormatFactory & factory)
{
factory.registerOutputFormat("Pretty", [](
WriteBuffer & buf,
const Block & sample,
const RowOutputFormatParams &,
const FormatSettings & format_settings)
{
return std::make_shared<PrettyBlockOutputFormat>(buf, sample, format_settings);
});
factory.markOutputFormatSupportsParallelFormatting("Pretty");
factory.registerOutputFormat("PrettyNoEscapes", [](
WriteBuffer & buf,
const Block & sample,
const RowOutputFormatParams &,
const FormatSettings & format_settings)
{
FormatSettings changed_settings = format_settings;
changed_settings.pretty.color = false;
return std::make_shared<PrettyBlockOutputFormat>(buf, sample, changed_settings);
});
factory.markOutputFormatSupportsParallelFormatting("PrettyNoEscapes");
registerPrettyFormatWithNoEscapesAndMonoBlock<PrettyBlockOutputFormat>(factory, "Pretty");
}
}

40
src/Processors/Formats/Impl/PrettyBlockOutputFormat.h
View File

@ -3,6 +3,7 @@
#include <Core/Block.h>
#include <Processors/Formats/IOutputFormat.h>
#include <Formats/FormatSettings.h>
#include <Formats/FormatFactory.h>
namespace DB
@ -18,7 +19,7 @@ class PrettyBlockOutputFormat : public IOutputFormat
{
public:
/// no_escapes - do not use ANSI escape sequences - to display in the browser, not in the console.
PrettyBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_);
PrettyBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_, bool mono_block_);
String getName() const override { return "PrettyBlockOutputFormat"; }
@ -38,7 +39,9 @@ protected:
using Widths = PODArray<size_t>;
using WidthsPerColumn = std::vector<Widths>;
virtual void write(Chunk chunk, PortKind port_kind);
void write(Chunk chunk, PortKind port_kind);
virtual void writeChunk(const Chunk & chunk, PortKind port_kind);
void writeMonoChunkIfNeeded();
void writeSuffix() override;
void onRowsReadBeforeUpdate() override { total_rows = getRowsReadBefore(); }
@ -50,6 +53,39 @@ protected:
void writeValueWithPadding(
const IColumn & column, const ISerialization & serialization, size_t row_num,
size_t value_width, size_t pad_to_width, bool align_right);
private:
bool mono_block;
/// For mono_block == true only
Chunk mono_chunk;
};
template <class OutputFormat>
void registerPrettyFormatWithNoEscapesAndMonoBlock(FormatFactory & factory, const String & base_name)
{
auto creator = [&](FormatFactory & fact, const String & name, bool no_escapes, bool mono_block)
{
fact.registerOutputFormat(name, [no_escapes, mono_block](
WriteBuffer & buf,
const Block & sample,
const RowOutputFormatParams &,
const FormatSettings & format_settings)
{
if (no_escapes)
{
FormatSettings changed_settings = format_settings;
changed_settings.pretty.color = false;
return std::make_shared<OutputFormat>(buf, sample, changed_settings, mono_block);
}
return std::make_shared<OutputFormat>(buf, sample, format_settings, mono_block);
});
if (!mono_block)
factory.markOutputFormatSupportsParallelFormatting(name);
};
creator(factory, base_name, false, false);
creator(factory, base_name + "NoEscapes", true, false);
creator(factory, base_name + "MonoBlock", false, true);
creator(factory, base_name + "NoEscapesMonoBlock", true, true);
}
}

72
src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp
View File

@ -49,22 +49,10 @@ GridSymbols ascii_grid_symbols {
}
PrettyCompactBlockOutputFormat::PrettyCompactBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings_, bool mono_block_)
: PrettyBlockOutputFormat(out_, header, format_settings_)
, mono_block(mono_block_)
: PrettyBlockOutputFormat(out_, header, format_settings_, mono_block_)
{
}
void PrettyCompactBlockOutputFormat::writeSuffix()
{
if (mono_chunk)
{
writeChunk(mono_chunk, PortKind::Main);
mono_chunk.clear();
}
PrettyBlockOutputFormat::writeSuffix();
}
void PrettyCompactBlockOutputFormat::writeHeader(
const Block & block,
const Widths & max_widths,
@ -186,38 +174,6 @@ void PrettyCompactBlockOutputFormat::writeRow(
writeCString("\n", out);
}
void PrettyCompactBlockOutputFormat::write(Chunk chunk, PortKind port_kind)
{
UInt64 max_rows = format_settings.pretty.max_rows;
if (total_rows >= max_rows)
{
total_rows += chunk.getNumRows();
return;
}
if (mono_block)
{
if (port_kind == PortKind::Main)
{
if (!mono_chunk)
{
mono_chunk = std::move(chunk);
return;
}
mono_chunk.append(chunk);
return;
}
else
{
/// Should be written from writeSuffix()
assert(!mono_chunk);
}
}
writeChunk(chunk, port_kind);
}
void PrettyCompactBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port_kind)
{
UInt64 max_rows = format_settings.pretty.max_rows;
@ -244,31 +200,7 @@ void PrettyCompactBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind po
void registerOutputFormatPrettyCompact(FormatFactory & factory)
{
for (const auto & [name, mono_block] : {std::make_pair("PrettyCompact", false), std::make_pair("PrettyCompactMonoBlock", true)})
{
factory.registerOutputFormat(name, [mono_block = mono_block](
WriteBuffer & buf,
const Block & sample,
const RowOutputFormatParams &,
const FormatSettings & format_settings)
{
return std::make_shared<PrettyCompactBlockOutputFormat>(buf, sample, format_settings, mono_block);
});
}
factory.markOutputFormatSupportsParallelFormatting("PrettyCompact");
factory.registerOutputFormat("PrettyCompactNoEscapes", [](
WriteBuffer & buf,
const Block & sample,
const RowOutputFormatParams &,
const FormatSettings & format_settings)
{
FormatSettings changed_settings = format_settings;
changed_settings.pretty.color = false;
return std::make_shared<PrettyCompactBlockOutputFormat>(buf, sample, changed_settings, false /* mono_block */);
});
factory.markOutputFormatSupportsParallelFormatting("PrettyCompactNoEscapes");
registerPrettyFormatWithNoEscapesAndMonoBlock<PrettyCompactBlockOutputFormat>(factory, "PrettyCompact");
}
}

8
src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.h
View File

@ -17,7 +17,6 @@ public:
String getName() const override { return "PrettyCompactBlockOutputFormat"; }
private:
void write(Chunk chunk, PortKind port_kind) override;
void writeHeader(const Block & block, const Widths & max_widths, const Widths & name_widths);
void writeBottom(const Widths & max_widths);
void writeRow(
@ -27,12 +26,7 @@ private:
const WidthsPerColumn & widths,
const Widths & max_widths);
bool mono_block;
/// For mono_block == true only
Chunk mono_chunk;
void writeChunk(const Chunk & chunk, PortKind port_kind);
void writeSuffix() override;
void writeChunk(const Chunk & chunk, PortKind port_kind) override;
};
}

28
src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp
View File

@ -9,7 +9,7 @@ namespace DB
{
void PrettySpaceBlockOutputFormat::write(Chunk chunk, PortKind port_kind)
void PrettySpaceBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port_kind)
{
UInt64 max_rows = format_settings.pretty.max_rows;
@ -100,6 +100,8 @@ void PrettySpaceBlockOutputFormat::write(Chunk chunk, PortKind port_kind)
void PrettySpaceBlockOutputFormat::writeSuffix()
{
writeMonoChunkIfNeeded();
if (total_rows >= format_settings.pretty.max_rows)
{
writeCString("\nShowed first ", out);
@ -111,29 +113,7 @@ void PrettySpaceBlockOutputFormat::writeSuffix()
void registerOutputFormatPrettySpace(FormatFactory & factory)
{
factory.registerOutputFormat("PrettySpace", [](
WriteBuffer & buf,
const Block & sample,
const RowOutputFormatParams &,
const FormatSettings & format_settings)
{
return std::make_shared<PrettySpaceBlockOutputFormat>(buf, sample, format_settings);
});
factory.markOutputFormatSupportsParallelFormatting("PrettySpace");
factory.registerOutputFormat("PrettySpaceNoEscapes", [](
WriteBuffer & buf,
const Block & sample,
const RowOutputFormatParams &,
const FormatSettings & format_settings)
{
FormatSettings changed_settings = format_settings;
changed_settings.pretty.color = false;
return std::make_shared<PrettySpaceBlockOutputFormat>(buf, sample, changed_settings);
});
factory.markOutputFormatSupportsParallelFormatting("PrettySpaceNoEscapes");
registerPrettyFormatWithNoEscapesAndMonoBlock<PrettySpaceBlockOutputFormat>(factory, "PrettySpace");
}
}

6
src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.h
View File

@ -11,13 +11,13 @@ namespace DB
class PrettySpaceBlockOutputFormat : public PrettyBlockOutputFormat
{
public:
PrettySpaceBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings_)
: PrettyBlockOutputFormat(out_, header, format_settings_) {}
PrettySpaceBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings_, bool mono_block_)
: PrettyBlockOutputFormat(out_, header, format_settings_, mono_block_) {}
String getName() const override { return "PrettySpaceBlockOutputFormat"; }
private:
void write(Chunk chunk, PortKind port_kind) override;
void writeChunk(const Chunk & chunk, PortKind port_kind) override;
void writeSuffix() override;
};

78
src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
View File

@ -42,37 +42,50 @@ static bool filterColumnIsNotAmongAggregatesArguments(const AggregateDescription
return true;
}
static size_t
tryAddNewFilterStep(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, const Names & allowed_inputs,
bool can_remove_filter = true, size_t child_idx = 0)
/// Assert that `node->children` has at least `child_num` elements
static void checkChildrenSize(QueryPlan::Node * node, size_t child_num)
{
auto & child = node->step;
if (child_num > child->getInputStreams().size() || child_num > node->children.size())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong number of children: expected at least {}, got {} children and {} streams",
child_num, child->getInputStreams().size(), node->children.size());
}
static ActionsDAGPtr splitFilter(QueryPlan::Node * parent_node, const Names & allowed_inputs, size_t child_idx = 0)
{
QueryPlan::Node * child_node = parent_node->children.front();
checkChildrenSize(child_node, child_idx + 1);
auto & parent = parent_node->step;
auto & child = child_node->step;
auto * filter = static_cast<FilterStep *>(parent.get());
auto * filter = assert_cast<FilterStep *>(parent.get());
const auto & expression = filter->getExpression();
const auto & filter_column_name = filter->getFilterColumnName();
bool removes_filter = filter->removesFilterColumn();
// std::cerr << "Filter: \n" << expression->dumpDAG() << std::endl;
if (child_idx >= child->getInputStreams().size() || child_idx >= child_node->children.size())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Child index {} is out of range (streams: {}, children: {})",
child_idx, child->getInputStreams().size(), child_node->children.size());
const auto & all_inputs = child->getInputStreams()[child_idx].header.getColumnsWithTypeAndName();
auto split_filter = expression->cloneActionsForFilterPushDown(filter_column_name, removes_filter, allowed_inputs, all_inputs);
if (!split_filter)
return 0;
return split_filter;
}
// std::cerr << "===============\n" << expression->dumpDAG() << std::endl;
// std::cerr << "---------------\n" << split_filter->dumpDAG() << std::endl;
static size_t
tryAddNewFilterStep(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, const ActionsDAGPtr & split_filter,
bool can_remove_filter = true, size_t child_idx = 0)
{
QueryPlan::Node * child_node = parent_node->children.front();
checkChildrenSize(child_node, child_idx + 1);
auto & parent = parent_node->step;
auto & child = child_node->step;
auto * filter = assert_cast<FilterStep *>(parent.get());
const auto & expression = filter->getExpression();
const auto & filter_column_name = filter->getFilterColumnName();
const auto * filter_node = expression->tryFindInIndex(filter_column_name);
if (!filter_node && !removes_filter)
if (!filter_node && !filter->removesFilterColumn())
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Filter column {} was removed from ActionsDAG but it is needed in result. DAG:\n{}",
filter_column_name, expression->dumpDAG());
@ -89,9 +102,9 @@ tryAddNewFilterStep(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, con
/// Expression/Filter -> Aggregating -> Filter -> Something
/// New filter column is the first one.
auto split_filter_column_name = (*split_filter->getIndex().begin())->result_name;
String split_filter_column_name = split_filter->getIndex().front()->result_name;
node.step = std::make_unique<FilterStep>(
node.children.at(0)->step->getOutputStream(), std::move(split_filter), std::move(split_filter_column_name), can_remove_filter);
node.children.at(0)->step->getOutputStream(), split_filter, std::move(split_filter_column_name), can_remove_filter);
if (auto * transforming_step = dynamic_cast<ITransformingStep *>(child.get()))
{
@ -118,6 +131,15 @@ tryAddNewFilterStep(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, con
return 3;
}
static size_t
tryAddNewFilterStep(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, const Names & allowed_inputs,
bool can_remove_filter = true)
{
if (auto split_filter = splitFilter(parent_node, allowed_inputs, 0))
return tryAddNewFilterStep(parent_node, nodes, split_filter, can_remove_filter, 0);
return 0;
}
size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes)
{
if (parent_node->children.size() != 1)
@ -248,12 +270,26 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes
allowed_keys.push_back(name);
}
const bool can_remove_filter
= std::find(source_columns.begin(), source_columns.end(), filter->getFilterColumnName()) == source_columns.end();
size_t updated_steps = tryAddNewFilterStep(parent_node, nodes, allowed_keys, can_remove_filter, is_left ? 0 : 1);
/// For left JOIN, push down to the first child; for right - to the second one.
const auto child_idx = is_left ? 0 : 1;
ActionsDAGPtr split_filter = splitFilter(parent_node, allowed_keys, child_idx);
if (!split_filter)
return 0;
/*
* We should check the presence of a split filter column name in `source_columns` to avoid removing the required column.
*
* Example:
* A filter expression is `a AND b = c`, but `b` and `c` belong to another side of the join and not in `allowed_keys`, so the final split filter is just `a`.
* In this case `a` can be in `source_columns` but not `and(a, equals(b, c))`.
*
* New filter column is the first one.
*/
const String & split_filter_column_name = split_filter->getIndex().front()->result_name;
bool can_remove_filter = source_columns.end() == std::find(source_columns.begin(), source_columns.end(), split_filter_column_name);
const size_t updated_steps = tryAddNewFilterStep(parent_node, nodes, split_filter, can_remove_filter, child_idx);
if (updated_steps > 0)
{
LOG_DEBUG(&Poco::Logger::get("tryPushDownFilter"), "Pushed down filter to {} side of join", kind);
LOG_DEBUG(&Poco::Logger::get("QueryPlanOptimizations"), "Pushed down filter to {} side of join", kind);
}
return updated_steps;
};

44
src/Storages/FileLog/StorageFileLog.cpp
View File

@ -37,6 +37,7 @@ namespace ErrorCodes
extern const int CANNOT_READ_ALL_DATA;
extern const int LOGICAL_ERROR;
extern const int TABLE_METADATA_ALREADY_EXISTS;
extern const int DIRECTORY_DOESNT_EXIST;
extern const int CANNOT_SELECT;
extern const int QUERY_NOT_ALLOWED;
}
@ -72,6 +73,25 @@ StorageFileLog::StorageFileLog(
try
{
if (!attach)
{
std::error_code ec;
std::filesystem::create_directories(metadata_base_path, ec);
if (ec)
{
if (ec == std::make_error_code(std::errc::file_exists))
{
throw Exception(ErrorCodes::TABLE_METADATA_ALREADY_EXISTS,
"Metadata files already exist by path: {}, remove them manually if it is intended",
metadata_base_path);
}
else
throw Exception(ErrorCodes::DIRECTORY_DOESNT_EXIST,
"Could not create directory {}, reason: {}", metadata_base_path, ec.message());
}
}
loadMetaFiles(attach);
loadFiles();
@ -117,19 +137,6 @@ void StorageFileLog::loadMetaFiles(bool attach)
/// Load all meta info to file_infos;
deserialize();
}
/// Create table, just create meta data directory
else
{
if (std::filesystem::exists(metadata_base_path))
{
throw Exception(
ErrorCodes::TABLE_METADATA_ALREADY_EXISTS,
"Metadata files already exist by path: {}, remove them manually if it is intended",
metadata_base_path);
}
/// We do not create the metadata_base_path directory at creation time, create it at the moment of serializing
/// meta files, such that can avoid unnecessarily create this directory if create table failed.
}
}
void StorageFileLog::loadFiles()
@ -218,10 +225,6 @@ void StorageFileLog::loadFiles()
void StorageFileLog::serialize() const
{
if (!std::filesystem::exists(metadata_base_path))
{
std::filesystem::create_directories(metadata_base_path);
}
for (const auto & [inode, meta] : file_infos.meta_by_inode)
{
auto full_name = getFullMetaPath(meta.file_name);
@ -242,10 +245,6 @@ void StorageFileLog::serialize() const
void StorageFileLog::serialize(UInt64 inode, const FileMeta & file_meta) const
{
if (!std::filesystem::exists(metadata_base_path))
{
std::filesystem::create_directories(metadata_base_path);
}
auto full_name = getFullMetaPath(file_meta.file_name);
if (!std::filesystem::exists(full_name))
{
@ -379,8 +378,7 @@ void StorageFileLog::drop()
{
try
{
if (std::filesystem::exists(metadata_base_path))
std::filesystem::remove_all(metadata_base_path);
std::filesystem::remove_all(metadata_base_path);
}
catch (...)
{

2
src/Storages/MergeTree/DataPartsExchange.cpp
View File

@ -719,7 +719,7 @@ void Fetcher::downloadBaseOrProjectionPartToDisk(
"This may happen if we are trying to download part from malicious replica or logical error.",
absolute_file_path, data_part_storage_builder->getRelativePath());
auto file_out = data_part_storage_builder->writeFile(file_name, file_size, {});
auto file_out = data_part_storage_builder->writeFile(file_name, std::min<UInt64>(file_size, DBMS_DEFAULT_BUFFER_SIZE), {});
HashingWriteBuffer hashing_out(*file_out);
copyDataWithThrottler(in, hashing_out, file_size, blocker.getCounter(), throttler);

8
src/Storages/NATS/NATSConnection.cpp
View File

@ -9,7 +9,6 @@
namespace DB
{
//static const auto CONNECT_SLEEP = 200;
static const auto RETRIES_MAX = 20;
static const auto CONNECTED_TO_BUFFER_SIZE = 256;
@ -19,6 +18,10 @@ NATSConnectionManager::NATSConnectionManager(const NATSConfiguration & configura
, log(log_)
, event_handler(loop.getLoop(), log)
{
const char * val = std::getenv("CLICKHOUSE_NATS_TLS_SECURE");
std::string tls_secure = val == nullptr ? std::string("1") : std::string(val);
if (tls_secure == "0")
skip_verification = true;
}
@ -92,6 +95,9 @@ void NATSConnectionManager::connectImpl()
if (configuration.secure)
{
natsOptions_SetSecure(options, true);
}
if (skip_verification)
{
natsOptions_SkipServerVerification(options, true);
}
if (!configuration.url.empty())

3
src/Storages/NATS/NATSConnection.h
View File

@ -65,6 +65,9 @@ private:
// true if at any point successfully connected to NATS
bool has_connection = false;
// use CLICKHOUSE_NATS_TLS_SECURE=0 env var to skip TLS verification of server cert
bool skip_verification = false;
std::mutex mutex;
};

88
src/Storages/StorageFile.cpp
View File

@ -444,36 +444,24 @@ public:
using FilesInfoPtr = std::shared_ptr<FilesInfo>;
static Block getHeader(const StorageMetadataPtr & metadata_snapshot, bool need_path_column, bool need_file_column)
static Block getBlockForSource(const Block & block_for_format, const FilesInfoPtr & files_info)
{
auto header = metadata_snapshot->getSampleBlock();
/// Note: AddingDefaultsTransform doesn't change header.
if (need_path_column)
header.insert(
auto res = block_for_format;
if (files_info->need_path_column)
{
res.insert(
{DataTypeLowCardinality{std::make_shared<DataTypeString>()}.createColumn(),
std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()),
"_path"});
if (need_file_column)
header.insert(
}
if (files_info->need_file_column)
{
res.insert(
{DataTypeLowCardinality{std::make_shared<DataTypeString>()}.createColumn(),
std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()),
"_file"});
return header;
}
static Block getBlockForSource(
const StorageFilePtr & storage,
const StorageSnapshotPtr & storage_snapshot,
const ColumnsDescription & columns_description,
const FilesInfoPtr & files_info)
{
if (storage->supportsSubsetOfColumns())
return storage_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical());
else
return getHeader(storage_snapshot->metadata, files_info->need_path_column, files_info->need_file_column);
}
return res;
}
StorageFileSource(
@ -483,13 +471,15 @@ public:
UInt64 max_block_size_,
FilesInfoPtr files_info_,
ColumnsDescription columns_description_,
const Block & block_for_format_,
std::unique_ptr<ReadBuffer> read_buf_)
: ISource(getBlockForSource(storage_, storage_snapshot_, columns_description_, files_info_))
: ISource(getBlockForSource(block_for_format_, files_info_))
, storage(std::move(storage_))
, storage_snapshot(storage_snapshot_)
, files_info(std::move(files_info_))
, read_buf(std::move(read_buf_))
, columns_description(std::move(columns_description_))
, block_for_format(block_for_format_)
, context(context_)
, max_block_size(max_block_size_)
{
@ -533,15 +523,8 @@ public:
if (!read_buf)
read_buf = createReadBuffer(current_path, storage->use_table_fd, storage->getName(), storage->table_fd, storage->compression_method, context);
auto get_block_for_format = [&]() -> Block
{
if (storage->supportsSubsetOfColumns())
return storage_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical());
return storage_snapshot->metadata->getSampleBlock();
};
auto format = context->getInputFormat(
storage->format_name, *read_buf, get_block_for_format(), max_block_size, storage->format_settings);
auto format
= context->getInputFormat(storage->format_name, *read_buf, block_for_format, max_block_size, storage->format_settings);
QueryPipelineBuilder builder;
builder.init(Pipe(format));
@ -627,6 +610,7 @@ private:
std::unique_ptr<PullingPipelineExecutor> reader;
ColumnsDescription columns_description;
Block block_for_format;
ContextPtr context; /// TODO Untangle potential issues with context lifetime.
UInt64 max_block_size;
@ -693,13 +677,30 @@ Pipe StorageFile::read(
for (size_t i = 0; i < num_streams; ++i)
{
const auto get_columns_for_format = [&]() -> ColumnsDescription
ColumnsDescription columns_description;
Block block_for_format;
if (supportsSubsetOfColumns())
{
if (supportsSubsetOfColumns())
return storage_snapshot->getDescriptionForColumns(column_names);
else
return storage_snapshot->metadata->getColumns();
};
auto fetch_columns = column_names;
const auto & virtuals = getVirtuals();
std::erase_if(
fetch_columns,
[&](const String & col)
{
return std::any_of(
virtuals.begin(), virtuals.end(), [&](const NameAndTypePair & virtual_col) { return col == virtual_col.name; });
});
if (fetch_columns.empty())
fetch_columns.push_back(ExpressionActions::getSmallestColumn(storage_snapshot->metadata->getColumns().getAllPhysical()));
columns_description = storage_snapshot->getDescriptionForColumns(fetch_columns);
}
else
{
columns_description = storage_snapshot->metadata->getColumns();
}
block_for_format = storage_snapshot->getSampleBlockForColumns(columns_description.getNamesOfPhysical());
/// In case of reading from fd we have to check whether we have already created
/// the read buffer from it in Storage constructor (for schema inference) or not.
@ -710,7 +711,14 @@ Pipe StorageFile::read(
read_buffer = std::move(peekable_read_buffer_from_fd);
pipes.emplace_back(std::make_shared<StorageFileSource>(
this_ptr, storage_snapshot, context, max_block_size, files_info, get_columns_for_format(), std::move(read_buffer)));
this_ptr,
storage_snapshot,
context,
max_block_size,
files_info,
columns_description,
block_for_format,
std::move(read_buffer)));
}
return Pipe::unitePipes(std::move(pipes));

2
tests/ci/ci_config.py
View File

@ -352,4 +352,6 @@ REQUIRED_CHECKS = [
"Style Check",
"ClickHouse build check",
"ClickHouse special build check",
"Stateful tests (release)",
"Stateless tests (release)",
]

4
tests/ci/functional_test_check.py
View File

@ -6,6 +6,7 @@ import logging
import os
import subprocess
import sys
import atexit
from github import Github
@ -22,6 +23,7 @@ from commit_status_helper import (
get_commit,
override_status,
post_commit_status_to_file,
update_mergeable_check,
)
from clickhouse_helper import (
ClickHouseHelper,
@ -209,6 +211,8 @@ if __name__ == "__main__":
pr_info = PRInfo(need_changed_files=run_changed_tests)
atexit.register(update_mergeable_check, gh, pr_info, check_name)
if not os.path.exists(temp_path):
os.makedirs(temp_path)

93
tests/ci/workflow_approve_rerun_lambda/app.py
View File

@ -3,15 +3,12 @@
from collections import namedtuple
import fnmatch
import json
import os
import time
import jwt
import requests # type: ignore
import boto3 # type: ignore
API_URL = os.getenv("API_URL", "https://api.github.com/repos/ClickHouse/ClickHouse")
SUSPICIOUS_CHANGED_FILES_NUMBER = 200
SUSPICIOUS_PATTERNS = [
@ -27,7 +24,7 @@ SUSPICIOUS_PATTERNS = [
MAX_RETRY = 5
# Number of times a check can re-run as a whole.
# It is needed, because we are using AWS "spot" instances, that are terminated very frequently.
# It is needed, because we are using AWS "spot" instances, that are terminated often
MAX_WORKFLOW_RERUN = 20
WorkflowDescription = namedtuple(
@ -46,6 +43,7 @@ WorkflowDescription = namedtuple(
"rerun_url",
"jobs_url",
"attempt",
"repo_url",
"url",
],
)
@ -189,10 +187,11 @@ def is_trusted_contributor(pr_user_login, pr_user_orgs):
return False
def _exec_get_with_retry(url):
def _exec_get_with_retry(url, token):
headers = {"Authorization": f"token {token}"}
for i in range(MAX_RETRY):
try:
response = requests.get(url)
response = requests.get(url, headers=headers)
response.raise_for_status()
return response.json()
except Exception as ex:
@ -228,9 +227,9 @@ def _exec_post_with_retry(url, token, data=None):
raise Exception("Cannot execute POST request with retry")
def _get_pull_requests_from(owner, branch):
url = f"{API_URL}/pulls?head={owner}:{branch}"
return _exec_get_with_retry(url)
def _get_pull_requests_from(repo_url, owner, branch, token):
url = f"{repo_url}/pulls?head={owner}:{branch}"
return _exec_get_with_retry(url, token)
def get_workflow_description_from_event(event):
@ -248,6 +247,7 @@ def get_workflow_description_from_event(event):
rerun_url = event["workflow_run"]["rerun_url"]
url = event["workflow_run"]["html_url"]
api_url = event["workflow_run"]["url"]
repo_url = event["repository"]["url"]
return WorkflowDescription(
name=name,
action=action,
@ -262,24 +262,24 @@ def get_workflow_description_from_event(event):
jobs_url=jobs_url,
rerun_url=rerun_url,
url=url,
repo_url=repo_url,
api_url=api_url,
)
def get_pr_author_and_orgs(pull_request):
def get_pr_author_and_orgs(pull_request, token):
author = pull_request["user"]["login"]
orgs = _exec_get_with_retry(pull_request["user"]["organizations_url"])
orgs = _exec_get_with_retry(pull_request["user"]["organizations_url"], token)
return author, [org["id"] for org in orgs]
def get_changed_files_for_pull_request(pull_request):
number = pull_request["number"]
def get_changed_files_for_pull_request(pull_request, token):
url = pull_request["url"]
changed_files = set([])
for i in range(1, 31):
print("Requesting changed files page", i)
url = f"{API_URL}/pulls/{number}/files?page={i}&per_page=100"
data = _exec_get_with_retry(url)
data = _exec_get_with_retry(f"{url}/files?page={i}&per_page=100", token)
print(f"Got {len(data)} changed files")
if len(data) == 0:
print("No more changed files")
@ -317,14 +317,13 @@ def check_suspicious_changed_files(changed_files):
return False
def approve_run(run_id, token):
url = f"{API_URL}/actions/runs/{run_id}/approve"
def approve_run(workflow_description: WorkflowDescription, token):
url = f"{workflow_description.api_url}/approve"
_exec_post_with_retry(url, token)
def label_manual_approve(pull_request, token):
number = pull_request["number"]
url = f"{API_URL}/issues/{number}/labels"
url = f"{pull_request['url']}/labels"
data = {"labels": "manual approve"}
_exec_post_with_retry(url, token, data)
@ -343,14 +342,14 @@ def get_token_from_aws():
return get_access_token(encoded_jwt, installation_id)
def get_workflow_jobs(workflow_description):
def get_workflow_jobs(workflow_description, token):
jobs_url = (
workflow_description.api_url + f"/attempts/{workflow_description.attempt}/jobs"
)
jobs = []
i = 1
while True:
got_jobs = _exec_get_with_retry(jobs_url + f"?page={i}")
got_jobs = _exec_get_with_retry(jobs_url + f"?page={i}", token)
if len(got_jobs["jobs"]) == 0:
break
@ -360,7 +359,7 @@ def get_workflow_jobs(workflow_description):
return jobs
def check_need_to_rerun(workflow_description):
def check_need_to_rerun(workflow_description, token):
if workflow_description.attempt >= MAX_WORKFLOW_RERUN:
print(
"Not going to rerun workflow because it's already tried more than two times"
@ -368,7 +367,7 @@ def check_need_to_rerun(workflow_description):
return False
print("Going to check jobs")
jobs = get_workflow_jobs(workflow_description)
jobs = get_workflow_jobs(workflow_description, token)
print("Got jobs", len(jobs))
for job in jobs:
if job["conclusion"] not in ("success", "skipped"):
@ -395,13 +394,9 @@ def rerun_workflow(workflow_description, token):
_exec_post_with_retry(workflow_description.rerun_url, token)
def main(event):
token = get_token_from_aws()
event_data = json.loads(event["body"])
print("The body received:", event["body"])
workflow_description = get_workflow_description_from_event(event_data)
print("Got workflow description", workflow_description)
def check_workflow_completed(
event_data, workflow_description: WorkflowDescription, token: str
) -> bool:
if workflow_description.action == "completed":
attempt = 0
# Nice and reliable GH API sends from time to time such events, e.g:
@ -411,7 +406,7 @@ def main(event):
progressive_sleep = 3 * sum(i + 1 for i in range(attempt))
time.sleep(progressive_sleep)
event_data["workflow_run"] = _exec_get_with_retry(
workflow_description.api_url
workflow_description.api_url, token
)
workflow_description = get_workflow_description_from_event(event_data)
attempt += 1
@ -421,7 +416,7 @@ def main(event):
"Workflow finished with status "
f"{workflow_description.conclusion}, exiting"
)
return
return True
print(
"Workflow",
@ -435,11 +430,24 @@ def main(event):
workflow_description.name,
"not in list of rerunable workflows",
)
return
return True
if check_need_to_rerun(workflow_description):
if check_need_to_rerun(workflow_description, token):
rerun_workflow(workflow_description, token)
return
return True
return False
def main(event):
token = get_token_from_aws()
event_data = json.loads(event["body"])
print("The body received:", event["body"])
workflow_description = get_workflow_description_from_event(event_data)
print("Got workflow description", workflow_description)
if check_workflow_completed(event_data, workflow_description, token):
return
if workflow_description.action != "requested":
print("Exiting, event action is", workflow_description.action)
@ -447,11 +455,14 @@ def main(event):
if workflow_description.workflow_id in TRUSTED_WORKFLOW_IDS:
print("Workflow in trusted list, approving run")
approve_run(workflow_description.run_id, token)
approve_run(workflow_description, token)
return
pull_requests = _get_pull_requests_from(
workflow_description.fork_owner_login, workflow_description.fork_branch
workflow_description.repo_url,
workflow_description.fork_owner_login,
workflow_description.fork_branch,
token,
)
print("Got pull requests for workflow", len(pull_requests))
@ -462,13 +473,13 @@ def main(event):
pull_request = pull_requests[0]
print("Pull request for workflow number", pull_request["number"])
author, author_orgs = get_pr_author_and_orgs(pull_request)
author, author_orgs = get_pr_author_and_orgs(pull_request, token)
if is_trusted_contributor(author, author_orgs):
print("Contributor is trusted, approving run")
approve_run(workflow_description.run_id, token)
approve_run(workflow_description, token)
return
changed_files = get_changed_files_for_pull_request(pull_request)
changed_files = get_changed_files_for_pull_request(pull_request, token)
print(f"Totally have {len(changed_files)} changed files in PR:", changed_files)
if check_suspicious_changed_files(changed_files):
print(
@ -478,7 +489,7 @@ def main(event):
label_manual_approve(pull_request, token)
else:
print(f"Pull Request {pull_request['number']} has no suspicious changes")
approve_run(workflow_description.run_id, token)
approve_run(workflow_description, token)
def handler(event, _):

3
tests/integration/README.md
View File

@ -31,6 +31,7 @@ sudo -H pip install \
kafka-python \
kazoo \
minio \
lz4 \
protobuf \
psycopg2-binary \
pymongo \
@ -147,7 +148,7 @@ will automagically detect the types of variables and only the small diff of two
### Troubleshooting
If tests failing for misterious reasons, this may help:
If tests failing for mysterious reasons, this may help:
```
sudo service docker stop

54
tests/integration/helpers/cluster.py
View File

@ -30,6 +30,7 @@ try:
import pymongo
import pymysql
import nats
import ssl
import meilisearch
from confluent_kafka.avro.cached_schema_registry_client import (
CachedSchemaRegistryClient,
@ -215,9 +216,27 @@ def check_rabbitmq_is_available(rabbitmq_id):
return p.returncode == 0
async def check_nats_is_available(nats_ip):
nc = await nats.connect("{}:4444".format(nats_ip), user="click", password="house")
return nc.is_connected
async def check_nats_is_available(nats_port, ssl_ctx=None):
nc = await nats_connect_ssl(
nats_port, user="click", password="house", ssl_ctx=ssl_ctx
)
available = nc.is_connected
await nc.close()
return available
async def nats_connect_ssl(nats_port, user, password, ssl_ctx=None):
if not ssl_ctx:
ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE
nc = await nats.connect(
"tls://localhost:{}".format(nats_port),
user=user,
password=password,
tls=ssl_ctx,
)
return nc
def enable_consistent_hash_plugin(rabbitmq_id):
@ -333,6 +352,7 @@ class ClickHouseCluster:
self.env_variables = {}
self.env_variables["TSAN_OPTIONS"] = "second_deadlock_stack=1"
self.env_variables["CLICKHOUSE_WATCHDOG_ENABLE"] = "0"
self.env_variables["CLICKHOUSE_NATS_TLS_SECURE"] = "0"
self.up_called = False
custom_dockerd_host = custom_dockerd_host or os.environ.get(
@ -461,9 +481,11 @@ class ClickHouseCluster:
self.rabbitmq_logs_dir = os.path.join(self.rabbitmq_dir, "logs")
self.nats_host = "nats1"
self.nats_ip = None
self.nats_port = 4444
self.nats_docker_id = None
self.nats_dir = p.abspath(p.join(self.instances_dir, "nats"))
self.nats_cert_dir = os.path.join(self.nats_dir, "cert")
self.nats_ssl_context = None
# available when with_nginx == True
self.nginx_host = "nginx"
@ -1082,6 +1104,7 @@ class ClickHouseCluster:
env_variables["NATS_HOST"] = self.nats_host
env_variables["NATS_INTERNAL_PORT"] = "4444"
env_variables["NATS_EXTERNAL_PORT"] = str(self.nats_port)
env_variables["NATS_CERT_DIR"] = self.nats_cert_dir
self.base_cmd.extend(
["--file", p.join(docker_compose_yml_dir, "docker_compose_nats.yml")]
@ -2003,10 +2026,12 @@ class ClickHouseCluster:
raise Exception("Cannot wait RabbitMQ container")
return False
def wait_nats_is_available(self, nats_ip, max_retries=5):
def wait_nats_is_available(self, max_retries=5):
retries = 0
while True:
if asyncio.run(check_nats_is_available(nats_ip)):
if asyncio.run(
check_nats_is_available(self.nats_port, ssl_ctx=self.nats_ssl_context)
):
break
else:
retries += 1
@ -2481,11 +2506,24 @@ class ClickHouseCluster:
if self.with_nats and self.base_nats_cmd:
logging.debug("Setup NATS")
os.makedirs(self.nats_cert_dir)
env = os.environ.copy()
env["NATS_CERT_DIR"] = self.nats_cert_dir
run_and_check(
p.join(self.base_dir, "nats_certs.sh"),
env=env,
detach=False,
nothrow=False,
)
self.nats_ssl_context = ssl.create_default_context()
self.nats_ssl_context.load_verify_locations(
p.join(self.nats_cert_dir, "ca", "ca-cert.pem")
)
subprocess_check_call(self.base_nats_cmd + common_opts)
self.nats_docker_id = self.get_instance_docker_id("nats1")
self.up_called = True
self.nats_ip = self.get_instance_ip("nats1")
self.wait_nats_is_available(self.nats_ip)
self.wait_nats_is_available()
if self.with_hdfs and self.base_hdfs_cmd:
logging.debug("Setup HDFS")

2
tests/integration/runner
View File

@ -345,7 +345,7 @@ if __name__ == "__main__":
f"docker volume create {VOLUME_NAME}_volume", shell=True
)
except Exception as ex:
print("Volume creationg failed, probably it already exists, exception", ex)
print("Volume creation failed, probably it already exists, exception", ex)
# TODO: this part cleans out stale volumes produced by container name
# randomizer, we should remove it after Sep 2022
try:

0
tests/integration/test_fetch_memory_usage/__init__.py Normal file
View File

3
tests/integration/test_fetch_memory_usage/configs/config.xml Normal file
View File

@ -0,0 +1,3 @@
<clickhouse>
<max_server_memory_usage>1000000000</max_server_memory_usage>
</clickhouse>

59
tests/integration/test_fetch_memory_usage/test.py Normal file
View File

@ -0,0 +1,59 @@
import pytest
from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__)
node = cluster.add_instance(
"node", main_configs=["configs/config.xml"], with_zookeeper=True
)
@pytest.fixture(scope="module")
def started_cluster():
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
def test_huge_column(started_cluster):
if (
node.is_built_with_thread_sanitizer()
or node.is_built_with_memory_sanitizer()
or node.is_built_with_address_sanitizer()
):
pytest.skip("sanitizer build has higher memory consumption; also it is slow")
# max_server_memory_usage is set to 1GB
# Added column should be 1e6 * 2000 ~= 2GB
# ALTER ADD COLUMN + MATRIALIZE COLUMN should not cause big memory consumption
node.query(
"""
create table test_fetch (x UInt64) engine = ReplicatedMergeTree('/clickhouse/tables/test_fetch', 'r1') order by x settings index_granularity=1024;
insert into test_fetch select number from numbers(1e6);
set mutations_sync=1;
alter table test_fetch add column y String default repeat(' ', 2000) CODEC(NONE);
alter table test_fetch materialize column y;
create table test_fetch2 (x UInt64, y String default repeat(' ', 2000) CODEC(NONE)) engine = ReplicatedMergeTree('/clickhouse/tables/test_fetch', 'r2') order by x settings index_granularity=1024;
"""
)
# Here we just check that fetch has started.
node.query(
"""
set receive_timeout=1;
system sync replica test_fetch2;
""",
ignore_error=True,
)
# Here we check that fetch did not use too much memory.
# See https://github.com/ClickHouse/ClickHouse/issues/39915
maybe_exception = node.query(
"select last_exception from system.replication_queue where last_exception like '%Memory limit%';"
)
assert maybe_exception == ""

13
tests/integration/test_storage_nats/nats_certs.sh Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
set -euxo pipefail
mkdir -p "${NATS_CERT_DIR}/ca"
mkdir -p "${NATS_CERT_DIR}/nats"
openssl req -newkey rsa:4096 -x509 -days 365 -nodes -batch -keyout "${NATS_CERT_DIR}/ca/ca-key.pem" -out "${NATS_CERT_DIR}/ca/ca-cert.pem" -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=ca"
openssl req -newkey rsa:4096 -nodes -batch -keyout "${NATS_CERT_DIR}/nats/server-key.pem" -out "${NATS_CERT_DIR}/nats/server-req.pem" -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=server"
openssl x509 -req -days 365 -in "${NATS_CERT_DIR}/nats/server-req.pem" -CA "${NATS_CERT_DIR}/ca/ca-cert.pem" -CAkey "${NATS_CERT_DIR}/ca/ca-key.pem" -CAcreateserial -out "${NATS_CERT_DIR}/nats/server-cert.pem" -extfile <(
cat <<-EOF
subjectAltName = DNS:localhost, DNS:nats1
EOF
)
rm -f "${NATS_CERT_DIR}/nats/server-req.pem"

92
tests/integration/test_storage_nats/test.py
View File

@ -9,11 +9,10 @@ from random import randrange
import math
import asyncio
import nats
import pytest
from google.protobuf.internal.encoder import _VarintBytes
from helpers.client import QueryRuntimeException
from helpers.cluster import ClickHouseCluster, check_nats_is_available
from helpers.cluster import ClickHouseCluster, check_nats_is_available, nats_connect_ssl
from helpers.test_tools import TSV
from . import nats_pb2
@ -35,11 +34,11 @@ instance = cluster.add_instance(
# Helpers
def wait_nats_to_start(nats_ip, timeout=180):
def wait_nats_to_start(nats_port, ssl_ctx=None, timeout=180):
start = time.time()
while time.time() - start < timeout:
try:
if asyncio.run(check_nats_is_available(nats_ip)):
if asyncio.run(check_nats_is_available(nats_port, ssl_ctx=ssl_ctx)):
logging.debug("NATS is available")
return
time.sleep(0.5)
@ -63,10 +62,10 @@ def kill_nats(nats_id):
return p.returncode == 0
def revive_nats(nats_id, nats_ip):
def revive_nats(nats_id, nats_port):
p = subprocess.Popen(("docker", "start", nats_id), stdout=subprocess.PIPE)
p.communicate()
wait_nats_to_start(nats_ip)
wait_nats_to_start(nats_port)
# Fixtures
@ -96,8 +95,13 @@ def nats_setup_teardown():
# Tests
async def nats_produce_messages(ip, subject, messages=(), bytes=None):
nc = await nats.connect("{}:4444".format(ip), user="click", password="house")
async def nats_produce_messages(cluster_inst, subject, messages=(), bytes=None):
nc = await nats_connect_ssl(
cluster_inst.nats_port,
user="click",
password="house",
ssl_ctx=cluster_inst.nats_ssl_context,
)
logging.debug("NATS connection status: " + str(nc.is_connected))
for message in messages:
@ -136,7 +140,7 @@ def test_nats_select(nats_cluster):
messages = []
for i in range(50):
messages.append(json.dumps({"key": i, "value": i}))
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "select", messages))
asyncio.run(nats_produce_messages(nats_cluster, "select", messages))
# The order of messages in select * from test.nats is not guaranteed, so sleep to collect everything in one select
time.sleep(1)
@ -186,13 +190,13 @@ def test_nats_json_without_delimiter(nats_cluster):
messages += json.dumps({"key": i, "value": i}) + "\n"
all_messages = [messages]
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "json", all_messages))
asyncio.run(nats_produce_messages(nats_cluster, "json", all_messages))
messages = ""
for i in range(25, 50):
messages += json.dumps({"key": i, "value": i}) + "\n"
all_messages = [messages]
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "json", all_messages))
asyncio.run(nats_produce_messages(nats_cluster, "json", all_messages))
time.sleep(1)
@ -229,7 +233,7 @@ def test_nats_csv_with_delimiter(nats_cluster):
for i in range(50):
messages.append("{i}, {i}".format(i=i))
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "csv", messages))
asyncio.run(nats_produce_messages(nats_cluster, "csv", messages))
time.sleep(1)
@ -268,7 +272,7 @@ def test_nats_tsv_with_delimiter(nats_cluster):
for i in range(50):
messages.append("{i}\t{i}".format(i=i))
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "tsv", messages))
asyncio.run(nats_produce_messages(nats_cluster, "tsv", messages))
result = ""
for _ in range(60):
@ -299,7 +303,7 @@ def test_nats_macros(nats_cluster):
message = ""
for i in range(50):
message += json.dumps({"key": i, "value": i}) + "\n"
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "macro", [message]))
asyncio.run(nats_produce_messages(nats_cluster, "macro", [message]))
time.sleep(1)
@ -344,7 +348,7 @@ def test_nats_materialized_view(nats_cluster):
for i in range(50):
messages.append(json.dumps({"key": i, "value": i}))
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "mv", messages))
asyncio.run(nats_produce_messages(nats_cluster, "mv", messages))
time_limit_sec = 60
deadline = time.monotonic() + time_limit_sec
@ -389,7 +393,7 @@ def test_nats_materialized_view_with_subquery(nats_cluster):
messages = []
for i in range(50):
messages.append(json.dumps({"key": i, "value": i}))
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "mvsq", messages))
asyncio.run(nats_produce_messages(nats_cluster, "mvsq", messages))
time_limit_sec = 60
deadline = time.monotonic() + time_limit_sec
@ -434,7 +438,7 @@ def test_nats_many_materialized_views(nats_cluster):
messages = []
for i in range(50):
messages.append(json.dumps({"key": i, "value": i}))
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "mmv", messages))
asyncio.run(nats_produce_messages(nats_cluster, "mmv", messages))
time_limit_sec = 60
deadline = time.monotonic() + time_limit_sec
@ -485,7 +489,7 @@ def test_nats_protobuf(nats_cluster):
msg.value = str(i)
serialized_msg = msg.SerializeToString()
data = data + _VarintBytes(len(serialized_msg)) + serialized_msg
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "pb", bytes=data))
asyncio.run(nats_produce_messages(nats_cluster, "pb", bytes=data))
data = b""
for i in range(20, 21):
msg = nats_pb2.ProtoKeyValue()
@ -493,7 +497,7 @@ def test_nats_protobuf(nats_cluster):
msg.value = str(i)
serialized_msg = msg.SerializeToString()
data = data + _VarintBytes(len(serialized_msg)) + serialized_msg
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "pb", bytes=data))
asyncio.run(nats_produce_messages(nats_cluster, "pb", bytes=data))
data = b""
for i in range(21, 50):
msg = nats_pb2.ProtoKeyValue()
@ -501,7 +505,7 @@ def test_nats_protobuf(nats_cluster):
msg.value = str(i)
serialized_msg = msg.SerializeToString()
data = data + _VarintBytes(len(serialized_msg)) + serialized_msg
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "pb", bytes=data))
asyncio.run(nats_produce_messages(nats_cluster, "pb", bytes=data))
result = ""
time_limit_sec = 60
@ -542,7 +546,7 @@ def test_nats_big_message(nats_cluster):
logging.debug("Table test.nats is not yet ready")
time.sleep(0.5)
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "big", messages))
asyncio.run(nats_produce_messages(nats_cluster, "big", messages))
while True:
result = instance.query("SELECT count() FROM test.view")
@ -600,7 +604,7 @@ def test_nats_mv_combo(nats_cluster):
for _ in range(messages_num):
messages.append(json.dumps({"key": i[0], "value": i[0]}))
i[0] += 1
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "combo", messages))
asyncio.run(nats_produce_messages(nats_cluster, "combo", messages))
threads = []
threads_num = 20
@ -662,8 +666,11 @@ def test_nats_insert(nats_cluster):
insert_messages = []
async def sub_to_nats():
nc = await nats.connect(
"{}:4444".format(nats_cluster.nats_ip), user="click", password="house"
nc = await nats_connect_ssl(
nats_cluster.nats_port,
user="click",
password="house",
ssl_ctx=nats_cluster.nats_ssl_context,
)
sub = await nc.subscribe("insert")
await sub.unsubscribe(50)
@ -771,8 +778,11 @@ def test_nats_many_subjects_insert_right(nats_cluster):
insert_messages = []
async def sub_to_nats():
nc = await nats.connect(
"{}:4444".format(nats_cluster.nats_ip), user="click", password="house"
nc = await nats_connect_ssl(
nats_cluster.nats_port,
user="click",
password="house",
ssl_ctx=nats_cluster.nats_ssl_context,
)
sub = await nc.subscribe("right_insert1")
await sub.unsubscribe(50)
@ -1003,7 +1013,7 @@ def test_nats_virtual_column(nats_cluster):
messages.append(json.dumps({"key": i, "value": i}))
i += 1
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "virtuals", messages))
asyncio.run(nats_produce_messages(nats_cluster, "virtuals", messages))
while True:
result = instance.query("SELECT count() FROM test.view")
@ -1067,7 +1077,7 @@ def test_nats_virtual_column_with_materialized_view(nats_cluster):
messages.append(json.dumps({"key": i, "value": i}))
i += 1
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "virtuals_mv", messages))
asyncio.run(nats_produce_messages(nats_cluster, "virtuals_mv", messages))
while True:
result = instance.query("SELECT count() FROM test.view")
@ -1147,9 +1157,7 @@ def test_nats_many_consumers_to_each_queue(nats_cluster):
for _ in range(messages_num):
messages.append(json.dumps({"key": i[0], "value": i[0]}))
i[0] += 1
asyncio.run(
nats_produce_messages(nats_cluster.nats_ip, "many_consumers", messages)
)
asyncio.run(nats_produce_messages(nats_cluster, "many_consumers", messages))
threads = []
threads_num = 20
@ -1243,7 +1251,7 @@ def test_nats_restore_failed_connection_without_losses_on_write(nats_cluster):
kill_nats(nats_cluster.nats_docker_id)
time.sleep(4)
revive_nats(nats_cluster.nats_docker_id, nats_cluster.nats_ip)
revive_nats(nats_cluster.nats_docker_id, nats_cluster.nats_port)
while True:
result = instance.query("SELECT count(DISTINCT key) FROM test.view")
@ -1310,7 +1318,7 @@ def test_nats_no_connection_at_startup_2(nats_cluster):
messages = []
for i in range(messages_num):
messages.append(json.dumps({"key": i, "value": i}))
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "cs", messages))
asyncio.run(nats_produce_messages(nats_cluster, "cs", messages))
for _ in range(20):
result = instance.query("SELECT count() FROM test.view")
@ -1353,9 +1361,7 @@ def test_nats_format_factory_settings(nats_cluster):
"""SELECT parseDateTimeBestEffort(CAST('2021-01-19T14:42:33.1829214Z', 'String'))"""
)
asyncio.run(
nats_produce_messages(nats_cluster.nats_ip, "format_settings", [message])
)
asyncio.run(nats_produce_messages(nats_cluster, "format_settings", [message]))
while True:
result = instance.query("SELECT date FROM test.format_settings")
@ -1372,9 +1378,7 @@ def test_nats_format_factory_settings(nats_cluster):
"""
)
asyncio.run(
nats_produce_messages(nats_cluster.nats_ip, "format_settings", [message])
)
asyncio.run(nats_produce_messages(nats_cluster, "format_settings", [message]))
while True:
result = instance.query("SELECT date FROM test.view")
if result == expected:
@ -1424,13 +1428,13 @@ def test_nats_drop_mv(nats_cluster):
messages = []
for i in range(20):
messages.append(json.dumps({"key": i, "value": i}))
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "mv", messages))
asyncio.run(nats_produce_messages(nats_cluster, "mv", messages))
instance.query("DROP VIEW test.consumer")
messages = []
for i in range(20, 40):
messages.append(json.dumps({"key": i, "value": i}))
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "mv", messages))
asyncio.run(nats_produce_messages(nats_cluster, "mv", messages))
instance.query(
"""
@ -1441,7 +1445,7 @@ def test_nats_drop_mv(nats_cluster):
messages = []
for i in range(40, 50):
messages.append(json.dumps({"key": i, "value": i}))
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "mv", messages))
asyncio.run(nats_produce_messages(nats_cluster, "mv", messages))
while True:
result = instance.query("SELECT * FROM test.view ORDER BY key")
@ -1454,7 +1458,7 @@ def test_nats_drop_mv(nats_cluster):
messages = []
for i in range(50, 60):
messages.append(json.dumps({"key": i, "value": i}))
asyncio.run(nats_produce_messages(nats_cluster.nats_ip, "mv", messages))
asyncio.run(nats_produce_messages(nats_cluster, "mv", messages))
count = 0
while True:
@ -1477,7 +1481,7 @@ def test_nats_predefined_configuration(nats_cluster):
asyncio.run(
nats_produce_messages(
nats_cluster.nats_ip, "named", [json.dumps({"key": 1, "value": 2})]
nats_cluster, "named", [json.dumps({"key": 1, "value": 2})]
)
)
while True:

78
tests/queries/0_stateless/02352_rwlock.sh
View File

@ -1,4 +1,6 @@
#!/usr/bin/env bash
# Tags: no-parallel
# Tag no-parallel -- to avoid running it in parallel, this will avoid possible issues due to high pressure
# Test that ensures that WRITE lock failure notifies READ.
# In other words to ensure that after WRITE lock failure (DROP),
@ -13,31 +15,73 @@ function wait_query_by_id_started()
local query_id=$1 && shift
# wait for query to be started
while [ "$($CLICKHOUSE_CLIENT "$@" -q "select count() from system.processes where query_id = '$query_id'")" -ne 1 ]; do
sleep 0.1
if [ "$(
$CLICKHOUSE_CLIENT -nm -q "
system flush logs;
select count() from system.query_log
where
event_date >= yesterday() and
current_database = '$CLICKHOUSE_DATABASE' and
type = 'QueryFinish' and
query_id = '$query_id'
"
)" -eq 1 ]; then
return 1
else
sleep 0.1
fi
done
return 0
}
# to avoid removal via separate thread
$CLICKHOUSE_CLIENT -q "CREATE DATABASE ${CLICKHOUSE_DATABASE}_ordinary Engine=Ordinary" --allow_deprecated_database_ordinary=1
$CLICKHOUSE_CLIENT -q "CREATE TABLE ${CLICKHOUSE_DATABASE}_ordinary.data_02352 (key Int) Engine=Null()"
query_id="insert-$(random_str 10)"
# 20 seconds sleep
$CLICKHOUSE_CLIENT --query_id "$query_id" -q "INSERT INTO ${CLICKHOUSE_DATABASE}_ordinary.data_02352 SELECT sleepEachRow(1) FROM numbers(20) GROUP BY number" &
wait_query_by_id_started "$query_id"
# It is possible that the subsequent after INSERT query will be processed
# only after INSERT finished, it is unlikely, but it happens few times in
# debug build on CI, so if this will happen, then DROP query will be
# finished instantly, and to avoid flakiness we will retry in this case
while :; do
$CLICKHOUSE_CLIENT -nm -q "
DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}_ordinary.data_02352;
CREATE TABLE ${CLICKHOUSE_DATABASE}_ordinary.data_02352 (key Int) Engine=Null();
"
query_id="drop-$(random_str 10)"
# 10 second wait
$CLICKHOUSE_CLIENT --query_id "$query_id" -q "DROP TABLE ${CLICKHOUSE_DATABASE}_ordinary.data_02352 SYNC" --lock_acquire_timeout 10 > >(grep -m1 -o 'WRITE locking attempt on ".*" has timed out') 2>&1 &
wait_query_by_id_started "$query_id"
insert_query_id="insert-$(random_str 10)"
# 20 seconds sleep
$CLICKHOUSE_CLIENT --query_id "$insert_query_id" -q "INSERT INTO ${CLICKHOUSE_DATABASE}_ordinary.data_02352 SELECT sleepEachRow(1) FROM numbers(20) GROUP BY number" &
if ! wait_query_by_id_started "$insert_query_id"; then
wait
continue
fi
# NOTE: we need to run SELECT after DROP and
# if the bug is there, then the query will wait 20 seconds (INSERT), instead of 10 (DROP) and will fail
#
# 11 seconds wait (DROP + 1 second lag)
$CLICKHOUSE_CLIENT -q "SELECT * FROM ${CLICKHOUSE_DATABASE}_ordinary.data_02352" --lock_acquire_timeout 11
drop_query_id="drop-$(random_str 10)"
# 10 second wait
$CLICKHOUSE_CLIENT --query_id "$drop_query_id" -q "DROP TABLE ${CLICKHOUSE_DATABASE}_ordinary.data_02352 SYNC" --lock_acquire_timeout 10 > >(
grep -m1 -o 'WRITE locking attempt on ".*" has timed out'
) 2>&1 &
if ! wait_query_by_id_started "$drop_query_id"; then
wait
continue
fi
# Check INSERT query again, and retry if it does not exist.
if ! wait_query_by_id_started "$insert_query_id"; then
wait
continue
fi
# wait DROP and INSERT
wait
# NOTE: we need to run SELECT after DROP and
# if the bug is there, then the query will wait 20 seconds (INSERT), instead of 10 (DROP) and will fail
#
# 11 seconds wait (DROP + 1 second lag)
$CLICKHOUSE_CLIENT -q "SELECT * FROM ${CLICKHOUSE_DATABASE}_ordinary.data_02352" --lock_acquire_timeout 11
# wait DROP and INSERT
wait
break
done | uniq
$CLICKHOUSE_CLIENT -q "DROP DATABASE ${CLICKHOUSE_DATABASE}_ordinary"

2
tests/queries/0_stateless/02354_distributed_with_external_aggregation_memory_usage.sql
View File

@ -19,4 +19,4 @@ select a, b, c, sum(a) as s
from remote('127.0.0.{1,2}', currentDatabase(), t_2354_dist_with_external_aggr)
group by a, b, c
format Null
settings max_memory_usage = '4G';
settings max_memory_usage = '4Gi';

2
tests/queries/0_stateless/02360_small_notation_h_for_hour_interval.reference Normal file
View File

@ -0,0 +1,2 @@
25
2018-01-02 02:00:00

2
tests/queries/0_stateless/02360_small_notation_h_for_hour_interval.sql Normal file
View File

@ -0,0 +1,2 @@
SELECT dateDiff('h', toDateTime('2018-01-01 22:00:00'), toDateTime('2018-01-02 23:00:00'));
SELECT toDateTime('2018-01-01 22:00:00') + INTERVAL 4 h

4
tests/queries/0_stateless/02367_join_pushdown_column_not_found.reference Normal file
View File

@ -0,0 +1,4 @@
1
1
1
1

16
tests/queries/0_stateless/02367_join_pushdown_column_not_found.sql.j2 Normal file
View File

@ -0,0 +1,16 @@
{% for join_algorithm in ['default', 'full_sorting_merge', 'hash', 'partial_merge'] -%}
SET join_algorithm = '{{ join_algorithm }}';
SELECT deleted
FROM (
SELECT 1 AS deleted, 'k' AS a, 'v' AS b
) AS q
INNER JOIN (
SELECT 'k' AS a, 'v' AS c
) AS s
ON q.a = s.a
WHERE deleted AND (b = c);
{% endfor -%}

118
tests/queries/0_stateless/02375_pretty_formats.reference Normal file
View File

@ -0,0 +1,118 @@
Pretty
┏━━━┳━━━┓
┃ x ┃ y ┃
┡━━━╇━━━┩
│ 0 │ 1 │
├───┼───┤
│ 1 │ 2 │
└───┴───┘
┏━━━┳━━━┓
┃ x ┃ y ┃
┡━━━╇━━━┩
│ 2 │ 3 │
├───┼───┤
│ 3 │ 4 │
└───┴───┘
PrettyNoEscapes
┏━━━┳━━━┓
┃ x ┃ y ┃
┡━━━╇━━━┩
│ 0 │ 1 │
├───┼───┤
│ 1 │ 2 │
└───┴───┘
┏━━━┳━━━┓
┃ x ┃ y ┃
┡━━━╇━━━┩
│ 2 │ 3 │
├───┼───┤
│ 3 │ 4 │
└───┴───┘
PrettyMonoBlock
┏━━━┳━━━┓
┃ x ┃ y ┃
┡━━━╇━━━┩
│ 0 │ 1 │
├───┼───┤
│ 1 │ 2 │
├───┼───┤
│ 2 │ 3 │
├───┼───┤
│ 3 │ 4 │
└───┴───┘
PrettyNoEscapesMonoBlock
┏━━━┳━━━┓
┃ x ┃ y ┃
┡━━━╇━━━┩
│ 0 │ 1 │
├───┼───┤
│ 1 │ 2 │
├───┼───┤
│ 2 │ 3 │
├───┼───┤
│ 3 │ 4 │
└───┴───┘
PrettyCompact
┌─x─┬─y─┐
│ 0 │ 1 │
│ 1 │ 2 │
└───┴───┘
┌─x─┬─y─┐
│ 2 │ 3 │
│ 3 │ 4 │
└───┴───┘
PrettyCompactNoEscapes
┌─x─┬─y─┐
│ 0 │ 1 │
│ 1 │ 2 │
└───┴───┘
┌─x─┬─y─┐
│ 2 │ 3 │
│ 3 │ 4 │
└───┴───┘
PrettyCompactMonoBlock
┌─x─┬─y─┐
│ 0 │ 1 │
│ 1 │ 2 │
│ 2 │ 3 │
│ 3 │ 4 │
└───┴───┘
PrettyCompactNoEscapesMonoBlock
┌─x─┬─y─┐
│ 0 │ 1 │
│ 1 │ 2 │
│ 2 │ 3 │
│ 3 │ 4 │
└───┴───┘
PrettySpace
x y
0 1
1 2
x y
2 3
3 4
PrettySpaceNoEscapes
x y
0 1
1 2
x y
2 3
3 4
PrettySpaceMonoBlock
x y
0 1
1 2
2 3
3 4
PrettySpaceNoEscapesMonoBlock
x y
0 1
1 2
2 3
3 4

8
tests/queries/0_stateless/02375_pretty_formats.sql.j2 Normal file
View File

@ -0,0 +1,8 @@
{% for format in ['Pretty', 'PrettyNoEscapes', 'PrettyMonoBlock', 'PrettyNoEscapesMonoBlock', 'PrettyCompact', 'PrettyCompactNoEscapes',
'PrettyCompactMonoBlock', 'PrettyCompactNoEscapesMonoBlock', 'PrettySpace', 'PrettySpaceNoEscapes', 'PrettySpaceMonoBlock',
'PrettySpaceNoEscapesMonoBlock'] -%}
select '{{ format }}';
select number as x, number + 1 as y from numbers(4) settings max_block_size=2 format {{ format }};
{% endfor -%}

3
tests/queries/0_stateless/02377_fix_file_virtual_column.reference Normal file
View File

@ -0,0 +1,3 @@
1 s
2 x
3 y

7
tests/queries/0_stateless/02377_fix_file_virtual_column.sql Normal file
View File

@ -0,0 +1,7 @@
drop table if exists test_02377;
create table test_02377 (n UInt32, s String) engine=File(CSVWithNames);
insert into test_02377 values(1, 's') (2, 'x') (3, 'y');
select * from test_02377 order by n;
select *, _path, _file from test_02377 format Null;
select _path, _file from test_02377 format Null;
drop table test_02377;

2
tests/queries/0_stateless/02380_insert_mv_race.sh
View File

@ -1,5 +1,5 @@
#!/usr/bin/env bash
# Tags: long, race
# Tags: long, race, no-ordinary-database
# Regression test for INSERT into table with MV attached,
# to avoid possible errors if some table will disappears,

10
utils/check-style/aspell-ignore/en/aspell-dict.txt
View File

@ -108,6 +108,11 @@ PrettyCompactNoEscapes
PrettyNoEscapes
PrettySpace
PrettySpaceNoEscapes
PrettyCompactNoEscapesMonoBlock
PrettyMonoBlock
PrettyNoEscapesMonoBlock
PrettySpaceMonoBlock
PrettySpaceNoEscapesMonoBlock
Protobuf
ProtobufSingle
QTCreator
@ -374,6 +379,11 @@ prettycompactnoescapes
prettynoescapes
prettyspace
prettyspacenoescapes
prettycompactnoescapesmonoblock
prettymonoblock
prettynoescapesmonoblock
prettyspacemonoblock
prettyspacenoescapesmonoblock
prlimit
prometheus
proto

1
utils/list-versions/version_date.tsv
View File

@ -1,5 +1,6 @@
v22.7.2.15-stable 2022-08-03
v22.7.1.2484-stable 2022-07-21
v22.6.5.22-stable 2022-08-09
v22.6.4.35-stable 2022-07-25
v22.6.3.35-stable 2022-07-06
v22.6.2.12-stable 2022-06-29

1 v22.7.2.15-stable 2022-08-03
2 v22.7.1.2484-stable 2022-07-21
3 v22.6.5.22-stable 2022-08-09
4 v22.6.4.35-stable 2022-07-25
5 v22.6.3.35-stable 2022-07-06
6 v22.6.2.12-stable 2022-06-29

114
utils/self-extracting-executable/decompressor.cpp
View File

@ -21,11 +21,16 @@
#endif
#if defined OS_DARWIN
# include <mach-o/dyld.h>
# include <libkern/OSByteOrder.h>
// define 64 bit macros
# define le64toh(x) OSSwapLittleToHostInt64(x)
#endif
#if defined(OS_FREEBSD)
# include <sys/sysctl.h>
#endif
#include "types.h"
/// decompress part
@ -82,7 +87,7 @@ int decompress(char * input, char * output, off_t start, off_t end, size_t max_n
pid = fork();
if (-1 == pid)
{
perror(nullptr);
perror("fork");
/// If fork failed just decompress data in main process.
if (0 != doDecompress(input, output, in_pointer, out_pointer, size, decompressed_size, dctx))
{
@ -170,7 +175,7 @@ int decompressFiles(int input_fd, char * path, char * name, bool & have_compress
struct stat info_in;
if (0 != fstat(input_fd, &info_in))
{
perror(nullptr);
perror("fstat");
return 1;
}
@ -178,7 +183,7 @@ int decompressFiles(int input_fd, char * path, char * name, bool & have_compress
char * input = static_cast<char*>(mmap(nullptr, info_in.st_size, PROT_READ, MAP_PRIVATE, input_fd, 0));
if (input == MAP_FAILED)
{
perror(nullptr);
perror("mmap");
return 1;
}
@ -202,9 +207,9 @@ int decompressFiles(int input_fd, char * path, char * name, bool & have_compress
struct statfs fs_info;
if (0 != fstatfs(input_fd, &fs_info))
{
perror(nullptr);
perror("fstatfs");
if (0 != munmap(input, info_in.st_size))
perror(nullptr);
perror("munmap");
return 1;
}
if (fs_info.f_blocks * info_in.st_blksize < decompressed_full_size)
@ -241,7 +246,7 @@ int decompressFiles(int input_fd, char * path, char * name, bool & have_compress
int fd = mkstemp(file_name);
if (fd == -1)
{
perror(nullptr);
perror("mkstemp");
return 1;
}
close(fd);
@ -254,18 +259,18 @@ int decompressFiles(int input_fd, char * path, char * name, bool & have_compress
if (output_fd == -1)
{
perror(nullptr);
perror("open");
if (0 != munmap(input, info_in.st_size))
perror(nullptr);
perror("munmap");
return 1;
}
/// Prepare output file
if (0 != ftruncate(output_fd, le64toh(file_info.uncompressed_size)))
{
perror(nullptr);
perror("ftruncate");
if (0 != munmap(input, info_in.st_size))
perror(nullptr);
perror("munmap");
return 1;
}
@ -278,9 +283,9 @@ int decompressFiles(int input_fd, char * path, char * name, bool & have_compress
);
if (output == MAP_FAILED)
{
perror(nullptr);
perror("mmap");
if (0 != munmap(input, info_in.st_size))
perror(nullptr);
perror("munmap");
return 1;
}
@ -288,29 +293,70 @@ int decompressFiles(int input_fd, char * path, char * name, bool & have_compress
if (0 != decompress(input, output, le64toh(file_info.start), le64toh(file_info.end)))
{
if (0 != munmap(input, info_in.st_size))
perror(nullptr);
perror("munmap");
if (0 != munmap(output, le64toh(file_info.uncompressed_size)))
perror(nullptr);
perror("munmap");
return 1;
}
if (0 != fsync(output_fd))
perror(nullptr);
perror("fsync");
if (0 != close(output_fd))
perror(nullptr);
perror("close");
}
if (0 != munmap(input, info_in.st_size))
perror(nullptr);
perror("munmap");
return 0;
}
#if defined(OS_DARWIN)
int read_exe_path(char *exe, size_t buf_sz)
{
uint32_t size = buf_sz;
char apple[size];
if (_NSGetExecutablePath(apple, &size) != 0)
return 1;
if (realpath(apple, exe) == nullptr)
return 1;
return 0;
}
#elif defined(OS_FREEBSD)
int read_exe_path(char *exe, size_t buf_sz)
{
int name[] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 };
size_t length = buf_sz;
int error = sysctl(name, 4, exe, &length, nullptr, 0);
if (error < 0 || length <= 1)
return 1;
return 0;
}
#else
int read_exe_path(char *exe, size_t/* buf_sz*/)
{
if (realpath("/proc/self/exe", exe) == nullptr)
return 1;
return 0;
}
#endif
int main(int/* argc*/, char* argv[])
{
char file_path[strlen(argv[0]) + 1];
memset(file_path, 0, sizeof(file_path));
strcpy(file_path, argv[0]);
char self[4096] = {0};
if (read_exe_path(self, 4096) == -1)
{
perror("read_exe_path");
return 1;
}
char file_path[strlen(self) + 1];
strcpy(file_path, self);
char * path = nullptr;
char * name = strrchr(file_path, '/');
@ -323,10 +369,10 @@ int main(int/* argc*/, char* argv[])
else
name = file_path;
int input_fd = open(argv[0], O_RDONLY);
int input_fd = open(self, O_RDONLY);
if (input_fd == -1)
{
perror(nullptr);
perror("open");
return 1;
}
@ -339,16 +385,16 @@ int main(int/* argc*/, char* argv[])
{
printf("Error happened during decompression.\n");
if (0 != close(input_fd))
perror(nullptr);
perror("close");
return 1;
}
if (0 != close(input_fd))
perror(nullptr);
perror("close");
if (unlink(argv[0]))
if (unlink(self))
{
perror(nullptr);
perror("unlink");
return 1;
}
@ -357,34 +403,34 @@ int main(int/* argc*/, char* argv[])
else
{
const char * const decompressed_name_fmt = "%s.decompressed.%s";
int decompressed_name_len = snprintf(nullptr, 0, decompressed_name_fmt, argv[0], decompressed_suffix);
int decompressed_name_len = snprintf(nullptr, 0, decompressed_name_fmt, self, decompressed_suffix);
char decompressed_name[decompressed_name_len + 1];
(void)snprintf(decompressed_name, decompressed_name_len + 1, decompressed_name_fmt, argv[0], decompressed_suffix);
(void)snprintf(decompressed_name, decompressed_name_len + 1, decompressed_name_fmt, self, decompressed_suffix);
std::error_code ec;
std::filesystem::copy_file(static_cast<char *>(decompressed_name), argv[0], ec);
std::filesystem::copy_file(static_cast<char *>(decompressed_name), static_cast<char *>(self), ec);
if (ec)
{
std::cerr << ec.message() << std::endl;
return 1;
}
if (chmod(argv[0], decompressed_umask))
if (chmod(self, decompressed_umask))
{
perror(nullptr);
perror("chmod");
return 1;
}
if (unlink(decompressed_name))
{
perror(nullptr);
perror("unlink");
return 1;
}
execv(argv[0], argv);
execv(self, argv);
/// This part of code will be reached only if error happened
perror(nullptr);
perror("execv");
return 1;
}
}