thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-10-18 14:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into database_atomic

Browse Source
This commit is contained in:
Alexander Tokmakov 2020-01-22 14:30:55 +03:00
commit c4e3f333ce
225 changed files with 4570 additions and 1661 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.github/PULL_REQUEST_TEMPLATE.md vendored
View File

@ -7,16 +7,20 @@ Changelog category (leave one):
- Performance Improvement
- Backward Incompatible Change
- Build/Testing/Packaging Improvement
- Documentation
- Documentation (changelog entry is not required)
- Other
- Non-significant (changelog entry is not needed)
- Non-significant (changelog entry is not required)
Changelog entry (up to few sentences, required except for Non-significant/Documentation categories):
Changelog entry (a user-readable short description of the changes that goes to CHANGELOG.md):
...
Detailed description (optional):
Detailed description / Documentation draft:
...
By adding documentation, you'll allow users to try your new feature immediately, not when someone else will have time to document it later. Documentation is necessary for all features that affect user experience in any way. You can add brief documentation draft above, or add documentation right into your patch as Markdown files in [docs](https://github.com/ClickHouse/ClickHouse/tree/master/docs) folder.
If you are doing this for the first time, it's recommended to read the lightweight [Contributing to ClickHouse Documentation](https://github.com/ClickHouse/ClickHouse/tree/master/docs/README.md) guide first.

3
.gitmodules vendored
View File

@ -134,6 +134,9 @@
[submodule "contrib/libc-headers"]
path = contrib/libc-headers
url = https://github.com/ClickHouse-Extras/libc-headers.git
[submodule "contrib/replxx"]
path = contrib/replxx
url = https://github.com/AmokHuginnsson/replxx.git
[submodule "contrib/ryu"]
path = contrib/ryu
url = https://github.com/ClickHouse-Extras/ryu.git

2
CMakeLists.txt
View File

@ -328,7 +328,6 @@ include (cmake/find/xxhash.cmake)
include (cmake/find/sparsehash.cmake)
include (cmake/find/rt.cmake)
include (cmake/find/execinfo.cmake)
include (cmake/find/readline_edit.cmake)
include (cmake/find/re2.cmake)
include (cmake/find/libgsasl.cmake)
include (cmake/find/rdkafka.cmake)
@ -353,6 +352,7 @@ include (cmake/find/simdjson.cmake)
include (cmake/find/rapidjson.cmake)
include (cmake/find/fastops.cmake)
include (cmake/find/orc.cmake)
include (cmake/find/replxx.cmake)
find_contrib_lib(cityhash)
find_contrib_lib(farmhash)

14
CONTRIBUTING.md
View File

@ -1,10 +1,16 @@
# Contributing to ClickHouse
## Technical info
Developer guide for writing code for ClickHouse is published on official website alongside the usage and operations documentation:
https://clickhouse.yandex/docs/en/development/architecture/
ClickHouse is an open project, and you can contribute to it in many ways. You can help with ideas, code, or documentation. We appreciate any efforts that help us to make the project better.
## Legal info
Thank you.
## Technical Info
We have a [developer's guide](https://clickhouse.yandex/docs/en/development/developer_instruction/) for writing code for ClickHouse. Besides this guide, you can find [Overview of ClickHouse Architecture](https://clickhouse.yandex/docs/en/development/architecture/) and instructions on how to build ClickHouse in different environments.
If you want to contribute to documentation, read the [Contributing to ClickHouse Documentation](docs/README.md) guide.
## Legal Info
In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here:
1) https://yandex.ru/legal/cla/?lang=en (in English) and

1
cmake/find/base64.cmake
View File

@ -11,7 +11,6 @@ if (ENABLE_BASE64)
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64")
message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
else()
set (BASE64_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64/include)
set (BASE64_LIBRARY base64)
set (USE_BASE64 1)
endif()

60
cmake/find/readline_edit.cmake
View File

@ -1,60 +0,0 @@
include (CMakePushCheckState)
cmake_push_check_state ()
option (ENABLE_READLINE "Enable readline" ${ENABLE_LIBRARIES})
if (ENABLE_READLINE)
set (READLINE_PATHS "/usr/local/opt/readline/lib")
# First try find custom lib for macos users (default lib without history support)
find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS} NO_DEFAULT_PATH)
if (NOT READLINE_LIB)
find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS})
endif ()
list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.2)
find_library (EDIT_LIB NAMES edit)
set(READLINE_INCLUDE_PATHS "/usr/local/opt/readline/include")
if (READLINE_LIB AND TERMCAP_LIBRARY)
find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS} NO_DEFAULT_PATH)
if (NOT READLINE_INCLUDE_DIR)
find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS})
endif ()
if (READLINE_INCLUDE_DIR AND READLINE_LIB)
set (USE_READLINE 1)
set (LINE_EDITING_LIBS ${READLINE_LIB} ${TERMCAP_LIBRARY})
message (STATUS "Using line editing libraries (readline): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}")
endif ()
elseif (EDIT_LIB AND TERMCAP_LIBRARY)
find_library (CURSES_LIB NAMES curses)
find_path (READLINE_INCLUDE_DIR NAMES editline/readline.h PATHS ${READLINE_INCLUDE_PATHS})
if (CURSES_LIB AND READLINE_INCLUDE_DIR)
set (USE_LIBEDIT 1)
set (LINE_EDITING_LIBS ${EDIT_LIB} ${CURSES_LIB} ${TERMCAP_LIBRARY})
message (STATUS "Using line editing libraries (edit): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}")
endif ()
endif ()
endif ()
if (LINE_EDITING_LIBS AND READLINE_INCLUDE_DIR)
include (CheckCXXSourceRuns)
set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${LINE_EDITING_LIBS})
set (CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES} ${READLINE_INCLUDE_DIR})
check_cxx_source_runs ("
#include <stdio.h>
#include <readline/readline.h>
#include <readline/history.h>
int main() {
add_history(NULL);
append_history(1,NULL);
return 0;
}
" HAVE_READLINE_HISTORY)
else ()
message (STATUS "Not using any library for line editing.")
endif ()
cmake_pop_check_state ()

40
cmake/find/replxx.cmake Normal file
View File

@ -0,0 +1,40 @@
option (ENABLE_REPLXX "Enable replxx support" ${NOT_UNBUNDLED})
if (ENABLE_REPLXX)
option (USE_INTERNAL_REPLXX "Use internal replxx library" ${NOT_UNBUNDLED})
if (USE_INTERNAL_REPLXX AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/replxx/README.md")
message (WARNING "submodule contrib/replxx is missing. to fix try run: \n git submodule update --init --recursive")
set (USE_INTERNAL_REPLXX 0)
endif ()
if (NOT USE_INTERNAL_REPLXX)
find_library(LIBRARY_REPLXX NAMES replxx replxx-static)
find_path(INCLUDE_REPLXX replxx.hxx)
add_library(replxx UNKNOWN IMPORTED)
set_property(TARGET replxx PROPERTY IMPORTED_LOCATION ${LIBRARY_REPLXX})
target_include_directories(replxx PUBLIC ${INCLUDE_REPLXX})
set(CMAKE_REQUIRED_LIBRARIES replxx)
check_cxx_source_compiles(
"
#include <replxx.hxx>
int main() {
replxx::Replxx rx;
}
"
EXTERNAL_REPLXX_WORKS
)
if (NOT EXTERNAL_REPLXX_WORKS)
message (FATAL_ERROR "replxx is unusable: ${LIBRARY_REPLXX} ${INCLUDE_REPLXX}")
endif ()
endif ()
set(USE_REPLXX 1)
message (STATUS "Using replxx")
else ()
set(USE_REPLXX 0)
endif ()

1
cmake/sanitize.cmake
View File

@ -48,7 +48,6 @@ if (SANITIZE)
set (ENABLE_EMBEDDED_COMPILER 0 CACHE BOOL "")
set (USE_INTERNAL_CAPNP_LIBRARY 0 CACHE BOOL "")
set (USE_SIMDJSON 0 CACHE BOOL "")
set (ENABLE_READLINE 0 CACHE BOOL "")
set (ENABLE_ORC 0 CACHE BOOL "")
set (ENABLE_PARQUET 0 CACHE BOOL "")
set (USE_CAPNP 0 CACHE BOOL "")

1
cmake/target.cmake
View File

@ -15,7 +15,6 @@ if (CMAKE_CROSSCOMPILING)
set (USE_SNAPPY OFF CACHE INTERNAL "")
set (ENABLE_PROTOBUF OFF CACHE INTERNAL "")
set (ENABLE_PARQUET OFF CACHE INTERNAL "")
set (ENABLE_READLINE OFF CACHE INTERNAL "")
set (ENABLE_ICU OFF CACHE INTERNAL "")
set (ENABLE_FASTOPS OFF CACHE INTERNAL "")
elseif (OS_LINUX)

4
contrib/CMakeLists.txt vendored
View File

@ -331,3 +331,7 @@ endif()
if (USE_FASTOPS)
add_subdirectory (fastops-cmake)
endif()
if (USE_INTERNAL_REPLXX)
add_subdirectory (replxx-cmake)
endif()

3
contrib/aws-s3-cmake/CMakeLists.txt
View File

@ -74,7 +74,6 @@ file(GLOB S3_UNIFIED_SRC
)
set(S3_INCLUDES
"${CMAKE_CURRENT_SOURCE_DIR}/include/"
"${AWS_COMMON_LIBRARY_DIR}/include/"
"${AWS_EVENT_STREAM_LIBRARY_DIR}/include/"
"${AWS_S3_LIBRARY_DIR}/include/"
@ -96,7 +95,7 @@ target_compile_definitions(aws_s3 PUBLIC -DENABLE_CURL_CLIENT)
target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MAJOR=1")
target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MINOR=7")
target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_PATCH=231")
target_include_directories(aws_s3 PUBLIC ${S3_INCLUDES} "${CMAKE_BINARY_DIR}/install")
target_include_directories(aws_s3 PUBLIC ${S3_INCLUDES})
if (OPENSSL_FOUND)
target_compile_definitions(aws_s3 PUBLIC -DENABLE_OPENSSL_ENCRYPTION)

1
contrib/replxx vendored Submodule

@ -0,0 +1 @@
Subproject commit 37582f0bb8c52513c6c6b76797c02d852d701dad

18
contrib/replxx-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,18 @@
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx")
set(SRCS
${LIBRARY_DIR}/src/conversion.cxx
${LIBRARY_DIR}/src/escape.cxx
${LIBRARY_DIR}/src/history.cxx
${LIBRARY_DIR}/src/io.cxx
${LIBRARY_DIR}/src/prompt.cxx
${LIBRARY_DIR}/src/replxx.cxx
${LIBRARY_DIR}/src/replxx_impl.cxx
${LIBRARY_DIR}/src/util.cxx
${LIBRARY_DIR}/src/wcwidth.cpp
${LIBRARY_DIR}/src/ConvertUTF.cpp
)
add_library(replxx ${SRCS})
target_include_directories(replxx PUBLIC ${LIBRARY_DIR}/include)
target_compile_options(replxx PUBLIC -Wno-documentation)

8
dbms/CMakeLists.txt
View File

@ -142,10 +142,10 @@ elseif (COMPILER_GCC)
add_cxx_compile_options(-Wmaybe-uninitialized)
# Warn when the indentation of the code does not reflect the block structure
add_cxx_compile_options(-Wmisleading-indentation)
# Warn if a global function is defined without a previous declaration
# Warn if a global function is defined without a previous declaration - disabled because of build times
# add_cxx_compile_options(-Wmissing-declarations)
# Warn if a user-supplied include directory does not exist
# add_cxx_compile_options(-Wmissing-include-dirs)
add_cxx_compile_options(-Wmissing-include-dirs)
# Obvious
add_cxx_compile_options(-Wnon-virtual-dtor)
# Obvious
@ -177,7 +177,7 @@ elseif (COMPILER_GCC)
# Warn for suspicious length parameters to certain string and memory built-in functions if the argument uses sizeof
add_cxx_compile_options(-Wsizeof-pointer-memaccess)
# Warn about overriding virtual functions that are not marked with the override keyword
# add_cxx_compile_options(-Wsuggest-override)
add_cxx_compile_options(-Wsuggest-override)
# Warn whenever a switch statement has an index of boolean type and the case values are outside the range of a boolean type
add_cxx_compile_options(-Wswitch-bool)
# Warn if a self-comparison always evaluates to true or false
@ -563,7 +563,7 @@ if (USE_JEMALLOC)
endif()
endif ()
dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/Formats/include)
dbms_target_include_directories (PUBLIC ${DBMS_INCLUDE_DIR})
target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR})
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR})

10
dbms/programs/benchmark/Benchmark.cpp
View File

@ -101,7 +101,7 @@ public:
}
void initialize(Poco::Util::Application & self [[maybe_unused]])
void initialize(Poco::Util::Application & self [[maybe_unused]]) override
{
std::string home_path;
const char * home_path_cstr = getenv("HOME");
@ -111,7 +111,7 @@ public:
configReadClient(config(), home_path);
}
int main(const std::vector<std::string> &)
int main(const std::vector<std::string> &) override
{
if (!json_path.empty() && Poco::File(json_path).exists()) /// Clear file with previous results
Poco::File(json_path).remove();
@ -418,7 +418,7 @@ private:
std::cerr << percent << "%\t\t";
for (const auto & info : infos)
{
std::cerr << info->sampler.quantileInterpolated(percent / 100.0) << " sec." << "\t";
std::cerr << info->sampler.quantileNearest(percent / 100.0) << " sec." << "\t";
}
std::cerr << "\n";
};
@ -453,7 +453,7 @@ private:
auto print_percentile = [&json_out](Stats & info, auto percent, bool with_comma = true)
{
json_out << "\"" << percent << "\"" << ": " << info.sampler.quantileInterpolated(percent / 100.0) << (with_comma ? ",\n" : "\n");
json_out << "\"" << percent << "\"" << ": " << info.sampler.quantileNearest(percent / 100.0) << (with_comma ? ",\n" : "\n");
};
json_out << "{\n";
@ -492,7 +492,7 @@ private:
public:
~Benchmark()
~Benchmark() override
{
shutdown = true;
}

6
dbms/programs/client/CMakeLists.txt
View File

@ -1,14 +1,10 @@
set(CLICKHOUSE_CLIENT_SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/Client.cpp
${CMAKE_CURRENT_SOURCE_DIR}/ConnectionParameters.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Suggest.cpp
)
set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
set(CLICKHOUSE_CLIENT_INCLUDE PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include)
if (READLINE_INCLUDE_DIR)
set(CLICKHOUSE_CLIENT_INCLUDE ${CLICKHOUSE_CLIENT_INCLUDE} SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
endif ()
include(CheckSymbolExists)
check_symbol_exists(readpassphrase readpassphrase.h HAVE_READPASSPHRASE)

278
dbms/programs/client/Client.cpp
View File

@ -1,7 +1,7 @@
#include "TestHint.h"
#include "ConnectionParameters.h"
#include "Suggest.h"
#include <port/unistd.h>
#include <stdlib.h>
#include <fcntl.h>
#include <signal.h>
@ -18,8 +18,9 @@
#include <Poco/String.h>
#include <Poco/File.h>
#include <Poco/Util/Application.h>
#include <common/readline_use.h>
#include <common/find_symbols.h>
#include <common/config_common.h>
#include <common/LineReader.h>
#include <Common/ClickHouseRevision.h>
#include <Common/Stopwatch.h>
#include <Common/Exception.h>
@ -69,10 +70,6 @@
#include <common/argsToConfig.h>
#include <Common/TerminalSize.h>
#if USE_READLINE
#include "Suggest.h"
#endif
#ifndef __clang__
#pragma GCC optimize("-fno-var-tracking-assignments")
#endif
@ -89,39 +86,6 @@
#define DISABLE_LINE_WRAPPING "\033[?7l"
#define ENABLE_LINE_WRAPPING "\033[?7h"
#if USE_READLINE && RL_VERSION_MAJOR >= 7
#define BRACK_PASTE_PREF "\033[200~"
#define BRACK_PASTE_SUFF "\033[201~"
#define BRACK_PASTE_LAST '~'
#define BRACK_PASTE_SLEN 6
/// This handler bypasses some unused macro/event checkings.
static int clickhouse_rl_bracketed_paste_begin(int /* count */, int /* key */)
{
std::string buf;
buf.reserve(128);
RL_SETSTATE(RL_STATE_MOREINPUT);
SCOPE_EXIT(RL_UNSETSTATE(RL_STATE_MOREINPUT));
int c;
while ((c = rl_read_key()) >= 0)
{
if (c == '\r')
c = '\n';
buf.push_back(c);
if (buf.size() >= BRACK_PASTE_SLEN && c == BRACK_PASTE_LAST && buf.substr(buf.size() - BRACK_PASTE_SLEN) == BRACK_PASTE_SUFF)
{
buf.resize(buf.size() - BRACK_PASTE_SLEN);
break;
}
}
return static_cast<size_t>(rl_insert_text(buf.c_str())) == buf.size() ? 0 : 1;
}
#endif
namespace DB
{
@ -136,7 +100,6 @@ namespace ErrorCodes
extern const int UNEXPECTED_PACKET_FROM_SERVER;
extern const int CLIENT_OUTPUT_FORMAT_SPECIFIED;
extern const int CANNOT_SET_SIGNAL_HANDLER;
extern const int CANNOT_READLINE;
extern const int SYSTEM_ERROR;
extern const int INVALID_USAGE_OF_INPUT;
}
@ -157,7 +120,7 @@ private:
"учшеж", "йгшеж", "дщпщгеж",
"q", "й", "\\q", "\\Q", "\\й", "\\Й", ":q", "Жй"
};
bool is_interactive = true; /// Use either readline interface or batch mode.
bool is_interactive = true; /// Use either interactive line editing interface or batch mode.
bool need_render_progress = true; /// Render query execution progress.
bool echo_queries = false; /// Print queries before execution in batch mode.
bool ignore_error = false; /// In case of errors, don't print error message, continue to next query. Only applicable for non-interactive mode.
@ -242,7 +205,7 @@ private:
ConnectionParameters connection_parameters;
void initialize(Poco::Util::Application & self)
void initialize(Poco::Util::Application & self) override
{
Poco::Util::Application::initialize(self);
@ -270,7 +233,7 @@ private:
}
int main(const std::vector<std::string> & /*args*/)
int main(const std::vector<std::string> & /*args*/) override
{
try
{
@ -514,26 +477,10 @@ private:
if (print_time_to_stderr)
throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS);
#if USE_READLINE
SCOPE_EXIT({ Suggest::instance().finalize(); });
if (server_revision >= Suggest::MIN_SERVER_REVISION
&& !config().getBool("disable_suggestion", false))
{
if (server_revision >= Suggest::MIN_SERVER_REVISION && !config().getBool("disable_suggestion", false))
/// Load suggestion data from the server.
Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit"));
/// Added '.' to the default list. Because it is used to separate database and table.
rl_basic_word_break_characters = " \t\n\r\"\\'`@$><=;|&{(.";
/// Not append whitespace after single suggestion. Because whitespace after function name is meaningless.
rl_completion_append_character = '\0';
rl_completion_entry_function = Suggest::generator;
}
else
/// Turn tab completion off.
rl_bind_key('\t', rl_insert);
#endif
/// Load command history if present.
if (config().has("history_file"))
history_file = config().getString("history_file");
@ -546,70 +493,45 @@ private:
history_file = home_path + "/.clickhouse-client-history";
}
if (!history_file.empty())
{
if (Poco::File(history_file).exists())
{
#if USE_READLINE
int res = read_history(history_file.c_str());
if (res)
std::cerr << "Cannot read history from file " + history_file + ": "+ errnoToString(ErrorCodes::CANNOT_READ_HISTORY);
#endif
}
else /// Create history file.
if (!history_file.empty() && !Poco::File(history_file).exists())
Poco::File(history_file).createFile();
LineReader lr(&Suggest::instance(), history_file, '\\', config().has("multiline") ? ';' : 0);
do
{
auto input = lr.readLine(prompt(), ":-] ");
if (input.empty())
break;
try
{
if (!process(input))
break;
}
catch (const Exception & e)
{
actual_client_error = e.code();
if (!actual_client_error || actual_client_error != expected_client_error)
{
std::cerr << std::endl
<< "Exception on client:" << std::endl
<< "Code: " << e.code() << ". " << e.displayText() << std::endl;
if (config().getBool("stacktrace", false))
std::cerr << "Stack trace:" << std::endl << e.getStackTraceString() << std::endl;
std::cerr << std::endl;
}
#if USE_READLINE
/// Install Ctrl+C signal handler that will be used in interactive mode.
if (rl_initialize())
throw Exception("Cannot initialize readline", ErrorCodes::CANNOT_READLINE);
#if RL_VERSION_MAJOR >= 7
/// Enable bracketed-paste-mode only when multiquery is enabled and multiline is
/// disabled, so that we are able to paste and execute multiline queries in a whole
/// instead of erroring out, while be less intrusive.
if (config().has("multiquery") && !config().has("multiline"))
{
/// When bracketed paste mode is set, pasted text is bracketed with control sequences so
/// that the program can differentiate pasted text from typed-in text. This helps
/// clickhouse-client so that without -m flag, one can still paste multiline queries, and
/// possibly get better pasting performance. See https://cirw.in/blog/bracketed-paste for
/// more details.
rl_variable_bind("enable-bracketed-paste", "on");
/// Use our bracketed paste handler to get better user experience. See comments above.
rl_bind_keyseq(BRACK_PASTE_PREF, clickhouse_rl_bracketed_paste_begin);
/// Client-side exception during query execution can result in the loss of
/// sync in the connection protocol.
/// So we reconnect and allow to enter the next query.
connect();
}
#endif
auto clear_prompt_or_exit = [](int)
{
/// This is signal safe.
ssize_t res = write(STDOUT_FILENO, "\n", 1);
/// Allow to quit client while query is in progress by pressing Ctrl+C twice.
/// (First press to Ctrl+C will try to cancel query by InterruptListener).
if (res == 1 && rl_line_buffer[0] && !RL_ISSTATE(RL_STATE_DONE))
{
rl_replace_line("", 0);
if (rl_forced_update_display())
_exit(0);
}
else
{
/// A little dirty, but we struggle to find better way to correctly
/// force readline to exit after returning from the signal handler.
_exit(0);
}
};
if (signal(SIGINT, clear_prompt_or_exit) == SIG_ERR)
throwFromErrno("Cannot set signal handler.", ErrorCodes::CANNOT_SET_SIGNAL_HANDLER);
#endif
loop();
while (true);
if (isNewYearMode())
std::cout << "Happy new year." << std::endl;
@ -621,17 +543,6 @@ private:
}
else
{
/// This is intended for testing purposes.
if (config().getBool("always_load_suggestion_data", false))
{
#if USE_READLINE
SCOPE_EXIT({ Suggest::instance().finalize(); });
Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit"));
#else
throw Exception("Command line suggestions cannot work without readline", ErrorCodes::BAD_ARGUMENTS);
#endif
}
query_id = config().getString("query_id", "");
nonInteractive();
@ -706,111 +617,11 @@ private:
}
/// Check if multi-line query is inserted from the paste buffer.
/// Allows delaying the start of query execution until the entirety of query is inserted.
static bool hasDataInSTDIN()
{
timeval timeout = { 0, 0 };
fd_set fds;
FD_ZERO(&fds);
FD_SET(STDIN_FILENO, &fds);
return select(1, &fds, nullptr, nullptr, &timeout) == 1;
}
inline const String prompt() const
{
return boost::replace_all_copy(prompt_by_server_display_name, "{database}", config().getString("database", "default"));
}
void loop()
{
String input;
String prev_input;
while (char * line_ = readline(input.empty() ? prompt().c_str() : ":-] "))
{
String line = line_;
free(line_);
size_t ws = line.size();
while (ws > 0 && isWhitespaceASCII(line[ws - 1]))
--ws;
if (ws == 0 || line.empty())
continue;
bool ends_with_semicolon = line[ws - 1] == ';';
bool ends_with_backslash = line[ws - 1] == '\\';
has_vertical_output_suffix = (ws >= 2) && (line[ws - 2] == '\\') && (line[ws - 1] == 'G');
if (ends_with_backslash)
line = line.substr(0, ws - 1);
input += line;
if (!ends_with_backslash && (ends_with_semicolon || has_vertical_output_suffix || (!config().has("multiline") && !hasDataInSTDIN())))
{
// TODO: should we do sensitive data masking on client too? History file can be source of secret leaks.
if (input != prev_input)
{
/// Replace line breaks with spaces to prevent the following problem.
/// Every line of multi-line query is saved to history file as a separate line.
/// If the user restarts the client then after pressing the "up" button
/// every line of the query will be displayed separately.
std::string logged_query = input;
if (config().has("multiline"))
std::replace(logged_query.begin(), logged_query.end(), '\n', ' ');
add_history(logged_query.c_str());
#if USE_READLINE && HAVE_READLINE_HISTORY
if (!history_file.empty() && append_history(1, history_file.c_str()))
std::cerr << "Cannot append history to file " + history_file + ": " + errnoToString(ErrorCodes::CANNOT_APPEND_HISTORY);
#endif
prev_input = input;
}
if (has_vertical_output_suffix)
input = input.substr(0, input.length() - 2);
try
{
if (!process(input))
break;
}
catch (const Exception & e)
{
actual_client_error = e.code();
if (!actual_client_error || actual_client_error != expected_client_error)
{
std::cerr << std::endl
<< "Exception on client:" << std::endl
<< "Code: " << e.code() << ". " << e.displayText() << std::endl;
if (config().getBool("stacktrace", false))
std::cerr << "Stack trace:" << std::endl
<< e.getStackTraceString() << std::endl;
std::cerr << std::endl;
}
/// Client-side exception during query execution can result in the loss of
/// sync in the connection protocol.
/// So we reconnect and allow to enter the next query.
connect();
}
input = "";
}
else
{
input += '\n';
}
}
}
void nonInteractive()
{
@ -2001,13 +1812,6 @@ public:
server_logs_file = options["server_logs_file"].as<std::string>();
if (options.count("disable_suggestion"))
config().setBool("disable_suggestion", true);
if (options.count("always_load_suggestion_data"))
{
if (options.count("disable_suggestion"))
throw Exception("Command line parameters disable_suggestion (-A) and always_load_suggestion_data cannot be specified simultaneously",
ErrorCodes::BAD_ARGUMENTS);
config().setBool("always_load_suggestion_data", true);
}
if (options.count("suggestion_limit"))
config().setInt("suggestion_limit", options["suggestion_limit"].as<int>());

144
dbms/programs/client/Suggest.cpp Normal file
View File

@ -0,0 +1,144 @@
#include "Suggest.h"
#include <Columns/ColumnString.h>
#include <Common/typeid_cast.h>
namespace DB
{
void Suggest::load(const ConnectionParameters & connection_parameters, size_t suggestion_limit)
{
loading_thread = std::thread([connection_parameters, suggestion_limit, this]
{
try
{
Connection connection(
connection_parameters.host,
connection_parameters.port,
connection_parameters.default_database,
connection_parameters.user,
connection_parameters.password,
"client",
connection_parameters.compression,
connection_parameters.security);
loadImpl(connection, connection_parameters.timeouts, suggestion_limit);
}
catch (...)
{
std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n";
}
/// Note that keyword suggestions are available even if we cannot load data from server.
std::sort(words.begin(), words.end());
ready = true;
});
}
Suggest::Suggest()
{
/// Keywords may be not up to date with ClickHouse parser.
words = {"CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT",
"MATERIALIZED", "ALIAS", "ENGINE", "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP",
"RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", "COPY", "PROJECT",
"PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO",
"OUTFILE", "FORMAT", "TABLES", "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE",
"END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", "INSERT", "VALUES",
"SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER",
"LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY",
"WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC",
"IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE"};
}
void Suggest::loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit)
{
std::stringstream query;
query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM ("
"SELECT name FROM system.functions"
" UNION ALL "
"SELECT name FROM system.table_engines"
" UNION ALL "
"SELECT name FROM system.formats"
" UNION ALL "
"SELECT name FROM system.table_functions"
" UNION ALL "
"SELECT name FROM system.data_type_families"
" UNION ALL "
"SELECT name FROM system.settings"
" UNION ALL "
"SELECT cluster FROM system.clusters"
" UNION ALL "
"SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate";
/// The user may disable loading of databases, tables, columns by setting suggestion_limit to zero.
if (suggestion_limit > 0)
{
String limit_str = toString(suggestion_limit);
query <<
" UNION ALL "
"SELECT name FROM system.databases LIMIT " << limit_str
<< " UNION ALL "
"SELECT DISTINCT name FROM system.tables LIMIT " << limit_str
<< " UNION ALL "
"SELECT DISTINCT name FROM system.columns LIMIT " << limit_str;
}
query << ") WHERE notEmpty(res)";
fetch(connection, timeouts, query.str());
}
void Suggest::fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query)
{
connection.sendQuery(timeouts, query);
while (true)
{
Packet packet = connection.receivePacket();
switch (packet.type)
{
case Protocol::Server::Data:
fillWordsFromBlock(packet.block);
continue;
case Protocol::Server::Progress:
continue;
case Protocol::Server::ProfileInfo:
continue;
case Protocol::Server::Totals:
continue;
case Protocol::Server::Extremes:
continue;
case Protocol::Server::Log:
continue;
case Protocol::Server::Exception:
packet.exception->rethrow();
return;
case Protocol::Server::EndOfStream:
return;
default:
throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
}
}
}
void Suggest::fillWordsFromBlock(const Block & block)
{
if (!block)
return;
if (block.columns() != 1)
throw Exception("Wrong number of columns received for query to read words for suggestion", ErrorCodes::LOGICAL_ERROR);
const ColumnString & column = typeid_cast<const ColumnString &>(*block.getByPosition(0).column);
size_t rows = block.rows();
for (size_t i = 0; i < rows; ++i)
words.emplace_back(column.getDataAt(i).toString());
}
}

207
dbms/programs/client/Suggest.h
View File

@ -2,18 +2,9 @@
#include "ConnectionParameters.h"
#include <string>
#include <sstream>
#include <string.h>
#include <vector>
#include <algorithm>
#include <common/readline_use.h>
#include <Common/typeid_cast.h>
#include <Columns/ColumnString.h>
#include <Client/Connection.h>
#include <IO/ConnectionTimeouts.h>
#include <common/LineReader.h>
namespace DB
@ -24,141 +15,8 @@ namespace ErrorCodes
extern const int UNKNOWN_PACKET_FROM_SERVER;
}
class Suggest : private boost::noncopyable
class Suggest : public LineReader::Suggest, boost::noncopyable
{
private:
/// The vector will be filled with completion words from the server and sorted.
using Words = std::vector<std::string>;
/// Keywords may be not up to date with ClickHouse parser.
Words words
{
"CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", "MATERIALIZED", "ALIAS", "ENGINE",
"AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP", "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER",
"COPY", "PROJECT", "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO", "OUTFILE", "FORMAT", "TABLES",
"DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE", "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE",
"INSERT", "VALUES", "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER", "LEFT", "RIGHT", "FULL", "OUTER",
"CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY", "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", "IN",
"KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE"
};
/// Words are fetched asynchronously.
std::thread loading_thread;
std::atomic<bool> ready{false};
/// Points to current word to suggest.
Words::const_iterator pos;
/// Points after the last possible match.
Words::const_iterator end;
/// Set iterators to the matched range of words if any.
void findRange(const char * prefix, size_t prefix_length)
{
std::string prefix_str(prefix);
std::tie(pos, end) = std::equal_range(words.begin(), words.end(), prefix_str,
[prefix_length](const std::string & s, const std::string & prefix_searched) { return strncmp(s.c_str(), prefix_searched.c_str(), prefix_length) < 0; });
}
/// Iterates through matched range.
char * nextMatch()
{
if (pos >= end)
return nullptr;
/// readline will free memory by itself.
char * word = strdup(pos->c_str());
++pos;
return word;
}
void loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit)
{
std::stringstream query;
query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM ("
"SELECT name FROM system.functions"
" UNION ALL "
"SELECT name FROM system.table_engines"
" UNION ALL "
"SELECT name FROM system.formats"
" UNION ALL "
"SELECT name FROM system.table_functions"
" UNION ALL "
"SELECT name FROM system.data_type_families"
" UNION ALL "
"SELECT name FROM system.settings"
" UNION ALL "
"SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate";
/// The user may disable loading of databases, tables, columns by setting suggestion_limit to zero.
if (suggestion_limit > 0)
{
String limit_str = toString(suggestion_limit);
query <<
" UNION ALL "
"SELECT name FROM system.databases LIMIT " << limit_str
<< " UNION ALL "
"SELECT DISTINCT name FROM system.tables LIMIT " << limit_str
<< " UNION ALL "
"SELECT DISTINCT name FROM system.columns LIMIT " << limit_str;
}
query << ") WHERE notEmpty(res)";
fetch(connection, timeouts, query.str());
}
void fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query)
{
connection.sendQuery(timeouts, query);
while (true)
{
Packet packet = connection.receivePacket();
switch (packet.type)
{
case Protocol::Server::Data:
fillWordsFromBlock(packet.block);
continue;
case Protocol::Server::Progress:
continue;
case Protocol::Server::ProfileInfo:
continue;
case Protocol::Server::Totals:
continue;
case Protocol::Server::Extremes:
continue;
case Protocol::Server::Log:
continue;
case Protocol::Server::Exception:
packet.exception->rethrow();
return;
case Protocol::Server::EndOfStream:
return;
default:
throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);
}
}
}
void fillWordsFromBlock(const Block & block)
{
if (!block)
return;
if (block.columns() != 1)
throw Exception("Wrong number of columns received for query to read words for suggestion", ErrorCodes::LOGICAL_ERROR);
const ColumnString & column = typeid_cast<const ColumnString &>(*block.getByPosition(0).column);
size_t rows = block.rows();
for (size_t i = 0; i < rows; ++i)
words.emplace_back(column.getDataAt(i).toString());
}
public:
static Suggest & instance()
{
@ -166,64 +24,25 @@ public:
return instance;
}
/// More old server versions cannot execute the query above.
void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit);
/// Older server versions cannot execute the query above.
static constexpr int MIN_SERVER_REVISION = 54406;
void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit)
{
loading_thread = std::thread([connection_parameters, suggestion_limit, this]
{
try
{
Connection connection(
connection_parameters.host,
connection_parameters.port,
connection_parameters.default_database,
connection_parameters.user,
connection_parameters.password,
"client",
connection_parameters.compression,
connection_parameters.security);
loadImpl(connection, connection_parameters.timeouts, suggestion_limit);
}
catch (...)
{
std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n";
}
/// Note that keyword suggestions are available even if we cannot load data from server.
std::sort(words.begin(), words.end());
ready = true;
});
}
void finalize()
private:
Suggest();
~Suggest()
{
if (loading_thread.joinable())
loading_thread.join();
}
/// A function for readline.
static char * generator(const char * text, int state)
{
Suggest & suggest = Suggest::instance();
if (!suggest.ready)
return nullptr;
if (state == 0)
suggest.findRange(text, strlen(text));
void loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit);
void fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query);
void fillWordsFromBlock(const Block & block);
/// Do not append whitespace after word. For unknown reason, rl_completion_append_character = '\0' does not work.
rl_completion_suppress_append = 1;
return suggest.nextMatch();
}
~Suggest()
{
finalize();
}
/// Words are fetched asynchronously.
std::thread loading_thread;
};
}

2
dbms/programs/client/readpassphrase/CMakeLists.txt
View File

@ -10,4 +10,4 @@ set_target_properties(readpassphrase
PROPERTIES LINKER_LANGUAGE C
)
# . to allow #include <readpassphrase.h>
target_include_directories(readpassphrase PUBLIC . ${CMAKE_CURRENT_BINARY_DIR}/include ${CMAKE_CURRENT_BINARY_DIR}/../include)
target_include_directories(readpassphrase PUBLIC . ${CMAKE_CURRENT_BINARY_DIR}/include)

12
dbms/programs/odbc-bridge/CMakeLists.txt
View File

@ -15,20 +15,24 @@ set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemo
if (USE_POCO_SQLODBC)
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQLODBC_LIBRARY})
set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_SQLODBC_INCLUDE_DIR})
# Wouldnt work anyway because of the way list variable got expanded in `target_include_directories`
# set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_SQLODBC_INCLUDE_DIR})
endif ()
if (Poco_SQL_FOUND)
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQL_LIBRARY})
set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR})
# Wouldnt work anyway because of the way list variable got expanded in `target_include_directories`
# set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR})
endif ()
if (USE_POCO_DATAODBC)
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_DataODBC_LIBRARY})
set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_DataODBC_INCLUDE_DIR})
# Wouldnt work anyway because of the way list variable got expanded in `target_include_directories`
# set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRS} ${Poco_DataODBC_INCLUDE_DIR})
endif()
if (Poco_Data_FOUND)
set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_Data_LIBRARY})
set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR})
# Wouldnt work anyway because of the way list variable got expanded in `target_include_directories`
# set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR})
endif ()
clickhouse_program_add_library(odbc-bridge)

4
dbms/programs/server/InterserverIOHTTPHandler.cpp
View File

@ -61,6 +61,10 @@ void InterserverIOHTTPHandler::processQuery(Poco::Net::HTTPServerRequest & reque
ReadBufferFromIStream body(request.stream());
auto endpoint = server.context().getInterserverIOHandler().getEndpoint(endpoint_name);
/// Locked for read while query processing
std::shared_lock lock(endpoint->rwlock);
if (endpoint->blocker.isCancelled())
throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED);
if (compress)
{

20
dbms/programs/server/MySQLHandler.cpp
View File

@ -79,21 +79,19 @@ void MySQLHandler::run()
if (!connection_context.mysql.max_packet_size)
connection_context.mysql.max_packet_size = MAX_PACKET_LENGTH;
/* LOG_TRACE(log, "Capabilities: " << handshake_response.capability_flags
<< "\nmax_packet_size: "
LOG_TRACE(log, "Capabilities: " << handshake_response.capability_flags
<< ", max_packet_size: "
<< handshake_response.max_packet_size
<< "\ncharacter_set: "
<< handshake_response.character_set
<< "\nuser: "
<< ", character_set: "
<< static_cast<int>(handshake_response.character_set)
<< ", user: "
<< handshake_response.username
<< "\nauth_response length: "
<< ", auth_response length: "
<< handshake_response.auth_response.length()
<< "\nauth_response: "
<< handshake_response.auth_response
<< "\ndatabase: "
<< ", database: "
<< handshake_response.database
<< "\nauth_plugin_name: "
<< handshake_response.auth_plugin_name);*/
<< ", auth_plugin_name: "
<< handshake_response.auth_plugin_name);
client_capability_flags = handshake_response.capability_flags;
if (!(client_capability_flags & CLIENT_PROTOCOL_41))

6
dbms/programs/server/MySQLHandlerFactory.cpp
View File

@ -34,7 +34,7 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_)
}
catch (...)
{
LOG_INFO(log, "Failed to create SSL context. SSL will be disabled. Error: " << getCurrentExceptionMessage(false));
LOG_TRACE(log, "Failed to create SSL context. SSL will be disabled. Error: " << getCurrentExceptionMessage(false));
ssl_enabled = false;
}
#endif
@ -47,7 +47,7 @@ MySQLHandlerFactory::MySQLHandlerFactory(IServer & server_)
}
catch (...)
{
LOG_WARNING(log, "Failed to read RSA keys. Error: " << getCurrentExceptionMessage(false));
LOG_TRACE(log, "Failed to read RSA key pair from server certificate. Error: " << getCurrentExceptionMessage(false));
generateRSAKeys();
}
#endif
@ -104,7 +104,7 @@ void MySQLHandlerFactory::readRSAKeys()
void MySQLHandlerFactory::generateRSAKeys()
{
LOG_INFO(log, "Generating new RSA key.");
LOG_TRACE(log, "Generating new RSA key pair.");
public_key.reset(RSA_new());
if (!public_key)
throw Exception("Failed to allocate RSA key. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);

9
dbms/programs/server/Server.cpp
View File

@ -432,8 +432,10 @@ int Server::main(const std::vector<std::string> & /*args*/)
main_config_zk_changed_event,
[&](ConfigurationPtr config)
{
setTextLog(global_context->getTextLog());
buildLoggers(*config, logger());
// FIXME logging-related things need synchronization -- see the 'Logger * log' saved
// in a lot of places. For now, disable updating log configuration without server restart.
//setTextLog(global_context->getTextLog());
//buildLoggers(*config, logger());
global_context->setClustersConfig(config);
global_context->setMacros(std::make_unique<Macros>(*config, "macros"));
@ -858,6 +860,9 @@ int Server::main(const std::vector<std::string> & /*args*/)
for (auto & server : servers)
server->start();
setTextLog(global_context->getTextLog());
buildLoggers(config(), logger());
main_config_reloader->start();
users_config_reloader->start();
if (dns_cache_updater)

2
dbms/programs/server/TCPHandler.cpp
View File

@ -606,7 +606,7 @@ void TCPHandler::processOrdinaryQueryWithProcessors(size_t num_threads)
/// If exception was thrown during pipeline execution, skip it while processing other exception.
}
pipeline = QueryPipeline()
/// pipeline = QueryPipeline()
);
while (true)

2
dbms/programs/server/TCPHandler.h
View File

@ -111,7 +111,7 @@ public:
server_display_name = server.config().getString("display_name", getFQDNOrHostName());
}
void run();
void run() override;
/// This method is called right before the query execution.
virtual void customizeContext(DB::Context & /*context*/) {}

1
dbms/programs/server/config.xml
View File

@ -34,6 +34,7 @@
<!--display_name>production</display_name--> <!-- It is the name that will be shown in the client -->
<http_port>8123</http_port>
<tcp_port>9000</tcp_port>
<mysql_port>9004</mysql_port>
<!-- For HTTPS and SSL over native protocol. -->
<!--
<https_port>8443</https_port>

2
dbms/programs/server/users.xml
View File

@ -49,7 +49,7 @@
In first line will be password and in second - corresponding SHA256.
How to generate double SHA1:
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | openssl dgst -sha1 -binary | openssl dgst -sha1
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-'
In first line will be password and in second - corresponding double SHA1.
-->
<password></password>

1
dbms/src/Common/ErrorCodes.cpp
View File

@ -387,7 +387,6 @@ namespace ErrorCodes
extern const int PTHREAD_ERROR = 411;
extern const int NETLINK_ERROR = 412;
extern const int CANNOT_SET_SIGNAL_HANDLER = 413;
extern const int CANNOT_READLINE = 414;
extern const int ALL_REPLICAS_LOST = 415;
extern const int REPLICA_STATUS_CHANGED = 416;
extern const int EXPECTED_ALL_OR_ANY = 417;

2
dbms/src/Common/FileChecker.cpp
View File

@ -112,7 +112,7 @@ void FileChecker::save() const
out->next();
}
disk->moveFile(tmp_files_info_path, files_info_path);
disk->replaceFile(tmp_files_info_path, files_info_path);
}
void FileChecker::load(Map & local_map, const String & path) const

2
dbms/src/Core/ExternalTable.h
View File

@ -99,7 +99,7 @@ class ExternalTablesHandler : public Poco::Net::PartHandler, BaseExternalTable
public:
ExternalTablesHandler(Context & context_, const Poco::Net::NameValueCollection & params_) : context(context_), params(params_) {}
void handlePart(const Poco::Net::MessageHeader & header, std::istream & stream);
void handlePart(const Poco::Net::MessageHeader & header, std::istream & stream) override;
private:
Context & context;

6
dbms/src/Core/MySQLProtocol.h
View File

@ -1030,6 +1030,7 @@ public:
LOG_TRACE(log, "Authentication method match.");
}
bool sent_public_key = false;
if (auth_response == "\1")
{
LOG_TRACE(log, "Client requests public key.");
@ -1050,6 +1051,7 @@ public:
AuthMoreData data(pem);
packet_sender->sendPacket(data, true);
sent_public_key = true;
AuthSwitchResponse response;
packet_sender->receivePacket(response);
@ -1069,13 +1071,15 @@ public:
*/
if (!is_secure_connection && !auth_response->empty() && auth_response != String("\0", 1))
{
LOG_TRACE(log, "Received nonempty password");
LOG_TRACE(log, "Received nonempty password.");
auto ciphertext = reinterpret_cast<unsigned char *>(auth_response->data());
unsigned char plaintext[RSA_size(&private_key)];
int plaintext_size = RSA_private_decrypt(auth_response->size(), ciphertext, plaintext, &private_key, RSA_PKCS1_OAEP_PADDING);
if (plaintext_size == -1)
{
if (!sent_public_key)
LOG_WARNING(log, "Client could have encrypted password with different public key since it didn't request it from server.");
throw Exception("Failed to decrypt auth data. Error: " + getOpenSSLErrors(), ErrorCodes::OPENSSL_ERROR);
}

20
dbms/src/Core/Settings.h
View File

@ -64,14 +64,14 @@ struct Settings : public SettingsCollection<Settings>
M(SettingSeconds, send_timeout, DBMS_DEFAULT_SEND_TIMEOUT_SEC, "", 0) \
M(SettingSeconds, tcp_keep_alive_timeout, 0, "The time in seconds the connection needs to remain idle before TCP starts sending keepalive probes", 0) \
M(SettingMilliseconds, queue_max_wait_ms, 0, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.", 0) \
M(SettingMilliseconds, connection_pool_max_wait_ms, 0, "The wait time when connection pool is full.", 0) \
M(SettingMilliseconds, connection_pool_max_wait_ms, 0, "The wait time when the connection pool is full.", 0) \
M(SettingMilliseconds, replace_running_query_max_wait_ms, 5000, "The wait time for running query with the same query_id to finish when setting 'replace_running_query' is active.", 0) \
M(SettingMilliseconds, kafka_max_wait_ms, 5000, "The wait time for reading from Kafka before retry.", 0) \
M(SettingUInt64, poll_interval, DBMS_DEFAULT_POLL_INTERVAL, "Block at the query wait loop on the server for the specified number of seconds.", 0) \
M(SettingUInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.", 0) \
M(SettingUInt64, distributed_connections_pool_size, DBMS_DEFAULT_DISTRIBUTED_CONNECTIONS_POOL_SIZE, "Maximum number of connections with one remote server in the pool.", 0) \
M(SettingUInt64, connections_with_failover_max_tries, DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, "The maximum number of attempts to connect to replicas.", 0) \
M(SettingUInt64, s3_min_upload_part_size, 512*1024*1024, "The mininum size of part to upload during multipart upload to S3.", 0) \
M(SettingUInt64, s3_min_upload_part_size, 512*1024*1024, "The minimum size of part to upload during multipart upload to S3.", 0) \
M(SettingBool, extremes, false, "Calculate minimums and maximums of the result columns. They can be output in JSON-formats.", IMPORTANT) \
M(SettingBool, use_uncompressed_cache, true, "Whether to use the cache of uncompressed blocks.", 0) \
M(SettingBool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \
@ -183,8 +183,8 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, input_format_tsv_empty_as_default, false, "Treat empty fields in TSV input as default values.", 0) \
M(SettingBool, input_format_null_as_default, false, "For text input formats initialize null fields with default values if data type of this field is not nullable", 0) \
\
M(SettingBool, input_format_values_interpret_expressions, true, "For Values format: if field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.", 0) \
M(SettingBool, input_format_values_deduce_templates_of_expressions, true, "For Values format: if field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \
M(SettingBool, input_format_values_interpret_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.", 0) \
M(SettingBool, input_format_values_deduce_templates_of_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \
M(SettingBool, input_format_values_accurate_types_of_literals, true, "For Values format: when parsing and interpreting expressions using template, check actual type of literal to avoid possible overflow and precision issues.", 0) \
\
M(SettingBool, output_format_json_quote_64bit_integers, true, "Controls quoting of 64-bit integers in JSON output format.", 0) \
@ -212,7 +212,7 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, join_use_nulls, 0, "Use NULLs for non-joined rows of outer JOINs for types that can be inside Nullable. If false, use default value of corresponding columns data type.", IMPORTANT) \
\
M(SettingJoinStrictness, join_default_strictness, JoinStrictness::ALL, "Set default strictness in JOIN query. Possible values: empty string, 'ANY', 'ALL'. If empty, query without strictness will throw exception.", 0) \
M(SettingBool, any_join_distinct_right_table_keys, false, "Enable old ANY JOIN logic with many-to-one left-to-right table keys mapping for all ANY JOINs. It leads to confusing not equal results for 't1 ANY LEFT JOIN t2' and 't2 ANY RIGHT JOIN t1'. ANY RIGHT JOIN needs one-to-many keys maping to be consistent with LEFT one.", IMPORTANT) \
M(SettingBool, any_join_distinct_right_table_keys, false, "Enable old ANY JOIN logic with many-to-one left-to-right table keys mapping for all ANY JOINs. It leads to confusing not equal results for 't1 ANY LEFT JOIN t2' and 't2 ANY RIGHT JOIN t1'. ANY RIGHT JOIN needs one-to-many keys mapping to be consistent with LEFT one.", IMPORTANT) \
\
M(SettingUInt64, preferred_block_size_bytes, 1000000, "", 0) \
\
@ -249,8 +249,8 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, empty_result_for_aggregation_by_empty_set, false, "Return empty result when aggregating without keys on empty set.", 0) \
M(SettingBool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \
M(SettingUInt64, odbc_max_field_size, 1024, "Max size of filed can be read from ODBC dictionary. Long strings are truncated.", 0) \
M(SettingUInt64, query_profiler_real_time_period_ns, 1000000000, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
M(SettingUInt64, query_profiler_cpu_time_period_ns, 1000000000, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
M(SettingUInt64, query_profiler_real_time_period_ns, 1000000000, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
M(SettingUInt64, query_profiler_cpu_time_period_ns, 1000000000, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
\
\
/** Limits during query execution are part of the settings. \
@ -310,9 +310,9 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, join_any_take_last_row, false, "When disabled (default) ANY JOIN will take the first found row for a key. When enabled, it will take the last row seen if there are multiple rows for the same key.", IMPORTANT) \
M(SettingBool, partial_merge_join, false, "Use partial merge join instead of hash join for LEFT and INNER JOINs.", 0) \
M(SettingBool, partial_merge_join_optimizations, false, "Enable optimizations in partial merge join", 0) \
M(SettingUInt64, default_max_bytes_in_join, 100000000, "Maximum size of right-side table if limit's required but max_bytes_in_join is not set.", 0) \
M(SettingUInt64, default_max_bytes_in_join, 100000000, "Maximum size of right-side table if limit is required but max_bytes_in_join is not set.", 0) \
M(SettingUInt64, partial_merge_join_rows_in_right_blocks, 10000, "Split right-hand joining data in blocks of specified size. It's a portion of data indexed by min-max values and possibly unloaded on disk.", 0) \
M(SettingUInt64, partial_merge_join_rows_in_left_blocks, 10000, "Group left-hand joining data in bigger blocks. Setting it to a bigger value increase JOIN performance and memory usage.", 0) \
M(SettingUInt64, partial_merge_join_rows_in_left_blocks, 10000, "Group left-hand joining data in bigger blocks. Setting it to a bigger value increases JOIN performance and memory usage.", 0) \
\
M(SettingUInt64, max_rows_to_transfer, 0, "Maximum size (in rows) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \
M(SettingUInt64, max_bytes_to_transfer, 0, "Maximum size (in uncompressed bytes) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \
@ -371,7 +371,7 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, allow_drop_detached, false, "Allow ALTER TABLE ... DROP DETACHED PART[ITION] ... queries", 0) \
\
M(SettingSeconds, distributed_replica_error_half_life, DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_DECREASE_ERROR_PERIOD, "Time period reduces replica error counter by 2 times.", 0) \
M(SettingUInt64, distributed_replica_error_cap, DBMS_CONNECTION_POOL_WITH_FAILOVER_MAX_ERROR_COUNT, "Max number of errors per replica, prevents piling up increadible amount of errors if replica was offline for some time and allows it to be reconsidered in a shorter amount of time.", 0) \
M(SettingUInt64, distributed_replica_error_cap, DBMS_CONNECTION_POOL_WITH_FAILOVER_MAX_ERROR_COUNT, "Max number of errors per replica, prevents piling up an incredible amount of errors if replica was offline for some time and allows it to be reconsidered in a shorter amount of time.", 0) \
\
M(SettingBool, allow_experimental_live_view, false, "Enable LIVE VIEW. Not mature enough.", 0) \
M(SettingSeconds, live_view_heartbeat_interval, DEFAULT_LIVE_VIEW_HEARTBEAT_INTERVAL_SEC, "The heartbeat interval in seconds to indicate live query is alive.", 0) \

2
dbms/src/DataStreams/BlockIO.h
View File

@ -66,6 +66,8 @@ struct BlockIO
finish_callback = rhs.finish_callback;
exception_callback = rhs.exception_callback;
null_format = rhs.null_format;
return *this;
}
};

23
dbms/src/DataStreams/ConvertingBlockInputStream.cpp
View File

@ -5,7 +5,6 @@
#include <Common/quoteString.h>
#include <Parsers/IAST.h>
namespace DB
{
@ -65,8 +64,18 @@ ConvertingBlockInputStream::ConvertingBlockInputStream(
throw Exception("Cannot find column " + backQuote(res_elem.name) + " in source stream",
ErrorCodes::THERE_IS_NO_COLUMN);
break;
case MatchColumnsMode::NameOrDefault:
if (input_header.has(res_elem.name))
conversion[result_col_num] = input_header.getPositionByName(res_elem.name);
else
conversion[result_col_num] = USE_DEFAULT;
break;
}
if (conversion[result_col_num] == USE_DEFAULT)
continue;
const auto & src_elem = input_header.getByPosition(conversion[result_col_num]);
/// Check constants.
@ -100,9 +109,18 @@ Block ConvertingBlockInputStream::readImpl()
Block res = header.cloneEmpty();
for (size_t res_pos = 0, size = conversion.size(); res_pos < size; ++res_pos)
{
const auto & src_elem = src.getByPosition(conversion[res_pos]);
auto & res_elem = res.getByPosition(res_pos);
if (conversion[res_pos] == USE_DEFAULT)
{
// Create a column with default values
auto column_with_defaults = res_elem.type->createColumn()->cloneResized(src.rows());
res_elem.column = std::move(column_with_defaults);
continue;
}
const auto & src_elem = src.getByPosition(conversion[res_pos]);
ColumnPtr converted = castColumnWithDiagnostic(src_elem, res_elem, context);
if (isColumnConst(*src_elem.column) && !isColumnConst(*res_elem.column))
@ -114,3 +132,4 @@ Block ConvertingBlockInputStream::readImpl()
}
}

5
dbms/src/DataStreams/ConvertingBlockInputStream.h
View File

@ -28,7 +28,9 @@ public:
/// Require same number of columns in source and result. Match columns by corresponding positions, regardless to names.
Position,
/// Find columns in source by their names. Allow excessive columns in source.
Name
Name,
/// Find columns in source by their names if present else use the default. Allow excessive columns in source.
NameOrDefault
};
ConvertingBlockInputStream(
@ -48,6 +50,7 @@ private:
/// How to construct result block. Position in source block, where to get each column.
using Conversion = std::vector<size_t>;
const size_t USE_DEFAULT = static_cast<size_t>(-1);
Conversion conversion;
};

2
dbms/src/DataStreams/PushingToViewsBlockOutputStream.cpp
View File

@ -230,7 +230,7 @@ void PushingToViewsBlockOutputStream::process(const Block & block, size_t view_n
/// and two-level aggregation is triggered).
in = std::make_shared<SquashingBlockInputStream>(
in, context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes);
in = std::make_shared<ConvertingBlockInputStream>(context, in, view.out->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::Name);
in = std::make_shared<ConvertingBlockInputStream>(context, in, view.out->getHeader(), ConvertingBlockInputStream::MatchColumnsMode::NameOrDefault);
}
else
in = std::make_shared<OneBlockInputStream>(block);

4
dbms/src/DataTypes/DataTypeSet.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <DataTypes/IDataTypeDummy.h>
#include <Columns/ColumnSet.h>
namespace DB
@ -18,6 +19,9 @@ public:
bool equals(const IDataType & rhs) const override { return typeid(rhs) == typeid(*this); }
bool isParametric() const override { return true; }
// Used for expressions analysis.
MutableColumnPtr createColumn() const override { return ColumnSet::create(0, nullptr); }
// Used only for debugging, making it DUMPABLE
Field getDefault() const override { return Tuple(); }
};

2
dbms/src/DataTypes/tests/gtest_data_type_get_common_type.cpp
View File

@ -60,7 +60,7 @@ std::ostream & operator<<(std::ostream & ostr, const TypesTestCase & test_case)
class TypeTest : public ::testing::TestWithParam<TypesTestCase>
{
public:
void SetUp()
void SetUp() override
{
const auto & p = GetParam();
from_types = typesFromString(p.from_types);

2
dbms/src/Databases/DatabaseLazy.cpp
View File

@ -120,7 +120,7 @@ StoragePtr DatabaseLazy::tryGetTable(
std::lock_guard lock(mutex);
auto it = tables_cache.find(table_name);
if (it == tables_cache.end())
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
return {};
if (it->second.table)
{

1
dbms/src/Databases/DatabaseMemory.cpp
View File

@ -9,6 +9,7 @@ namespace DB
DatabaseMemory::DatabaseMemory(const String & name_)
: DatabaseWithOwnTablesBase(name_, "DatabaseMemory(" + name_ + ")")
, data_path("data/" + escapeForFileName(database_name) + "/")
{}
void DatabaseMemory::createTable(

12
dbms/src/Databases/DatabaseMemory.h
View File

@ -1,6 +1,8 @@
#pragma once
#include <Databases/DatabasesCommon.h>
#include <Common/escapeForFileName.h>
#include <Parsers/ASTCreateQuery.h>
namespace Poco { class Logger; }
@ -32,6 +34,16 @@ public:
const String & table_name) override;
ASTPtr getCreateDatabaseQuery(const Context & /*context*/) const override;
/// DatabaseMemory allows to create tables, which store data on disk.
/// It's needed to create such tables in default database of clickhouse-local.
/// TODO May be it's better to use DiskMemory for such tables.
/// To save data on disk it's possible to explicitly CREATE DATABASE db ENGINE=Ordinary in clickhouse-local.
String getTableDataPath(const String & table_name) const override { return data_path + escapeForFileName(table_name) + "/"; }
String getTableDataPath(const ASTCreateQuery & query) const override { return getTableDataPath(query.table); }
private:
String data_path;
};
}

8
dbms/src/Databases/IDatabase.h
View File

@ -57,13 +57,13 @@ public:
DatabaseTablesSnapshotIterator(Tables && tables_) : tables(tables_), it(tables.begin()) {}
void next() { ++it; }
void next() override { ++it; }
bool isValid() const { return it != tables.end(); }
bool isValid() const override { return it != tables.end(); }
const String & name() const { return it->first; }
const String & name() const override { return it->first; }
const StoragePtr & table() const { return it->second; }
const StoragePtr & table() const override { return it->second; }
};
/// Copies list of dictionaries and iterates through such snapshot.

71
dbms/src/Disks/DiskLocal.cpp
View File

@ -15,7 +15,57 @@ namespace ErrorCodes
extern const int PATH_ACCESS_DENIED;
}
std::mutex DiskLocal::mutex;
std::mutex DiskLocal::reservation_mutex;
using DiskLocalPtr = std::shared_ptr<DiskLocal>;
class DiskLocalReservation : public IReservation
{
public:
DiskLocalReservation(const DiskLocalPtr & disk_, UInt64 size_)
: disk(disk_), size(size_), metric_increment(CurrentMetrics::DiskSpaceReservedForMerge, size_)
{
}
UInt64 getSize() const override { return size; }
DiskPtr getDisk() const override { return disk; }
void update(UInt64 new_size) override;
~DiskLocalReservation() override;
private:
DiskLocalPtr disk;
UInt64 size;
CurrentMetrics::Increment metric_increment;
};
class DiskLocalDirectoryIterator : public IDiskDirectoryIterator
{
public:
explicit DiskLocalDirectoryIterator(const String & disk_path_, const String & dir_path_) :
dir_path(dir_path_), iter(disk_path_ + dir_path_) {}
void next() override { ++iter; }
bool isValid() const override { return iter != Poco::DirectoryIterator(); }
String path() const override
{
if (iter->isDirectory())
return dir_path + iter.name() + '/';
else
return dir_path + iter.name();
}
private:
String dir_path;
Poco::DirectoryIterator iter;
};
ReservationPtr DiskLocal::reserve(UInt64 bytes)
{
@ -26,7 +76,7 @@ ReservationPtr DiskLocal::reserve(UInt64 bytes)
bool DiskLocal::tryReserve(UInt64 bytes)
{
std::lock_guard lock(mutex);
std::lock_guard lock(DiskLocal::reservation_mutex);
if (bytes == 0)
{
LOG_DEBUG(&Logger::get("DiskLocal"), "Reserving 0 bytes on disk " << backQuote(name));
@ -71,7 +121,7 @@ UInt64 DiskLocal::getAvailableSpace() const
UInt64 DiskLocal::getUnreservedSpace() const
{
std::lock_guard lock(mutex);
std::lock_guard lock(DiskLocal::reservation_mutex);
auto available_space = getAvailableSpace();
available_space -= std::min(available_space, reserved_bytes);
return available_space;
@ -161,20 +211,31 @@ std::unique_ptr<WriteBuffer> DiskLocal::writeFile(const String & path, size_t bu
return std::make_unique<WriteBufferFromFile>(disk_path + path, buf_size, flags);
}
void DiskLocal::remove(const String & path)
{
Poco::File(disk_path + path).remove(false);
}
void DiskLocal::removeRecursive(const String & path)
{
Poco::File(disk_path + path).remove(true);
}
void DiskLocalReservation::update(UInt64 new_size)
{
std::lock_guard lock(DiskLocal::mutex);
std::lock_guard lock(DiskLocal::reservation_mutex);
disk->reserved_bytes -= size;
size = new_size;
disk->reserved_bytes += size;
}
DiskLocalReservation::~DiskLocalReservation()
{
try
{
std::lock_guard lock(DiskLocal::mutex);
std::lock_guard lock(DiskLocal::reservation_mutex);
if (disk->reserved_bytes < size)
{
disk->reserved_bytes = 0;

60
dbms/src/Disks/DiskLocal.h
View File

@ -4,7 +4,6 @@
#include <IO/ReadBufferFromFile.h>
#include <IO/WriteBufferFromFile.h>
#include <mutex>
#include <Poco/DirectoryIterator.h>
#include <Poco/File.h>
@ -71,6 +70,10 @@ public:
std::unique_ptr<WriteBuffer> writeFile(const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, WriteMode mode = WriteMode::Rewrite) override;
void remove(const String & path) override;
void removeRecursive(const String & path) override;
private:
bool tryReserve(UInt64 bytes);
@ -79,61 +82,10 @@ private:
const String disk_path;
const UInt64 keep_free_space_bytes;
/// Used for reservation counters modification
static std::mutex mutex;
UInt64 reserved_bytes = 0;
UInt64 reservation_count = 0;
static std::mutex reservation_mutex;
};
using DiskLocalPtr = std::shared_ptr<DiskLocal>;
class DiskLocalDirectoryIterator : public IDiskDirectoryIterator
{
public:
explicit DiskLocalDirectoryIterator(const String & disk_path_, const String & dir_path_) :
dir_path(dir_path_), iter(disk_path_ + dir_path_) {}
void next() override { ++iter; }
bool isValid() const override { return iter != Poco::DirectoryIterator(); }
String path() const override
{
if (iter->isDirectory())
return dir_path + iter.name() + '/';
else
return dir_path + iter.name();
}
private:
String dir_path;
Poco::DirectoryIterator iter;
};
class DiskLocalReservation : public IReservation
{
public:
DiskLocalReservation(const DiskLocalPtr & disk_, UInt64 size_)
: disk(disk_), size(size_), metric_increment(CurrentMetrics::DiskSpaceReservedForMerge, size_)
{
}
UInt64 getSize() const override { return size; }
DiskPtr getDisk() const override { return disk; }
void update(UInt64 new_size) override;
~DiskLocalReservation() override;
private:
DiskLocalPtr disk;
UInt64 size;
CurrentMetrics::Increment metric_increment;
};
class DiskFactory;
void registerDiskLocal(DiskFactory & factory);
}

77
dbms/src/Disks/DiskMemory.cpp
View File

@ -10,14 +10,33 @@ namespace DB
{
namespace ErrorCodes
{
extern const int UNKNOWN_ELEMENT_IN_CONFIG;
extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
extern const int FILE_DOESNT_EXIST;
extern const int FILE_ALREADY_EXISTS;
extern const int DIRECTORY_DOESNT_EXIST;
extern const int CANNOT_DELETE_DIRECTORY;
}
class DiskMemoryDirectoryIterator : public IDiskDirectoryIterator
{
public:
explicit DiskMemoryDirectoryIterator(std::vector<String> && dir_file_paths_)
: dir_file_paths(std::move(dir_file_paths_)), iter(dir_file_paths.begin())
{
}
void next() override { ++iter; }
bool isValid() const override { return iter != dir_file_paths.end(); }
String path() const override { return *iter; }
private:
std::vector<String> dir_file_paths;
std::vector<String>::iterator iter;
};
ReservationPtr DiskMemory::reserve(UInt64 /*bytes*/)
{
throw Exception("Method reserve is not implemented for memory disks", ErrorCodes::NOT_IMPLEMENTED);
@ -73,7 +92,7 @@ size_t DiskMemory::getFileSize(const String & path) const
auto iter = files.find(path);
if (iter == files.end())
throw Exception("File " + path + " does not exist", ErrorCodes::FILE_DOESNT_EXIST);
throw Exception("File '" + path + "' does not exist", ErrorCodes::FILE_DOESNT_EXIST);
return iter->second.data.size();
}
@ -88,7 +107,7 @@ void DiskMemory::createDirectory(const String & path)
String parent_path = parentPath(path);
if (!parent_path.empty() && files.find(parent_path) == files.end())
throw Exception(
"Failed to create directory " + path + ". Parent directory " + parent_path + " does not exist",
"Failed to create directory '" + path + "'. Parent directory " + parent_path + " does not exist",
ErrorCodes::DIRECTORY_DOESNT_EXIST);
files.emplace(path, FileData{FileType::Directory});
@ -118,7 +137,7 @@ void DiskMemory::clearDirectory(const String & path)
std::lock_guard lock(mutex);
if (files.find(path) == files.end())
throw Exception("Directory " + path + " does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST);
throw Exception("Directory '" + path + "' does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST);
for (auto iter = files.begin(); iter != files.end();)
{
@ -130,7 +149,7 @@ void DiskMemory::clearDirectory(const String & path)
if (iter->second.type == FileType::Directory)
throw Exception(
"Failed to clear directory " + path + ". " + iter->first + " is a directory", ErrorCodes::CANNOT_DELETE_DIRECTORY);
"Failed to clear directory '" + path + "'. " + iter->first + " is a directory", ErrorCodes::CANNOT_DELETE_DIRECTORY);
files.erase(iter++);
}
@ -146,7 +165,7 @@ DiskDirectoryIteratorPtr DiskMemory::iterateDirectory(const String & path)
std::lock_guard lock(mutex);
if (!path.empty() && files.find(path) == files.end())
throw Exception("Directory " + path + " does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST);
throw Exception("Directory '" + path + "' does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST);
std::vector<String> dir_file_paths;
for (const auto & file : files)
@ -205,7 +224,7 @@ std::unique_ptr<ReadBuffer> DiskMemory::readFile(const String & path, size_t /*b
auto iter = files.find(path);
if (iter == files.end())
throw Exception("File " + path + " does not exist", ErrorCodes::FILE_DOESNT_EXIST);
throw Exception("File '" + path + "' does not exist", ErrorCodes::FILE_DOESNT_EXIST);
return std::make_unique<ReadBufferFromString>(iter->second.data);
}
@ -220,7 +239,7 @@ std::unique_ptr<WriteBuffer> DiskMemory::writeFile(const String & path, size_t /
String parent_path = parentPath(path);
if (!parent_path.empty() && files.find(parent_path) == files.end())
throw Exception(
"Failed to create file " + path + ". Directory " + parent_path + " does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST);
"Failed to create file '" + path + "'. Directory " + parent_path + " does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST);
iter = files.emplace(path, FileData{FileType::File}).first;
}
@ -231,6 +250,46 @@ std::unique_ptr<WriteBuffer> DiskMemory::writeFile(const String & path, size_t /
return std::make_unique<WriteBufferFromString>(iter->second.data);
}
void DiskMemory::remove(const String & path)
{
std::lock_guard lock(mutex);
auto file_it = files.find(path);
if (file_it == files.end())
throw Exception("File '" + path + "' doesn't exist", ErrorCodes::FILE_DOESNT_EXIST);
if (file_it->second.type == FileType::Directory)
{
files.erase(file_it);
if (std::any_of(files.begin(), files.end(), [path](const auto & file) { return parentPath(file.first) == path; }))
throw Exception("Directory '" + path + "' is not empty", ErrorCodes::CANNOT_DELETE_DIRECTORY);
}
else
{
files.erase(file_it);
}
}
void DiskMemory::removeRecursive(const String & path)
{
std::lock_guard lock(mutex);
auto file_it = files.find(path);
if (file_it == files.end())
throw Exception("File '" + path + "' doesn't exist", ErrorCodes::FILE_DOESNT_EXIST);
for (auto iter = files.begin(); iter != files.end();)
{
if (iter->first.size() >= path.size() && std::string_view(iter->first.data(), path.size()) == path)
iter = files.erase(iter);
else
++iter;
}
}
using DiskMemoryPtr = std::shared_ptr<DiskMemory>;
void registerDiskMemory(DiskFactory & factory)
{

47
dbms/src/Disks/DiskMemory.h
View File

@ -1,19 +1,24 @@
#pragma once
#include <Disks/IDisk.h>
#include <IO/ReadBuffer.h>
#include <IO/WriteBuffer.h>
#include <mutex>
#include <memory>
#include <unordered_map>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
class ReadBuffer;
class WriteBuffer;
/** Implementation of Disk intended only for testing purposes.
* All filesystem objects are stored in memory and lost on server restart.
*
* NOTE Work in progress. Currently the interface is not viable enough to support MergeTree or even StripeLog tables.
* Please delete this interface if it will not be finished after 2020-06-18.
*/
class DiskMemory : public IDisk
{
public:
@ -62,6 +67,10 @@ public:
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
WriteMode mode = WriteMode::Rewrite) override;
void remove(const String & path) override;
void removeRecursive(const String & path) override;
private:
void createDirectoriesImpl(const String & path);
void replaceFileImpl(const String & from_path, const String & to_path);
@ -88,30 +97,4 @@ private:
mutable std::mutex mutex;
};
using DiskMemoryPtr = std::shared_ptr<DiskMemory>;
class DiskMemoryDirectoryIterator : public IDiskDirectoryIterator
{
public:
explicit DiskMemoryDirectoryIterator(std::vector<String> && dir_file_paths_)
: dir_file_paths(std::move(dir_file_paths_)), iter(dir_file_paths.begin())
{
}
void next() override { ++iter; }
bool isValid() const override { return iter != dir_file_paths.end(); }
String path() const override { return *iter; }
private:
std::vector<String> dir_file_paths;
std::vector<String>::iterator iter;
};
class DiskFactory;
void registerDiskMemory(DiskFactory & factory);
}

439
dbms/src/Disks/DiskS3.cpp Normal file
View File

@ -0,0 +1,439 @@
#include "DiskS3.h"
#if USE_AWS_S3
# include "DiskFactory.h"
# include <random>
# include <IO/S3Common.h>
# include <IO/ReadBufferFromS3.h>
# include <IO/WriteBufferFromS3.h>
# include <IO/ReadBufferFromFile.h>
# include <IO/WriteBufferFromFile.h>
# include <IO/ReadHelpers.h>
# include <IO/WriteHelpers.h>
# include <Poco/File.h>
# include <Common/checkStackSize.h>
# include <Common/quoteString.h>
# include <Common/thread_local_rng.h>
# include <aws/s3/model/CopyObjectRequest.h>
# include <aws/s3/model/DeleteObjectRequest.h>
# include <aws/s3/model/GetObjectRequest.h>
namespace DB
{
namespace ErrorCodes
{
extern const int FILE_ALREADY_EXISTS;
extern const int FILE_DOESNT_EXIST;
extern const int PATH_ACCESS_DENIED;
}
namespace
{
template <typename Result, typename Error>
void throwIfError(Aws::Utils::Outcome<Result, Error> && response)
{
if (!response.IsSuccess())
{
const auto & err = response.GetError();
throw Exception(err.GetMessage(), static_cast<int>(err.GetErrorType()));
}
}
String readKeyFromFile(const String & path)
{
String key;
ReadBufferFromFile buf(path, 1024); /* reasonable buffer size for small file */
readStringUntilEOF(key, buf);
return key;
}
void writeKeyToFile(const String & key, const String & path)
{
WriteBufferFromFile buf(path, 1024);
writeString(key, buf);
buf.next();
}
/// Stores data in S3 and the object key in file in local filesystem.
class WriteIndirectBufferFromS3 : public WriteBufferFromS3
{
public:
WriteIndirectBufferFromS3(
std::shared_ptr<Aws::S3::S3Client> & client_ptr_,
const String & bucket_,
const String & metadata_path_,
const String & s3_path_,
size_t buf_size_)
: WriteBufferFromS3(client_ptr_, bucket_, s3_path_, DEFAULT_BLOCK_SIZE, buf_size_)
, metadata_path(metadata_path_)
, s3_path(s3_path_)
{
}
void finalize() override
{
WriteBufferFromS3::finalize();
writeKeyToFile(s3_path, metadata_path);
finalized = true;
}
~WriteIndirectBufferFromS3() override
{
if (finalized)
return;
try
{
finalize();
}
catch (...)
{
tryLogCurrentException(__PRETTY_FUNCTION__);
}
}
private:
bool finalized = false;
const String metadata_path;
const String s3_path;
};
}
class DiskS3DirectoryIterator : public IDiskDirectoryIterator
{
public:
DiskS3DirectoryIterator(const String & full_path, const String & folder_path_) : iter(full_path), folder_path(folder_path_) {}
void next() override { ++iter; }
bool isValid() const override { return iter != Poco::DirectoryIterator(); }
String path() const override
{
if (iter->isDirectory())
return folder_path + iter.name() + '/';
else
return folder_path + iter.name();
}
private:
Poco::DirectoryIterator iter;
String folder_path;
};
using DiskS3Ptr = std::shared_ptr<DiskS3>;
class DiskS3Reservation : public IReservation
{
public:
DiskS3Reservation(const DiskS3Ptr & disk_, UInt64 size_)
: disk(disk_), size(size_), metric_increment(CurrentMetrics::DiskSpaceReservedForMerge, size_)
{
}
UInt64 getSize() const override { return size; }
DiskPtr getDisk() const override { return disk; }
void update(UInt64 new_size) override
{
std::lock_guard lock(disk->reservation_mutex);
disk->reserved_bytes -= size;
size = new_size;
disk->reserved_bytes += size;
}
~DiskS3Reservation() override;
private:
DiskS3Ptr disk;
UInt64 size;
CurrentMetrics::Increment metric_increment;
};
DiskS3::DiskS3(String name_, std::shared_ptr<Aws::S3::S3Client> client_, String bucket_, String s3_root_path_, String metadata_path_)
: name(std::move(name_))
, client(std::move(client_))
, bucket(std::move(bucket_))
, s3_root_path(std::move(s3_root_path_))
, metadata_path(std::move(metadata_path_))
{
}
ReservationPtr DiskS3::reserve(UInt64 bytes)
{
if (!tryReserve(bytes))
return {};
return std::make_unique<DiskS3Reservation>(std::static_pointer_cast<DiskS3>(shared_from_this()), bytes);
}
bool DiskS3::exists(const String & path) const
{
return Poco::File(metadata_path + path).exists();
}
bool DiskS3::isFile(const String & path) const
{
return Poco::File(metadata_path + path).isFile();
}
bool DiskS3::isDirectory(const String & path) const
{
return Poco::File(metadata_path + path).isDirectory();
}
size_t DiskS3::getFileSize(const String & path) const
{
Aws::S3::Model::GetObjectRequest request;
request.SetBucket(bucket);
request.SetKey(getS3Path(path));
auto outcome = client->GetObject(request);
if (!outcome.IsSuccess())
{
auto & err = outcome.GetError();
throw Exception(err.GetMessage(), static_cast<int>(err.GetErrorType()));
}
else
{
return outcome.GetResult().GetContentLength();
}
}
void DiskS3::createDirectory(const String & path)
{
Poco::File(metadata_path + path).createDirectory();
}
void DiskS3::createDirectories(const String & path)
{
Poco::File(metadata_path + path).createDirectories();
}
DiskDirectoryIteratorPtr DiskS3::iterateDirectory(const String & path)
{
return std::make_unique<DiskS3DirectoryIterator>(metadata_path + path, path);
}
void DiskS3::clearDirectory(const String & path)
{
for (auto it{iterateDirectory(path)}; it->isValid(); it->next())
if (isFile(it->path()))
remove(it->path());
}
void DiskS3::moveFile(const String & from_path, const String & to_path)
{
if (exists(to_path))
throw Exception("File already exists " + to_path, ErrorCodes::FILE_ALREADY_EXISTS);
Poco::File(metadata_path + from_path).renameTo(metadata_path + to_path);
}
void DiskS3::replaceFile(const String & from_path, const String & to_path)
{
Poco::File from_file(metadata_path + from_path);
Poco::File to_file(metadata_path + to_path);
if (to_file.exists())
{
Poco::File tmp_file(metadata_path + to_path + ".old");
to_file.renameTo(tmp_file.path());
from_file.renameTo(metadata_path + to_path);
remove(to_path + ".old");
}
else
from_file.renameTo(to_file.path());
}
void DiskS3::copyFile(const String & from_path, const String & to_path)
{
if (exists(to_path))
remove(to_path);
String s3_from_path = readKeyFromFile(metadata_path + from_path);
String s3_to_path = s3_root_path + getRandomName();
Aws::S3::Model::CopyObjectRequest req;
req.SetBucket(bucket);
req.SetCopySource(s3_from_path);
req.SetKey(s3_to_path);
throwIfError(client->CopyObject(req));
writeKeyToFile(s3_to_path, metadata_path + to_path);
}
std::unique_ptr<ReadBuffer> DiskS3::readFile(const String & path, size_t buf_size) const
{
return std::make_unique<ReadBufferFromS3>(client, bucket, getS3Path(path), buf_size);
}
std::unique_ptr<WriteBuffer> DiskS3::writeFile(const String & path, size_t buf_size, WriteMode mode)
{
if (!exists(path) || mode == WriteMode::Rewrite)
{
String new_s3_path = s3_root_path + getRandomName();
return std::make_unique<WriteIndirectBufferFromS3>(client, bucket, metadata_path + path, new_s3_path, buf_size);
}
else
{
auto old_s3_path = getS3Path(path);
ReadBufferFromS3 read_buffer(client, bucket, old_s3_path, buf_size);
auto writeBuffer = std::make_unique<WriteIndirectBufferFromS3>(client, bucket, metadata_path + path, old_s3_path, buf_size);
std::vector<char> buffer(buf_size);
while (!read_buffer.eof())
writeBuffer->write(buffer.data(), read_buffer.read(buffer.data(), buf_size));
return writeBuffer;
}
}
void DiskS3::remove(const String & path)
{
Poco::File file(metadata_path + path);
if (file.isFile())
{
Aws::S3::Model::DeleteObjectRequest request;
request.SetBucket(bucket);
request.SetKey(getS3Path(path));
throwIfError(client->DeleteObject(request));
}
file.remove();
}
void DiskS3::removeRecursive(const String & path)
{
checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks.
Poco::File file(metadata_path + path);
if (file.isFile())
{
Aws::S3::Model::DeleteObjectRequest request;
request.SetBucket(bucket);
request.SetKey(getS3Path(path));
throwIfError(client->DeleteObject(request));
}
else
{
for (auto it{iterateDirectory(path)}; it->isValid(); it->next())
removeRecursive(it->path());
}
file.remove();
}
String DiskS3::getS3Path(const String & path) const
{
if (!exists(path))
throw Exception("File not found: " + path, ErrorCodes::FILE_DOESNT_EXIST);
return readKeyFromFile(metadata_path + path);
}
String DiskS3::getRandomName() const
{
std::uniform_int_distribution<int> distribution('a', 'z');
String res(32, ' '); /// The number of bits of entropy should be not less than 128.
for (auto & c : res)
c = distribution(thread_local_rng);
return res;
}
bool DiskS3::tryReserve(UInt64 bytes)
{
std::lock_guard lock(reservation_mutex);
if (bytes == 0)
{
LOG_DEBUG(&Logger::get("DiskS3"), "Reserving 0 bytes on s3 disk " << backQuote(name));
++reservation_count;
return true;
}
auto available_space = getAvailableSpace();
UInt64 unreserved_space = available_space - std::min(available_space, reserved_bytes);
if (unreserved_space >= bytes)
{
LOG_DEBUG(
&Logger::get("DiskS3"),
"Reserving " << formatReadableSizeWithBinarySuffix(bytes) << " on disk " << backQuote(name) << ", having unreserved "
<< formatReadableSizeWithBinarySuffix(unreserved_space) << ".");
++reservation_count;
reserved_bytes += bytes;
return true;
}
return false;
}
DiskS3Reservation::~DiskS3Reservation()
{
try
{
std::lock_guard lock(disk->reservation_mutex);
if (disk->reserved_bytes < size)
{
disk->reserved_bytes = 0;
LOG_ERROR(&Logger::get("DiskLocal"), "Unbalanced reservations size for disk '" + disk->getName() + "'.");
}
else
{
disk->reserved_bytes -= size;
}
if (disk->reservation_count == 0)
LOG_ERROR(&Logger::get("DiskLocal"), "Unbalanced reservation count for disk '" + disk->getName() + "'.");
else
--disk->reservation_count;
}
catch (...)
{
tryLogCurrentException(__PRETTY_FUNCTION__);
}
}
void registerDiskS3(DiskFactory & factory)
{
auto creator = [](const String & name,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
const Context & context) -> DiskPtr {
Poco::File disk{context.getPath() + "disks/" + name};
disk.createDirectories();
S3::URI uri(Poco::URI(config.getString(config_prefix + ".endpoint")));
auto client = S3::ClientFactory::instance().create(
uri.endpoint,
config.getString(config_prefix + ".access_key_id", ""),
config.getString(config_prefix + ".secret_access_key", ""));
if (uri.key.back() != '/')
throw Exception("S3 path must ends with '/', but '" + uri.key + "' doesn't.", ErrorCodes::LOGICAL_ERROR);
String metadata_path = context.getPath() + "disks/" + name + "/";
auto s3disk = std::make_shared<DiskS3>(name, client, uri.bucket, uri.key, metadata_path);
/// This code is used only to check access to the corresponding disk.
{
auto file = s3disk->writeFile("test_acl", DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite);
file->write("test", 4);
}
{
auto file = s3disk->readFile("test_acl", DBMS_DEFAULT_BUFFER_SIZE);
String buf(4, '0');
file->readStrict(buf.data(), 4);
if (buf != "test")
throw Exception("No read accecss to S3 bucket in disk " + name, ErrorCodes::PATH_ACCESS_DENIED);
}
{
s3disk->remove("test_acl");
}
return s3disk;
};
factory.registerDiskType("s3", creator);
}
}
#endif

93
dbms/src/Disks/DiskS3.h Normal file
View File

@ -0,0 +1,93 @@
#pragma once
#include <Common/config.h>
#if USE_AWS_S3
# include "DiskFactory.h"
# include <aws/s3/S3Client.h>
# include <Poco/DirectoryIterator.h>
namespace DB
{
/**
* Storage for persisting data in S3 and metadata on the local disk.
* Files are represented by file in local filesystem (clickhouse_root/disks/disk_name/path/to/file)
* that contains S3 object key with actual data.
*/
class DiskS3 : public IDisk
{
public:
friend class DiskS3Reservation;
DiskS3(String name_, std::shared_ptr<Aws::S3::S3Client> client_, String bucket_, String s3_root_path_, String metadata_path_);
const String & getName() const override { return name; }
const String & getPath() const override { return s3_root_path; }
ReservationPtr reserve(UInt64 bytes) override;
UInt64 getTotalSpace() const override { return std::numeric_limits<UInt64>::max(); }
UInt64 getAvailableSpace() const override { return std::numeric_limits<UInt64>::max(); }
UInt64 getUnreservedSpace() const override { return std::numeric_limits<UInt64>::max(); }
UInt64 getKeepingFreeSpace() const override { return 0; }
bool exists(const String & path) const override;
bool isFile(const String & path) const override;
bool isDirectory(const String & path) const override;
size_t getFileSize(const String & path) const override;
void createDirectory(const String & path) override;
void createDirectories(const String & path) override;
void clearDirectory(const String & path) override;
void moveDirectory(const String & from_path, const String & to_path) override { moveFile(from_path, to_path); }
DiskDirectoryIteratorPtr iterateDirectory(const String & path) override;
void moveFile(const String & from_path, const String & to_path) override;
void replaceFile(const String & from_path, const String & to_path) override;
void copyFile(const String & from_path, const String & to_path) override;
std::unique_ptr<ReadBuffer> readFile(const String & path, size_t buf_size) const override;
std::unique_ptr<WriteBuffer> writeFile(const String & path, size_t buf_size, WriteMode mode) override;
void remove(const String & path) override;
void removeRecursive(const String & path) override;
private:
String getS3Path(const String & path) const;
String getRandomName() const;
bool tryReserve(UInt64 bytes);
private:
const String name;
std::shared_ptr<Aws::S3::S3Client> client;
const String bucket;
const String s3_root_path;
const String metadata_path;
UInt64 reserved_bytes = 0;
UInt64 reservation_count = 0;
std::mutex reservation_mutex;
};
}
#endif

1
dbms/src/Disks/IDisk.cpp
View File

@ -2,6 +2,7 @@
namespace DB
{
bool IDisk::isDirectoryEmpty(const String & path)
{
return !iterateDirectory(path)->isValid();

11
dbms/src/Disks/IDisk.h
View File

@ -6,6 +6,7 @@
#include <Common/Exception.h>
#include <memory>
#include <mutex>
#include <utility>
#include <boost/noncopyable.hpp>
#include <Poco/Path.h>
@ -97,7 +98,7 @@ public:
/// Create directory and all parent directories if necessary.
virtual void createDirectories(const String & path) = 0;
/// Remove all files from the directory.
/// Remove all files from the directory. Directories are not removed.
virtual void clearDirectory(const String & path) = 0;
/// Move directory from `from_path` to `to_path`.
@ -125,6 +126,12 @@ public:
/// Open the file for write and return WriteBuffer object.
virtual std::unique_ptr<WriteBuffer> writeFile(const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, WriteMode mode = WriteMode::Rewrite) = 0;
/// Remove file or directory. Throws exception if file doesn't exists or if directory is not empty.
virtual void remove(const String & path) = 0;
/// Remove file or directory with all children. Use with extra caution. Throws exception if file doesn't exists.
virtual void removeRecursive(const String & path) = 0;
};
using DiskPtr = std::shared_ptr<IDisk>;
@ -151,7 +158,7 @@ public:
/**
* Information about reserved size on particular disk.
*/
class IReservation
class IReservation : boost::noncopyable
{
public:
/// Get reservation size.

11
dbms/src/Disks/registerDisks.cpp
View File

@ -1,10 +1,16 @@
#include "DiskFactory.h"
#include "registerDisks.h"
#include "DiskFactory.h"
#include <Common/config.h>
namespace DB
{
void registerDiskLocal(DiskFactory & factory);
void registerDiskMemory(DiskFactory & factory);
#if USE_AWS_S3
void registerDiskS3(DiskFactory & factory);
#endif
void registerDisks()
{
@ -12,6 +18,9 @@ void registerDisks()
registerDiskLocal(factory);
registerDiskMemory(factory);
#if USE_AWS_S3
registerDiskS3(factory);
#endif
}
}

1
dbms/src/Disks/registerDisks.h
View File

@ -3,5 +3,4 @@
namespace DB
{
void registerDisks();
}

4
dbms/src/Disks/tests/gtest_disk.cpp
View File

@ -5,6 +5,10 @@
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#if !__clang__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsuggest-override"
#endif
template <typename T>
DB::DiskPtr createDisk();

1
dbms/src/Functions/CMakeLists.txt
View File

@ -32,7 +32,6 @@ if (OPENSSL_CRYPTO_LIBRARY)
target_link_libraries(clickhouse_functions PUBLIC ${OPENSSL_CRYPTO_LIBRARY})
endif()
target_include_directories(clickhouse_functions PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include)
target_include_directories(clickhouse_functions SYSTEM PRIVATE ${DIVIDE_INCLUDE_DIR} ${METROHASH_INCLUDE_DIR} ${SPARSEHASH_INCLUDE_DIR})
if (CONSISTENT_HASHING_INCLUDE_DIR)

65
dbms/src/Functions/FunctionsLogical.cpp
View File

@ -282,28 +282,31 @@ template <
typename Op, template <typename, size_t> typename OperationApplierImpl, size_t N = 10>
struct OperationApplier
{
template <typename Columns, typename ResultColumn>
static void apply(Columns & in, ResultColumn & result)
template <typename Columns, typename ResultData>
static void apply(Columns & in, ResultData & result_data, bool use_result_data_as_input = false)
{
while (in.size() > 1)
{
doBatchedApply(in, result->getData());
in.push_back(result.get());
}
if (!use_result_data_as_input)
doBatchedApply<false>(in, result_data);
while (in.size() > 0)
doBatchedApply<true>(in, result_data);
}
template <typename Columns, typename ResultData>
template <bool CarryResult, typename Columns, typename ResultData>
static void NO_INLINE doBatchedApply(Columns & in, ResultData & result_data)
{
if (N > in.size())
{
OperationApplier<Op, OperationApplierImpl, N - 1>::doBatchedApply(in, result_data);
OperationApplier<Op, OperationApplierImpl, N - 1>
::template doBatchedApply<CarryResult>(in, result_data);
return;
}
const OperationApplierImpl<Op, N> operationApplierImpl(in);
size_t i = 0;
for (auto & res : result_data)
if constexpr (CarryResult)
res = Op::apply(res, operationApplierImpl.apply(i++));
else
res = operationApplierImpl.apply(i++);
in.erase(in.end() - N, in.end());
@ -312,9 +315,9 @@ struct OperationApplier
template <
typename Op, template <typename, size_t> typename OperationApplierImpl>
struct OperationApplier<Op, OperationApplierImpl, 1>
struct OperationApplier<Op, OperationApplierImpl, 0>
{
template <typename Columns, typename Result>
template <bool, typename Columns, typename Result>
static void NO_INLINE doBatchedApply(Columns &, Result &)
{
throw Exception(
@ -332,7 +335,7 @@ static void executeForTernaryLogicImpl(ColumnRawPtrs arguments, ColumnWithTypeAn
const bool has_consts = extractConstColumnsTernary<Op>(arguments, const_3v_value);
/// If the constant value uniquely determines the result, return it.
if (has_consts && (arguments.empty() || (Op::isSaturable() && Op::isSaturatedValue(const_3v_value))))
if (has_consts && (arguments.empty() || Op::isSaturatedValue(const_3v_value)))
{
result_info.column = ColumnConst::create(
convertFromTernaryData(UInt8Container({const_3v_value}), result_info.type->isNullable()),
@ -341,16 +344,10 @@ static void executeForTernaryLogicImpl(ColumnRawPtrs arguments, ColumnWithTypeAn
return;
}
const auto result_column = ColumnUInt8::create(input_rows_count);
MutableColumnPtr const_column_holder;
if (has_consts)
{
const_column_holder =
convertFromTernaryData(UInt8Container(input_rows_count, const_3v_value), const_3v_value == Ternary::Null);
arguments.push_back(const_column_holder.get());
}
const auto result_column = has_consts ?
ColumnUInt8::create(input_rows_count, const_3v_value) : ColumnUInt8::create(input_rows_count);
OperationApplier<Op, AssociativeGenericApplierImpl>::apply(arguments, result_column);
OperationApplier<Op, AssociativeGenericApplierImpl>::apply(arguments, result_column->getData(), has_consts);
result_info.column = convertFromTernaryData(result_column->getData(), result_info.type->isNullable());
}
@ -425,19 +422,8 @@ static void basicExecuteImpl(ColumnRawPtrs arguments, ColumnWithTypeAndName & re
if (has_consts && Op::apply(const_val, 0) == 0 && Op::apply(const_val, 1) == 1)
has_consts = false;
UInt8ColumnPtrs uint8_args;
auto col_res = ColumnUInt8::create();
UInt8Container & vec_res = col_res->getData();
if (has_consts)
{
vec_res.assign(input_rows_count, const_val);
uint8_args.push_back(col_res.get());
}
else
{
vec_res.resize(input_rows_count);
}
auto col_res = has_consts ?
ColumnUInt8::create(input_rows_count, const_val) : ColumnUInt8::create(input_rows_count);
/// FastPath detection goes in here
if (arguments.size() == (has_consts ? 1 : 2))
@ -452,7 +438,8 @@ static void basicExecuteImpl(ColumnRawPtrs arguments, ColumnWithTypeAndName & re
}
/// Convert all columns to UInt8
Columns converted_columns;
UInt8ColumnPtrs uint8_args;
Columns converted_columns_holder;
for (const IColumn * column : arguments)
{
if (auto uint8_column = checkAndGetColumn<ColumnUInt8>(column))
@ -462,15 +449,11 @@ static void basicExecuteImpl(ColumnRawPtrs arguments, ColumnWithTypeAndName & re
auto converted_column = ColumnUInt8::create(input_rows_count);
convertColumnToUInt8(column, converted_column->getData());
uint8_args.push_back(converted_column.get());
converted_columns.emplace_back(std::move(converted_column));
converted_columns_holder.emplace_back(std::move(converted_column));
}
}
OperationApplier<Op, AssociativeApplierImpl>::apply(uint8_args, col_res);
/// This is possible if there is exactly one non-constant among the arguments, and it is of type UInt8.
if (uint8_args[0] != col_res.get())
vec_res.assign(uint8_args[0]->getData());
OperationApplier<Op, AssociativeApplierImpl>::apply(uint8_args, col_res->getData(), has_consts);
result_info.column = std::move(col_res);
}

7
dbms/src/Functions/FunctionsLogical.h
View File

@ -83,12 +83,7 @@ struct XorImpl
static inline constexpr bool isSaturable() { return false; }
static inline constexpr bool isSaturatedValue(bool) { return false; }
/** Considering that CH uses UInt8 for representation of boolean values this function
* returns 255 as "true" but the current implementation of logical functions suggests that
* any nonzero value is "true" as well. Also the current code provides no guarantee
* for "true" to be represented with the value of 1.
*/
static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return (a != b) ? Ternary::True : Ternary::False; }
static inline constexpr ResultType apply(UInt8 a, UInt8 b) { return !!a != !!b; }
static inline constexpr bool specialImplementationForNulls() { return false; }
#if USE_EMBEDDED_COMPILER

9
dbms/src/Functions/GatherUtils/Sources.h
View File

@ -121,6 +121,11 @@ struct NumericArraySource : public ArraySourceImpl<NumericArraySource<T>>
}
};
#if !__clang__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsuggest-override"
#endif
template <typename Base>
struct ConstSource : public Base
{
@ -199,6 +204,10 @@ struct ConstSource : public Base
}
};
#if !__clang__
#pragma GCC diagnostic pop
#endif
struct StringSource
{
using Slice = NumericArraySlice<UInt8>;

1
dbms/src/Functions/URL/CMakeLists.txt
View File

@ -2,7 +2,6 @@ include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
add_headers_and_sources(clickhouse_functions_url .)
add_library(clickhouse_functions_url ${clickhouse_functions_url_sources} ${clickhouse_functions_url_headers})
target_link_libraries(clickhouse_functions_url PRIVATE dbms)
target_include_directories(clickhouse_functions_url PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../include) # ${CMAKE_CURRENT_BINARY_DIR}/include
if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL")
# Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size.

39
dbms/src/Functions/array/arrayZip.cpp
View File

@ -14,6 +14,7 @@ namespace ErrorCodes
{
extern const int SIZES_OF_ARRAYS_DOESNT_MATCH;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int ILLEGAL_COLUMN;
}
/// arrayZip(['a', 'b', 'c'], ['d', 'e', 'f']) = [('a', 'd'), ('b', 'e'), ('c', 'f')]
@ -44,9 +45,8 @@ public:
const DataTypeArray * array_type = checkAndGetDataType<DataTypeArray>(arguments[index].type.get());
if (!array_type)
throw Exception(
"Argument " + toString(index + 1) + " of function must be array. Found " + arguments[0].type->getName() + " instead.",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
throw Exception("Argument " + toString(index + 1) + " of function " + getName()
+ " must be array. Found " + arguments[0].type->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
arguments_types.emplace_back(array_type->getNestedType());
}
@ -56,26 +56,37 @@ public:
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override
{
auto first_argument = block.getByPosition(arguments[0]);
const auto & first_array_column = checkAndGetColumn<ColumnArray>(first_argument.column.get());
size_t num_arguments = arguments.size();
Columns res_tuple_columns(arguments.size());
res_tuple_columns[0] = first_array_column->getDataPtr();
ColumnPtr first_array_column;
Columns tuple_columns(num_arguments);
for (size_t index = 1; index < arguments.size(); ++index)
for (size_t i = 0; i < num_arguments; ++i)
{
const auto & argument_type_and_column = block.getByPosition(arguments[index]);
const auto & argument_array_column = checkAndGetColumn<ColumnArray>(argument_type_and_column.column.get());
/// Constant columns cannot be inside tuple. It's only possible to have constant tuple as a whole.
ColumnPtr holder = block.getByPosition(arguments[i]).column->convertToFullColumnIfConst();
if (!first_array_column->hasEqualOffsets(*argument_array_column))
throw Exception("The argument 1 and argument " + toString(index + 1) + " of function have different array sizes",
const ColumnArray * column_array = checkAndGetColumn<ColumnArray>(holder.get());
if (!column_array)
throw Exception("Argument " + toString(i + 1) + " of function " + getName() + " must be array."
" Found column " + holder->getName() + " instead.", ErrorCodes::ILLEGAL_COLUMN);
if (i == 0)
{
first_array_column = holder;
}
else if (!column_array->hasEqualOffsets(static_cast<const ColumnArray &>(*first_array_column)))
{
throw Exception("The argument 1 and argument " + toString(i + 1) + " of function " + getName() + " have different array sizes",
ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH);
}
res_tuple_columns[index] = argument_array_column->getDataPtr();
tuple_columns[i] = column_array->getDataPtr();
}
block.getByPosition(result).column = ColumnArray::create(
ColumnTuple::create(res_tuple_columns), first_array_column->getOffsetsPtr());
ColumnTuple::create(tuple_columns), static_cast<const ColumnArray &>(*first_array_column).getOffsetsPtr());
}
};

54
dbms/src/Functions/bitCount.cpp Normal file
View File

@ -0,0 +1,54 @@
#include <ext/bit_cast.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionUnaryArithmetic.h>
namespace DB
{
template <typename A>
struct BitCountImpl
{
using ResultType = UInt8;
static inline ResultType apply(A a)
{
/// We count bits in the value representation in memory. For example, we support floats.
/// We need to avoid sign-extension when converting signed numbers to larger type. So, uint8_t(-1) has 8 bits.
if constexpr (std::is_same_v<A, UInt64> || std::is_same_v<A, Int64>)
return __builtin_popcountll(a);
if constexpr (std::is_same_v<A, UInt32> || std::is_same_v<A, Int32> || std::is_unsigned_v<A>)
return __builtin_popcount(a);
if constexpr (std::is_same_v<A, Int16>)
return __builtin_popcount(static_cast<UInt16>(a));
if constexpr (std::is_same_v<A, Int8>)
return __builtin_popcount(static_cast<UInt8>(a));
else
return __builtin_popcountll(ext::bit_cast<unsigned long long>(a));
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = false;
#endif
};
struct NameBitCount { static constexpr auto name = "bitCount"; };
using FunctionBitCount = FunctionUnaryArithmetic<BitCountImpl, NameBitCount, false /* is injective */>;
/// The function has no ranges of monotonicity.
template <> struct FunctionUnaryArithmeticMonotonicity<NameBitCount>
{
static bool has() { return false; }
static IFunction::Monotonicity get(const Field &, const Field &)
{
return {};
}
};
void registerFunctionBitCount(FunctionFactory & factory)
{
factory.registerFunction<FunctionBitCount>();
}
}

73
dbms/src/Functions/ifNotFinite.cpp Normal file
View File

@ -0,0 +1,73 @@
#include <Functions/IFunctionImpl.h>
#include <Functions/FunctionHelpers.h>
#include <Functions/FunctionFactory.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/getLeastSupertype.h>
#include <Core/ColumnNumbers.h>
namespace DB
{
/// ifNotFinite(x, y) is equivalent to isFinite(x) ? x : y.
class FunctionIfNotFinite : public IFunction
{
public:
static constexpr auto name = "ifNotFinite";
FunctionIfNotFinite(const Context & context_) : context(context_) {}
static FunctionPtr create(const Context & context)
{
return std::make_shared<FunctionIfNotFinite>(context);
}
std::string getName() const override
{
return name;
}
size_t getNumberOfArguments() const override { return 2; }
bool useDefaultImplementationForNulls() const override { return false; }
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const override { return {0}; }
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
auto is_finite_type = FunctionFactory::instance().get("isFinite", context)->build({arguments[0]})->getReturnType();
auto if_type = FunctionFactory::instance().get("if", context)->build({{nullptr, is_finite_type, ""}, arguments[0], arguments[1]})->getReturnType();
return if_type;
}
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
{
Block temp_block = block;
auto is_finite = FunctionFactory::instance().get("isFinite", context)->build(
{temp_block.getByPosition(arguments[0])});
size_t is_finite_pos = temp_block.columns();
temp_block.insert({nullptr, is_finite->getReturnType(), ""});
auto func_if = FunctionFactory::instance().get("if", context)->build(
{temp_block.getByPosition(is_finite_pos), temp_block.getByPosition(arguments[0]), temp_block.getByPosition(arguments[1])});
is_finite->execute(temp_block, {arguments[0]}, is_finite_pos, input_rows_count);
func_if->execute(temp_block, {is_finite_pos, arguments[0], arguments[1]}, result, input_rows_count);
block.getByPosition(result).column = std::move(temp_block.getByPosition(result).column);
}
private:
const Context & context;
};
void registerFunctionIfNotFinite(FunctionFactory & factory)
{
factory.registerFunction<FunctionIfNotFinite>();
}
}

2
dbms/src/Functions/registerFunctionsArithmetic.cpp
View File

@ -20,6 +20,7 @@ void registerFunctionBitShiftLeft(FunctionFactory & factory);
void registerFunctionBitShiftRight(FunctionFactory & factory);
void registerFunctionBitRotateLeft(FunctionFactory & factory);
void registerFunctionBitRotateRight(FunctionFactory & factory);
void registerFunctionBitCount(FunctionFactory & factory);
void registerFunctionLeast(FunctionFactory & factory);
void registerFunctionGreatest(FunctionFactory & factory);
void registerFunctionBitTest(FunctionFactory & factory);
@ -58,6 +59,7 @@ void registerFunctionsArithmetic(FunctionFactory & factory)
registerFunctionBitShiftRight(factory);
registerFunctionBitRotateLeft(factory);
registerFunctionBitRotateRight(factory);
registerFunctionBitCount(factory);
registerFunctionLeast(factory);
registerFunctionGreatest(factory);
registerFunctionBitTest(factory);

2
dbms/src/Functions/registerFunctionsMiscellaneous.cpp
View File

@ -36,6 +36,7 @@ void registerFunctionHasColumnInTable(FunctionFactory &);
void registerFunctionIsFinite(FunctionFactory &);
void registerFunctionIsInfinite(FunctionFactory &);
void registerFunctionIsNaN(FunctionFactory &);
void registerFunctionIfNotFinite(FunctionFactory &);
void registerFunctionThrowIf(FunctionFactory &);
void registerFunctionVersion(FunctionFactory &);
void registerFunctionUptime(FunctionFactory &);
@ -93,6 +94,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory)
registerFunctionIsFinite(factory);
registerFunctionIsInfinite(factory);
registerFunctionIsNaN(factory);
registerFunctionIfNotFinite(factory);
registerFunctionThrowIf(factory);
registerFunctionVersion(factory);
registerFunctionUptime(factory);

7
dbms/src/IO/S3Common.cpp
View File

@ -93,11 +93,10 @@ namespace S3
if (!endpoint.empty())
cfg.endpointOverride = endpoint;
auto cred_provider = std::make_shared<Aws::Auth::SimpleAWSCredentialsProvider>(access_key_id,
secret_access_key);
Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key);
return std::make_shared<Aws::S3::S3Client>(
std::move(cred_provider), // Credentials provider.
credentials, // Aws credentials.
std::move(cfg), // Client configuration.
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, // Sign policy.
endpoint.empty() // Use virtual addressing only if endpoint is not specified.
@ -105,7 +104,7 @@ namespace S3
}
URI::URI(Poco::URI & uri_)
URI::URI(const Poco::URI & uri_)
{
static const std::regex BUCKET_KEY_PATTERN("([^/]+)/(.*)");

2
dbms/src/IO/S3Common.h
View File

@ -49,7 +49,7 @@ struct URI
String bucket;
String key;
explicit URI (Poco::URI & uri_);
explicit URI(const Poco::URI & uri_);
};
}

2
dbms/src/IO/WriteBufferFromHTTP.h
View File

@ -29,7 +29,7 @@ public:
size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE);
/// Receives response from the server after sending all data.
void finalize();
void finalize() override;
};
}

1
dbms/src/IO/WriteBufferFromS3.cpp
View File

@ -69,6 +69,7 @@ void WriteBufferFromS3::nextImpl()
void WriteBufferFromS3::finalize()
{
next();
temporary_buffer->finalize();
if (!buffer_string.empty())
{

8
dbms/src/IO/readDecimalText.h
View File

@ -101,7 +101,13 @@ inline bool readDigits(ReadBuffer & buf, T & x, unsigned int & digits, int & exp
{
++buf.position();
Int32 addition_exp = 0;
readIntText(addition_exp, buf);
if (!tryReadIntText(addition_exp, buf))
{
if constexpr (_throw_on_error)
throw Exception("Cannot parse exponent while reading decimal", ErrorCodes::CANNOT_PARSE_NUMBER);
else
return false;
}
exponent += addition_exp;
stop = true;
continue;

1
dbms/src/IO/tests/ryu_test.cpp
View File

@ -1,5 +1,6 @@
#include <string>
#include <iostream>
#include <cstring>
#include <ryu/ryu.h>

10
dbms/src/Interpreters/Aggregator.cpp
View File

@ -781,7 +781,8 @@ Block Aggregator::mergeAndConvertOneBucketToBlock(
ManyAggregatedDataVariants & variants,
Arena * arena,
bool final,
size_t bucket) const
size_t bucket,
std::atomic<bool> * is_cancelled) const
{
auto & merged_data = *variants[0];
auto method = merged_data.type;
@ -792,6 +793,8 @@ Block Aggregator::mergeAndConvertOneBucketToBlock(
else if (method == AggregatedDataVariants::Type::NAME) \
{ \
mergeBucketImpl<decltype(merged_data.NAME)::element_type>(variants, bucket, arena); \
if (is_cancelled && is_cancelled->load(std::memory_order_seq_cst)) \
return {}; \
block = convertOneBucketToBlock(merged_data, *merged_data.NAME, final, bucket); \
}
@ -1482,12 +1485,15 @@ void NO_INLINE Aggregator::mergeSingleLevelDataImpl(
template <typename Method>
void NO_INLINE Aggregator::mergeBucketImpl(
ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena) const
ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena, std::atomic<bool> * is_cancelled) const
{
/// We merge all aggregation results to the first.
AggregatedDataVariantsPtr & res = data[0];
for (size_t result_num = 1, size = data.size(); result_num < size; ++result_num)
{
if (is_cancelled && is_cancelled->load(std::memory_order_seq_cst))
return;
AggregatedDataVariants & current = *data[result_num];
mergeDataImpl<Method>(

5
dbms/src/Interpreters/Aggregator.h
View File

@ -1170,7 +1170,8 @@ protected:
ManyAggregatedDataVariants & variants,
Arena * arena,
bool final,
size_t bucket) const;
size_t bucket,
std::atomic<bool> * is_cancelled = nullptr) const;
Block prepareBlockAndFillWithoutKey(AggregatedDataVariants & data_variants, bool final, bool is_overflows) const;
Block prepareBlockAndFillSingleLevel(AggregatedDataVariants & data_variants, bool final) const;
@ -1206,7 +1207,7 @@ protected:
template <typename Method>
void mergeBucketImpl(
ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena) const;
ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena, std::atomic<bool> * is_cancelled = nullptr) const;
template <typename Method>
void convertBlockToTwoLevelImpl(

3
dbms/src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -1950,7 +1950,8 @@ void InterpreterSelectQuery::executeAggregation(QueryPipeline & pipeline, const
if (pipeline.getNumStreams() > 1)
{
/// Add resize transform to uniformly distribute data between aggregating streams.
pipeline.resize(pipeline.getNumStreams(), true);
if (!(storage && storage->hasEvenlyDistributedRead()))
pipeline.resize(pipeline.getNumStreams(), true, true);
auto many_data = std::make_shared<ManyAggregatedData>(pipeline.getNumStreams());
auto merge_threads = settings.aggregation_memory_efficient_merge_threads

80
dbms/src/Interpreters/InterserverIOHandler.h
View File

@ -11,6 +11,7 @@
#include <map>
#include <atomic>
#include <utility>
#include <shared_mutex>
#include <Poco/Net/HTMLForm.h>
namespace Poco { namespace Net { class HTTPServerResponse; } }
@ -24,42 +25,6 @@ namespace ErrorCodes
extern const int NO_SUCH_INTERSERVER_IO_ENDPOINT;
}
/** Location of the service.
*/
struct InterserverIOEndpointLocation
{
public:
InterserverIOEndpointLocation(const std::string & name_, const std::string & host_, UInt16 port_)
: name(name_), host(host_), port(port_)
{
}
/// Creates a location based on its serialized representation.
InterserverIOEndpointLocation(const std::string & serialized_location)
{
ReadBufferFromString buf(serialized_location);
readBinary(name, buf);
readBinary(host, buf);
readBinary(port, buf);
assertEOF(buf);
}
/// Serializes the location.
std::string toString() const
{
WriteBufferFromOwnString buf;
writeBinary(name, buf);
writeBinary(host, buf);
writeBinary(port, buf);
return buf.str();
}
public:
std::string name;
std::string host;
UInt16 port;
};
/** Query processor from other servers.
*/
class InterserverIOEndpoint
@ -71,6 +36,7 @@ public:
/// You need to stop the data transfer if blocker is activated.
ActionBlocker blocker;
std::shared_mutex rwlock;
};
using InterserverIOEndpointPtr = std::shared_ptr<InterserverIOEndpoint>;
@ -90,11 +56,10 @@ public:
throw Exception("Duplicate interserver IO endpoint: " + name, ErrorCodes::DUPLICATE_INTERSERVER_IO_ENDPOINT);
}
void removeEndpoint(const String & name)
bool removeEndpointIfExists(const String & name)
{
std::lock_guard lock(mutex);
if (!endpoint_map.erase(name))
throw Exception("No interserver IO endpoint named " + name, ErrorCodes::NO_SUCH_INTERSERVER_IO_ENDPOINT);
return endpoint_map.erase(name);
}
InterserverIOEndpointPtr getEndpoint(const String & name)
@ -115,41 +80,4 @@ private:
std::mutex mutex;
};
/// In the constructor calls `addEndpoint`, in the destructor - `removeEndpoint`.
class InterserverIOEndpointHolder
{
public:
InterserverIOEndpointHolder(const String & name_, InterserverIOEndpointPtr endpoint_, InterserverIOHandler & handler_)
: name(name_), endpoint(std::move(endpoint_)), handler(handler_)
{
handler.addEndpoint(name, endpoint);
}
InterserverIOEndpointPtr getEndpoint()
{
return endpoint;
}
~InterserverIOEndpointHolder()
try
{
handler.removeEndpoint(name);
/// After destroying the object, `endpoint` can still live, since its ownership is acquired during the processing of the request,
/// see InterserverIOHTTPHandler.cpp
}
catch (...)
{
tryLogCurrentException("~InterserverIOEndpointHolder");
}
ActionBlocker & getBlocker() { return endpoint->blocker; }
private:
String name;
InterserverIOEndpointPtr endpoint;
InterserverIOHandler & handler;
};
using InterserverIOEndpointHolderPtr = std::shared_ptr<InterserverIOEndpointHolder>;
}

96
dbms/src/Parsers/ExpressionElementParsers.h
View File

@ -10,8 +10,8 @@ namespace DB
class ParserArray : public IParserBase
{
protected:
const char * getName() const { return "array"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "array"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -22,8 +22,8 @@ protected:
class ParserParenthesisExpression : public IParserBase
{
protected:
const char * getName() const { return "parenthesized expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "parenthesized expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -32,8 +32,8 @@ protected:
class ParserSubquery : public IParserBase
{
protected:
const char * getName() const { return "SELECT subquery"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "SELECT subquery"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -42,8 +42,8 @@ protected:
class ParserIdentifier : public IParserBase
{
protected:
const char * getName() const { return "identifier"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "identifier"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -52,8 +52,8 @@ protected:
class ParserCompoundIdentifier : public IParserBase
{
protected:
const char * getName() const { return "compound identifier"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "compound identifier"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
struct StorageID;
@ -66,8 +66,8 @@ bool parseStorageID(IParser::Pos & pos, StorageID & res, Expected & expected);
class ParserAsterisk : public IParserBase
{
protected:
const char * getName() const { return "asterisk"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "asterisk"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
/** Something like t.* or db.table.*
@ -75,8 +75,8 @@ protected:
class ParserQualifiedAsterisk : public IParserBase
{
protected:
const char * getName() const { return "qualified asterisk"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "qualified asterisk"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
/** COLUMNS('<regular expression>')
@ -84,8 +84,8 @@ protected:
class ParserColumnsMatcher : public IParserBase
{
protected:
const char * getName() const { return "COLUMNS matcher"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "COLUMNS matcher"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
/** A function, for example, f(x, y + 1, g(z)).
@ -99,16 +99,16 @@ class ParserFunction : public IParserBase
public:
ParserFunction(bool allow_function_parameters_ = true) : allow_function_parameters(allow_function_parameters_) {}
protected:
const char * getName() const { return "function"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "function"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
bool allow_function_parameters;
};
class ParserCodecDeclarationList : public IParserBase
{
protected:
const char * getName() const { return "codec declaration list"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "codec declaration list"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
/** Parse compression codec
@ -117,8 +117,8 @@ protected:
class ParserCodec : public IParserBase
{
protected:
const char * getName() const { return "codec"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "codec"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
class ParserCastExpression : public IParserBase
@ -182,8 +182,8 @@ protected:
class ParserNull : public IParserBase
{
protected:
const char * getName() const { return "NULL"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "NULL"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -192,8 +192,8 @@ protected:
class ParserNumber : public IParserBase
{
protected:
const char * getName() const { return "number"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "number"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
/** Unsigned integer, used in right hand side of tuple access operator (x.1).
@ -201,8 +201,8 @@ protected:
class ParserUnsignedInteger : public IParserBase
{
protected:
const char * getName() const { return "unsigned integer"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "unsigned integer"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -211,8 +211,8 @@ protected:
class ParserStringLiteral : public IParserBase
{
protected:
const char * getName() const { return "string literal"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "string literal"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -225,8 +225,8 @@ protected:
class ParserArrayOfLiterals : public IParserBase
{
protected:
const char * getName() const { return "array"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "array"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -235,8 +235,8 @@ protected:
class ParserLiteral : public IParserBase
{
protected:
const char * getName() const { return "literal"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "literal"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -252,8 +252,8 @@ private:
bool allow_alias_without_as_keyword;
const char * getName() const { return "alias"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "alias"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -263,8 +263,8 @@ private:
class ParserSubstitution : public IParserBase
{
protected:
const char * getName() const { return "substitution"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "substitution"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -273,8 +273,8 @@ protected:
class ParserExpressionElement : public IParserBase
{
protected:
const char * getName() const { return "element of expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "element of expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -289,8 +289,8 @@ protected:
ParserPtr elem_parser;
bool allow_alias_without_as_keyword;
const char * getName() const { return "element of expression with optional alias"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "element of expression with optional alias"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -302,8 +302,8 @@ protected:
class ParserOrderByElement : public IParserBase
{
protected:
const char * getName() const { return "element of ORDER BY expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "element of ORDER BY expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
/** Parser for function with arguments like KEY VALUE (space separated)
@ -322,8 +322,8 @@ protected:
class ParserIdentifierWithOptionalParameters : public IParserBase
{
protected:
const char * getName() const { return "identifier with optional parameters"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override{ return "identifier with optional parameters"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
/** Element of TTL expression - same as expression element, but in addition,
@ -332,8 +332,8 @@ protected:
class ParserTTLElement : public IParserBase
{
protected:
const char * getName() const { return "element of TTL expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "element of TTL expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
}

92
dbms/src/Parsers/ExpressionListParsers.h
View File

@ -27,8 +27,8 @@ public:
{
}
protected:
const char * getName() const { return "list of elements"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "list of elements"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
private:
ParserPtr elem_parser;
ParserPtr separator_parser;
@ -63,9 +63,9 @@ public:
}
protected:
const char * getName() const { return "list, delimited by binary operators"; }
const char * getName() const override { return "list, delimited by binary operators"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -86,9 +86,9 @@ public:
}
protected:
const char * getName() const { return "list, delimited by operator of variable arity"; }
const char * getName() const override { return "list, delimited by operator of variable arity"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -110,8 +110,8 @@ public:
}
protected:
const char * getName() const { return "expression with prefix unary operator"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "expression with prefix unary operator"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -121,9 +121,9 @@ private:
static const char * operators[];
protected:
const char * getName() const { return "array element expression"; }
const char * getName() const override{ return "array element expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -133,9 +133,9 @@ private:
static const char * operators[];
protected:
const char * getName() const { return "tuple element expression"; }
const char * getName() const override { return "tuple element expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -146,9 +146,9 @@ private:
ParserPrefixUnaryOperatorExpression operator_parser {operators, std::make_unique<ParserTupleElementExpression>()};
protected:
const char * getName() const { return "unary minus expression"; }
const char * getName() const override { return "unary minus expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -159,9 +159,9 @@ private:
ParserLeftAssociativeBinaryOperatorList operator_parser {operators, std::make_unique<ParserUnaryMinusExpression>()};
protected:
const char * getName() const { return "multiplicative expression"; }
const char * getName() const override{ return "multiplicative expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
{
return operator_parser.parse(pos, node, expected);
}
@ -174,8 +174,8 @@ class ParserIntervalOperatorExpression : public IParserBase
protected:
ParserMultiplicativeExpression next_parser;
const char * getName() const { return "INTERVAL operator expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override{ return "INTERVAL operator expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -186,9 +186,9 @@ private:
ParserLeftAssociativeBinaryOperatorList operator_parser {operators, std::make_unique<ParserIntervalOperatorExpression>()};
protected:
const char * getName() const { return "additive expression"; }
const char * getName() const override{ return "additive expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
{
return operator_parser.parse(pos, node, expected);
}
@ -200,9 +200,9 @@ class ParserConcatExpression : public IParserBase
ParserVariableArityOperatorList operator_parser {"||", "concat", std::make_unique<ParserAdditiveExpression>()};
protected:
const char * getName() const { return "string concatenation expression"; }
const char * getName() const override { return "string concatenation expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
{
return operator_parser.parse(pos, node, expected);
}
@ -215,9 +215,9 @@ private:
ParserConcatExpression elem_parser;
protected:
const char * getName() const { return "BETWEEN expression"; }
const char * getName() const override { return "BETWEEN expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -228,9 +228,9 @@ private:
ParserLeftAssociativeBinaryOperatorList operator_parser {operators, std::make_unique<ParserBetweenExpression>()};
protected:
const char * getName() const { return "comparison expression"; }
const char * getName() const override{ return "comparison expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
{
return operator_parser.parse(pos, node, expected);
}
@ -257,9 +257,9 @@ private:
ParserPrefixUnaryOperatorExpression operator_parser {operators, std::make_unique<ParserNullityChecking>()};
protected:
const char * getName() const { return "logical-NOT expression"; }
const char * getName() const override{ return "logical-NOT expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
{
return operator_parser.parse(pos, node, expected);
}
@ -272,9 +272,9 @@ private:
ParserVariableArityOperatorList operator_parser {"AND", "and", std::make_unique<ParserLogicalNotExpression>()};
protected:
const char * getName() const { return "logical-AND expression"; }
const char * getName() const override { return "logical-AND expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
{
return operator_parser.parse(pos, node, expected);
}
@ -287,9 +287,9 @@ private:
ParserVariableArityOperatorList operator_parser {"OR", "or", std::make_unique<ParserLogicalAndExpression>()};
protected:
const char * getName() const { return "logical-OR expression"; }
const char * getName() const override { return "logical-OR expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
{
return operator_parser.parse(pos, node, expected);
}
@ -305,9 +305,9 @@ private:
ParserLogicalOrExpression elem_parser;
protected:
const char * getName() const { return "expression with ternary operator"; }
const char * getName() const override { return "expression with ternary operator"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -317,9 +317,9 @@ private:
ParserTernaryOperatorExpression elem_parser;
protected:
const char * getName() const { return "lambda expression"; }
const char * getName() const override { return "lambda expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -333,9 +333,9 @@ public:
protected:
ParserPtr impl;
const char * getName() const { return "expression with optional alias"; }
const char * getName() const override { return "expression with optional alias"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
{
return impl->parse(pos, node, expected);
}
@ -352,8 +352,8 @@ public:
protected:
bool allow_alias_without_as_keyword;
const char * getName() const { return "list of expressions"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "list of expressions"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -365,16 +365,16 @@ public:
private:
ParserExpressionList nested_parser;
protected:
const char * getName() const { return "not empty list of expressions"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "not empty list of expressions"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
class ParserOrderByExpressionList : public IParserBase
{
protected:
const char * getName() const { return "order by expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "order by expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -399,8 +399,8 @@ protected:
class ParserTTLExpressionList : public IParserBase
{
protected:
const char * getName() const { return "ttl expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "ttl expression"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
}

2
dbms/src/Parsers/IParserBase.h
View File

@ -35,7 +35,7 @@ public:
return res;
}
bool parse(Pos & pos, ASTPtr & node, Expected & expected);
bool parse(Pos & pos, ASTPtr & node, Expected & expected) override;
protected:
virtual bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) = 0;

16
dbms/src/Parsers/ParserAlterQuery.h
View File

@ -27,16 +27,16 @@ namespace DB
class ParserAlterQuery : public IParserBase
{
protected:
const char * getName() const { return "ALTER query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override{ return "ALTER query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
class ParserAlterCommandList : public IParserBase
{
protected:
const char * getName() const { return "a list of ALTER commands"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override{ return "a list of ALTER commands"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
public:
bool is_live_view;
@ -48,8 +48,8 @@ public:
class ParserAlterCommand : public IParserBase
{
protected:
const char * getName() const { return "ALTER command"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override{ return "ALTER command"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
public:
bool is_live_view;
@ -62,8 +62,8 @@ public:
class ParserAssignment : public IParserBase
{
protected:
const char * getName() const { return "column assignment"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override{ return "column assignment"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
}

4
dbms/src/Parsers/ParserCheckQuery.h
View File

@ -10,8 +10,8 @@ namespace DB
class ParserCheckQuery : public IParserBase
{
protected:
const char * getName() const { return "ALTER query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override{ return "ALTER query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
}

52
dbms/src/Parsers/ParserCreateQuery.h
View File

@ -19,8 +19,8 @@ namespace DB
class ParserNestedTable : public IParserBase
{
protected:
const char * getName() const { return "nested table"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "nested table"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -33,16 +33,16 @@ protected:
class ParserIdentifierWithParameters : public IParserBase
{
protected:
const char * getName() const { return "identifier with parameters"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "identifier with parameters"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
template <typename NameParser>
class IParserNameTypePair : public IParserBase
{
protected:
const char * getName() const { return "name and type pair"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override{ return "name and type pair"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
/** The name and type are separated by a space. For example, URL String. */
@ -75,16 +75,16 @@ bool IParserNameTypePair<NameParser>::parseImpl(Pos & pos, ASTPtr & node, Expect
class ParserNameTypePairList : public IParserBase
{
protected:
const char * getName() const { return "name and type pair list"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "name and type pair list"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
/** List of table names. */
class ParserNameList : public IParserBase
{
protected:
const char * getName() const { return "name list"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "name list"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -99,9 +99,9 @@ public:
protected:
using ASTDeclarePtr = std::shared_ptr<ASTColumnDeclaration>;
const char * getName() const { return "column declaration"; }
const char * getName() const override{ return "column declaration"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
bool require_type = true;
};
@ -224,8 +224,8 @@ bool IParserColumnDeclaration<NameParser>::parseImpl(Pos & pos, ASTPtr & node, E
class ParserColumnDeclarationList : public IParserBase
{
protected:
const char * getName() const { return "column declaration list"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "column declaration list"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -284,8 +284,8 @@ protected:
class ParserStorage : public IParserBase
{
protected:
const char * getName() const { return "storage definition"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "storage definition"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
/** Query like this:
@ -308,32 +308,32 @@ protected:
class ParserCreateTableQuery : public IParserBase
{
protected:
const char * getName() const { return "CREATE TABLE or ATTACH TABLE query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "CREATE TABLE or ATTACH TABLE query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
/// CREATE|ATTACH LIVE VIEW [IF NOT EXISTS] [db.]name [UUID 'uuid'] [TO [db.]name] AS SELECT ...
class ParserCreateLiveViewQuery : public IParserBase
{
protected:
const char * getName() const { return "CREATE LIVE VIEW query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "CREATE LIVE VIEW query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
/// CREATE|ATTACH DATABASE db [ENGINE = engine]
class ParserCreateDatabaseQuery : public IParserBase
{
protected:
const char * getName() const { return "CREATE DATABASE query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "CREATE DATABASE query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
/// CREATE[OR REPLACE]|ATTACH [[MATERIALIZED] VIEW] | [VIEW]] [IF NOT EXISTS] [db.]name [UUID 'uuid'] [TO [db.]name] [ENGINE = engine] [POPULATE] AS SELECT ...
class ParserCreateViewQuery : public IParserBase
{
protected:
const char * getName() const { return "CREATE VIEW query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "CREATE VIEW query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
/// Parses complete dictionary create query. Uses ParserDictionary and
@ -372,8 +372,8 @@ protected:
class ParserCreateQuery : public IParserBase
{
protected:
const char * getName() const { return "CREATE TABLE or ATTACH TABLE query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "CREATE TABLE or ATTACH TABLE query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
}

4
dbms/src/Parsers/ParserDescribeTableQuery.h
View File

@ -13,8 +13,8 @@ namespace DB
class ParserDescribeTableQuery : public IParserBase
{
protected:
const char * getName() const { return "DESCRIBE query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "DESCRIBE query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
}

4
dbms/src/Parsers/ParserDictionaryAttributeDeclaration.h
View File

@ -25,8 +25,8 @@ protected:
class ParserDictionaryAttributeDeclarationList : public IParserBase
{
protected:
const char * getName() const { return "attribute declaration list"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override{ return "attribute declaration list"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
}

4
dbms/src/Parsers/ParserDropQuery.h
View File

@ -19,8 +19,8 @@ namespace DB
class ParserDropQuery : public IParserBase
{
protected:
const char * getName() const { return "DROP query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override{ return "DROP query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
bool parseDropQuery(Pos & pos, ASTPtr & node, Expected & expected);
bool parseDetachQuery(Pos & pos, ASTPtr & node, Expected & expected);

4
dbms/src/Parsers/ParserOptimizeQuery.h
View File

@ -12,8 +12,8 @@ namespace DB
class ParserOptimizeQuery : public IParserBase
{
protected:
const char * getName() const { return "OPTIMIZE query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "OPTIMIZE query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
}

4
dbms/src/Parsers/ParserRenameQuery.h
View File

@ -14,8 +14,8 @@ namespace DB
class ParserRenameQuery : public IParserBase
{
protected:
const char * getName() const { return "RENAME query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override{ return "RENAME query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
}

4
dbms/src/Parsers/ParserRoleList.h
View File

@ -11,8 +11,8 @@ namespace DB
class ParserRoleList : public IParserBase
{
protected:
const char * getName() const { return "RoleList"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "RoleList"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
}

4
dbms/src/Parsers/ParserSampleRatio.h
View File

@ -12,8 +12,8 @@ namespace DB
class ParserSampleRatio : public IParserBase
{
protected:
const char * getName() const { return "Sample ratio or offset"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "Sample ratio or offset"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
}

4
dbms/src/Parsers/ParserShowProcesslistQuery.h
View File

@ -14,9 +14,9 @@ namespace DB
class ParserShowProcesslistQuery : public IParserBase
{
protected:
const char * getName() const { return "SHOW PROCESSLIST query"; }
const char * getName() const override { return "SHOW PROCESSLIST query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
{
auto query = std::make_shared<ASTShowProcesslistQuery>();

4
dbms/src/Parsers/ParserShowTablesQuery.h
View File

@ -14,8 +14,8 @@ namespace DB
class ParserShowTablesQuery : public IParserBase
{
protected:
const char * getName() const { return "SHOW [TEMPORARY] TABLES|DATABASES [[NOT] LIKE 'str'] [LIMIT expr]"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "SHOW [TEMPORARY] TABLES|DATABASES [[NOT] LIKE 'str'] [LIMIT expr]"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
}

16
dbms/src/Parsers/ParserTablesInSelectQuery.h
View File

@ -11,8 +11,8 @@ namespace DB
class ParserTablesInSelectQuery : public IParserBase
{
protected:
const char * getName() const { return "table, table function, subquery or list of joined tables"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "table, table function, subquery or list of joined tables"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
@ -22,8 +22,8 @@ public:
ParserTablesInSelectQueryElement(bool is_first_) : is_first(is_first_) {}
protected:
const char * getName() const { return "table, table function, subquery or list of joined tables"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "table, table function, subquery or list of joined tables"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
private:
bool is_first;
@ -33,16 +33,16 @@ private:
class ParserTableExpression : public IParserBase
{
protected:
const char * getName() const { return "table or subquery or table function"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "table or subquery or table function"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
class ParserArrayJoin : public IParserBase
{
protected:
const char * getName() const { return "array join"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "array join"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};

4
dbms/src/Parsers/ParserUseQuery.h
View File

@ -11,8 +11,8 @@ namespace DB
class ParserUseQuery : public IParserBase
{
protected:
const char * getName() const { return "USE query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override{ return "USE query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
}

4
dbms/src/Parsers/ParserWatchQuery.h
View File

@ -23,8 +23,8 @@ namespace DB
class ParserWatchQuery : public IParserBase
{
protected:
const char * getName() const { return "WATCH query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
const char * getName() const override { return "WATCH query"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};
}

23
dbms/src/Processors/Executors/PipelineExecutor.cpp
View File

@ -259,7 +259,6 @@ bool PipelineExecutor::prepareProcessor(UInt64 pid, size_t thread_number, Queue
/// In this method we have ownership on node.
auto & node = graph[pid];
bool need_traverse = false;
bool need_expand_pipeline = false;
std::vector<Edge *> updated_back_edges;
@ -290,13 +289,11 @@ bool PipelineExecutor::prepareProcessor(UInt64 pid, size_t thread_number, Queue
case IProcessor::Status::NeedData:
case IProcessor::Status::PortFull:
{
need_traverse = true;
node.status = ExecStatus::Idle;
break;
}
case IProcessor::Status::Finished:
{
need_traverse = true;
node.status = ExecStatus::Finished;
break;
}
@ -325,7 +322,6 @@ bool PipelineExecutor::prepareProcessor(UInt64 pid, size_t thread_number, Queue
}
}
if (need_traverse)
{
for (auto & edge_id : node.post_updated_input_ports)
{
@ -346,7 +342,6 @@ bool PipelineExecutor::prepareProcessor(UInt64 pid, size_t thread_number, Queue
}
}
if (need_traverse)
{
for (auto & edge : updated_direct_edges)
{
@ -543,7 +538,13 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads
if (!task_queue.empty() && !threads_queue.empty() /*&& task_queue.quota() > threads_queue.size()*/)
{
auto thread_to_wake = threads_queue.pop_any();
auto thread_to_wake = task_queue.getAnyThreadWithTasks(thread_num + 1 == num_threads ? 0 : (thread_num + 1));
if (threads_queue.has(thread_to_wake))
threads_queue.pop(thread_to_wake);
else
thread_to_wake = threads_queue.pop_any();
lock.unlock();
wake_up_executor(thread_to_wake);
}
@ -627,9 +628,15 @@ void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads
queue.pop();
}
if (!threads_queue.empty() /* && task_queue.quota() > threads_queue.size()*/)
if (!threads_queue.empty() && !finished /* && task_queue.quota() > threads_queue.size()*/)
{
auto thread_to_wake = threads_queue.pop_any();
auto thread_to_wake = task_queue.getAnyThreadWithTasks(thread_num + 1 == num_threads ? 0 : (thread_num + 1));
if (threads_queue.has(thread_to_wake))
threads_queue.pop(thread_to_wake);
else
thread_to_wake = threads_queue.pop_any();
lock.unlock();
wake_up_executor(thread_to_wake);

31
dbms/src/Processors/Executors/PipelineExecutor.h
View File

@ -149,17 +149,30 @@ private:
++quota_;
}
ExecutionState * pop(size_t thread_num)
size_t getAnyThreadWithTasks(size_t from_thread = 0)
{
if (size_ == 0)
throw Exception("TaskQueue is not empty.", ErrorCodes::LOGICAL_ERROR);
throw Exception("TaskQueue is empty.", ErrorCodes::LOGICAL_ERROR);
for (size_t i = 0; i < queues.size(); ++i)
{
if (!queues[thread_num].empty())
if (!queues[from_thread].empty())
return from_thread;
++from_thread;
if (from_thread >= queues.size())
from_thread = 0;
}
throw Exception("TaskQueue is empty.", ErrorCodes::LOGICAL_ERROR);
}
ExecutionState * pop(size_t thread_num)
{
ExecutionState * state = queues[thread_num].front();
queues[thread_num].pop();
auto thread_with_tasks = getAnyThreadWithTasks(thread_num);
ExecutionState * state = queues[thread_with_tasks].front();
queues[thread_with_tasks].pop();
--size_;
@ -169,14 +182,6 @@ private:
return state;
}
++thread_num;
if (thread_num >= queues.size())
thread_num = 0;
}
throw Exception("TaskQueue is not empty.", ErrorCodes::LOGICAL_ERROR);
}
size_t size() const { return size_; }
bool empty() const { return size_ == 0; }
size_t quota() const { return quota_; }

11
dbms/src/Processors/ISimpleTransform.cpp
View File

@ -33,6 +33,10 @@ ISimpleTransform::Status ISimpleTransform::prepare()
{
output.pushData(std::move(current_data));
transformed = false;
if (!no_more_data_needed)
return Status::PortFull;
}
/// Stop if don't need more data.
@ -52,12 +56,13 @@ ISimpleTransform::Status ISimpleTransform::prepare()
return Status::Finished;
}
input.setNeeded();
if (!input.hasData())
{
input.setNeeded();
return Status::NeedData;
}
current_data = input.pullData();
current_data = input.pullData(true);
has_input = true;
if (current_data.exception)

20
dbms/src/Processors/Port.h
View File

@ -161,12 +161,17 @@ protected:
throw Exception("Cannot push block to port which already has data.", ErrorCodes::LOGICAL_ERROR);
}
void ALWAYS_INLINE pull(DataPtr & data_, std::uintptr_t & flags)
void ALWAYS_INLINE pull(DataPtr & data_, std::uintptr_t & flags, bool set_not_needed = false)
{
flags = data_.swap(data, 0, HAS_DATA);
uintptr_t mask = HAS_DATA;
if (set_not_needed)
mask |= IS_NEEDED;
flags = data_.swap(data, 0, mask);
/// It's ok to check because this flag can be changed only by pulling thread.
if (unlikely((flags & IS_NEEDED) == 0))
if (unlikely((flags & IS_NEEDED) == 0) && !set_not_needed)
throw Exception("Cannot pull block from port which is not needed.", ErrorCodes::LOGICAL_ERROR);
if (unlikely((flags & HAS_DATA) == 0))
@ -266,14 +271,15 @@ private:
public:
using Port::Port;
Data ALWAYS_INLINE pullData()
Data ALWAYS_INLINE pullData(bool set_not_needed = false)
{
if (!set_not_needed)
updateVersion();
assumeConnected();
std::uintptr_t flags = 0;
state->pull(data, flags);
state->pull(data, flags, set_not_needed);
is_finished = flags & State::IS_FINISHED;
@ -293,9 +299,9 @@ public:
return std::move(*data);
}
Chunk ALWAYS_INLINE pull()
Chunk ALWAYS_INLINE pull(bool set_not_needed = false)
{
auto data_ = pullData();
auto data_ = pullData(set_not_needed);
if (data_.exception)
std::rethrow_exception(data_.exception);

10
dbms/src/Processors/QueryPipeline.cpp
View File

@ -234,7 +234,7 @@ void QueryPipeline::addDelayedStream(ProcessorPtr source)
addPipe({ std::move(processor) });
}
void QueryPipeline::resize(size_t num_streams, bool force)
void QueryPipeline::resize(size_t num_streams, bool force, bool strict)
{
checkInitialized();
@ -243,7 +243,13 @@ void QueryPipeline::resize(size_t num_streams, bool force)
has_resize = true;
auto resize = std::make_shared<ResizeProcessor>(current_header, getNumStreams(), num_streams);
ProcessorPtr resize;
if (strict)
resize = std::make_shared<StrictResizeProcessor>(current_header, getNumStreams(), num_streams);
else
resize = std::make_shared<ResizeProcessor>(current_header, getNumStreams(), num_streams);
auto stream = streams.begin();
for (auto & input : resize->getInputs())
connect(**(stream++), input);

2
dbms/src/Processors/QueryPipeline.h
View File

@ -61,7 +61,7 @@ public:
/// Check if resize transform was used. (In that case another distinct transform will be added).
bool hasMixedStreams() const { return has_resize || hasMoreThanOneStream(); }
void resize(size_t num_streams, bool force = false);
void resize(size_t num_streams, bool force = false, bool strict = false);
void enableQuotaForCurrentStreams();

140
dbms/src/Processors/ResizeProcessor.cpp
View File

@ -1,5 +1,5 @@
#include <Processors/ResizeProcessor.h>
#include <iostream>
namespace DB
{
@ -257,5 +257,143 @@ IProcessor::Status ResizeProcessor::prepare(const PortNumbers & updated_inputs,
return Status::PortFull;
}
IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_inputs, const PortNumbers & updated_outputs)
{
if (!initialized)
{
initialized = true;
for (auto & input : inputs)
input_ports.push_back({.port = &input, .status = InputStatus::NotActive, .waiting_output = -1});
for (UInt64 i = 0; i < input_ports.size(); ++i)
disabled_input_ports.push(i);
for (auto & output : outputs)
output_ports.push_back({.port = &output, .status = OutputStatus::NotActive});
}
for (auto & output_number : updated_outputs)
{
auto & output = output_ports[output_number];
if (output.port->isFinished())
{
if (output.status != OutputStatus::Finished)
{
++num_finished_outputs;
output.status = OutputStatus::Finished;
}
continue;
}
if (output.port->canPush())
{
if (output.status != OutputStatus::NeedData)
{
output.status = OutputStatus::NeedData;
waiting_outputs.push(output_number);
}
}
}
if (num_finished_outputs == outputs.size())
{
for (auto & input : inputs)
input.close();
return Status::Finished;
}
std::queue<UInt64> inputs_with_data;
for (auto & input_number : updated_inputs)
{
auto & input = input_ports[input_number];
if (input.port->isFinished())
{
if (input.status != InputStatus::Finished)
{
input.status = InputStatus::Finished;
++num_finished_inputs;
waiting_outputs.push(input.waiting_output);
}
continue;
}
if (input.port->hasData())
{
if (input.status != InputStatus::NotActive)
{
input.status = InputStatus::NotActive;
inputs_with_data.push(input_number);
}
}
}
while (!inputs_with_data.empty())
{
auto input_number = inputs_with_data.front();
auto & input_with_data = input_ports[input_number];
inputs_with_data.pop();
if (input_with_data.waiting_output == -1)
throw Exception("No associated output for input with data.", ErrorCodes::LOGICAL_ERROR);
auto & waiting_output = output_ports[input_with_data.waiting_output];
if (waiting_output.status != OutputStatus::NeedData)
throw Exception("Invalid status for associated output.", ErrorCodes::LOGICAL_ERROR);
waiting_output.port->pushData(input_with_data.port->pullData(/* set_not_deeded = */ true));
waiting_output.status = OutputStatus::NotActive;
if (input_with_data.port->isFinished())
{
input_with_data.status = InputStatus::Finished;
++num_finished_inputs;
}
else
disabled_input_ports.push(input_number);
}
if (num_finished_inputs == inputs.size())
{
for (auto & output : outputs)
output.finish();
return Status::Finished;
}
/// Enable more inputs if needed.
while (!disabled_input_ports.empty() && !waiting_outputs.empty())
{
auto & input = input_ports[disabled_input_ports.front()];
disabled_input_ports.pop();
input.port->setNeeded();
input.status = InputStatus::NeedData;
input.waiting_output = waiting_outputs.front();
waiting_outputs.pop();
}
while (!waiting_outputs.empty())
{
auto & output = output_ports[waiting_outputs.front()];
waiting_outputs.pop();
output.status = OutputStatus::Finished;
output.port->finish();
++num_finished_outputs;
}
if (disabled_input_ports.empty())
return Status::NeedData;
return Status::PortFull;
}
}

Some files were not shown because too many files have changed in this diff Show More