thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into pr-local-plan

Browse Source
This commit is contained in:
Igor Nikonov 2024-06-28 14:02:28 +00:00
commit 9b694e611c
68 changed files with 749 additions and 691 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/backport_branches.yml vendored
View File

@ -167,10 +167,16 @@ jobs:
steps: steps:
- name: Check out repository code - name: Check out repository code
uses: ClickHouse/checkout@v1 uses: ClickHouse/checkout@v1
- name: Download reports
run: |
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --pre --job-name Builds
- name: Builds report - name: Builds report
run: | run: |
cd "$GITHUB_WORKSPACE/tests/ci" cd "$GITHUB_WORKSPACE/tests/ci"
python3 ./build_report_check.py --reports package_release package_aarch64 package_asan package_tsan package_debug binary_darwin binary_darwin_aarch64 python3 ./build_report_check.py --reports package_release package_aarch64 package_asan package_tsan package_debug binary_darwin binary_darwin_aarch64
- name: Set status
run: |
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --post --job-name Builds
############################################################################################ ############################################################################################
#################################### INSTALL PACKAGES ###################################### #################################### INSTALL PACKAGES ######################################
############################################################################################ ############################################################################################

6
.github/workflows/release_branches.yml vendored
View File

@ -184,10 +184,16 @@ jobs:
steps: steps:
- name: Check out repository code - name: Check out repository code
uses: ClickHouse/checkout@v1 uses: ClickHouse/checkout@v1
- name: Download reports
run: |
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --pre --job-name Builds
- name: Builds report - name: Builds report
run: | run: |
cd "$GITHUB_WORKSPACE/tests/ci" cd "$GITHUB_WORKSPACE/tests/ci"
python3 ./build_report_check.py --reports package_release package_aarch64 package_asan package_msan package_ubsan package_tsan package_debug binary_darwin binary_darwin_aarch64 python3 ./build_report_check.py --reports package_release package_aarch64 package_asan package_msan package_ubsan package_tsan package_debug binary_darwin binary_darwin_aarch64
- name: Set status
run: |
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(needs.RunConfig.outputs.data) }} --post --job-name Builds
MarkReleaseReady: MarkReleaseReady:
if: ${{ !failure() && !cancelled() }} if: ${{ !failure() && !cancelled() }}
needs: needs:

4
base/base/getFQDNOrHostName.cpp
View File

@ -6,6 +6,9 @@ namespace
{ {
std::string getFQDNOrHostNameImpl() std::string getFQDNOrHostNameImpl()
{ {
#if defined(OS_DARWIN)
return Poco::Net::DNS::hostName();
#else
try try
{ {
return Poco::Net::DNS::thisHost().name(); return Poco::Net::DNS::thisHost().name();
@ -14,6 +17,7 @@ namespace
{ {
return Poco::Net::DNS::hostName(); return Poco::Net::DNS::hostName();
} }
#endif
} }
} }

4
contrib/jemalloc-cmake/CMakeLists.txt
View File

@ -34,9 +34,9 @@ if (OS_LINUX)
# avoid spurious latencies and additional work associated with # avoid spurious latencies and additional work associated with
# MADV_DONTNEED. See # MADV_DONTNEED. See
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000") set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true")
else() else()
set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000") set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true")
endif() endif()
# CACHE variable is empty to allow changing defaults without the necessity # CACHE variable is empty to allow changing defaults without the necessity
# to purge cache # to purge cache

4
docs/en/operations/named-collections.md
View File

@ -5,6 +5,10 @@ sidebar_label: "Named collections"
title: "Named collections" title: "Named collections"
--- ---
import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge';
<CloudNotSupportedBadge />
Named collections provide a way to store collections of key-value pairs to be Named collections provide a way to store collections of key-value pairs to be
used to configure integrations with external sources. You can use named collections with used to configure integrations with external sources. You can use named collections with
dictionaries, tables, table functions, and object storage. dictionaries, tables, table functions, and object storage.

2
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -498,6 +498,8 @@ Default: 0.9
Interval in seconds during which the server's maximum allowed memory consumption is adjusted by the corresponding threshold in cgroups. (see Interval in seconds during which the server's maximum allowed memory consumption is adjusted by the corresponding threshold in cgroups. (see
settings `cgroup_memory_watcher_hard_limit_ratio` and `cgroup_memory_watcher_soft_limit_ratio`). settings `cgroup_memory_watcher_hard_limit_ratio` and `cgroup_memory_watcher_soft_limit_ratio`).
To disable the cgroup observer, set this value to `0`.
Type: UInt64 Type: UInt64
Default: 15 Default: 15

4
docs/en/operations/utilities/clickhouse-local.md
View File

@ -236,10 +236,10 @@ Read 2 rows, 32.00 B in 0.000 sec., 5182 rows/sec., 80.97 KiB/sec.
Previous example is the same as: Previous example is the same as:
``` bash ``` bash
$ echo -e "1,2\n3,4" | clickhouse-local --query " $ echo -e "1,2\n3,4" | clickhouse-local -n --query "
CREATE TABLE table (a Int64, b Int64) ENGINE = File(CSV, stdin); CREATE TABLE table (a Int64, b Int64) ENGINE = File(CSV, stdin);
SELECT a, b FROM table; SELECT a, b FROM table;
DROP TABLE table" DROP TABLE table;"
Read 2 rows, 32.00 B in 0.000 sec., 4987 rows/sec., 77.93 KiB/sec. Read 2 rows, 32.00 B in 0.000 sec., 4987 rows/sec., 77.93 KiB/sec.
1 2 1 2
3 4 3 4

4
docs/en/sql-reference/statements/alter/named-collection.md
View File

@ -3,6 +3,10 @@ slug: /en/sql-reference/statements/alter/named-collection
sidebar_label: NAMED COLLECTION sidebar_label: NAMED COLLECTION
--- ---
import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge';
<CloudNotSupportedBadge />
# ALTER NAMED COLLECTION # ALTER NAMED COLLECTION
This query intends to modify already existing named collections. This query intends to modify already existing named collections.

4
docs/en/sql-reference/statements/alter/view.md
View File

@ -134,8 +134,8 @@ PRIMARY KEY (event_type, ts)
ORDER BY (event_type, ts, browser) ORDER BY (event_type, ts, browser)
SETTINGS index_granularity = 8192 SETTINGS index_granularity = 8192
-- !!! The columns' definition is unchanged but it does not matter, we are not quering -- !!! The columns' definition is unchanged but it does not matter, we are not querying
-- MATERIALIZED VIEW, we are quering TO (storage) table. -- MATERIALIZED VIEW, we are querying TO (storage) table.
-- SELECT section is updated. -- SELECT section is updated.
SHOW CREATE TABLE mv FORMAT TSVRaw; SHOW CREATE TABLE mv FORMAT TSVRaw;

4
docs/en/sql-reference/statements/create/named-collection.md
View File

@ -3,6 +3,10 @@ slug: /en/sql-reference/statements/create/named-collection
sidebar_label: NAMED COLLECTION sidebar_label: NAMED COLLECTION
--- ---
import CloudNotSupportedBadge from '@theme/badges/CloudNotSupportedBadge';
<CloudNotSupportedBadge />
# CREATE NAMED COLLECTION # CREATE NAMED COLLECTION
Creates a new named collection. Creates a new named collection.

8
programs/client/Client.cpp
View File

@ -248,6 +248,10 @@ std::vector<String> Client::loadWarningMessages()
} }
} }
Poco::Util::LayeredConfiguration & Client::getClientConfiguration()
{
return config();
}
void Client::initialize(Poco::Util::Application & self) void Client::initialize(Poco::Util::Application & self)
{ {
@ -697,9 +701,7 @@ bool Client::processWithFuzzing(const String & full_query)
const char * begin = full_query.data(); const char * begin = full_query.data();
orig_ast = parseQuery(begin, begin + full_query.size(), orig_ast = parseQuery(begin, begin + full_query.size(),
global_context->getSettingsRef(), global_context->getSettingsRef(),
/*allow_multi_statements=*/ true, /*allow_multi_statements=*/ true);
/*is_interactive=*/ is_interactive,
/*ignore_error=*/ ignore_error);
} }
catch (const Exception & e) catch (const Exception & e)
{ {

3
programs/client/Client.h
View File

@ -16,6 +16,9 @@ public:
int main(const std::vector<String> & /*args*/) override; int main(const std::vector<String> & /*args*/) override;
protected: protected:
Poco::Util::LayeredConfiguration & getClientConfiguration() override;
bool processWithFuzzing(const String & full_query) override; bool processWithFuzzing(const String & full_query) override;
std::optional<bool> processFuzzingStep(const String & query_to_execute, const ASTPtr & parsed_query); std::optional<bool> processFuzzingStep(const String & query_to_execute, const ASTPtr & parsed_query);

2
programs/library-bridge/CMakeLists.txt
View File

@ -11,7 +11,6 @@ set (CLICKHOUSE_LIBRARY_BRIDGE_SOURCES
LibraryBridgeHandlers.cpp LibraryBridgeHandlers.cpp
SharedLibrary.cpp SharedLibrary.cpp
library-bridge.cpp library-bridge.cpp
createFunctionBaseCast.cpp
) )
clickhouse_add_executable(clickhouse-library-bridge ${CLICKHOUSE_LIBRARY_BRIDGE_SOURCES}) clickhouse_add_executable(clickhouse-library-bridge ${CLICKHOUSE_LIBRARY_BRIDGE_SOURCES})
@ -20,6 +19,7 @@ target_link_libraries(clickhouse-library-bridge PRIVATE
daemon daemon
dbms dbms
bridge bridge
clickhouse_functions
) )
set_target_properties(clickhouse-library-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) set_target_properties(clickhouse-library-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)

207
programs/local/LocalServer.cpp
View File

@ -82,6 +82,11 @@ void applySettingsOverridesForLocal(ContextMutablePtr context)
context->setSettings(settings); context->setSettings(settings);
} }
Poco::Util::LayeredConfiguration & LocalServer::getClientConfiguration()
{
return config();
}
void LocalServer::processError(const String &) const void LocalServer::processError(const String &) const
{ {
if (ignore_error) if (ignore_error)
@ -117,19 +122,19 @@ void LocalServer::initialize(Poco::Util::Application & self)
Poco::Util::Application::initialize(self); Poco::Util::Application::initialize(self);
/// Load config files if exists /// Load config files if exists
if (config().has("config-file") || fs::exists("config.xml")) if (getClientConfiguration().has("config-file") || fs::exists("config.xml"))
{ {
const auto config_path = config().getString("config-file", "config.xml"); const auto config_path = getClientConfiguration().getString("config-file", "config.xml");
ConfigProcessor config_processor(config_path, false, true); ConfigProcessor config_processor(config_path, false, true);
ConfigProcessor::setConfigPath(fs::path(config_path).parent_path()); ConfigProcessor::setConfigPath(fs::path(config_path).parent_path());
auto loaded_config = config_processor.loadConfig(); auto loaded_config = config_processor.loadConfig();
config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); getClientConfiguration().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
} }
GlobalThreadPool::initialize( GlobalThreadPool::initialize(
config().getUInt("max_thread_pool_size", 10000), getClientConfiguration().getUInt("max_thread_pool_size", 10000),
config().getUInt("max_thread_pool_free_size", 1000), getClientConfiguration().getUInt("max_thread_pool_free_size", 1000),
config().getUInt("thread_pool_queue_size", 10000) getClientConfiguration().getUInt("thread_pool_queue_size", 10000)
); );
#if USE_AZURE_BLOB_STORAGE #if USE_AZURE_BLOB_STORAGE
@ -141,18 +146,18 @@ void LocalServer::initialize(Poco::Util::Application & self)
#endif #endif
getIOThreadPool().initialize( getIOThreadPool().initialize(
config().getUInt("max_io_thread_pool_size", 100), getClientConfiguration().getUInt("max_io_thread_pool_size", 100),
config().getUInt("max_io_thread_pool_free_size", 0), getClientConfiguration().getUInt("max_io_thread_pool_free_size", 0),
config().getUInt("io_thread_pool_queue_size", 10000)); getClientConfiguration().getUInt("io_thread_pool_queue_size", 10000));
const size_t active_parts_loading_threads = config().getUInt("max_active_parts_loading_thread_pool_size", 64); const size_t active_parts_loading_threads = getClientConfiguration().getUInt("max_active_parts_loading_thread_pool_size", 64);
getActivePartsLoadingThreadPool().initialize( getActivePartsLoadingThreadPool().initialize(
active_parts_loading_threads, active_parts_loading_threads,
0, // We don't need any threads one all the parts will be loaded 0, // We don't need any threads one all the parts will be loaded
active_parts_loading_threads); active_parts_loading_threads);
const size_t outdated_parts_loading_threads = config().getUInt("max_outdated_parts_loading_thread_pool_size", 32); const size_t outdated_parts_loading_threads = getClientConfiguration().getUInt("max_outdated_parts_loading_thread_pool_size", 32);
getOutdatedPartsLoadingThreadPool().initialize( getOutdatedPartsLoadingThreadPool().initialize(
outdated_parts_loading_threads, outdated_parts_loading_threads,
0, // We don't need any threads one all the parts will be loaded 0, // We don't need any threads one all the parts will be loaded
@ -160,7 +165,7 @@ void LocalServer::initialize(Poco::Util::Application & self)
getOutdatedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads); getOutdatedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads);
const size_t unexpected_parts_loading_threads = config().getUInt("max_unexpected_parts_loading_thread_pool_size", 32); const size_t unexpected_parts_loading_threads = getClientConfiguration().getUInt("max_unexpected_parts_loading_thread_pool_size", 32);
getUnexpectedPartsLoadingThreadPool().initialize( getUnexpectedPartsLoadingThreadPool().initialize(
unexpected_parts_loading_threads, unexpected_parts_loading_threads,
0, // We don't need any threads one all the parts will be loaded 0, // We don't need any threads one all the parts will be loaded
@ -168,7 +173,7 @@ void LocalServer::initialize(Poco::Util::Application & self)
getUnexpectedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads); getUnexpectedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads);
const size_t cleanup_threads = config().getUInt("max_parts_cleaning_thread_pool_size", 128); const size_t cleanup_threads = getClientConfiguration().getUInt("max_parts_cleaning_thread_pool_size", 128);
getPartsCleaningThreadPool().initialize( getPartsCleaningThreadPool().initialize(
cleanup_threads, cleanup_threads,
0, // We don't need any threads one all the parts will be deleted 0, // We don't need any threads one all the parts will be deleted
@ -201,10 +206,10 @@ void LocalServer::tryInitPath()
{ {
std::string path; std::string path;
if (config().has("path")) if (getClientConfiguration().has("path"))
{ {
// User-supplied path. // User-supplied path.
path = config().getString("path"); path = getClientConfiguration().getString("path");
Poco::trimInPlace(path); Poco::trimInPlace(path);
if (path.empty()) if (path.empty())
@ -263,13 +268,13 @@ void LocalServer::tryInitPath()
global_context->setUserFilesPath(""); /// user's files are everywhere global_context->setUserFilesPath(""); /// user's files are everywhere
std::string user_scripts_path = config().getString("user_scripts_path", fs::path(path) / "user_scripts/"); std::string user_scripts_path = getClientConfiguration().getString("user_scripts_path", fs::path(path) / "user_scripts/");
global_context->setUserScriptsPath(user_scripts_path); global_context->setUserScriptsPath(user_scripts_path);
/// top_level_domains_lists /// top_level_domains_lists
const std::string & top_level_domains_path = config().getString("top_level_domains_path", fs::path(path) / "top_level_domains/"); const std::string & top_level_domains_path = getClientConfiguration().getString("top_level_domains_path", fs::path(path) / "top_level_domains/");
if (!top_level_domains_path.empty()) if (!top_level_domains_path.empty())
TLDListsHolder::getInstance().parseConfig(fs::path(top_level_domains_path) / "", config()); TLDListsHolder::getInstance().parseConfig(fs::path(top_level_domains_path) / "", getClientConfiguration());
} }
@ -311,14 +316,14 @@ void LocalServer::cleanup()
std::string LocalServer::getInitialCreateTableQuery() std::string LocalServer::getInitialCreateTableQuery()
{ {
if (!config().has("table-structure") && !config().has("table-file") && !config().has("table-data-format") && (!isRegularFile(STDIN_FILENO) || queries.empty())) if (!getClientConfiguration().has("table-structure") && !getClientConfiguration().has("table-file") && !getClientConfiguration().has("table-data-format") && (!isRegularFile(STDIN_FILENO) || queries.empty()))
return {}; return {};
auto table_name = backQuoteIfNeed(config().getString("table-name", "table")); auto table_name = backQuoteIfNeed(getClientConfiguration().getString("table-name", "table"));
auto table_structure = config().getString("table-structure", "auto"); auto table_structure = getClientConfiguration().getString("table-structure", "auto");
String table_file; String table_file;
if (!config().has("table-file") || config().getString("table-file") == "-") if (!getClientConfiguration().has("table-file") || getClientConfiguration().getString("table-file") == "-")
{ {
/// Use Unix tools stdin naming convention /// Use Unix tools stdin naming convention
table_file = "stdin"; table_file = "stdin";
@ -326,7 +331,7 @@ std::string LocalServer::getInitialCreateTableQuery()
else else
{ {
/// Use regular file /// Use regular file
auto file_name = config().getString("table-file"); auto file_name = getClientConfiguration().getString("table-file");
table_file = quoteString(file_name); table_file = quoteString(file_name);
} }
@ -374,18 +379,18 @@ void LocalServer::setupUsers()
ConfigurationPtr users_config; ConfigurationPtr users_config;
auto & access_control = global_context->getAccessControl(); auto & access_control = global_context->getAccessControl();
access_control.setNoPasswordAllowed(config().getBool("allow_no_password", true)); access_control.setNoPasswordAllowed(getClientConfiguration().getBool("allow_no_password", true));
access_control.setPlaintextPasswordAllowed(config().getBool("allow_plaintext_password", true)); access_control.setPlaintextPasswordAllowed(getClientConfiguration().getBool("allow_plaintext_password", true));
if (config().has("config-file") || fs::exists("config.xml")) if (getClientConfiguration().has("config-file") || fs::exists("config.xml"))
{ {
String config_path = config().getString("config-file", ""); String config_path = getClientConfiguration().getString("config-file", "");
bool has_user_directories = config().has("user_directories"); bool has_user_directories = getClientConfiguration().has("user_directories");
const auto config_dir = fs::path{config_path}.remove_filename().string(); const auto config_dir = fs::path{config_path}.remove_filename().string();
String users_config_path = config().getString("users_config", ""); String users_config_path = getClientConfiguration().getString("users_config", "");
if (users_config_path.empty() && has_user_directories) if (users_config_path.empty() && has_user_directories)
{ {
users_config_path = config().getString("user_directories.users_xml.path"); users_config_path = getClientConfiguration().getString("user_directories.users_xml.path");
if (fs::path(users_config_path).is_relative() && fs::exists(fs::path(config_dir) / users_config_path)) if (fs::path(users_config_path).is_relative() && fs::exists(fs::path(config_dir) / users_config_path))
users_config_path = fs::path(config_dir) / users_config_path; users_config_path = fs::path(config_dir) / users_config_path;
} }
@ -409,10 +414,10 @@ void LocalServer::setupUsers()
void LocalServer::connect() void LocalServer::connect()
{ {
connection_parameters = ConnectionParameters(config(), "localhost"); connection_parameters = ConnectionParameters(getClientConfiguration(), "localhost");
ReadBuffer * in; ReadBuffer * in;
auto table_file = config().getString("table-file", "-"); auto table_file = getClientConfiguration().getString("table-file", "-");
if (table_file == "-" || table_file == "stdin") if (table_file == "-" || table_file == "stdin")
{ {
in = &std_in; in = &std_in;
@ -433,7 +438,7 @@ try
UseSSL use_ssl; UseSSL use_ssl;
thread_status.emplace(); thread_status.emplace();
StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true)); StackTrace::setShowAddresses(getClientConfiguration().getBool("show_addresses_in_stack_traces", true));
setupSignalHandler(); setupSignalHandler();
@ -448,7 +453,7 @@ try
if (rlim.rlim_cur < rlim.rlim_max) if (rlim.rlim_cur < rlim.rlim_max)
{ {
rlim.rlim_cur = config().getUInt("max_open_files", static_cast<unsigned>(rlim.rlim_max)); rlim.rlim_cur = getClientConfiguration().getUInt("max_open_files", static_cast<unsigned>(rlim.rlim_max));
int rc = setrlimit(RLIMIT_NOFILE, &rlim); int rc = setrlimit(RLIMIT_NOFILE, &rlim);
if (rc != 0) if (rc != 0)
std::cerr << fmt::format("Cannot set max number of file descriptors to {}. Try to specify max_open_files according to your system limits. error: {}", rlim.rlim_cur, errnoToString()) << '\n'; std::cerr << fmt::format("Cannot set max number of file descriptors to {}. Try to specify max_open_files according to your system limits. error: {}", rlim.rlim_cur, errnoToString()) << '\n';
@ -456,8 +461,8 @@ try
} }
is_interactive = stdin_is_a_tty is_interactive = stdin_is_a_tty
&& (config().hasOption("interactive") && (getClientConfiguration().hasOption("interactive")
|| (queries.empty() && !config().has("table-structure") && queries_files.empty() && !config().has("table-file"))); || (queries.empty() && !getClientConfiguration().has("table-structure") && queries_files.empty() && !getClientConfiguration().has("table-file")));
if (!is_interactive) if (!is_interactive)
{ {
@ -481,7 +486,7 @@ try
SCOPE_EXIT({ cleanup(); }); SCOPE_EXIT({ cleanup(); });
initTTYBuffer(toProgressOption(config().getString("progress", "default"))); initTTYBuffer(toProgressOption(getClientConfiguration().getString("progress", "default")));
ASTAlterCommand::setFormatAlterCommandsWithParentheses(true); ASTAlterCommand::setFormatAlterCommandsWithParentheses(true);
applyCmdSettings(global_context); applyCmdSettings(global_context);
@ -489,7 +494,7 @@ try
/// try to load user defined executable functions, throw on error and die /// try to load user defined executable functions, throw on error and die
try try
{ {
global_context->loadOrReloadUserDefinedExecutableFunctions(config()); global_context->loadOrReloadUserDefinedExecutableFunctions(getClientConfiguration());
} }
catch (...) catch (...)
{ {
@ -530,7 +535,7 @@ try
} }
catch (const DB::Exception & e) catch (const DB::Exception & e)
{ {
bool need_print_stack_trace = config().getBool("stacktrace", false); bool need_print_stack_trace = getClientConfiguration().getBool("stacktrace", false);
std::cerr << getExceptionMessage(e, need_print_stack_trace, true) << std::endl; std::cerr << getExceptionMessage(e, need_print_stack_trace, true) << std::endl;
return e.code() ? e.code() : -1; return e.code() ? e.code() : -1;
} }
@ -542,42 +547,42 @@ catch (...)
void LocalServer::updateLoggerLevel(const String & logs_level) void LocalServer::updateLoggerLevel(const String & logs_level)
{ {
config().setString("logger.level", logs_level); getClientConfiguration().setString("logger.level", logs_level);
updateLevels(config(), logger()); updateLevels(getClientConfiguration(), logger());
} }
void LocalServer::processConfig() void LocalServer::processConfig()
{ {
if (!queries.empty() && config().has("queries-file")) if (!queries.empty() && getClientConfiguration().has("queries-file"))
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Options '--query' and '--queries-file' cannot be specified at the same time"); throw Exception(ErrorCodes::BAD_ARGUMENTS, "Options '--query' and '--queries-file' cannot be specified at the same time");
if (config().has("multiquery")) if (getClientConfiguration().has("multiquery"))
is_multiquery = true; is_multiquery = true;
pager = config().getString("pager", ""); pager = getClientConfiguration().getString("pager", "");
delayed_interactive = config().has("interactive") && (!queries.empty() || config().has("queries-file")); delayed_interactive = getClientConfiguration().has("interactive") && (!queries.empty() || getClientConfiguration().has("queries-file"));
if (!is_interactive || delayed_interactive) if (!is_interactive || delayed_interactive)
{ {
echo_queries = config().hasOption("echo") || config().hasOption("verbose"); echo_queries = getClientConfiguration().hasOption("echo") || getClientConfiguration().hasOption("verbose");
ignore_error = config().getBool("ignore-error", false); ignore_error = getClientConfiguration().getBool("ignore-error", false);
} }
print_stack_trace = config().getBool("stacktrace", false); print_stack_trace = getClientConfiguration().getBool("stacktrace", false);
const std::string clickhouse_dialect{"clickhouse"}; const std::string clickhouse_dialect{"clickhouse"};
load_suggestions = (is_interactive || delayed_interactive) && !config().getBool("disable_suggestion", false) load_suggestions = (is_interactive || delayed_interactive) && !getClientConfiguration().getBool("disable_suggestion", false)
&& config().getString("dialect", clickhouse_dialect) == clickhouse_dialect; && getClientConfiguration().getString("dialect", clickhouse_dialect) == clickhouse_dialect;
wait_for_suggestions_to_load = config().getBool("wait_for_suggestions_to_load", false); wait_for_suggestions_to_load = getClientConfiguration().getBool("wait_for_suggestions_to_load", false);
auto logging = (config().has("logger.console") auto logging = (getClientConfiguration().has("logger.console")
|| config().has("logger.level") || getClientConfiguration().has("logger.level")
|| config().has("log-level") || getClientConfiguration().has("log-level")
|| config().has("send_logs_level") || getClientConfiguration().has("send_logs_level")
|| config().has("logger.log")); || getClientConfiguration().has("logger.log"));
auto level = config().getString("log-level", "trace"); auto level = getClientConfiguration().getString("log-level", "trace");
if (config().has("server_logs_file")) if (getClientConfiguration().has("server_logs_file"))
{ {
auto poco_logs_level = Poco::Logger::parseLevel(level); auto poco_logs_level = Poco::Logger::parseLevel(level);
Poco::Logger::root().setLevel(poco_logs_level); Poco::Logger::root().setLevel(poco_logs_level);
@ -587,10 +592,10 @@ void LocalServer::processConfig()
} }
else else
{ {
config().setString("logger", "logger"); getClientConfiguration().setString("logger", "logger");
auto log_level_default = logging ? level : "fatal"; auto log_level_default = logging ? level : "fatal";
config().setString("logger.level", config().getString("log-level", config().getString("send_logs_level", log_level_default))); getClientConfiguration().setString("logger.level", getClientConfiguration().getString("log-level", getClientConfiguration().getString("send_logs_level", log_level_default)));
buildLoggers(config(), logger(), "clickhouse-local"); buildLoggers(getClientConfiguration(), logger(), "clickhouse-local");
} }
shared_context = Context::createShared(); shared_context = Context::createShared();
@ -604,13 +609,13 @@ void LocalServer::processConfig()
LoggerRawPtr log = &logger(); LoggerRawPtr log = &logger();
/// Maybe useless /// Maybe useless
if (config().has("macros")) if (getClientConfiguration().has("macros"))
global_context->setMacros(std::make_unique<Macros>(config(), "macros", log)); global_context->setMacros(std::make_unique<Macros>(getClientConfiguration(), "macros", log));
setDefaultFormatsAndCompressionFromConfiguration(); setDefaultFormatsAndCompressionFromConfiguration();
/// Sets external authenticators config (LDAP, Kerberos). /// Sets external authenticators config (LDAP, Kerberos).
global_context->setExternalAuthenticatorsConfig(config()); global_context->setExternalAuthenticatorsConfig(getClientConfiguration());
setupUsers(); setupUsers();
@ -619,12 +624,12 @@ void LocalServer::processConfig()
global_context->getProcessList().setMaxSize(0); global_context->getProcessList().setMaxSize(0);
const size_t physical_server_memory = getMemoryAmount(); const size_t physical_server_memory = getMemoryAmount();
const double cache_size_to_ram_max_ratio = config().getDouble("cache_size_to_ram_max_ratio", 0.5); const double cache_size_to_ram_max_ratio = getClientConfiguration().getDouble("cache_size_to_ram_max_ratio", 0.5);
const size_t max_cache_size = static_cast<size_t>(physical_server_memory * cache_size_to_ram_max_ratio); const size_t max_cache_size = static_cast<size_t>(physical_server_memory * cache_size_to_ram_max_ratio);
String uncompressed_cache_policy = config().getString("uncompressed_cache_policy", DEFAULT_UNCOMPRESSED_CACHE_POLICY); String uncompressed_cache_policy = getClientConfiguration().getString("uncompressed_cache_policy", DEFAULT_UNCOMPRESSED_CACHE_POLICY);
size_t uncompressed_cache_size = config().getUInt64("uncompressed_cache_size", DEFAULT_UNCOMPRESSED_CACHE_MAX_SIZE); size_t uncompressed_cache_size = getClientConfiguration().getUInt64("uncompressed_cache_size", DEFAULT_UNCOMPRESSED_CACHE_MAX_SIZE);
double uncompressed_cache_size_ratio = config().getDouble("uncompressed_cache_size_ratio", DEFAULT_UNCOMPRESSED_CACHE_SIZE_RATIO); double uncompressed_cache_size_ratio = getClientConfiguration().getDouble("uncompressed_cache_size_ratio", DEFAULT_UNCOMPRESSED_CACHE_SIZE_RATIO);
if (uncompressed_cache_size > max_cache_size) if (uncompressed_cache_size > max_cache_size)
{ {
uncompressed_cache_size = max_cache_size; uncompressed_cache_size = max_cache_size;
@ -632,9 +637,9 @@ void LocalServer::processConfig()
} }
global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size, uncompressed_cache_size_ratio); global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size, uncompressed_cache_size_ratio);
String mark_cache_policy = config().getString("mark_cache_policy", DEFAULT_MARK_CACHE_POLICY); String mark_cache_policy = getClientConfiguration().getString("mark_cache_policy", DEFAULT_MARK_CACHE_POLICY);
size_t mark_cache_size = config().getUInt64("mark_cache_size", DEFAULT_MARK_CACHE_MAX_SIZE); size_t mark_cache_size = getClientConfiguration().getUInt64("mark_cache_size", DEFAULT_MARK_CACHE_MAX_SIZE);
double mark_cache_size_ratio = config().getDouble("mark_cache_size_ratio", DEFAULT_MARK_CACHE_SIZE_RATIO); double mark_cache_size_ratio = getClientConfiguration().getDouble("mark_cache_size_ratio", DEFAULT_MARK_CACHE_SIZE_RATIO);
if (!mark_cache_size) if (!mark_cache_size)
LOG_ERROR(log, "Too low mark cache size will lead to severe performance degradation."); LOG_ERROR(log, "Too low mark cache size will lead to severe performance degradation.");
if (mark_cache_size > max_cache_size) if (mark_cache_size > max_cache_size)
@ -644,9 +649,9 @@ void LocalServer::processConfig()
} }
global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio); global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio);
String index_uncompressed_cache_policy = config().getString("index_uncompressed_cache_policy", DEFAULT_INDEX_UNCOMPRESSED_CACHE_POLICY); String index_uncompressed_cache_policy = getClientConfiguration().getString("index_uncompressed_cache_policy", DEFAULT_INDEX_UNCOMPRESSED_CACHE_POLICY);
size_t index_uncompressed_cache_size = config().getUInt64("index_uncompressed_cache_size", DEFAULT_INDEX_UNCOMPRESSED_CACHE_MAX_SIZE); size_t index_uncompressed_cache_size = getClientConfiguration().getUInt64("index_uncompressed_cache_size", DEFAULT_INDEX_UNCOMPRESSED_CACHE_MAX_SIZE);
double index_uncompressed_cache_size_ratio = config().getDouble("index_uncompressed_cache_size_ratio", DEFAULT_INDEX_UNCOMPRESSED_CACHE_SIZE_RATIO); double index_uncompressed_cache_size_ratio = getClientConfiguration().getDouble("index_uncompressed_cache_size_ratio", DEFAULT_INDEX_UNCOMPRESSED_CACHE_SIZE_RATIO);
if (index_uncompressed_cache_size > max_cache_size) if (index_uncompressed_cache_size > max_cache_size)
{ {
index_uncompressed_cache_size = max_cache_size; index_uncompressed_cache_size = max_cache_size;
@ -654,9 +659,9 @@ void LocalServer::processConfig()
} }
global_context->setIndexUncompressedCache(index_uncompressed_cache_policy, index_uncompressed_cache_size, index_uncompressed_cache_size_ratio); global_context->setIndexUncompressedCache(index_uncompressed_cache_policy, index_uncompressed_cache_size, index_uncompressed_cache_size_ratio);
String index_mark_cache_policy = config().getString("index_mark_cache_policy", DEFAULT_INDEX_MARK_CACHE_POLICY); String index_mark_cache_policy = getClientConfiguration().getString("index_mark_cache_policy", DEFAULT_INDEX_MARK_CACHE_POLICY);
size_t index_mark_cache_size = config().getUInt64("index_mark_cache_size", DEFAULT_INDEX_MARK_CACHE_MAX_SIZE); size_t index_mark_cache_size = getClientConfiguration().getUInt64("index_mark_cache_size", DEFAULT_INDEX_MARK_CACHE_MAX_SIZE);
double index_mark_cache_size_ratio = config().getDouble("index_mark_cache_size_ratio", DEFAULT_INDEX_MARK_CACHE_SIZE_RATIO); double index_mark_cache_size_ratio = getClientConfiguration().getDouble("index_mark_cache_size_ratio", DEFAULT_INDEX_MARK_CACHE_SIZE_RATIO);
if (index_mark_cache_size > max_cache_size) if (index_mark_cache_size > max_cache_size)
{ {
index_mark_cache_size = max_cache_size; index_mark_cache_size = max_cache_size;
@ -664,7 +669,7 @@ void LocalServer::processConfig()
} }
global_context->setIndexMarkCache(index_mark_cache_policy, index_mark_cache_size, index_mark_cache_size_ratio); global_context->setIndexMarkCache(index_mark_cache_policy, index_mark_cache_size, index_mark_cache_size_ratio);
size_t mmap_cache_size = config().getUInt64("mmap_cache_size", DEFAULT_MMAP_CACHE_MAX_SIZE); size_t mmap_cache_size = getClientConfiguration().getUInt64("mmap_cache_size", DEFAULT_MMAP_CACHE_MAX_SIZE);
if (mmap_cache_size > max_cache_size) if (mmap_cache_size > max_cache_size)
{ {
mmap_cache_size = max_cache_size; mmap_cache_size = max_cache_size;
@ -676,8 +681,8 @@ void LocalServer::processConfig()
global_context->setQueryCache(0, 0, 0, 0); global_context->setQueryCache(0, 0, 0, 0);
#if USE_EMBEDDED_COMPILER #if USE_EMBEDDED_COMPILER
size_t compiled_expression_cache_max_size_in_bytes = config().getUInt64("compiled_expression_cache_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_SIZE); size_t compiled_expression_cache_max_size_in_bytes = getClientConfiguration().getUInt64("compiled_expression_cache_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_SIZE);
size_t compiled_expression_cache_max_elements = config().getUInt64("compiled_expression_cache_elements_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_ENTRIES); size_t compiled_expression_cache_max_elements = getClientConfiguration().getUInt64("compiled_expression_cache_elements_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_ENTRIES);
CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_max_size_in_bytes, compiled_expression_cache_max_elements); CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_max_size_in_bytes, compiled_expression_cache_max_elements);
#endif #endif
@ -689,16 +694,16 @@ void LocalServer::processConfig()
applyCmdOptions(global_context); applyCmdOptions(global_context);
/// Load global settings from default_profile and system_profile. /// Load global settings from default_profile and system_profile.
global_context->setDefaultProfiles(config()); global_context->setDefaultProfiles(getClientConfiguration());
/// We load temporary database first, because projections need it. /// We load temporary database first, because projections need it.
DatabaseCatalog::instance().initializeAndLoadTemporaryDatabase(); DatabaseCatalog::instance().initializeAndLoadTemporaryDatabase();
std::string default_database = config().getString("default_database", "default"); std::string default_database = getClientConfiguration().getString("default_database", "default");
DatabaseCatalog::instance().attachDatabase(default_database, createClickHouseLocalDatabaseOverlay(default_database, global_context)); DatabaseCatalog::instance().attachDatabase(default_database, createClickHouseLocalDatabaseOverlay(default_database, global_context));
global_context->setCurrentDatabase(default_database); global_context->setCurrentDatabase(default_database);
if (config().has("path")) if (getClientConfiguration().has("path"))
{ {
String path = global_context->getPath(); String path = global_context->getPath();
fs::create_directories(fs::path(path)); fs::create_directories(fs::path(path));
@ -713,7 +718,7 @@ void LocalServer::processConfig()
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)); attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
waitLoad(TablesLoaderForegroundPoolId, startup_system_tasks); waitLoad(TablesLoaderForegroundPoolId, startup_system_tasks);
if (!config().has("only-system-tables")) if (!getClientConfiguration().has("only-system-tables"))
{ {
DatabaseCatalog::instance().createBackgroundTasks(); DatabaseCatalog::instance().createBackgroundTasks();
waitLoad(loadMetadata(global_context)); waitLoad(loadMetadata(global_context));
@ -725,15 +730,15 @@ void LocalServer::processConfig()
LOG_DEBUG(log, "Loaded metadata."); LOG_DEBUG(log, "Loaded metadata.");
} }
else if (!config().has("no-system-tables")) else if (!getClientConfiguration().has("no-system-tables"))
{ {
attachSystemTablesServer(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE), false); attachSystemTablesServer(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE), false);
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA)); attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA));
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)); attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
} }
server_display_name = config().getString("display_name", ""); server_display_name = getClientConfiguration().getString("display_name", "");
prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name.default", ":) "); prompt_by_server_display_name = getClientConfiguration().getRawString("prompt_by_server_display_name.default", ":) ");
global_context->setQueryKindInitial(); global_context->setQueryKindInitial();
global_context->setQueryKind(query_kind); global_context->setQueryKind(query_kind);
@ -811,7 +816,7 @@ void LocalServer::applyCmdSettings(ContextMutablePtr context)
void LocalServer::applyCmdOptions(ContextMutablePtr context) void LocalServer::applyCmdOptions(ContextMutablePtr context)
{ {
context->setDefaultFormat(config().getString("output-format", config().getString("format", is_interactive ? "PrettyCompact" : "TSV"))); context->setDefaultFormat(getClientConfiguration().getString("output-format", getClientConfiguration().getString("format", is_interactive ? "PrettyCompact" : "TSV")));
applyCmdSettings(context); applyCmdSettings(context);
} }
@ -819,33 +824,33 @@ void LocalServer::applyCmdOptions(ContextMutablePtr context)
void LocalServer::processOptions(const OptionsDescription &, const CommandLineOptions & options, const std::vector<Arguments> &, const std::vector<Arguments> &) void LocalServer::processOptions(const OptionsDescription &, const CommandLineOptions & options, const std::vector<Arguments> &, const std::vector<Arguments> &)
{ {
if (options.count("table")) if (options.count("table"))
config().setString("table-name", options["table"].as<std::string>()); getClientConfiguration().setString("table-name", options["table"].as<std::string>());
if (options.count("file")) if (options.count("file"))
config().setString("table-file", options["file"].as<std::string>()); getClientConfiguration().setString("table-file", options["file"].as<std::string>());
if (options.count("structure")) if (options.count("structure"))
config().setString("table-structure", options["structure"].as<std::string>()); getClientConfiguration().setString("table-structure", options["structure"].as<std::string>());
if (options.count("no-system-tables")) if (options.count("no-system-tables"))
config().setBool("no-system-tables", true); getClientConfiguration().setBool("no-system-tables", true);
if (options.count("only-system-tables")) if (options.count("only-system-tables"))
config().setBool("only-system-tables", true); getClientConfiguration().setBool("only-system-tables", true);
if (options.count("database")) if (options.count("database"))
config().setString("default_database", options["database"].as<std::string>()); getClientConfiguration().setString("default_database", options["database"].as<std::string>());
if (options.count("input-format")) if (options.count("input-format"))
config().setString("table-data-format", options["input-format"].as<std::string>()); getClientConfiguration().setString("table-data-format", options["input-format"].as<std::string>());
if (options.count("output-format")) if (options.count("output-format"))
config().setString("output-format", options["output-format"].as<std::string>()); getClientConfiguration().setString("output-format", options["output-format"].as<std::string>());
if (options.count("logger.console")) if (options.count("logger.console"))
config().setBool("logger.console", options["logger.console"].as<bool>()); getClientConfiguration().setBool("logger.console", options["logger.console"].as<bool>());
if (options.count("logger.log")) if (options.count("logger.log"))
config().setString("logger.log", options["logger.log"].as<std::string>()); getClientConfiguration().setString("logger.log", options["logger.log"].as<std::string>());
if (options.count("logger.level")) if (options.count("logger.level"))
config().setString("logger.level", options["logger.level"].as<std::string>()); getClientConfiguration().setString("logger.level", options["logger.level"].as<std::string>());
if (options.count("send_logs_level")) if (options.count("send_logs_level"))
config().setString("send_logs_level", options["send_logs_level"].as<std::string>()); getClientConfiguration().setString("send_logs_level", options["send_logs_level"].as<std::string>());
if (options.count("wait_for_suggestions_to_load")) if (options.count("wait_for_suggestions_to_load"))
config().setBool("wait_for_suggestions_to_load", true); getClientConfiguration().setBool("wait_for_suggestions_to_load", true);
} }
void LocalServer::readArguments(int argc, char ** argv, Arguments & common_arguments, std::vector<Arguments> &, std::vector<Arguments> &) void LocalServer::readArguments(int argc, char ** argv, Arguments & common_arguments, std::vector<Arguments> &, std::vector<Arguments> &)

3
programs/local/LocalServer.h
View File

@ -30,6 +30,9 @@ public:
int main(const std::vector<String> & /*args*/) override; int main(const std::vector<String> & /*args*/) override;
protected: protected:
Poco::Util::LayeredConfiguration & getClientConfiguration() override;
void connect() override; void connect() override;
void processError(const String & query) const override; void processError(const String & query) const override;

2
programs/odbc-bridge/CMakeLists.txt
View File

@ -13,7 +13,6 @@ set (CLICKHOUSE_ODBC_BRIDGE_SOURCES
getIdentifierQuote.cpp getIdentifierQuote.cpp
odbc-bridge.cpp odbc-bridge.cpp
validateODBCConnectionString.cpp validateODBCConnectionString.cpp
createFunctionBaseCast.cpp
) )
clickhouse_add_executable(clickhouse-odbc-bridge ${CLICKHOUSE_ODBC_BRIDGE_SOURCES}) clickhouse_add_executable(clickhouse-odbc-bridge ${CLICKHOUSE_ODBC_BRIDGE_SOURCES})
@ -25,6 +24,7 @@ target_link_libraries(clickhouse-odbc-bridge PRIVATE
clickhouse_parsers clickhouse_parsers
ch_contrib::nanodbc ch_contrib::nanodbc
ch_contrib::unixodbc ch_contrib::unixodbc
clickhouse_functions
) )
set_target_properties(clickhouse-odbc-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) set_target_properties(clickhouse-odbc-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)

210
src/Client/ClientBase.cpp
View File

@ -302,8 +302,29 @@ public:
ClientBase::~ClientBase() = default; ClientBase::~ClientBase() = default;
ClientBase::ClientBase() = default; ClientBase::ClientBase(
int in_fd_,
int out_fd_,
int err_fd_,
std::istream & input_stream_,
std::ostream & output_stream_,
std::ostream & error_stream_
)
: std_in(in_fd_)
, std_out(out_fd_)
, progress_indication(output_stream_, in_fd_, err_fd_)
, in_fd(in_fd_)
, out_fd(out_fd_)
, err_fd(err_fd_)
, input_stream(input_stream_)
, output_stream(output_stream_)
, error_stream(error_stream_)
{
stdin_is_a_tty = isatty(in_fd);
stdout_is_a_tty = isatty(out_fd);
stderr_is_a_tty = isatty(err_fd);
terminal_width = getTerminalWidth(in_fd, err_fd);
}
void ClientBase::setupSignalHandler() void ClientBase::setupSignalHandler()
{ {
@ -330,7 +351,7 @@ void ClientBase::setupSignalHandler()
} }
ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, const Settings & settings, bool allow_multi_statements, bool is_interactive, bool ignore_error) ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, const Settings & settings, bool allow_multi_statements)
{ {
std::unique_ptr<IParserBase> parser; std::unique_ptr<IParserBase> parser;
ASTPtr res; ASTPtr res;
@ -359,7 +380,7 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, const Setting
if (!res) if (!res)
{ {
std::cerr << std::endl << message << std::endl << std::endl; error_stream << std::endl << message << std::endl << std::endl;
return nullptr; return nullptr;
} }
} }
@ -373,11 +394,11 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, const Setting
if (is_interactive) if (is_interactive)
{ {
std::cout << std::endl; output_stream << std::endl;
WriteBufferFromOStream res_buf(std::cout, 4096); WriteBufferFromOStream res_buf(output_stream, 4096);
formatAST(*res, res_buf); formatAST(*res, res_buf);
res_buf.finalize(); res_buf.finalize();
std::cout << std::endl << std::endl; output_stream << std::endl << std::endl;
} }
return res; return res;
@ -481,7 +502,7 @@ void ClientBase::onData(Block & block, ASTPtr parsed_query)
if (need_render_progress && tty_buf) if (need_render_progress && tty_buf)
{ {
if (select_into_file && !select_into_file_and_stdout) if (select_into_file && !select_into_file_and_stdout)
std::cerr << "\r"; error_stream << "\r";
progress_indication.writeProgress(*tty_buf); progress_indication.writeProgress(*tty_buf);
} }
} }
@ -741,17 +762,17 @@ bool ClientBase::isRegularFile(int fd)
void ClientBase::setDefaultFormatsAndCompressionFromConfiguration() void ClientBase::setDefaultFormatsAndCompressionFromConfiguration()
{ {
if (config().has("output-format")) if (getClientConfiguration().has("output-format"))
{ {
default_output_format = config().getString("output-format"); default_output_format = getClientConfiguration().getString("output-format");
is_default_format = false; is_default_format = false;
} }
else if (config().has("format")) else if (getClientConfiguration().has("format"))
{ {
default_output_format = config().getString("format"); default_output_format = getClientConfiguration().getString("format");
is_default_format = false; is_default_format = false;
} }
else if (config().has("vertical")) else if (getClientConfiguration().has("vertical"))
{ {
default_output_format = "Vertical"; default_output_format = "Vertical";
is_default_format = false; is_default_format = false;
@ -777,17 +798,17 @@ void ClientBase::setDefaultFormatsAndCompressionFromConfiguration()
default_output_format = "TSV"; default_output_format = "TSV";
} }
if (config().has("input-format")) if (getClientConfiguration().has("input-format"))
{ {
default_input_format = config().getString("input-format"); default_input_format = getClientConfiguration().getString("input-format");
} }
else if (config().has("format")) else if (getClientConfiguration().has("format"))
{ {
default_input_format = config().getString("format"); default_input_format = getClientConfiguration().getString("format");
} }
else if (config().getString("table-file", "-") != "-") else if (getClientConfiguration().getString("table-file", "-") != "-")
{ {
auto file_name = config().getString("table-file"); auto file_name = getClientConfiguration().getString("table-file");
std::optional<String> format_from_file_name = FormatFactory::instance().tryGetFormatFromFileName(file_name); std::optional<String> format_from_file_name = FormatFactory::instance().tryGetFormatFromFileName(file_name);
if (format_from_file_name) if (format_from_file_name)
default_input_format = *format_from_file_name; default_input_format = *format_from_file_name;
@ -803,7 +824,7 @@ void ClientBase::setDefaultFormatsAndCompressionFromConfiguration()
default_input_format = "TSV"; default_input_format = "TSV";
} }
format_max_block_size = config().getUInt64("format_max_block_size", format_max_block_size = getClientConfiguration().getUInt64("format_max_block_size",
global_context->getSettingsRef().max_block_size); global_context->getSettingsRef().max_block_size);
/// Setting value from cmd arg overrides one from config /// Setting value from cmd arg overrides one from config
@ -813,7 +834,7 @@ void ClientBase::setDefaultFormatsAndCompressionFromConfiguration()
} }
else else
{ {
insert_format_max_block_size = config().getUInt64("insert_format_max_block_size", insert_format_max_block_size = getClientConfiguration().getUInt64("insert_format_max_block_size",
global_context->getSettingsRef().max_insert_block_size); global_context->getSettingsRef().max_insert_block_size);
} }
} }
@ -924,9 +945,7 @@ void ClientBase::processTextAsSingleQuery(const String & full_query)
const char * begin = full_query.data(); const char * begin = full_query.data();
auto parsed_query = parseQuery(begin, begin + full_query.size(), auto parsed_query = parseQuery(begin, begin + full_query.size(),
global_context->getSettingsRef(), global_context->getSettingsRef(),
/*allow_multi_statements=*/ false, /*allow_multi_statements=*/ false);
is_interactive,
ignore_error);
if (!parsed_query) if (!parsed_query)
return; return;
@ -1100,7 +1119,7 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa
/// has been received yet. /// has been received yet.
if (processed_rows == 0 && e.code() == ErrorCodes::DEADLOCK_AVOIDED && --retries_left) if (processed_rows == 0 && e.code() == ErrorCodes::DEADLOCK_AVOIDED && --retries_left)
{ {
std::cerr << "Got a transient error from the server, will" error_stream << "Got a transient error from the server, will"
<< " retry (" << retries_left << " retries left)"; << " retry (" << retries_left << " retries left)";
} }
else else
@ -1154,7 +1173,7 @@ void ClientBase::receiveResult(ASTPtr parsed_query, Int32 signals_before_stop, b
double elapsed = receive_watch.elapsedSeconds(); double elapsed = receive_watch.elapsedSeconds();
if (break_on_timeout && elapsed > receive_timeout.totalSeconds()) if (break_on_timeout && elapsed > receive_timeout.totalSeconds())
{ {
std::cout << "Timeout exceeded while receiving data from server." output_stream << "Timeout exceeded while receiving data from server."
<< " Waited for " << static_cast<size_t>(elapsed) << " seconds," << " Waited for " << static_cast<size_t>(elapsed) << " seconds,"
<< " timeout is " << receive_timeout.totalSeconds() << " seconds." << std::endl; << " timeout is " << receive_timeout.totalSeconds() << " seconds." << std::endl;
@ -1189,7 +1208,7 @@ void ClientBase::receiveResult(ASTPtr parsed_query, Int32 signals_before_stop, b
if (cancelled && is_interactive) if (cancelled && is_interactive)
{ {
std::cout << "Query was cancelled." << std::endl; output_stream << "Query was cancelled." << std::endl;
cancelled_printed = true; cancelled_printed = true;
} }
} }
@ -1308,9 +1327,9 @@ void ClientBase::onEndOfStream()
if (is_interactive) if (is_interactive)
{ {
if (cancelled && !cancelled_printed) if (cancelled && !cancelled_printed)
std::cout << "Query was cancelled." << std::endl; output_stream << "Query was cancelled." << std::endl;
else if (!written_first_block) else if (!written_first_block)
std::cout << "Ok." << std::endl; output_stream << "Ok." << std::endl;
} }
} }
@ -1863,7 +1882,7 @@ void ClientBase::cancelQuery()
progress_indication.clearProgressOutput(*tty_buf); progress_indication.clearProgressOutput(*tty_buf);
if (is_interactive) if (is_interactive)
std::cout << "Cancelling query." << std::endl; output_stream << "Cancelling query." << std::endl;
cancelled = true; cancelled = true;
} }
@ -2026,7 +2045,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
{ {
const String & new_database = use_query->getDatabase(); const String & new_database = use_query->getDatabase();
/// If the client initiates the reconnection, it takes the settings from the config. /// If the client initiates the reconnection, it takes the settings from the config.
config().setString("database", new_database); getClientConfiguration().setString("database", new_database);
/// If the connection initiates the reconnection, it uses its variable. /// If the connection initiates the reconnection, it uses its variable.
connection->setDefaultDatabase(new_database); connection->setDefaultDatabase(new_database);
} }
@ -2046,21 +2065,21 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
if (is_interactive) if (is_interactive)
{ {
std::cout << std::endl; output_stream << std::endl;
if (!server_exception || processed_rows != 0) if (!server_exception || processed_rows != 0)
std::cout << processed_rows << " row" << (processed_rows == 1 ? "" : "s") << " in set. "; output_stream << processed_rows << " row" << (processed_rows == 1 ? "" : "s") << " in set. ";
std::cout << "Elapsed: " << progress_indication.elapsedSeconds() << " sec. "; output_stream << "Elapsed: " << progress_indication.elapsedSeconds() << " sec. ";
progress_indication.writeFinalProgress(); progress_indication.writeFinalProgress();
std::cout << std::endl << std::endl; output_stream << std::endl << std::endl;
} }
else if (print_time_to_stderr) else if (getClientConfiguration().getBool("print-time-to-stderr", false))
{ {
std::cerr << progress_indication.elapsedSeconds() << "\n"; error_stream << progress_indication.elapsedSeconds() << "\n";
} }
if (!is_interactive && print_num_processed_rows) if (!is_interactive && getClientConfiguration().getBool("print-num-processed-rows", false))
{ {
std::cout << "Processed rows: " << processed_rows << "\n"; output_stream << "Processed rows: " << processed_rows << "\n";
} }
if (have_error && report_error) if (have_error && report_error)
@ -2110,9 +2129,7 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
{ {
parsed_query = parseQuery(this_query_end, all_queries_end, parsed_query = parseQuery(this_query_end, all_queries_end,
global_context->getSettingsRef(), global_context->getSettingsRef(),
/*allow_multi_statements=*/ true, /*allow_multi_statements=*/ true);
is_interactive,
ignore_error);
} }
catch (const Exception & e) catch (const Exception & e)
{ {
@ -2428,12 +2445,12 @@ void ClientBase::initQueryIdFormats()
return; return;
/// Initialize query_id_formats if any /// Initialize query_id_formats if any
if (config().has("query_id_formats")) if (getClientConfiguration().has("query_id_formats"))
{ {
Poco::Util::AbstractConfiguration::Keys keys; Poco::Util::AbstractConfiguration::Keys keys;
config().keys("query_id_formats", keys); getClientConfiguration().keys("query_id_formats", keys);
for (const auto & name : keys) for (const auto & name : keys)
query_id_formats.emplace_back(name + ":", config().getString("query_id_formats." + name)); query_id_formats.emplace_back(name + ":", getClientConfiguration().getString("query_id_formats." + name));
} }
if (query_id_formats.empty()) if (query_id_formats.empty())
@ -2478,9 +2495,9 @@ bool ClientBase::addMergeTreeSettings(ASTCreateQuery & ast_create)
void ClientBase::runInteractive() void ClientBase::runInteractive()
{ {
if (config().has("query_id")) if (getClientConfiguration().has("query_id"))
throw Exception(ErrorCodes::BAD_ARGUMENTS, "query_id could be specified only in non-interactive mode"); throw Exception(ErrorCodes::BAD_ARGUMENTS, "query_id could be specified only in non-interactive mode");
if (print_time_to_stderr) if (getClientConfiguration().getBool("print-time-to-stderr", false))
throw Exception(ErrorCodes::BAD_ARGUMENTS, "time option could be specified only in non-interactive mode"); throw Exception(ErrorCodes::BAD_ARGUMENTS, "time option could be specified only in non-interactive mode");
initQueryIdFormats(); initQueryIdFormats();
@ -2493,9 +2510,9 @@ void ClientBase::runInteractive()
{ {
/// Load suggestion data from the server. /// Load suggestion data from the server.
if (global_context->getApplicationType() == Context::ApplicationType::CLIENT) if (global_context->getApplicationType() == Context::ApplicationType::CLIENT)
suggest->load<Connection>(global_context, connection_parameters, config().getInt("suggestion_limit"), wait_for_suggestions_to_load); suggest->load<Connection>(global_context, connection_parameters, getClientConfiguration().getInt("suggestion_limit"), wait_for_suggestions_to_load);
else if (global_context->getApplicationType() == Context::ApplicationType::LOCAL) else if (global_context->getApplicationType() == Context::ApplicationType::LOCAL)
suggest->load<LocalConnection>(global_context, connection_parameters, config().getInt("suggestion_limit"), wait_for_suggestions_to_load); suggest->load<LocalConnection>(global_context, connection_parameters, getClientConfiguration().getInt("suggestion_limit"), wait_for_suggestions_to_load);
} }
if (home_path.empty()) if (home_path.empty())
@ -2506,8 +2523,8 @@ void ClientBase::runInteractive()
} }
/// Load command history if present. /// Load command history if present.
if (config().has("history_file")) if (getClientConfiguration().has("history_file"))
history_file = config().getString("history_file"); history_file = getClientConfiguration().getString("history_file");
else else
{ {
auto * history_file_from_env = getenv("CLICKHOUSE_HISTORY_FILE"); // NOLINT(concurrency-mt-unsafe) auto * history_file_from_env = getenv("CLICKHOUSE_HISTORY_FILE"); // NOLINT(concurrency-mt-unsafe)
@ -2528,7 +2545,7 @@ void ClientBase::runInteractive()
{ {
if (e.getErrno() != EEXIST) if (e.getErrno() != EEXIST)
{ {
std::cerr << getCurrentExceptionMessage(false) << '\n'; error_stream << getCurrentExceptionMessage(false) << '\n';
} }
} }
} }
@ -2539,13 +2556,13 @@ void ClientBase::runInteractive()
#if USE_REPLXX #if USE_REPLXX
replxx::Replxx::highlighter_callback_t highlight_callback{}; replxx::Replxx::highlighter_callback_t highlight_callback{};
if (config().getBool("highlight", true)) if (getClientConfiguration().getBool("highlight", true))
highlight_callback = highlight; highlight_callback = highlight;
ReplxxLineReader lr( ReplxxLineReader lr(
*suggest, *suggest,
history_file, history_file,
config().has("multiline"), getClientConfiguration().has("multiline"),
query_extenders, query_extenders,
query_delimiters, query_delimiters,
word_break_characters, word_break_characters,
@ -2553,7 +2570,7 @@ void ClientBase::runInteractive()
#else #else
LineReader lr( LineReader lr(
history_file, history_file,
config().has("multiline"), getClientConfiguration().has("multiline"),
query_extenders, query_extenders,
query_delimiters, query_delimiters,
word_break_characters); word_break_characters);
@ -2633,7 +2650,7 @@ void ClientBase::runInteractive()
{ {
// If a separate connection loading suggestions failed to open a new session, // If a separate connection loading suggestions failed to open a new session,
// use the main session to receive them. // use the main session to receive them.
suggest->load(*connection, connection_parameters.timeouts, config().getInt("suggestion_limit"), global_context->getClientInfo()); suggest->load(*connection, connection_parameters.timeouts, getClientConfiguration().getInt("suggestion_limit"), global_context->getClientInfo());
} }
try try
@ -2648,7 +2665,7 @@ void ClientBase::runInteractive()
break; break;
/// We don't need to handle the test hints in the interactive mode. /// We don't need to handle the test hints in the interactive mode.
std::cerr << "Exception on client:" << std::endl << getExceptionMessage(e, print_stack_trace, true) << std::endl << std::endl; error_stream << "Exception on client:" << std::endl << getExceptionMessage(e, print_stack_trace, true) << std::endl << std::endl;
client_exception.reset(e.clone()); client_exception.reset(e.clone());
} }
@ -2665,11 +2682,11 @@ void ClientBase::runInteractive()
while (true); while (true);
if (isNewYearMode()) if (isNewYearMode())
std::cout << "Happy new year." << std::endl; output_stream << "Happy new year." << std::endl;
else if (isChineseNewYearMode(local_tz)) else if (isChineseNewYearMode(local_tz))
std::cout << "Happy Chinese new year. 春节快乐!" << std::endl; output_stream << "Happy Chinese new year. 春节快乐!" << std::endl;
else else
std::cout << "Bye." << std::endl; output_stream << "Bye." << std::endl;
} }
@ -2680,7 +2697,7 @@ bool ClientBase::processMultiQueryFromFile(const String & file_name)
ReadBufferFromFile in(file_name); ReadBufferFromFile in(file_name);
readStringUntilEOF(queries_from_file, in); readStringUntilEOF(queries_from_file, in);
if (!has_log_comment) if (!getClientConfiguration().has("log_comment"))
{ {
Settings settings = global_context->getSettings(); Settings settings = global_context->getSettings();
/// NOTE: cannot use even weakly_canonical() since it fails for /dev/stdin due to resolving of "pipe:[X]" /// NOTE: cannot use even weakly_canonical() since it fails for /dev/stdin due to resolving of "pipe:[X]"
@ -2789,13 +2806,13 @@ void ClientBase::clearTerminal()
/// It is needed if garbage is left in terminal. /// It is needed if garbage is left in terminal.
/// Show cursor. It can be left hidden by invocation of previous programs. /// Show cursor. It can be left hidden by invocation of previous programs.
/// A test for this feature: perl -e 'print "x"x100000'; echo -ne '\033[0;0H\033[?25l'; clickhouse-client /// A test for this feature: perl -e 'print "x"x100000'; echo -ne '\033[0;0H\033[?25l'; clickhouse-client
std::cout << "\033[0J" "\033[?25h"; output_stream << "\033[0J" "\033[?25h";
} }
void ClientBase::showClientVersion() void ClientBase::showClientVersion()
{ {
std::cout << VERSION_NAME << " " + getName() + " version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl; output_stream << VERSION_NAME << " " + getName() + " version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
} }
namespace namespace
@ -3080,18 +3097,18 @@ void ClientBase::init(int argc, char ** argv)
if (options.count("version-clean")) if (options.count("version-clean"))
{ {
std::cout << VERSION_STRING; output_stream << VERSION_STRING;
exit(0); // NOLINT(concurrency-mt-unsafe) exit(0); // NOLINT(concurrency-mt-unsafe)
} }
if (options.count("verbose")) if (options.count("verbose"))
config().setBool("verbose", true); getClientConfiguration().setBool("verbose", true);
/// Output of help message. /// Output of help message.
if (options.count("help") if (options.count("help")
|| (options.count("host") && options["host"].as<std::string>() == "elp")) /// If user writes -help instead of --help. || (options.count("host") && options["host"].as<std::string>() == "elp")) /// If user writes -help instead of --help.
{ {
if (config().getBool("verbose", false)) if (getClientConfiguration().getBool("verbose", false))
printHelpMessage(options_description, true); printHelpMessage(options_description, true);
else else
printHelpMessage(options_description_non_verbose, false); printHelpMessage(options_description_non_verbose, false);
@ -3099,72 +3116,75 @@ void ClientBase::init(int argc, char ** argv)
} }
/// Common options for clickhouse-client and clickhouse-local. /// Common options for clickhouse-client and clickhouse-local.
/// Output execution time to stderr in batch mode.
if (options.count("time")) if (options.count("time"))
print_time_to_stderr = true; getClientConfiguration().setBool("print-time-to-stderr", true);
if (options.count("query")) if (options.count("query"))
queries = options["query"].as<std::vector<std::string>>(); queries = options["query"].as<std::vector<std::string>>();
if (options.count("query_id")) if (options.count("query_id"))
config().setString("query_id", options["query_id"].as<std::string>()); getClientConfiguration().setString("query_id", options["query_id"].as<std::string>());
if (options.count("database")) if (options.count("database"))
config().setString("database", options["database"].as<std::string>()); getClientConfiguration().setString("database", options["database"].as<std::string>());
if (options.count("config-file")) if (options.count("config-file"))
config().setString("config-file", options["config-file"].as<std::string>()); getClientConfiguration().setString("config-file", options["config-file"].as<std::string>());
if (options.count("queries-file")) if (options.count("queries-file"))
queries_files = options["queries-file"].as<std::vector<std::string>>(); queries_files = options["queries-file"].as<std::vector<std::string>>();
if (options.count("multiline")) if (options.count("multiline"))
config().setBool("multiline", true); getClientConfiguration().setBool("multiline", true);
if (options.count("multiquery")) if (options.count("multiquery"))
config().setBool("multiquery", true); getClientConfiguration().setBool("multiquery", true);
if (options.count("ignore-error")) if (options.count("ignore-error"))
config().setBool("ignore-error", true); getClientConfiguration().setBool("ignore-error", true);
if (options.count("format")) if (options.count("format"))
config().setString("format", options["format"].as<std::string>()); getClientConfiguration().setString("format", options["format"].as<std::string>());
if (options.count("output-format")) if (options.count("output-format"))
config().setString("output-format", options["output-format"].as<std::string>()); getClientConfiguration().setString("output-format", options["output-format"].as<std::string>());
if (options.count("vertical")) if (options.count("vertical"))
config().setBool("vertical", true); getClientConfiguration().setBool("vertical", true);
if (options.count("stacktrace")) if (options.count("stacktrace"))
config().setBool("stacktrace", true); getClientConfiguration().setBool("stacktrace", true);
if (options.count("print-profile-events")) if (options.count("print-profile-events"))
config().setBool("print-profile-events", true); getClientConfiguration().setBool("print-profile-events", true);
if (options.count("profile-events-delay-ms")) if (options.count("profile-events-delay-ms"))
config().setUInt64("profile-events-delay-ms", options["profile-events-delay-ms"].as<UInt64>()); getClientConfiguration().setUInt64("profile-events-delay-ms", options["profile-events-delay-ms"].as<UInt64>());
/// Whether to print the number of processed rows at
if (options.count("processed-rows")) if (options.count("processed-rows"))
print_num_processed_rows = true; getClientConfiguration().setBool("print-num-processed-rows", true);
if (options.count("progress")) if (options.count("progress"))
{ {
switch (options["progress"].as<ProgressOption>()) switch (options["progress"].as<ProgressOption>())
{ {
case DEFAULT: case DEFAULT:
config().setString("progress", "default"); getClientConfiguration().setString("progress", "default");
break; break;
case OFF: case OFF:
config().setString("progress", "off"); getClientConfiguration().setString("progress", "off");
break; break;
case TTY: case TTY:
config().setString("progress", "tty"); getClientConfiguration().setString("progress", "tty");
break; break;
case ERR: case ERR:
config().setString("progress", "err"); getClientConfiguration().setString("progress", "err");
break; break;
} }
} }
if (options.count("echo")) if (options.count("echo"))
config().setBool("echo", true); getClientConfiguration().setBool("echo", true);
if (options.count("disable_suggestion")) if (options.count("disable_suggestion"))
config().setBool("disable_suggestion", true); getClientConfiguration().setBool("disable_suggestion", true);
if (options.count("wait_for_suggestions_to_load")) if (options.count("wait_for_suggestions_to_load"))
config().setBool("wait_for_suggestions_to_load", true); getClientConfiguration().setBool("wait_for_suggestions_to_load", true);
if (options.count("suggestion_limit")) if (options.count("suggestion_limit"))
config().setInt("suggestion_limit", options["suggestion_limit"].as<int>()); getClientConfiguration().setInt("suggestion_limit", options["suggestion_limit"].as<int>());
if (options.count("highlight")) if (options.count("highlight"))
config().setBool("highlight", options["highlight"].as<bool>()); getClientConfiguration().setBool("highlight", options["highlight"].as<bool>());
if (options.count("history_file")) if (options.count("history_file"))
config().setString("history_file", options["history_file"].as<std::string>()); getClientConfiguration().setString("history_file", options["history_file"].as<std::string>());
if (options.count("interactive")) if (options.count("interactive"))
config().setBool("interactive", true); getClientConfiguration().setBool("interactive", true);
if (options.count("pager")) if (options.count("pager"))
config().setString("pager", options["pager"].as<std::string>()); getClientConfiguration().setString("pager", options["pager"].as<std::string>());
if (options.count("log-level")) if (options.count("log-level"))
Poco::Logger::root().setLevel(options["log-level"].as<std::string>()); Poco::Logger::root().setLevel(options["log-level"].as<std::string>());
@ -3182,13 +3202,13 @@ void ClientBase::init(int argc, char ** argv)
alias_names.reserve(options_description.main_description->options().size()); alias_names.reserve(options_description.main_description->options().size());
for (const auto& option : options_description.main_description->options()) for (const auto& option : options_description.main_description->options())
alias_names.insert(option->long_name()); alias_names.insert(option->long_name());
argsToConfig(common_arguments, config(), 100, &alias_names); argsToConfig(common_arguments, getClientConfiguration(), 100, &alias_names);
} }
clearPasswordFromCommandLine(argc, argv); clearPasswordFromCommandLine(argc, argv);
/// Limit on total memory usage /// Limit on total memory usage
std::string max_client_memory_usage = config().getString("max_memory_usage_in_client", "0" /*default value*/); std::string max_client_memory_usage = getClientConfiguration().getString("max_memory_usage_in_client", "0" /*default value*/);
if (max_client_memory_usage != "0") if (max_client_memory_usage != "0")
{ {
UInt64 max_client_memory_usage_int = parseWithSizeSuffix<UInt64>(max_client_memory_usage.c_str(), max_client_memory_usage.length()); UInt64 max_client_memory_usage_int = parseWithSizeSuffix<UInt64>(max_client_memory_usage.c_str(), max_client_memory_usage.length());
@ -3197,8 +3217,6 @@ void ClientBase::init(int argc, char ** argv)
total_memory_tracker.setDescription("(total)"); total_memory_tracker.setDescription("(total)");
total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
} }
has_log_comment = config().has("log_comment");
} }
} }

35
src/Client/ClientBase.h
View File

@ -18,7 +18,6 @@
#include <Storages/SelectQueryInfo.h> #include <Storages/SelectQueryInfo.h>
#include <Storages/MergeTree/MergeTreeSettings.h> #include <Storages/MergeTree/MergeTreeSettings.h>
namespace po = boost::program_options; namespace po = boost::program_options;
@ -67,13 +66,22 @@ class ClientBase : public Poco::Util::Application, public IHints<2>
public: public:
using Arguments = std::vector<String>; using Arguments = std::vector<String>;
ClientBase(); explicit ClientBase
(
int in_fd_ = STDIN_FILENO,
int out_fd_ = STDOUT_FILENO,
int err_fd_ = STDERR_FILENO,
std::istream & input_stream_ = std::cin,
std::ostream & output_stream_ = std::cout,
std::ostream & error_stream_ = std::cerr
);
~ClientBase() override; ~ClientBase() override;
void init(int argc, char ** argv); void init(int argc, char ** argv);
std::vector<String> getAllRegisteredNames() const override { return cmd_options; } std::vector<String> getAllRegisteredNames() const override { return cmd_options; }
static ASTPtr parseQuery(const char *& pos, const char * end, const Settings & settings, bool allow_multi_statements, bool is_interactive, bool ignore_error); ASTPtr parseQuery(const char *& pos, const char * end, const Settings & settings, bool allow_multi_statements);
protected: protected:
void runInteractive(); void runInteractive();
@ -82,6 +90,9 @@ protected:
char * argv0 = nullptr; char * argv0 = nullptr;
void runLibFuzzer(); void runLibFuzzer();
/// This is the analogue of Poco::Application::config()
virtual Poco::Util::LayeredConfiguration & getClientConfiguration() = 0;
virtual bool processWithFuzzing(const String &) virtual bool processWithFuzzing(const String &)
{ {
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Query processing with fuzzing is not implemented"); throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Query processing with fuzzing is not implemented");
@ -107,7 +118,7 @@ protected:
String & query_to_execute, ASTPtr & parsed_query, const String & all_queries_text, String & query_to_execute, ASTPtr & parsed_query, const String & all_queries_text,
std::unique_ptr<Exception> & current_exception); std::unique_ptr<Exception> & current_exception);
static void clearTerminal(); void clearTerminal();
void showClientVersion(); void showClientVersion();
using ProgramOptionsDescription = boost::program_options::options_description; using ProgramOptionsDescription = boost::program_options::options_description;
@ -206,7 +217,6 @@ protected:
bool echo_queries = false; /// Print queries before execution in batch mode. bool echo_queries = false; /// Print queries before execution in batch mode.
bool ignore_error = false; /// In case of errors, don't print error message, continue to next query. Only applicable for non-interactive mode. bool ignore_error = false; /// In case of errors, don't print error message, continue to next query. Only applicable for non-interactive mode.
bool print_time_to_stderr = false; /// Output execution time to stderr in batch mode.
std::optional<Suggest> suggest; std::optional<Suggest> suggest;
bool load_suggestions = false; bool load_suggestions = false;
@ -251,9 +261,9 @@ protected:
ConnectionParameters connection_parameters; ConnectionParameters connection_parameters;
/// Buffer that reads from stdin in batch mode. /// Buffer that reads from stdin in batch mode.
ReadBufferFromFileDescriptor std_in{STDIN_FILENO}; ReadBufferFromFileDescriptor std_in;
/// Console output. /// Console output.
WriteBufferFromFileDescriptor std_out{STDOUT_FILENO}; WriteBufferFromFileDescriptor std_out;
std::unique_ptr<ShellCommand> pager_cmd; std::unique_ptr<ShellCommand> pager_cmd;
/// The user can specify to redirect query output to a file. /// The user can specify to redirect query output to a file.
@ -284,7 +294,6 @@ protected:
bool need_render_profile_events = true; bool need_render_profile_events = true;
bool written_first_block = false; bool written_first_block = false;
size_t processed_rows = 0; /// How many rows have been read or written. size_t processed_rows = 0; /// How many rows have been read or written.
bool print_num_processed_rows = false; /// Whether to print the number of processed rows at
bool print_stack_trace = false; bool print_stack_trace = false;
/// The last exception that was received from the server. Is used for the /// The last exception that was received from the server. Is used for the
@ -332,8 +341,14 @@ protected:
bool cancelled = false; bool cancelled = false;
bool cancelled_printed = false; bool cancelled_printed = false;
/// Does log_comment has specified by user? /// Unpacked descriptors and streams for the ease of use.
bool has_log_comment = false; int in_fd = STDIN_FILENO;
int out_fd = STDOUT_FILENO;
int err_fd = STDERR_FILENO;
std::istream & input_stream;
std::ostream & output_stream;
std::ostream & error_stream;
}; };
} }

35
src/Client/LineReader.cpp
View File

@ -23,14 +23,6 @@ void trim(String & s)
s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end()); s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end());
} }
/// Check if multi-line query is inserted from the paste buffer.
/// Allows delaying the start of query execution until the entirety of query is inserted.
bool hasInputData()
{
pollfd fd{STDIN_FILENO, POLLIN, 0};
return poll(&fd, 1, 0) == 1;
}
struct NoCaseCompare struct NoCaseCompare
{ {
bool operator()(const std::string & str1, const std::string & str2) bool operator()(const std::string & str1, const std::string & str2)
@ -63,6 +55,14 @@ void addNewWords(Words & to, const Words & from, Compare comp)
namespace DB namespace DB
{ {
/// Check if multi-line query is inserted from the paste buffer.
/// Allows delaying the start of query execution until the entirety of query is inserted.
bool LineReader::hasInputData() const
{
pollfd fd{in_fd, POLLIN, 0};
return poll(&fd, 1, 0) == 1;
}
replxx::Replxx::completions_t LineReader::Suggest::getCompletions(const String & prefix, size_t prefix_length, const char * word_break_characters) replxx::Replxx::completions_t LineReader::Suggest::getCompletions(const String & prefix, size_t prefix_length, const char * word_break_characters)
{ {
std::string_view last_word; std::string_view last_word;
@ -131,11 +131,22 @@ void LineReader::Suggest::addWords(Words && new_words) // NOLINT(cppcoreguidelin
} }
} }
LineReader::LineReader(const String & history_file_path_, bool multiline_, Patterns extenders_, Patterns delimiters_) LineReader::LineReader(
const String & history_file_path_,
bool multiline_,
Patterns extenders_,
Patterns delimiters_,
std::istream & input_stream_,
std::ostream & output_stream_,
int in_fd_
)
: history_file_path(history_file_path_) : history_file_path(history_file_path_)
, multiline(multiline_) , multiline(multiline_)
, extenders(std::move(extenders_)) , extenders(std::move(extenders_))
, delimiters(std::move(delimiters_)) , delimiters(std::move(delimiters_))
, input_stream(input_stream_)
, output_stream(output_stream_)
, in_fd(in_fd_)
{ {
/// FIXME: check extender != delimiter /// FIXME: check extender != delimiter
} }
@ -212,9 +223,9 @@ LineReader::InputStatus LineReader::readOneLine(const String & prompt)
input.clear(); input.clear();
{ {
std::cout << prompt; output_stream << prompt;
std::getline(std::cin, input); std::getline(input_stream, input);
if (!std::cin.good()) if (!input_stream.good())
return ABORT; return ABORT;
} }

19
src/Client/LineReader.h
View File

@ -1,5 +1,7 @@
#pragma once #pragma once
#include <iostream>
#include <unistd.h>
#include <mutex> #include <mutex>
#include <atomic> #include <atomic>
#include <vector> #include <vector>
@ -37,7 +39,16 @@ public:
using Patterns = std::vector<const char *>; using Patterns = std::vector<const char *>;
LineReader(const String & history_file_path, bool multiline, Patterns extenders, Patterns delimiters); LineReader(
const String & history_file_path,
bool multiline,
Patterns extenders,
Patterns delimiters,
std::istream & input_stream_ = std::cin,
std::ostream & output_stream_ = std::cout,
int in_fd_ = STDIN_FILENO
);
virtual ~LineReader() = default; virtual ~LineReader() = default;
/// Reads the whole line until delimiter (in multiline mode) or until the last line without extender. /// Reads the whole line until delimiter (in multiline mode) or until the last line without extender.
@ -56,6 +67,8 @@ public:
virtual void enableBracketedPaste() {} virtual void enableBracketedPaste() {}
virtual void disableBracketedPaste() {} virtual void disableBracketedPaste() {}
bool hasInputData() const;
protected: protected:
enum InputStatus enum InputStatus
{ {
@ -77,6 +90,10 @@ protected:
virtual InputStatus readOneLine(const String & prompt); virtual InputStatus readOneLine(const String & prompt);
virtual void addToHistory(const String &) {} virtual void addToHistory(const String &) {}
std::istream & input_stream;
std::ostream & output_stream;
int in_fd;
}; };
} }

29
src/Client/LocalConnection.cpp
View File

@ -16,7 +16,10 @@
#include <Storages/IStorage.h> #include <Storages/IStorage.h>
#include <Common/ConcurrentBoundedQueue.h> #include <Common/ConcurrentBoundedQueue.h>
#include <Common/CurrentThread.h> #include <Common/CurrentThread.h>
#include <Parsers/ParserQuery.h>
#include <Parsers/PRQL/ParserPRQLQuery.h>
#include <Parsers/Kusto/ParserKQLStatement.h>
#include <Parsers/Kusto/parseKQLQuery.h>
namespace DB namespace DB
{ {
@ -151,12 +154,26 @@ void LocalConnection::sendQuery(
state->block = sample; state->block = sample;
String current_format = "Values"; String current_format = "Values";
const auto & settings = context->getSettingsRef();
const char * begin = state->query.data(); const char * begin = state->query.data();
auto parsed_query = ClientBase::parseQuery(begin, begin + state->query.size(), const char * end = begin + state->query.size();
context->getSettingsRef(), const Dialect & dialect = settings.dialect;
/*allow_multi_statements=*/ false,
/*is_interactive=*/ false, std::unique_ptr<IParserBase> parser;
/*ignore_error=*/ false); if (dialect == Dialect::kusto)
parser = std::make_unique<ParserKQLStatement>(end, settings.allow_settings_after_format_in_insert);
else if (dialect == Dialect::prql)
parser = std::make_unique<ParserPRQLQuery>(settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks);
else
parser = std::make_unique<ParserQuery>(end, settings.allow_settings_after_format_in_insert);
ASTPtr parsed_query;
if (dialect == Dialect::kusto)
parsed_query = parseKQLQueryAndMovePosition(*parser, begin, end, "", /*allow_multi_statements*/false, settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks);
else
parsed_query = parseQueryAndMovePosition(*parser, begin, end, "", /*allow_multi_statements*/false, settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks);
if (const auto * insert = parsed_query->as<ASTInsertQuery>()) if (const auto * insert = parsed_query->as<ASTInsertQuery>())
{ {
if (!insert->format.empty()) if (!insert->format.empty())

17
src/Client/ReplxxLineReader.cpp
View File

@ -297,8 +297,15 @@ ReplxxLineReader::ReplxxLineReader(
Patterns extenders_, Patterns extenders_,
Patterns delimiters_, Patterns delimiters_,
const char word_break_characters_[], const char word_break_characters_[],
replxx::Replxx::highlighter_callback_t highlighter_) replxx::Replxx::highlighter_callback_t highlighter_,
: LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_)), highlighter(std::move(highlighter_)) [[ maybe_unused ]] std::istream & input_stream_,
[[ maybe_unused ]] std::ostream & output_stream_,
[[ maybe_unused ]] int in_fd_,
[[ maybe_unused ]] int out_fd_,
[[ maybe_unused ]] int err_fd_
)
: LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_), input_stream_, output_stream_, in_fd_)
, highlighter(std::move(highlighter_))
, word_break_characters(word_break_characters_) , word_break_characters(word_break_characters_)
, editor(getEditor()) , editor(getEditor())
{ {
@ -471,7 +478,7 @@ ReplxxLineReader::ReplxxLineReader(
ReplxxLineReader::~ReplxxLineReader() ReplxxLineReader::~ReplxxLineReader()
{ {
if (close(history_file_fd)) if (history_file_fd >= 0 && close(history_file_fd))
rx.print("Close of history file failed: %s\n", errnoToString().c_str()); rx.print("Close of history file failed: %s\n", errnoToString().c_str());
} }
@ -496,7 +503,7 @@ void ReplxxLineReader::addToHistory(const String & line)
// but replxx::Replxx::history_load() does not // but replxx::Replxx::history_load() does not
// and that is why flock() is added here. // and that is why flock() is added here.
bool locked = false; bool locked = false;
if (flock(history_file_fd, LOCK_EX)) if (history_file_fd >= 0 && flock(history_file_fd, LOCK_EX))
rx.print("Lock of history file failed: %s\n", errnoToString().c_str()); rx.print("Lock of history file failed: %s\n", errnoToString().c_str());
else else
locked = true; locked = true;
@ -507,7 +514,7 @@ void ReplxxLineReader::addToHistory(const String & line)
if (!rx.history_save(history_file_path)) if (!rx.history_save(history_file_path))
rx.print("Saving history failed: %s\n", errnoToString().c_str()); rx.print("Saving history failed: %s\n", errnoToString().c_str());
if (locked && 0 != flock(history_file_fd, LOCK_UN)) if (history_file_fd >= 0 && locked && 0 != flock(history_file_fd, LOCK_UN))
rx.print("Unlock of history file failed: %s\n", errnoToString().c_str()); rx.print("Unlock of history file failed: %s\n", errnoToString().c_str());
} }

15
src/Client/ReplxxLineReader.h
View File

@ -1,6 +1,7 @@
#pragma once #pragma once
#include "LineReader.h" #include <Client/LineReader.h>
#include <base/strong_typedef.h>
#include <replxx.hxx> #include <replxx.hxx>
namespace DB namespace DB
@ -9,14 +10,22 @@ namespace DB
class ReplxxLineReader : public LineReader class ReplxxLineReader : public LineReader
{ {
public: public:
ReplxxLineReader( ReplxxLineReader
(
Suggest & suggest, Suggest & suggest,
const String & history_file_path, const String & history_file_path,
bool multiline, bool multiline,
Patterns extenders_, Patterns extenders_,
Patterns delimiters_, Patterns delimiters_,
const char word_break_characters_[], const char word_break_characters_[],
replxx::Replxx::highlighter_callback_t highlighter_); replxx::Replxx::highlighter_callback_t highlighter_,
std::istream & input_stream_ = std::cin,
std::ostream & output_stream_ = std::cout,
int in_fd_ = STDIN_FILENO,
int out_fd_ = STDOUT_FILENO,
int err_fd_ = STDERR_FILENO
);
~ReplxxLineReader() override; ~ReplxxLineReader() override;
void enableBracketedPaste() override; void enableBracketedPaste() override;

10
src/Common/ProgressIndication.cpp
View File

@ -92,19 +92,19 @@ void ProgressIndication::writeFinalProgress()
if (progress.read_rows < 1000) if (progress.read_rows < 1000)
return; return;
std::cout << "Processed " << formatReadableQuantity(progress.read_rows) << " rows, " output_stream << "Processed " << formatReadableQuantity(progress.read_rows) << " rows, "
<< formatReadableSizeWithDecimalSuffix(progress.read_bytes); << formatReadableSizeWithDecimalSuffix(progress.read_bytes);
UInt64 elapsed_ns = getElapsedNanoseconds(); UInt64 elapsed_ns = getElapsedNanoseconds();
if (elapsed_ns) if (elapsed_ns)
std::cout << " (" << formatReadableQuantity(progress.read_rows * 1000000000.0 / elapsed_ns) << " rows/s., " output_stream << " (" << formatReadableQuantity(progress.read_rows * 1000000000.0 / elapsed_ns) << " rows/s., "
<< formatReadableSizeWithDecimalSuffix(progress.read_bytes * 1000000000.0 / elapsed_ns) << "/s.)"; << formatReadableSizeWithDecimalSuffix(progress.read_bytes * 1000000000.0 / elapsed_ns) << "/s.)";
else else
std::cout << ". "; output_stream << ". ";
auto peak_memory_usage = getMemoryUsage().peak; auto peak_memory_usage = getMemoryUsage().peak;
if (peak_memory_usage >= 0) if (peak_memory_usage >= 0)
std::cout << "\nPeak memory usage: " << formatReadableSizeWithBinarySuffix(peak_memory_usage) << "."; output_stream << "\nPeak memory usage: " << formatReadableSizeWithBinarySuffix(peak_memory_usage) << ".";
} }
void ProgressIndication::writeProgress(WriteBufferFromFileDescriptor & message) void ProgressIndication::writeProgress(WriteBufferFromFileDescriptor & message)
@ -125,7 +125,7 @@ void ProgressIndication::writeProgress(WriteBufferFromFileDescriptor & message)
const char * indicator = indicators[increment % 8]; const char * indicator = indicators[increment % 8];
size_t terminal_width = getTerminalWidth(); size_t terminal_width = getTerminalWidth(in_fd, err_fd);
if (!written_progress_chars) if (!written_progress_chars)
{ {

17
src/Common/ProgressIndication.h
View File

@ -32,6 +32,19 @@ using HostToTimesMap = std::unordered_map<String, ThreadEventData>;
class ProgressIndication class ProgressIndication
{ {
public: public:
explicit ProgressIndication
(
std::ostream & output_stream_ = std::cout,
int in_fd_ = STDIN_FILENO,
int err_fd_ = STDERR_FILENO
)
: output_stream(output_stream_),
in_fd(in_fd_),
err_fd(err_fd_)
{
}
/// Write progress bar. /// Write progress bar.
void writeProgress(WriteBufferFromFileDescriptor & message); void writeProgress(WriteBufferFromFileDescriptor & message);
void clearProgressOutput(WriteBufferFromFileDescriptor & message); void clearProgressOutput(WriteBufferFromFileDescriptor & message);
@ -103,6 +116,10 @@ private:
/// - hosts_data/cpu_usage_meter (guarded with profile_events_mutex) /// - hosts_data/cpu_usage_meter (guarded with profile_events_mutex)
mutable std::mutex profile_events_mutex; mutable std::mutex profile_events_mutex;
mutable std::mutex progress_mutex; mutable std::mutex progress_mutex;
std::ostream & output_stream;
int in_fd;
int err_fd;
}; };
} }

10
src/Common/TerminalSize.cpp
View File

@ -13,17 +13,17 @@ namespace DB::ErrorCodes
extern const int SYSTEM_ERROR; extern const int SYSTEM_ERROR;
} }
uint16_t getTerminalWidth() uint16_t getTerminalWidth(int in_fd, int err_fd)
{ {
struct winsize terminal_size {}; struct winsize terminal_size {};
if (isatty(STDIN_FILENO)) if (isatty(in_fd))
{ {
if (ioctl(STDIN_FILENO, TIOCGWINSZ, &terminal_size)) if (ioctl(in_fd, TIOCGWINSZ, &terminal_size))
throw DB::ErrnoException(DB::ErrorCodes::SYSTEM_ERROR, "Cannot obtain terminal window size (ioctl TIOCGWINSZ)"); throw DB::ErrnoException(DB::ErrorCodes::SYSTEM_ERROR, "Cannot obtain terminal window size (ioctl TIOCGWINSZ)");
} }
else if (isatty(STDERR_FILENO)) else if (isatty(err_fd))
{ {
if (ioctl(STDERR_FILENO, TIOCGWINSZ, &terminal_size)) if (ioctl(err_fd, TIOCGWINSZ, &terminal_size))
throw DB::ErrnoException(DB::ErrorCodes::SYSTEM_ERROR, "Cannot obtain terminal window size (ioctl TIOCGWINSZ)"); throw DB::ErrnoException(DB::ErrorCodes::SYSTEM_ERROR, "Cannot obtain terminal window size (ioctl TIOCGWINSZ)");
} }
/// Default - 0. /// Default - 0.

4
src/Common/TerminalSize.h
View File

@ -1,16 +1,16 @@
#pragma once #pragma once
#include <string> #include <string>
#include <unistd.h>
#include <boost/program_options.hpp> #include <boost/program_options.hpp>
namespace po = boost::program_options; namespace po = boost::program_options;
uint16_t getTerminalWidth(); uint16_t getTerminalWidth(int in_fd = STDIN_FILENO, int err_fd = STDERR_FILENO);
/** Creates po::options_description with name and an appropriate size for option displaying /** Creates po::options_description with name and an appropriate size for option displaying
* when program is called with option --help * when program is called with option --help
* */ * */
po::options_description createOptionsDescription(const std::string &caption, unsigned short terminal_width); /// NOLINT po::options_description createOptionsDescription(const std::string &caption, unsigned short terminal_width); /// NOLINT

3
src/Common/ZooKeeper/examples/CMakeLists.txt
View File

@ -1,15 +1,18 @@
clickhouse_add_executable(zkutil_test_commands zkutil_test_commands.cpp) clickhouse_add_executable(zkutil_test_commands zkutil_test_commands.cpp)
target_link_libraries(zkutil_test_commands PRIVATE target_link_libraries(zkutil_test_commands PRIVATE
clickhouse_common_zookeeper_no_log clickhouse_common_zookeeper_no_log
clickhouse_functions
dbms) dbms)
clickhouse_add_executable(zkutil_test_commands_new_lib zkutil_test_commands_new_lib.cpp) clickhouse_add_executable(zkutil_test_commands_new_lib zkutil_test_commands_new_lib.cpp)
target_link_libraries(zkutil_test_commands_new_lib PRIVATE target_link_libraries(zkutil_test_commands_new_lib PRIVATE
clickhouse_common_zookeeper_no_log clickhouse_common_zookeeper_no_log
clickhouse_compression clickhouse_compression
clickhouse_functions
dbms) dbms)
clickhouse_add_executable(zkutil_test_async zkutil_test_async.cpp) clickhouse_add_executable(zkutil_test_async zkutil_test_async.cpp)
target_link_libraries(zkutil_test_async PRIVATE target_link_libraries(zkutil_test_async PRIVATE
clickhouse_common_zookeeper_no_log clickhouse_common_zookeeper_no_log
clickhouse_functions
dbms) dbms)

6
src/Common/examples/CMakeLists.txt
View File

@ -11,10 +11,10 @@ clickhouse_add_executable (small_table small_table.cpp)
target_link_libraries (small_table PRIVATE clickhouse_common_io) target_link_libraries (small_table PRIVATE clickhouse_common_io)
clickhouse_add_executable (parallel_aggregation parallel_aggregation.cpp) clickhouse_add_executable (parallel_aggregation parallel_aggregation.cpp)
target_link_libraries (parallel_aggregation PRIVATE dbms) target_link_libraries (parallel_aggregation PRIVATE dbms clickhouse_functions)
clickhouse_add_executable (parallel_aggregation2 parallel_aggregation2.cpp) clickhouse_add_executable (parallel_aggregation2 parallel_aggregation2.cpp)
target_link_libraries (parallel_aggregation2 PRIVATE dbms) target_link_libraries (parallel_aggregation2 PRIVATE dbms clickhouse_functions)
clickhouse_add_executable (int_hashes_perf int_hashes_perf.cpp) clickhouse_add_executable (int_hashes_perf int_hashes_perf.cpp)
target_link_libraries (int_hashes_perf PRIVATE clickhouse_common_io) target_link_libraries (int_hashes_perf PRIVATE clickhouse_common_io)
@ -85,7 +85,7 @@ target_link_libraries (interval_tree PRIVATE dbms)
if (ENABLE_SSL) if (ENABLE_SSL)
clickhouse_add_executable (encrypt_decrypt encrypt_decrypt.cpp) clickhouse_add_executable (encrypt_decrypt encrypt_decrypt.cpp)
target_link_libraries (encrypt_decrypt PRIVATE dbms) target_link_libraries (encrypt_decrypt PRIVATE dbms clickhouse_functions)
endif() endif()
clickhouse_add_executable (check_pointer_valid check_pointer_valid.cpp) clickhouse_add_executable (check_pointer_valid check_pointer_valid.cpp)

1
src/Core/Settings.h
View File

@ -1093,6 +1093,7 @@ class IColumn;
M(Bool, input_format_json_defaults_for_missing_elements_in_named_tuple, true, "Insert default value in named tuple element if it's missing in json object", 0) \ M(Bool, input_format_json_defaults_for_missing_elements_in_named_tuple, true, "Insert default value in named tuple element if it's missing in json object", 0) \
M(Bool, input_format_json_throw_on_bad_escape_sequence, true, "Throw an exception if JSON string contains bad escape sequence in JSON input formats. If disabled, bad escape sequences will remain as is in the data", 0) \ M(Bool, input_format_json_throw_on_bad_escape_sequence, true, "Throw an exception if JSON string contains bad escape sequence in JSON input formats. If disabled, bad escape sequences will remain as is in the data", 0) \
M(Bool, input_format_json_ignore_unnecessary_fields, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields", 0) \ M(Bool, input_format_json_ignore_unnecessary_fields, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields", 0) \
M(Bool, input_format_json_ignore_key_case, false, "Ignore json key case while read json field from string", 0) \
M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \ M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \
M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \ M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \
M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \ M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \

1
src/Core/SettingsChangesHistory.h
View File

@ -117,6 +117,7 @@ static const std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges
{"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."}, {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
{"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."}, {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
{"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."}, {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
{"input_format_json_ignore_key_case", false, false, "Ignore json key case while read json field from string."},
{"parallel_replicas_local_plan", false, true, "Use local plan for local replica in a query with parallel replicas"}, {"parallel_replicas_local_plan", false, true, "Use local plan for local replica in a query with parallel replicas"},
}}, }},
{"24.5", {{"allow_deprecated_error_prone_window_functions", true, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)"}, {"24.5", {{"allow_deprecated_error_prone_window_functions", true, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)"},

1
src/Formats/FormatFactory.cpp
View File

@ -149,6 +149,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
format_settings.json.try_infer_objects_as_tuples = settings.input_format_json_try_infer_named_tuples_from_objects; format_settings.json.try_infer_objects_as_tuples = settings.input_format_json_try_infer_named_tuples_from_objects;
format_settings.json.throw_on_bad_escape_sequence = settings.input_format_json_throw_on_bad_escape_sequence; format_settings.json.throw_on_bad_escape_sequence = settings.input_format_json_throw_on_bad_escape_sequence;
format_settings.json.ignore_unnecessary_fields = settings.input_format_json_ignore_unnecessary_fields; format_settings.json.ignore_unnecessary_fields = settings.input_format_json_ignore_unnecessary_fields;
format_settings.json.ignore_key_case = settings.input_format_json_ignore_key_case;
format_settings.null_as_default = settings.input_format_null_as_default; format_settings.null_as_default = settings.input_format_null_as_default;
format_settings.force_null_for_omitted_fields = settings.input_format_force_null_for_omitted_fields; format_settings.force_null_for_omitted_fields = settings.input_format_force_null_for_omitted_fields;
format_settings.decimal_trailing_zeros = settings.output_format_decimal_trailing_zeros; format_settings.decimal_trailing_zeros = settings.output_format_decimal_trailing_zeros;

1
src/Formats/FormatSettings.h
View File

@ -228,6 +228,7 @@ struct FormatSettings
bool infer_incomplete_types_as_strings = true; bool infer_incomplete_types_as_strings = true;
bool throw_on_bad_escape_sequence = true; bool throw_on_bad_escape_sequence = true;
bool ignore_unnecessary_fields = true; bool ignore_unnecessary_fields = true;
bool ignore_key_case = false;
} json{}; } json{};
struct struct

26
src/Functions/CMakeLists.txt
View File

@ -3,35 +3,9 @@ add_subdirectory(divide)
include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake")
add_headers_and_sources(clickhouse_functions .) add_headers_and_sources(clickhouse_functions .)
set(DBMS_FUNCTIONS
IFunction.cpp
FunctionFactory.cpp
FunctionHelpers.cpp
extractTimeZoneFromFunctionArguments.cpp
FunctionsLogical.cpp
if.cpp
multiIf.cpp
multiMatchAny.cpp
checkHyperscanRegexp.cpp
array/has.cpp
CastOverloadResolver.cpp
# Provides dependency for cast - createFunctionBaseCast()
FunctionsConversion.cpp
)
extract_into_parent_list(clickhouse_functions_sources dbms_sources ${DBMS_FUNCTIONS})
extract_into_parent_list(clickhouse_functions_headers dbms_headers
IFunction.h
FunctionFactory.h
FunctionHelpers.h
extractTimeZoneFromFunctionArguments.h
FunctionsLogical.h
CastOverloadResolver.h
)
add_library(clickhouse_functions_obj OBJECT ${clickhouse_functions_headers} ${clickhouse_functions_sources}) add_library(clickhouse_functions_obj OBJECT ${clickhouse_functions_headers} ${clickhouse_functions_sources})
if (OMIT_HEAVY_DEBUG_SYMBOLS) if (OMIT_HEAVY_DEBUG_SYMBOLS)
target_compile_options(clickhouse_functions_obj PRIVATE "-g0") target_compile_options(clickhouse_functions_obj PRIVATE "-g0")
set_source_files_properties(${DBMS_FUNCTIONS} DIRECTORY .. PROPERTIES COMPILE_FLAGS "-g0")
endif() endif()
list (APPEND OBJECT_LIBS $<TARGET_OBJECTS:clickhouse_functions_obj>) list (APPEND OBJECT_LIBS $<TARGET_OBJECTS:clickhouse_functions_obj>)

15
src/Functions/parseDateTime.cpp
View File

@ -978,8 +978,7 @@ namespace
[[nodiscard]] [[nodiscard]]
static PosOrError mysqlAmericanDate(Pos cur, Pos end, const String & fragment, DateTime<error_handling> & date) static PosOrError mysqlAmericanDate(Pos cur, Pos end, const String & fragment, DateTime<error_handling> & date)
{ {
if (auto status = checkSpace(cur, end, 8, "mysqlAmericanDate requires size >= 8", fragment)) RETURN_ERROR_IF_FAILED(checkSpace(cur, end, 8, "mysqlAmericanDate requires size >= 8", fragment))
return tl::unexpected(status.error());
Int32 month; Int32 month;
ASSIGN_RESULT_OR_RETURN_ERROR(cur, (readNumber2<Int32, NeedCheckSpace::No>(cur, end, fragment, month))) ASSIGN_RESULT_OR_RETURN_ERROR(cur, (readNumber2<Int32, NeedCheckSpace::No>(cur, end, fragment, month)))
@ -993,7 +992,7 @@ namespace
Int32 year; Int32 year;
ASSIGN_RESULT_OR_RETURN_ERROR(cur, (readNumber2<Int32, NeedCheckSpace::No>(cur, end, fragment, year))) ASSIGN_RESULT_OR_RETURN_ERROR(cur, (readNumber2<Int32, NeedCheckSpace::No>(cur, end, fragment, year)))
RETURN_ERROR_IF_FAILED(date.setYear(year)) RETURN_ERROR_IF_FAILED(date.setYear(year + 2000))
return cur; return cur;
} }
@ -1015,8 +1014,7 @@ namespace
[[nodiscard]] [[nodiscard]]
static PosOrError mysqlISO8601Date(Pos cur, Pos end, const String & fragment, DateTime<error_handling> & date) static PosOrError mysqlISO8601Date(Pos cur, Pos end, const String & fragment, DateTime<error_handling> & date)
{ {
if (auto status = checkSpace(cur, end, 10, "mysqlISO8601Date requires size >= 10", fragment)) RETURN_ERROR_IF_FAILED(checkSpace(cur, end, 10, "mysqlISO8601Date requires size >= 10", fragment))
return tl::unexpected(status.error());
Int32 year; Int32 year;
Int32 month; Int32 month;
@ -1462,8 +1460,7 @@ namespace
[[nodiscard]] [[nodiscard]]
static PosOrError jodaDayOfWeekText(size_t /*min_represent_digits*/, Pos cur, Pos end, const String & fragment, DateTime<error_handling> & date) static PosOrError jodaDayOfWeekText(size_t /*min_represent_digits*/, Pos cur, Pos end, const String & fragment, DateTime<error_handling> & date)
{ {
if (auto result= checkSpace(cur, end, 3, "jodaDayOfWeekText requires size >= 3", fragment); !result.has_value()) RETURN_ERROR_IF_FAILED(checkSpace(cur, end, 3, "jodaDayOfWeekText requires size >= 3", fragment))
return tl::unexpected(result.error());
String text1(cur, 3); String text1(cur, 3);
boost::to_lower(text1); boost::to_lower(text1);
@ -1556,8 +1553,8 @@ namespace
Int32 day_of_month; Int32 day_of_month;
ASSIGN_RESULT_OR_RETURN_ERROR(cur, (readNumberWithVariableLength( ASSIGN_RESULT_OR_RETURN_ERROR(cur, (readNumberWithVariableLength(
cur, end, false, false, false, repetitions, std::max(repetitions, 2uz), fragment, day_of_month))) cur, end, false, false, false, repetitions, std::max(repetitions, 2uz), fragment, day_of_month)))
if (auto res = date.setDayOfMonth(day_of_month); !res.has_value()) RETURN_ERROR_IF_FAILED(date.setDayOfMonth(day_of_month))
return tl::unexpected(res.error());
return cur; return cur;
} }

354
src/Functions/tests/gtest_ternary_logic.cpp
View File

@ -1,354 +0,0 @@
#include <algorithm>
#include <cstring>
#include <vector>
#include <string>
#include <type_traits>
#include <gtest/gtest.h>
#include <Columns/ColumnNothing.h>
#include <Columns/ColumnsNumber.h>
#include <Functions/FunctionsLogical.h>
// I know that inclusion of .cpp is not good at all
#include <Functions/FunctionsLogical.cpp> // NOLINT
using namespace DB;
using TernaryValues = std::vector<Ternary::ResultType>;
struct LinearCongruentialGenerator
{
/// Constants from `man lrand48_r`.
static constexpr UInt64 a = 0x5DEECE66D;
static constexpr UInt64 c = 0xB;
/// And this is from `head -c8 /dev/urandom | xxd -p`
UInt64 current = 0x09826f4a081cee35ULL;
UInt32 next()
{
current = current * a + c;
return static_cast<UInt32>(current >> 16);
}
};
void generateRandomTernaryValue(LinearCongruentialGenerator & gen, Ternary::ResultType * output, size_t size, double false_ratio, double null_ratio)
{
/// The LinearCongruentialGenerator generates nonnegative integers uniformly distributed over the interval [0, 2^32).
/// See https://linux.die.net/man/3/nrand48
double false_percentile = false_ratio;
double null_percentile = false_ratio + null_ratio;
false_percentile = false_percentile > 1 ? 1 : false_percentile;
null_percentile = null_percentile > 1 ? 1 : null_percentile;
UInt32 false_threshold = static_cast<UInt32>(static_cast<double>(std::numeric_limits<UInt32>::max()) * false_percentile);
UInt32 null_threshold = static_cast<UInt32>(static_cast<double>(std::numeric_limits<UInt32>::max()) * null_percentile);
for (Ternary::ResultType * end = output + size; output != end; ++output)
{
UInt32 val = gen.next();
*output = val < false_threshold ? Ternary::False : (val < null_threshold ? Ternary::Null : Ternary::True);
}
}
template<typename T>
ColumnPtr createColumnNullable(const Ternary::ResultType * ternary_values, size_t size)
{
auto nested_column = ColumnVector<T>::create(size);
auto null_map = ColumnUInt8::create(size);
auto & nested_column_data = nested_column->getData();
auto & null_map_data = null_map->getData();
for (size_t i = 0; i < size; ++i)
{
if (ternary_values[i] == Ternary::Null)
{
null_map_data[i] = 1;
nested_column_data[i] = 0;
}
else if (ternary_values[i] == Ternary::True)
{
null_map_data[i] = 0;
nested_column_data[i] = 100;
}
else
{
null_map_data[i] = 0;
nested_column_data[i] = 0;
}
}
return ColumnNullable::create(std::move(nested_column), std::move(null_map));
}
template<typename T>
ColumnPtr createColumnVector(const Ternary::ResultType * ternary_values, size_t size)
{
auto column = ColumnVector<T>::create(size);
auto & column_data = column->getData();
for (size_t i = 0; i < size; ++i)
{
if (ternary_values[i] == Ternary::True)
{
column_data[i] = 100;
}
else
{
column_data[i] = 0;
}
}
return column;
}
template<typename ColumnType, typename T>
ColumnPtr createRandomColumn(LinearCongruentialGenerator & gen, TernaryValues & ternary_values)
{
size_t size = ternary_values.size();
Ternary::ResultType * ternary_data = ternary_values.data();
if constexpr (std::is_same_v<ColumnType, ColumnNullable>)
{
generateRandomTernaryValue(gen, ternary_data, size, 0.3, 0.7);
return createColumnNullable<T>(ternary_data, size);
}
else if constexpr (std::is_same_v<ColumnType, ColumnVector<UInt8>>)
{
generateRandomTernaryValue(gen, ternary_data, size, 0.5, 0);
return createColumnVector<T>(ternary_data, size);
}
else
{
auto nested_col = ColumnNothing::create(size);
auto null_map = ColumnUInt8::create(size);
memset(ternary_data, Ternary::Null, size);
return ColumnNullable::create(std::move(nested_col), std::move(null_map));
}
}
/* The truth table of ternary And and Or operations:
* +-------+-------+---------+--------+
* | a | b | a And b | a Or b |
* +-------+-------+---------+--------+
* | False | False | False | False |
* | False | Null | False | Null |
* | False | True | False | True |
* | Null | False | False | Null |
* | Null | Null | Null | Null |
* | Null | True | Null | True |
* | True | False | False | True |
* | True | Null | Null | True |
* | True | True | True | True |
* +-------+-------+---------+--------+
*
* https://en.wikibooks.org/wiki/Structured_Query_Language/NULLs_and_the_Three_Valued_Logic
*/
template <typename Op, typename T>
bool testTernaryLogicTruthTable()
{
constexpr size_t size = 9;
Ternary::ResultType col_a_ternary[] = {Ternary::False, Ternary::False, Ternary::False, Ternary::Null, Ternary::Null, Ternary::Null, Ternary::True, Ternary::True, Ternary::True};
Ternary::ResultType col_b_ternary[] = {Ternary::False, Ternary::Null, Ternary::True, Ternary::False, Ternary::Null, Ternary::True,Ternary::False, Ternary::Null, Ternary::True};
Ternary::ResultType and_expected_ternary[] = {Ternary::False, Ternary::False, Ternary::False, Ternary::False, Ternary::Null, Ternary::Null,Ternary::False, Ternary::Null, Ternary::True};
Ternary::ResultType or_expected_ternary[] = {Ternary::False, Ternary::Null, Ternary::True, Ternary::Null, Ternary::Null, Ternary::True,Ternary::True, Ternary::True, Ternary::True};
Ternary::ResultType * expected_ternary;
if constexpr (std::is_same_v<Op, AndImpl>)
{
expected_ternary = and_expected_ternary;
}
else
{
expected_ternary = or_expected_ternary;
}
auto col_a = createColumnNullable<T>(col_a_ternary, size);
auto col_b = createColumnNullable<T>(col_b_ternary, size);
ColumnRawPtrs arguments = {col_a.get(), col_b.get()};
auto col_res = ColumnUInt8::create(size);
auto & col_res_data = col_res->getData();
OperationApplier<Op, AssociativeGenericApplierImpl>::apply(arguments, col_res->getData(), false);
for (size_t i = 0; i < size; ++i)
{
if (col_res_data[i] != expected_ternary[i]) return false;
}
return true;
}
template <typename Op, typename LeftColumn, typename RightColumn>
bool testTernaryLogicOfTwoColumns(size_t size)
{
LinearCongruentialGenerator gen;
TernaryValues left_column_ternary(size);
TernaryValues right_column_ternary(size);
TernaryValues expected_ternary(size);
ColumnPtr left = createRandomColumn<LeftColumn, UInt8>(gen, left_column_ternary);
ColumnPtr right = createRandomColumn<RightColumn, UInt8>(gen, right_column_ternary);
for (size_t i = 0; i < size; ++i)
{
/// Given that False is less than Null and Null is less than True, the And operation can be implemented
/// with std::min, and the Or operation can be implemented with std::max.
if constexpr (std::is_same_v<Op, AndImpl>)
{
expected_ternary[i] = std::min(left_column_ternary[i], right_column_ternary[i]);
}
else
{
expected_ternary[i] = std::max(left_column_ternary[i], right_column_ternary[i]);
}
}
ColumnRawPtrs arguments = {left.get(), right.get()};
auto col_res = ColumnUInt8::create(size);
auto & col_res_data = col_res->getData();
OperationApplier<Op, AssociativeGenericApplierImpl>::apply(arguments, col_res->getData(), false);
for (size_t i = 0; i < size; ++i)
{
if (col_res_data[i] != expected_ternary[i]) return false;
}
return true;
}
TEST(TernaryLogicTruthTable, NestedUInt8)
{
bool test_1 = testTernaryLogicTruthTable<AndImpl, UInt8>();
bool test_2 = testTernaryLogicTruthTable<OrImpl, UInt8>();
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}
TEST(TernaryLogicTruthTable, NestedUInt16)
{
bool test_1 = testTernaryLogicTruthTable<AndImpl, UInt16>();
bool test_2 = testTernaryLogicTruthTable<OrImpl, UInt16>();
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}
TEST(TernaryLogicTruthTable, NestedUInt32)
{
bool test_1 = testTernaryLogicTruthTable<AndImpl, UInt32>();
bool test_2 = testTernaryLogicTruthTable<OrImpl, UInt32>();
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}
TEST(TernaryLogicTruthTable, NestedUInt64)
{
bool test_1 = testTernaryLogicTruthTable<AndImpl, UInt64>();
bool test_2 = testTernaryLogicTruthTable<OrImpl, UInt64>();
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}
TEST(TernaryLogicTruthTable, NestedInt8)
{
bool test_1 = testTernaryLogicTruthTable<AndImpl, Int8>();
bool test_2 = testTernaryLogicTruthTable<OrImpl, Int8>();
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}
TEST(TernaryLogicTruthTable, NestedInt16)
{
bool test_1 = testTernaryLogicTruthTable<AndImpl, Int16>();
bool test_2 = testTernaryLogicTruthTable<OrImpl, Int16>();
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}
TEST(TernaryLogicTruthTable, NestedInt32)
{
bool test_1 = testTernaryLogicTruthTable<AndImpl, Int32>();
bool test_2 = testTernaryLogicTruthTable<OrImpl, Int32>();
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}
TEST(TernaryLogicTruthTable, NestedInt64)
{
bool test_1 = testTernaryLogicTruthTable<AndImpl, Int64>();
bool test_2 = testTernaryLogicTruthTable<OrImpl, Int64>();
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}
TEST(TernaryLogicTruthTable, NestedFloat32)
{
bool test_1 = testTernaryLogicTruthTable<AndImpl, Float32>();
bool test_2 = testTernaryLogicTruthTable<OrImpl, Float32>();
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}
TEST(TernaryLogicTruthTable, NestedFloat64)
{
bool test_1 = testTernaryLogicTruthTable<AndImpl, Float64>();
bool test_2 = testTernaryLogicTruthTable<OrImpl, Float64>();
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}
TEST(TernaryLogicTwoColumns, TwoNullable)
{
bool test_1 = testTernaryLogicOfTwoColumns<AndImpl, ColumnNullable, ColumnNullable>(100 /*size*/);
bool test_2 = testTernaryLogicOfTwoColumns<OrImpl, ColumnNullable, ColumnNullable>(100 /*size*/);
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}
TEST(TernaryLogicTwoColumns, TwoVector)
{
bool test_1 = testTernaryLogicOfTwoColumns<AndImpl, ColumnUInt8, ColumnUInt8>(100 /*size*/);
bool test_2 = testTernaryLogicOfTwoColumns<OrImpl, ColumnUInt8, ColumnUInt8>(100 /*size*/);
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}
TEST(TernaryLogicTwoColumns, TwoNothing)
{
bool test_1 = testTernaryLogicOfTwoColumns<AndImpl, ColumnNothing, ColumnNothing>(100 /*size*/);
bool test_2 = testTernaryLogicOfTwoColumns<OrImpl, ColumnNothing, ColumnNothing>(100 /*size*/);
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}
TEST(TernaryLogicTwoColumns, NullableVector)
{
bool test_1 = testTernaryLogicOfTwoColumns<AndImpl, ColumnNullable, ColumnUInt8>(100 /*size*/);
bool test_2 = testTernaryLogicOfTwoColumns<OrImpl, ColumnNullable, ColumnUInt8>(100 /*size*/);
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}
TEST(TernaryLogicTwoColumns, NullableNothing)
{
bool test_1 = testTernaryLogicOfTwoColumns<AndImpl, ColumnNullable, ColumnNothing>(100 /*size*/);
bool test_2 = testTernaryLogicOfTwoColumns<OrImpl, ColumnNullable, ColumnNothing>(100 /*size*/);
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}
TEST(TernaryLogicTwoColumns, VectorNothing)
{
bool test_1 = testTernaryLogicOfTwoColumns<AndImpl, ColumnUInt8, ColumnNothing>(100 /*size*/);
bool test_2 = testTernaryLogicOfTwoColumns<OrImpl, ColumnUInt8, ColumnNothing>(100 /*size*/);
ASSERT_EQ(test_1, true);
ASSERT_EQ(test_2, true);
}

6
src/Interpreters/ExpressionActions.cpp
View File

@ -195,6 +195,10 @@ static void setLazyExecutionInfo(
} }
lazy_execution_info.short_circuit_ancestors_info[parent].insert(indexes.begin(), indexes.end()); lazy_execution_info.short_circuit_ancestors_info[parent].insert(indexes.begin(), indexes.end());
/// After checking arguments_with_disabled_lazy_execution, if there is no relation with parent,
/// disable the current node.
if (indexes.empty())
lazy_execution_info.can_be_lazy_executed = false;
} }
else else
/// If lazy execution is disabled for one of parents, we should disable it for current node. /// If lazy execution is disabled for one of parents, we should disable it for current node.
@ -292,9 +296,9 @@ static std::unordered_set<const ActionsDAG::Node *> processShortCircuitFunctions
/// Firstly, find all short-circuit functions and get their settings. /// Firstly, find all short-circuit functions and get their settings.
std::unordered_map<const ActionsDAG::Node *, IFunctionBase::ShortCircuitSettings> short_circuit_nodes; std::unordered_map<const ActionsDAG::Node *, IFunctionBase::ShortCircuitSettings> short_circuit_nodes;
IFunctionBase::ShortCircuitSettings short_circuit_settings;
for (const auto & node : nodes) for (const auto & node : nodes)
{ {
IFunctionBase::ShortCircuitSettings short_circuit_settings;
if (node.type == ActionsDAG::ActionType::FUNCTION && node.function_base->isShortCircuit(short_circuit_settings, node.children.size()) && !node.children.empty()) if (node.type == ActionsDAG::ActionType::FUNCTION && node.function_base->isShortCircuit(short_circuit_settings, node.children.size()) && !node.children.empty())
short_circuit_nodes[&node] = short_circuit_settings; short_circuit_nodes[&node] = short_circuit_settings;
} }

2
src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -898,6 +898,8 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti
assert(as_database_saved.empty() && as_table_saved.empty()); assert(as_database_saved.empty() && as_table_saved.empty());
std::swap(create.as_database, as_database_saved); std::swap(create.as_database, as_database_saved);
std::swap(create.as_table, as_table_saved); std::swap(create.as_table, as_table_saved);
if (!as_table_saved.empty())
create.is_create_empty = false;
return properties; return properties;
} }

15
src/Parsers/ASTCreateQuery.cpp
View File

@ -404,8 +404,18 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat
<< quoteString(toString(to_inner_uuid)); << quoteString(toString(to_inner_uuid));
} }
bool should_add_empty = is_create_empty;
auto add_empty_if_needed = [&]
{
if (!should_add_empty)
return;
should_add_empty = false;
settings.ostr << (settings.hilite ? hilite_keyword : "") << " EMPTY" << (settings.hilite ? hilite_none : "");
};
if (!as_table.empty()) if (!as_table.empty())
{ {
add_empty_if_needed();
settings.ostr settings.ostr
<< (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : "") << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : "")
<< (!as_database.empty() ? backQuoteIfNeed(as_database) + "." : "") << backQuoteIfNeed(as_table); << (!as_database.empty() ? backQuoteIfNeed(as_database) + "." : "") << backQuoteIfNeed(as_table);
@ -423,6 +433,7 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat
frame.expression_list_always_start_on_new_line = false; frame.expression_list_always_start_on_new_line = false;
} }
add_empty_if_needed();
settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : "");
as_table_function->formatImpl(settings, state, frame); as_table_function->formatImpl(settings, state, frame);
} }
@ -484,8 +495,8 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat
if (is_populate) if (is_populate)
settings.ostr << (settings.hilite ? hilite_keyword : "") << " POPULATE" << (settings.hilite ? hilite_none : ""); settings.ostr << (settings.hilite ? hilite_keyword : "") << " POPULATE" << (settings.hilite ? hilite_none : "");
else if (is_create_empty)
settings.ostr << (settings.hilite ? hilite_keyword : "") << " EMPTY" << (settings.hilite ? hilite_none : ""); add_empty_if_needed();
if (sql_security && supportSQLSecurity() && sql_security->as<ASTSQLSecurity &>().type.has_value()) if (sql_security && supportSQLSecurity() && sql_security->as<ASTSQLSecurity &>().type.has_value())
{ {

19
src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
View File

@ -46,6 +46,15 @@ JSONEachRowRowInputFormat::JSONEachRowRowInputFormat(
{ {
const auto & header = getPort().getHeader(); const auto & header = getPort().getHeader();
name_map = header.getNamesToIndexesMap(); name_map = header.getNamesToIndexesMap();
if (format_settings_.json.ignore_key_case)
{
for (auto & it : name_map)
{
StringRef key = it.first;
String lower_case_key = transformFieldNameToLowerCase(key);
lower_case_name_map[lower_case_key] = key;
}
}
if (format_settings_.import_nested_json) if (format_settings_.import_nested_json)
{ {
for (size_t i = 0; i != header.columns(); ++i) for (size_t i = 0; i != header.columns(); ++i)
@ -171,7 +180,15 @@ void JSONEachRowRowInputFormat::readJSONObject(MutableColumns & columns)
skipUnknownField(name_ref); skipUnknownField(name_ref);
continue; continue;
} }
const size_t column_index = columnIndex(name_ref, key_index); size_t column_index = 0;
if (format_settings.json.ignore_key_case)
{
String lower_case_name = transformFieldNameToLowerCase(name_ref);
StringRef field_name_ref = lower_case_name_map[lower_case_name];
column_index = columnIndex(field_name_ref, key_index);
}
else
column_index = columnIndex(name_ref, key_index);
if (unlikely(ssize_t(column_index) < 0)) if (unlikely(ssize_t(column_index) < 0))
{ {

11
src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h
View File

@ -55,7 +55,13 @@ private:
virtual void readRowStart(MutableColumns &) {} virtual void readRowStart(MutableColumns &) {}
virtual void skipRowStart() {} virtual void skipRowStart() {}
String transformFieldNameToLowerCase(const StringRef & field_name)
{
String field_name_str = field_name.toString();
std::transform(field_name_str.begin(), field_name_str.end(), field_name_str.begin(),
[](unsigned char c) { return std::tolower(c); });
return field_name_str;
}
/// Buffer for the read from the stream field name. Used when you have to copy it. /// Buffer for the read from the stream field name. Used when you have to copy it.
/// Also, if processing of Nested data is in progress, it holds the common prefix /// Also, if processing of Nested data is in progress, it holds the common prefix
/// of the nested column names (so that appending the field name to it produces /// of the nested column names (so that appending the field name to it produces
@ -74,7 +80,8 @@ private:
/// Hash table match `field name -> position in the block`. NOTE You can use perfect hash map. /// Hash table match `field name -> position in the block`. NOTE You can use perfect hash map.
Block::NameMap name_map; Block::NameMap name_map;
/// Hash table match `lower_case field name -> field name in the block`.
std::unordered_map<String, StringRef> lower_case_name_map;
/// Cached search results for previous row (keyed as index in JSON object) - used as a hint. /// Cached search results for previous row (keyed as index in JSON object) - used as a hint.
std::vector<Block::NameMap::const_iterator> prev_positions; std::vector<Block::NameMap::const_iterator> prev_positions;

2
src/Storages/examples/CMakeLists.txt
View File

@ -5,4 +5,4 @@ clickhouse_add_executable (merge_selector2 merge_selector2.cpp)
target_link_libraries (merge_selector2 PRIVATE dbms) target_link_libraries (merge_selector2 PRIVATE dbms)
clickhouse_add_executable (get_current_inserts_in_replicated get_current_inserts_in_replicated.cpp) clickhouse_add_executable (get_current_inserts_in_replicated get_current_inserts_in_replicated.cpp)
target_link_libraries (get_current_inserts_in_replicated PRIVATE dbms clickhouse_common_config clickhouse_common_zookeeper) target_link_libraries (get_current_inserts_in_replicated PRIVATE dbms clickhouse_common_config clickhouse_common_zookeeper clickhouse_functions)

8
tests/config/install.sh
View File

@ -183,13 +183,7 @@ elif [[ "$USE_AZURE_STORAGE_FOR_MERGE_TREE" == "1" ]]; then
fi fi
if [[ -n "$EXPORT_S3_STORAGE_POLICIES" ]]; then if [[ -n "$EXPORT_S3_STORAGE_POLICIES" ]]; then
if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then ln -sf $SRC_PATH/config.d/azure_storage_conf.xml $DEST_SERVER_PATH/config.d/
echo "Azure configuration will not be added"
else
echo "Adding azure configuration"
ln -sf $SRC_PATH/config.d/azure_storage_conf.xml $DEST_SERVER_PATH/config.d/
fi
ln -sf $SRC_PATH/config.d/storage_conf.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/storage_conf.xml $DEST_SERVER_PATH/config.d/
ln -sf $SRC_PATH/config.d/storage_conf_02944.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/storage_conf_02944.xml $DEST_SERVER_PATH/config.d/
ln -sf $SRC_PATH/config.d/storage_conf_02963.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/storage_conf_02963.xml $DEST_SERVER_PATH/config.d/

41
tests/integration/test_asynchronous_metric_jemalloc_profile_active/test.py
View File

@ -7,7 +7,6 @@ cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance( node1 = cluster.add_instance(
"node1", "node1",
main_configs=["configs/asynchronous_metrics_update_period_s.xml"], main_configs=["configs/asynchronous_metrics_update_period_s.xml"],
env_variables={"MALLOC_CONF": "background_thread:true,prof:true"},
) )
@ -29,26 +28,11 @@ def test_asynchronous_metric_jemalloc_profile_active(started_cluster):
if node1.is_built_with_sanitizer(): if node1.is_built_with_sanitizer():
pytest.skip("Disabled for sanitizers") pytest.skip("Disabled for sanitizers")
res_o = node1.query( res = node1.query(
"SELECT * FROM system.asynchronous_metrics WHERE metric ILIKE '%jemalloc.prof.active%' FORMAT Vertical;" "SELECT * FROM system.asynchronous_metrics WHERE metric ILIKE '%jemalloc.prof.active%' FORMAT Vertical;"
) )
assert ( assert (
res_o res
== """Row 1:
──────
metric: jemalloc.prof.active
value: 1
description: An internal metric of the low-level memory allocator (jemalloc). See https://jemalloc.net/jemalloc.3.html
"""
)
# disable
node1.query("SYSTEM JEMALLOC DISABLE PROFILE")
time.sleep(5)
res_t = node1.query(
"SELECT * FROM system.asynchronous_metrics WHERE metric ILIKE '%jemalloc.prof.active%' FORMAT Vertical;"
)
assert (
res_t
== """Row 1: == """Row 1:
────── ──────
metric: jemalloc.prof.active metric: jemalloc.prof.active
@ -58,16 +42,31 @@ description: An internal metric of the low-level memory allocator (jemalloc). Se
) )
# enable # enable
node1.query("SYSTEM JEMALLOC ENABLE PROFILE") node1.query("SYSTEM JEMALLOC ENABLE PROFILE")
time.sleep(5) node1.query("SYSTEM RELOAD ASYNCHRONOUS METRICS")
res_f = node1.query( res = node1.query(
"SELECT * FROM system.asynchronous_metrics WHERE metric ILIKE '%jemalloc.prof.active%' FORMAT Vertical;" "SELECT * FROM system.asynchronous_metrics WHERE metric ILIKE '%jemalloc.prof.active%' FORMAT Vertical;"
) )
assert ( assert (
res_f res
== """Row 1: == """Row 1:
────── ──────
metric: jemalloc.prof.active metric: jemalloc.prof.active
value: 1 value: 1
description: An internal metric of the low-level memory allocator (jemalloc). See https://jemalloc.net/jemalloc.3.html description: An internal metric of the low-level memory allocator (jemalloc). See https://jemalloc.net/jemalloc.3.html
"""
)
# disable
node1.query("SYSTEM JEMALLOC DISABLE PROFILE")
node1.query("SYSTEM RELOAD ASYNCHRONOUS METRICS")
res = node1.query(
"SELECT * FROM system.asynchronous_metrics WHERE metric ILIKE '%jemalloc.prof.active%' FORMAT Vertical;"
)
assert (
res
== """Row 1:
──────
metric: jemalloc.prof.active
value: 0
description: An internal metric of the low-level memory allocator (jemalloc). See https://jemalloc.net/jemalloc.3.html
""" """
) )

4
tests/integration/test_replicated_database/test.py
View File

@ -337,8 +337,12 @@ def test_alter_attach(started_cluster, attachable_part, engine):
main_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") main_node.query(f"SELECT CounterID FROM {database}.alter_attach_test")
== "123\n" == "123\n"
) )
# On the other node, data is replicated only if using a Replicated table engine # On the other node, data is replicated only if using a Replicated table engine
if engine == "ReplicatedMergeTree": if engine == "ReplicatedMergeTree":
dummy_node.query(
f"SYSTEM SYNC REPLICA {database}.alter_attach_test LIGHTWEIGHT"
)
assert ( assert (
dummy_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") dummy_node.query(f"SELECT CounterID FROM {database}.alter_attach_test")
== "123\n" == "123\n"

18
tests/queries/0_stateless/02240_system_filesystem_cache_table.reference
View File

@ -34,3 +34,21 @@ DOWNLOADED 0 79 80
DOWNLOADED 0 745 746 DOWNLOADED 0 745 746
2 2
Expect no cache Expect no cache
Using storage policy: azure_cache
0
Expect cache
DOWNLOADED 0 0 1
DOWNLOADED 0 79 80
DOWNLOADED 0 745 746
3
Expect cache
DOWNLOADED 0 0 1
DOWNLOADED 0 79 80
DOWNLOADED 0 745 746
3
Expect no cache
Expect cache
DOWNLOADED 0 79 80
DOWNLOADED 0 745 746
2
Expect no cache

2
tests/queries/0_stateless/02240_system_filesystem_cache_table.sh
View File

@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh # shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh . "$CUR_DIR"/../shell_config.sh
for STORAGE_POLICY in 's3_cache' 'local_cache'; do for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
echo "Using storage policy: $STORAGE_POLICY" echo "Using storage policy: $STORAGE_POLICY"
${CLICKHOUSE_CLIENT} --query "SYSTEM DROP FILESYSTEM CACHE" ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP FILESYSTEM CACHE"
${CLICKHOUSE_CLIENT} --query "SYSTEM DROP MARK CACHE" ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP MARK CACHE"

110
tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference
View File

@ -218,3 +218,113 @@ SELECT count() FROM test_02241
5010500 5010500
SELECT count() FROM test_02241 WHERE value LIKE '%010%' SELECT count() FROM test_02241 WHERE value LIKE '%010%'
18816 18816
Using storage policy: azure_cache
DROP TABLE IF EXISTS test_02241
CREATE TABLE test_02241 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='azure_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false, ratio_of_defaults_for_sparse_serialization = 1
SYSTEM STOP MERGES test_02241
SYSTEM DROP FILESYSTEM CACHE
SELECT file_segment_range_begin, file_segment_range_end, size, state
FROM
(
SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path
FROM
(
SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path
FROM system.remote_data_paths
) AS data_paths
INNER JOIN
system.filesystem_cache AS caches
ON data_paths.cache_path = caches.cache_path
)
WHERE endsWith(local_path, 'data.bin')
FORMAT Vertical
SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path
0
SELECT count(), sum(size) FROM system.filesystem_cache
0 0
INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100)
SELECT file_segment_range_begin, file_segment_range_end, size, state
FROM
(
SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path
FROM
(
SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path
FROM system.remote_data_paths
) AS data_paths
INNER JOIN
system.filesystem_cache AS caches
ON data_paths.cache_path = caches.cache_path
)
WHERE endsWith(local_path, 'data.bin')
FORMAT Vertical
Row 1:
──────
file_segment_range_begin: 0
file_segment_range_end: 745
size: 746
state: DOWNLOADED
SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path
8
SELECT count(), sum(size) FROM system.filesystem_cache
8 1100
SELECT count() FROM system.filesystem_cache WHERE cache_hits > 0
0
SELECT * FROM test_02241 FORMAT Null
SELECT count() FROM system.filesystem_cache WHERE cache_hits > 0
2
SELECT * FROM test_02241 FORMAT Null
SELECT count() FROM system.filesystem_cache WHERE cache_hits > 0
2
SELECT count(), sum(size) size FROM system.filesystem_cache
8 1100
SYSTEM DROP FILESYSTEM CACHE
INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100, 200)
SELECT file_segment_range_begin, file_segment_range_end, size, state
FROM
(
SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path
FROM
(
SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path
FROM system.remote_data_paths
) AS data_paths
INNER JOIN
system.filesystem_cache AS caches
ON data_paths.cache_path = caches.cache_path
)
WHERE endsWith(local_path, 'data.bin')
FORMAT Vertical;
Row 1:
──────
file_segment_range_begin: 0
file_segment_range_end: 1659
size: 1660
state: DOWNLOADED
SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path
8
SELECT count(), sum(size) FROM system.filesystem_cache
8 2014
SELECT count(), sum(size) FROM system.filesystem_cache
8 2014
INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100) SETTINGS enable_filesystem_cache_on_write_operations=0
SELECT count(), sum(size) FROM system.filesystem_cache
8 2014
INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100)
INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(300, 10000)
SELECT count(), sum(size) FROM system.filesystem_cache
24 84045
SYSTEM START MERGES test_02241
OPTIMIZE TABLE test_02241 FINAL
SELECT count(), sum(size) FROM system.filesystem_cache
32 167243
ALTER TABLE test_02241 UPDATE value = 'kek' WHERE key = 100
SELECT count(), sum(size) FROM system.filesystem_cache
41 250541
INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(5000000)
SYSTEM FLUSH LOGS
INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(5000000) 0
SELECT count() FROM test_02241
5010500
SELECT count() FROM test_02241 WHERE value LIKE '%010%'
18816

2
tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh
View File

@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh # shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh . "$CUR_DIR"/../shell_config.sh
for STORAGE_POLICY in 's3_cache' 'local_cache'; do for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
echo "Using storage policy: $STORAGE_POLICY" echo "Using storage policy: $STORAGE_POLICY"
$CLICKHOUSE_CLIENT --echo --query "DROP TABLE IF EXISTS test_02241" $CLICKHOUSE_CLIENT --echo --query "DROP TABLE IF EXISTS test_02241"

4
tests/queries/0_stateless/02242_system_filesystem_cache_log_table.reference
View File

@ -6,3 +6,7 @@ Using storage policy: local_cache
(0,519) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE (0,519) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE
(0,808110) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE (0,808110) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE
(0,808110) READ_FROM_CACHE (0,808110) READ_FROM_CACHE
Using storage policy: azure_cache
(0,519) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE
(0,808110) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE
(0,808110) READ_FROM_CACHE

2
tests/queries/0_stateless/02242_system_filesystem_cache_log_table.sh
View File

@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh # shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh . "$CUR_DIR"/../shell_config.sh
for STORAGE_POLICY in 's3_cache' 'local_cache'; do for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
echo "Using storage policy: $STORAGE_POLICY" echo "Using storage policy: $STORAGE_POLICY"
$CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE" $CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE"

9
tests/queries/0_stateless/02286_drop_filesystem_cache.reference
View File

@ -16,3 +16,12 @@ Using storage policy: local_cache
1 1
1 1
0 0
Using storage policy: azure_cache
0
2
0
1
1
1
1
0

2
tests/queries/0_stateless/02286_drop_filesystem_cache.sh
View File

@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh # shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh . "$CUR_DIR"/../shell_config.sh
for STORAGE_POLICY in 's3_cache' 'local_cache'; do for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
echo "Using storage policy: $STORAGE_POLICY" echo "Using storage policy: $STORAGE_POLICY"
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_02286" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_02286"

1
tests/queries/0_stateless/02313_filesystem_cache_seeks.reference
View File

@ -1,3 +1,4 @@
Using storage policy: s3_cache Using storage policy: s3_cache
Using storage policy: local_cache Using storage policy: local_cache
Using storage policy: s3_cache_multi Using storage policy: s3_cache_multi
Using storage policy: azure_cache

2
tests/queries/0_stateless/02313_filesystem_cache_seeks.sh
View File

@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CUR_DIR"/../shell_config.sh . "$CUR_DIR"/../shell_config.sh
for STORAGE_POLICY in 's3_cache' 'local_cache' 's3_cache_multi'; do for STORAGE_POLICY in 's3_cache' 'local_cache' 's3_cache_multi' 'azure_cache'; do
echo "Using storage policy: $STORAGE_POLICY" echo "Using storage policy: $STORAGE_POLICY"
$CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE" $CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE"

3
tests/queries/0_stateless/03013_json_key_ignore_case.reference Normal file
View File

@ -0,0 +1,3 @@
1 77328912 Ben
2 77328913 Jim
3 77328914 Bill

18
tests/queries/0_stateless/03013_json_key_ignore_case.sh Executable file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env bash
# NOTE: this sh wrapper is required because of shell_config
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
USER_FILES_PATH=$($CLICKHOUSE_CLIENT --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep -E '^Code: 107.*FILE_DOESNT_EXIST' | head -1 | awk '{gsub("/nonexist.txt","",$9); print $9}')
cp "$CURDIR"/data_json/key_ignore_case.json $USER_FILES_PATH/
$CLICKHOUSE_CLIENT -q "drop table if exists test_tbl"
$CLICKHOUSE_CLIENT -q "create table test_tbl (id UInt16, reqid UInt32, name String) engine=MergeTree order by id"
$CLICKHOUSE_CLIENT -q "INSERT INTO test_tbl SELECT * FROM file('key_ignore_case.json', 'JSONEachRow') SETTINGS input_format_json_ignore_key_case=true"
$CLICKHOUSE_CLIENT -q "select * from test_tbl"
$CLICKHOUSE_CLIENT -q "drop table test_tbl"

1
tests/queries/0_stateless/03071_fix_short_circuit_logic.reference Normal file
View File

@ -0,0 +1 @@
2024-01-02 16:54:59

62
tests/queries/0_stateless/03071_fix_short_circuit_logic.sql Normal file
View File

@ -0,0 +1,62 @@
CREATE FUNCTION IF NOT EXISTS unhexPrefixed AS value -> unhex(substring(value, 3));
CREATE FUNCTION IF NOT EXISTS hex2bytes AS address -> CAST(unhexPrefixed(address), 'FixedString(20)');
CREATE FUNCTION IF NOT EXISTS bytes2hex AS address -> concat('0x', lower(hex(address)));
CREATE TABLE test
(
`transfer_id` String,
`address` FixedString(20),
`value` UInt256,
`block_timestamp` DateTime('UTC'),
`token_address` FixedString(20)
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(block_timestamp)
PRIMARY KEY (address, block_timestamp)
ORDER BY (address, block_timestamp);
INSERT INTO test SELECT 'token-transfer-0x758f1bbabb160683e1c80ed52dcd24a32b599d40edf1cec91b5f1199c0e392a2-56', hex2bytes('0xd387a6e4e84a6c86bd90c158c6028a58cc8ac459'), 3000000000000000000000, '2024-01-02 16:54:59', 'abc';
CREATE TABLE token_data
(
token_address_hex String,
chain String,
is_blacklisted Bool
)
ENGINE = TinyLog;
INSERT INTO token_data SELECT bytes2hex('abc'), 'zksync', false;
CREATE DICTIONARY token_data_map
(
token_address_hex String,
chain String,
is_blacklisted Bool
)
PRIMARY KEY token_address_hex, chain
SOURCE(Clickhouse(table token_data))
LIFETIME(MIN 200 MAX 300)
LAYOUT(COMPLEX_KEY_HASHED_ARRAY());
SELECT block_timestamp
FROM
(
SELECT
block_timestamp,
bytes2hex(token_address) AS token_address_hex
FROM
(
SELECT
transfer_id,
address,
value,
block_timestamp,
token_address,
'zksync' AS chain
FROM test
)
WHERE (address = hex2bytes('0xd387a6e4e84a6c86bd90c158c6028a58cc8ac459')) AND (transfer_id NOT LIKE 'gas%') AND (value > 0) AND (dictGetOrDefault(token_data_map, 'is_blacklisted', (token_address_hex, 'zksync'), true))
)
SETTINGS max_threads = 1, short_circuit_function_evaluation = 'enable', allow_experimental_analyzer = 0;

1
tests/queries/0_stateless/03094_one_thousand_joins.sql
View File

@ -1,6 +1,7 @@
-- Tags: no-fasttest, no-tsan, long -- Tags: no-fasttest, no-tsan, long
-- (no-tsan because it has a small maximum stack size and the test would fail with TOO_DEEP_RECURSION) -- (no-tsan because it has a small maximum stack size and the test would fail with TOO_DEEP_RECURSION)
SET join_algorithm = 'default'; -- for 'full_sorting_merge' the query is 10x slower
SET allow_experimental_analyzer = 1; -- old analyzer returns TOO_DEEP_SUBQUERIES SET allow_experimental_analyzer = 1; -- old analyzer returns TOO_DEEP_SUBQUERIES
-- Bug 33446, marked as 'long' because it still runs around 10 sec -- Bug 33446, marked as 'long' because it still runs around 10 sec

5
tests/queries/0_stateless/03168_inconsistent_ast_formatting.sql
View File

@ -1,4 +1,7 @@
create table a (x `Null`); -- { clientError SYNTAX_ERROR } create table a (x `Null`); -- { clientError SYNTAX_ERROR }
create table a (x f(`Null`)); -- { clientError SYNTAX_ERROR } create table a (x f(`Null`)); -- { clientError SYNTAX_ERROR }
create table a (x Enum8(f(`Null`, 'World', 2))); -- { clientError SYNTAX_ERROR } create table a (x Enum8(f(`Null`, 'World', 2))); -- { clientError SYNTAX_ERROR }
create table a (`value2` Enum8('Hello' = 1, equals(`Null`, 'World', 2), '!' = 3)); -- { clientError SYNTAX_ERROR } create table a (`value2` Enum8('Hello' = 1, equals(`Null`, 'World', 2), '!' = 3)); -- { clientError SYNTAX_ERROR }
create table a (x Int8) engine Memory;
create table b empty as a;

2
tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.reference Normal file
View File

@ -0,0 +1,2 @@
2024-06-20 00:00:00
2024-06-20 00:00:00

2
tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.sql Normal file
View File

@ -0,0 +1,2 @@
SELECT parseDateTime('2024-06-20', '%F', 'UTC') AS x;
SELECT parseDateTime('06/20/24', '%D', 'UTC') AS x;

BIN
tests/queries/0_stateless/data_json/key_ignore_case.json Normal file
View File

Binary file not shown.

4
utils/check-style/check-style
View File

@ -322,10 +322,14 @@ std_cerr_cout_excludes=(
src/Client/LineReader.cpp src/Client/LineReader.cpp
src/Client/QueryFuzzer.cpp src/Client/QueryFuzzer.cpp
src/Client/Suggest.cpp src/Client/Suggest.cpp
src/Client/ClientBase.h
src/Client/LineReader.h
src/Client/ReplxxLineReader.h
src/Bridge/IBridge.cpp src/Bridge/IBridge.cpp
src/Daemon/BaseDaemon.cpp src/Daemon/BaseDaemon.cpp
src/Loggers/Loggers.cpp src/Loggers/Loggers.cpp
src/Common/GWPAsan.cpp src/Common/GWPAsan.cpp
src/Common/ProgressIndication.h
) )
sources_with_std_cerr_cout=( $( sources_with_std_cerr_cout=( $(
find $ROOT_PATH/{src,base} -name '*.h' -or -name '*.cpp' | \ find $ROOT_PATH/{src,base} -name '*.h' -or -name '*.cpp' | \

4
utils/zookeeper-cli/CMakeLists.txt
View File

@ -3,4 +3,6 @@ clickhouse_add_executable(clickhouse-zookeeper-cli
${ClickHouse_SOURCE_DIR}/src/Client/LineReader.cpp) ${ClickHouse_SOURCE_DIR}/src/Client/LineReader.cpp)
target_link_libraries(clickhouse-zookeeper-cli PRIVATE target_link_libraries(clickhouse-zookeeper-cli PRIVATE
clickhouse_common_zookeeper_no_log clickhouse_common_zookeeper_no_log
dbms) dbms
clickhouse_functions
)

1
utils/zookeeper-dump-tree/CMakeLists.txt
View File

@ -3,4 +3,5 @@ target_link_libraries(zookeeper-dump-tree PRIVATE
clickhouse_common_zookeeper_no_log clickhouse_common_zookeeper_no_log
clickhouse_common_io clickhouse_common_io
dbms dbms
clickhouse_functions
boost::program_options) boost::program_options)

1
utils/zookeeper-remove-by-list/CMakeLists.txt
View File

@ -2,4 +2,5 @@ clickhouse_add_executable (zookeeper-remove-by-list main.cpp ${SRCS})
target_link_libraries(zookeeper-remove-by-list PRIVATE target_link_libraries(zookeeper-remove-by-list PRIVATE
clickhouse_common_zookeeper_no_log clickhouse_common_zookeeper_no_log
dbms dbms
clickhouse_functions
boost::program_options) boost::program_options)