From 82ce2d76c31547539cdb6ed4bec6da065e13e7cb Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Fri, 3 Jun 2022 12:06:31 +0300 Subject: [PATCH 001/123] Add KerberosInit class; add kerberos_init console example; modify HDFSCommon.cpp --- src/Access/CMakeLists.txt | 3 + src/Access/KerberosInit.cpp | 162 ++++++++++++++++++++++++++ src/Access/KerberosInit.h | 37 ++++++ src/Access/examples/CMakeLists.txt | 4 + src/Access/examples/kerberos_init.cpp | 36 ++++++ src/Storages/HDFS/HDFSCommon.cpp | 15 ++- 6 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 src/Access/KerberosInit.cpp create mode 100644 src/Access/KerberosInit.h create mode 100644 src/Access/examples/CMakeLists.txt create mode 100644 src/Access/examples/kerberos_init.cpp diff --git a/src/Access/CMakeLists.txt b/src/Access/CMakeLists.txt index e69de29bb2d..83bbe418246 100644 --- a/src/Access/CMakeLists.txt +++ b/src/Access/CMakeLists.txt @@ -0,0 +1,3 @@ +if (ENABLE_EXAMPLES) + add_subdirectory(examples) +endif() diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp new file mode 100644 index 00000000000..8954eccf136 --- /dev/null +++ b/src/Access/KerberosInit.cpp @@ -0,0 +1,162 @@ +#include +#include + +#include +#include +#include + + +int KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) +{ + auto adqm_log = &Poco::Logger::get("ADQM"); + LOG_DEBUG(adqm_log,"KerberosInit: begin"); + + krb5_error_code ret; + + // todo: use deftype + //const char *deftype = nullptr; + int flags = 0; + principal_name = new char[256]; + std::copy(principal.begin(), principal.end(), principal_name); + principal_name[principal.size()] = '\0'; + + //memset(&k5, 0, sizeof(k5)); + memset(&k5d, 0, sizeof(k5d)); + k5 = &k5d; + //memset(k5, 0, sizeof(k5_data)); + // begin + ret = krb5_init_context(&k5->ctx); + if (ret) + throw DB::Exception(0, "Error while initializing Kerberos 5 library"); + + if (!cache_name.empty()) + { + ret = krb5_cc_resolve(k5->ctx, cache_name.c_str(), &k5->out_cc); + // todo: analyze return code + LOG_DEBUG(adqm_log,"Resolved cache"); + } + else + { + // Resolve the default ccache and get its type and default principal (if it is initialized). + ret = krb5_cc_default(k5->ctx, &defcache); + if (ret) + throw DB::Exception(0, "Error while getting default ccache"); + LOG_DEBUG(adqm_log,"Resolved default cache"); + // todo: deftype + /*deftype = */krb5_cc_get_type(k5->ctx, defcache); + if (krb5_cc_get_principal(k5->ctx, defcache, &defcache_princ) != 0) + defcache_princ = nullptr; + } + + // Use the specified principal name. + ret = krb5_parse_name_flags(k5->ctx, principal_name, flags, &k5->me); + if (ret) + throw DB::Exception(0, "Error when parsing principal name " + String(principal_name)); + + + // to-do: add more cache init commands + + ret = krb5_unparse_name(k5->ctx, k5->me, &k5->name); + if (ret) + throw DB::Exception(0, "Error when unparsing name"); + + LOG_DEBUG(adqm_log,"KerberosInit: Using principal: {}", k5->name); + + principal_name = k5->name; + + // init: + memset(&my_creds, 0, sizeof(my_creds)); + + ret = krb5_get_init_creds_opt_alloc(k5->ctx, &options); + if (ret) + throw DB::Exception(0, "Error in options allocation"); + + // todo +/* +#ifndef _WIN32 + if (strncmp(opts->keytab_name, "KDB:", 4) == 0) { + ret = kinit_kdb_init(&k5->ctx, k5->me->realm.data); + errctx = k5->ctx; + if (ret) { + com_err(progname, ret, + _("while setting up KDB keytab for realm %s"), + k5->me->realm.data); + goto cleanup; + } + } +#endif +*/ + // Resolve keytab + ret = krb5_kt_resolve(k5->ctx, keytab_file.c_str(), &keytab); + if (ret) + throw DB::Exception(0, "Error resolving keytab "+keytab_file); + + LOG_DEBUG(adqm_log,"KerberosInit: Using keytab: {}", keytab_file); + + // todo: num_pa_opts + + + // todo: in_cc / ccache + + // action: init or renew + // todo: doing only init action: + ret = krb5_get_init_creds_keytab(k5->ctx, &my_creds, k5->me, keytab, 0, nullptr, options); + if (ret) + LOG_DEBUG(adqm_log,"Getting initial credentials"); + + // todo: implement renew action + + + LOG_DEBUG(adqm_log,"Authenticated to Kerberos v5"); + LOG_DEBUG(adqm_log,"KerberosInit: end"); + return 0; +} + +KerberosInit::~KerberosInit() +{ + if (k5->ctx) + { + //begin. cleanup: + if (defcache) + krb5_cc_close(k5->ctx, defcache); + //todo + krb5_free_principal(k5->ctx, defcache_princ); + + // init. cleanup: + //todo: + /* + #ifndef _WIN32 + kinit_kdb_fini(); + #endif + */ + if (options) + krb5_get_init_creds_opt_free(k5->ctx, options); + if (my_creds.client == k5->me) + my_creds.client = nullptr; + /* + if (opts->pa_opts) { + free(opts->pa_opts); + opts->pa_opts = NULL; + opts->num_pa_opts = 0; + } + */ + krb5_free_cred_contents(k5->ctx, &my_creds); + if (keytab) + krb5_kt_close(k5->ctx, keytab); + + + // end: + krb5_free_unparsed_name(k5->ctx, k5->name); + krb5_free_principal(k5->ctx, k5->me); + /* + if (k5->in_cc != NULL) + krb5_cc_close(k5->ctx, k5->in_cc); + if (k5->out_cc != NULL) + krb5_cc_close(k5->ctx, k5->out_cc); + */ + krb5_free_context(k5->ctx); + } + memset(k5, 0, sizeof(*k5)); + + delete[] principal_name; +} diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h new file mode 100644 index 00000000000..ed64ba3b4a8 --- /dev/null +++ b/src/Access/KerberosInit.h @@ -0,0 +1,37 @@ +#pragma once + +#include "config_core.h" + +#include + +//#include +//#include "k5-platform.h" +#include +//#include + +struct k5_data +{ + krb5_context ctx; + krb5_ccache in_cc, out_cc; + krb5_principal me; + char *name; + krb5_boolean switch_to_cache; +}; + +class KerberosInit +{ +public: + int init(const String & keytab_file, const String & principal, const String & cache_name = ""); + ~KerberosInit(); +private: + struct k5_data * k5 = nullptr; + //struct k5_data k5; + struct k5_data k5d; + krb5_ccache defcache = nullptr; + krb5_get_init_creds_opt *options = nullptr; + krb5_creds my_creds; + krb5_keytab keytab = nullptr; + char * principal_name; + krb5_principal defcache_princ = nullptr; +}; + diff --git a/src/Access/examples/CMakeLists.txt b/src/Access/examples/CMakeLists.txt new file mode 100644 index 00000000000..07f75ca0b47 --- /dev/null +++ b/src/Access/examples/CMakeLists.txt @@ -0,0 +1,4 @@ +if (TARGET ch_contrib::krb5) + add_executable (kerberos_init kerberos_init.cpp) + target_link_libraries (kerberos_init PRIVATE dbms ch_contrib::krb5) +endif() diff --git a/src/Access/examples/kerberos_init.cpp b/src/Access/examples/kerberos_init.cpp new file mode 100644 index 00000000000..fb2575dc581 --- /dev/null +++ b/src/Access/examples/kerberos_init.cpp @@ -0,0 +1,36 @@ +#include +#include +#include +#include +#include +#include + +using namespace DB; + +int main(int argc, char ** argv) +{ + std::cout << "Kerberos Init" << "\n"; + + if (argc < 3) + { + std::cout << "Usage:" << "\n" << " kerberos_init keytab principal [cache]" << "\n"; + return 0; + } + + String cache_name = ""; + if (argc == 4) + cache_name = argv[3]; + + Poco::AutoPtr app_channel(new Poco::ConsoleChannel(std::cerr)); + Poco::Logger::root().setChannel(app_channel); + Poco::Logger::root().setLevel("trace"); + + KerberosInit k_init; + try { + k_init.init(argv[1], argv[2], cache_name); + } catch (const Exception & e) { + std::cout << "KerberosInit failure: " << getExceptionMessage(e, false) << "\n"; + } + std::cout << "Done" << "\n"; + return 0; +} diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 2f7b03790ee..71989bfd549 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB @@ -101,7 +102,7 @@ String HDFSBuilderWrapper::getKinitCmd() } void HDFSBuilderWrapper::runKinit() -{ +{ /* String cmd = getKinitCmd(); LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "running kinit: {}", cmd); @@ -113,6 +114,18 @@ void HDFSBuilderWrapper::runKinit() { throw Exception("kinit failure: " + cmd, ErrorCodes::BAD_ARGUMENTS); } + */ + LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: running KerberosInit"); + std::unique_lock lck(kinit_mtx); + KerberosInit k_init; + try { + k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal); + } catch (const DB::Exception & e) { + String msg = "KerberosInit failure: " + DB::getExceptionMessage(e, false); + LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: {}",msg); + throw Exception(0, msg); + } + LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: finished KerberosInit"); } HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::AbstractConfiguration & config) From 8b5bf0292748c5cfb0875b66c1d3a02118fea39d Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Fri, 3 Jun 2022 18:07:18 +0300 Subject: [PATCH 002/123] Add support of cache commands in KerberosInit --- src/Access/KerberosInit.cpp | 88 +++++++++++++++------------ src/Access/KerberosInit.h | 1 - src/Access/examples/kerberos_init.cpp | 3 +- src/Storages/HDFS/HDFSCommon.cpp | 2 +- 4 files changed, 53 insertions(+), 41 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 8954eccf136..0c643e27bd2 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -1,10 +1,9 @@ #include #include - #include #include #include - +#include int KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) { @@ -14,16 +13,14 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con krb5_error_code ret; // todo: use deftype - //const char *deftype = nullptr; + const char *deftype = nullptr; int flags = 0; - principal_name = new char[256]; - std::copy(principal.begin(), principal.end(), principal_name); - principal_name[principal.size()] = '\0'; - //memset(&k5, 0, sizeof(k5)); + if (!std::filesystem::exists(keytab_file)) + throw DB::Exception(0, "Error keytab file does not exist"); + memset(&k5d, 0, sizeof(k5d)); k5 = &k5d; - //memset(k5, 0, sizeof(k5_data)); // begin ret = krb5_init_context(&k5->ctx); if (ret) @@ -42,19 +39,46 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con if (ret) throw DB::Exception(0, "Error while getting default ccache"); LOG_DEBUG(adqm_log,"Resolved default cache"); - // todo: deftype - /*deftype = */krb5_cc_get_type(k5->ctx, defcache); + deftype = krb5_cc_get_type(k5->ctx, defcache); if (krb5_cc_get_principal(k5->ctx, defcache, &defcache_princ) != 0) defcache_princ = nullptr; } // Use the specified principal name. - ret = krb5_parse_name_flags(k5->ctx, principal_name, flags, &k5->me); + ret = krb5_parse_name_flags(k5->ctx, principal.c_str(), flags, &k5->me); if (ret) - throw DB::Exception(0, "Error when parsing principal name " + String(principal_name)); + throw DB::Exception(0, "Error when parsing principal name " + principal); + // Cache related commands + if (k5->out_cc == nullptr && krb5_cc_support_switch(k5->ctx, deftype)) + { + // Use an existing cache for the client principal if we can. + ret = krb5_cc_cache_match(k5->ctx, k5->me, &k5->out_cc); + if (ret && ret != KRB5_CC_NOTFOUND) + throw DB::Exception(0, "Error while searching for ccache for " + principal); + if (!ret) + { + LOG_DEBUG(adqm_log,"Using default cache: {}", krb5_cc_get_name(k5->ctx, k5->out_cc)); + k5->switch_to_cache = 1; + } + else if (defcache_princ != nullptr) + { + // Create a new cache to avoid overwriting the initialized default cache. + ret = krb5_cc_new_unique(k5->ctx, deftype, nullptr, &k5->out_cc); + if (ret) + throw DB::Exception(0, "Error while generating new ccache"); + LOG_DEBUG(adqm_log,"Using default cache: {}", krb5_cc_get_name(k5->ctx, k5->out_cc)); + k5->switch_to_cache = 1; + } + } - // to-do: add more cache init commands + // Use the default cache if we haven't picked one yet. + if (k5->out_cc == nullptr) + { + k5->out_cc = defcache; + defcache = nullptr; + LOG_DEBUG(adqm_log,"Using default cache: {}", krb5_cc_get_name(k5->ctx, k5->out_cc)); + } ret = krb5_unparse_name(k5->ctx, k5->me, &k5->name); if (ret) @@ -62,8 +86,6 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con LOG_DEBUG(adqm_log,"KerberosInit: Using principal: {}", k5->name); - principal_name = k5->name; - // init: memset(&my_creds, 0, sizeof(my_creds)); @@ -93,20 +115,23 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con LOG_DEBUG(adqm_log,"KerberosInit: Using keytab: {}", keytab_file); - // todo: num_pa_opts - - - // todo: in_cc / ccache + if (k5->in_cc) + { + ret = krb5_get_init_creds_opt_set_in_ccache(k5->ctx, options, k5->in_cc); + if (ret) + throw DB::Exception(0, "Error in setting input credential cache"); + } + ret = krb5_get_init_creds_opt_set_out_ccache(k5->ctx, options, k5->out_cc); + if (ret) + throw DB::Exception(0, "Error in setting output credential cache"); // action: init or renew + // todo: implement renew action // todo: doing only init action: ret = krb5_get_init_creds_keytab(k5->ctx, &my_creds, k5->me, keytab, 0, nullptr, options); if (ret) LOG_DEBUG(adqm_log,"Getting initial credentials"); - // todo: implement renew action - - LOG_DEBUG(adqm_log,"Authenticated to Kerberos v5"); LOG_DEBUG(adqm_log,"KerberosInit: end"); return 0; @@ -119,7 +144,6 @@ KerberosInit::~KerberosInit() //begin. cleanup: if (defcache) krb5_cc_close(k5->ctx, defcache); - //todo krb5_free_principal(k5->ctx, defcache_princ); // init. cleanup: @@ -133,30 +157,18 @@ KerberosInit::~KerberosInit() krb5_get_init_creds_opt_free(k5->ctx, options); if (my_creds.client == k5->me) my_creds.client = nullptr; - /* - if (opts->pa_opts) { - free(opts->pa_opts); - opts->pa_opts = NULL; - opts->num_pa_opts = 0; - } - */ krb5_free_cred_contents(k5->ctx, &my_creds); if (keytab) krb5_kt_close(k5->ctx, keytab); - - // end: + // end. cleanup: krb5_free_unparsed_name(k5->ctx, k5->name); krb5_free_principal(k5->ctx, k5->me); - /* - if (k5->in_cc != NULL) + if (k5->in_cc != nullptr) krb5_cc_close(k5->ctx, k5->in_cc); - if (k5->out_cc != NULL) + if (k5->out_cc != nullptr) krb5_cc_close(k5->ctx, k5->out_cc); - */ krb5_free_context(k5->ctx); } memset(k5, 0, sizeof(*k5)); - - delete[] principal_name; } diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index ed64ba3b4a8..07f619aaa0b 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -31,7 +31,6 @@ private: krb5_get_init_creds_opt *options = nullptr; krb5_creds my_creds; krb5_keytab keytab = nullptr; - char * principal_name; krb5_principal defcache_princ = nullptr; }; diff --git a/src/Access/examples/kerberos_init.cpp b/src/Access/examples/kerberos_init.cpp index fb2575dc581..554d57fb8b2 100644 --- a/src/Access/examples/kerberos_init.cpp +++ b/src/Access/examples/kerberos_init.cpp @@ -26,7 +26,8 @@ int main(int argc, char ** argv) Poco::Logger::root().setLevel("trace"); KerberosInit k_init; - try { + try + { k_init.init(argv[1], argv[2], cache_name); } catch (const Exception & e) { std::cout << "KerberosInit failure: " << getExceptionMessage(e, false) << "\n"; diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 71989bfd549..f2ec7af368a 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -119,7 +119,7 @@ void HDFSBuilderWrapper::runKinit() std::unique_lock lck(kinit_mtx); KerberosInit k_init; try { - k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal); + k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal,hadoop_security_kerberos_ticket_cache_path); } catch (const DB::Exception & e) { String msg = "KerberosInit failure: " + DB::getExceptionMessage(e, false); LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: {}",msg); From 323835f51d600f4b406492d5122304095b4b7cbb Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Mon, 6 Jun 2022 11:34:10 +0300 Subject: [PATCH 003/123] Add renew/init logic in KerberosInit --- src/Access/KerberosInit.cpp | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 0c643e27bd2..0142708c699 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -126,11 +126,35 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con throw DB::Exception(0, "Error in setting output credential cache"); // action: init or renew - // todo: implement renew action - // todo: doing only init action: - ret = krb5_get_init_creds_keytab(k5->ctx, &my_creds, k5->me, keytab, 0, nullptr, options); + LOG_DEBUG(adqm_log,"Trying to renew credentials"); + ret = krb5_get_renewed_creds(k5->ctx, &my_creds, k5->me, k5->out_cc, nullptr); if (ret) - LOG_DEBUG(adqm_log,"Getting initial credentials"); + { + LOG_DEBUG(adqm_log,"Renew failed, making init credentials"); + ret = krb5_get_init_creds_keytab(k5->ctx, &my_creds, k5->me, keytab, 0, nullptr, options); + if (ret) + throw DB::Exception(0, "Error in init"); + else + LOG_DEBUG(adqm_log,"Getting initial credentials"); + } + else + { + LOG_DEBUG(adqm_log,"Successfull reviewal"); + ret = krb5_cc_initialize(k5->ctx, k5->out_cc, k5->me); + if (ret) + throw DB::Exception(0, "Error when initializing cache"); + LOG_DEBUG(adqm_log,"Initialized cache"); + ret = krb5_cc_store_cred(k5->ctx, k5->out_cc, &my_creds); + if (ret) + LOG_DEBUG(adqm_log,"Error while storing credentials"); + LOG_DEBUG(adqm_log,"Stored credentials"); + } + + if (k5->switch_to_cache) { + ret = krb5_cc_switch(k5->ctx, k5->out_cc); + if (ret) + throw DB::Exception(0, "Error while switching to new ccache"); + } LOG_DEBUG(adqm_log,"Authenticated to Kerberos v5"); LOG_DEBUG(adqm_log,"KerberosInit: end"); From cb53aa15ec0a670eb98fd07738d95b74cbd1f73c Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Tue, 7 Jun 2022 12:06:22 +0300 Subject: [PATCH 004/123] Fix HDFSCommon and test_storage_kerberized_hdfs to make running integration tests --- src/Storages/HDFS/HDFSCommon.cpp | 4 +--- tests/integration/test_storage_kerberized_hdfs/test.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index f2ec7af368a..b13866d7a7b 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -121,9 +121,7 @@ void HDFSBuilderWrapper::runKinit() try { k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal,hadoop_security_kerberos_ticket_cache_path); } catch (const DB::Exception & e) { - String msg = "KerberosInit failure: " + DB::getExceptionMessage(e, false); - LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: {}",msg); - throw Exception(0, msg); + throw Exception("KerberosInit failure: "+ DB::getExceptionMessage(e, false), ErrorCodes::BAD_ARGUMENTS); } LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: finished KerberosInit"); } diff --git a/tests/integration/test_storage_kerberized_hdfs/test.py b/tests/integration/test_storage_kerberized_hdfs/test.py index fb00403b952..5ac8b4670f9 100644 --- a/tests/integration/test_storage_kerberized_hdfs/test.py +++ b/tests/integration/test_storage_kerberized_hdfs/test.py @@ -113,7 +113,7 @@ def test_read_table_expired(started_cluster): ) assert False, "Exception have to be thrown" except Exception as ex: - assert "DB::Exception: kinit failure:" in str(ex) + assert "DB::Exception: KerberosInit failure:" in str(ex) started_cluster.unpause_container("hdfskerberos") From a156a77890eeab8423f86a93e7a48e2784eeb0b0 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Tue, 7 Jun 2022 14:59:46 +0300 Subject: [PATCH 005/123] Add KerberosInit into StorageKafka --- src/Access/KerberosInit.cpp | 1 + src/Storages/HDFS/HDFSCommon.cpp | 3 ++- src/Storages/Kafka/StorageKafka.cpp | 21 +++++++++++++++++++ .../test_storage_kerberized_kafka/test.py | 4 +++- 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 0142708c699..724aa223c2b 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -9,6 +9,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con { auto adqm_log = &Poco::Logger::get("ADQM"); LOG_DEBUG(adqm_log,"KerberosInit: begin"); + //LOG_DEBUG(adqm_log,"KerberosInit: do nothing"); return 0; krb5_error_code ret; diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index b13866d7a7b..35319ce1d83 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -118,7 +118,8 @@ void HDFSBuilderWrapper::runKinit() LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: running KerberosInit"); std::unique_lock lck(kinit_mtx); KerberosInit k_init; - try { + try + { k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal,hadoop_security_kerberos_ticket_cache_path); } catch (const DB::Exception & e) { throw Exception("KerberosInit failure: "+ DB::getExceptionMessage(e, false), ErrorCodes::BAD_ARGUMENTS); diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 2409f8dcb6e..f6850f02511 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -44,6 +44,8 @@ #include #include +#include + namespace CurrentMetrics { @@ -515,6 +517,25 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) if (config.has(config_prefix)) loadFromConfig(conf, config, config_prefix); + if (conf.has_property("sasl.kerberos.keytab") && conf.has_property("sasl.kerberos.principal")) + { + LOG_DEBUG(log, "ADQM: preparing KerberosInit"); + String keytab = conf.get("sasl.kerberos.keytab"); + String principal = conf.get("sasl.kerberos.principal"); + LOG_DEBUG(log, "ADQM: keytab: {}, principal: {}", keytab, principal); + LOG_DEBUG(log, "ADQM: running KerberosInit"); + KerberosInit k_init; + try + { + k_init.init(keytab,principal); + } catch (const DB::Exception & e) { + LOG_ERROR(log, "ADQM: KerberosInit failure: {}", DB::getExceptionMessage(e, false)); + } + LOG_DEBUG(log, "ADQM: finished KerberosInit"); + conf.set("sasl.kerberos.kinit.cmd",""); + conf.set("sasl.kerberos.min.time.before.relogin","0"); + } + // Update consumer topic-specific configuration for (const auto & topic : topics) { diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index 6347ba89c16..98d39dd5d4f 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -122,6 +122,7 @@ def test_kafka_json_as_string(kafka_cluster): {"t": 124, "e": {"x": "test"} } {"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"} """ + logging.debug("ADQM: logs: %s", instance.grep_in_log("ADQM")) assert TSV(result) == TSV(expected) assert instance.contains_in_log( "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows" @@ -182,7 +183,8 @@ def test_kafka_json_as_string_no_kdc(kafka_cluster): assert TSV(result) == TSV(expected) assert instance.contains_in_log("StorageKafka (kafka_no_kdc): Nothing to commit") assert instance.contains_in_log("Ticket expired") - assert instance.contains_in_log("Kerberos ticket refresh failed") + #~ assert instance.contains_in_log("Kerberos ticket refresh failed") + assert instance.contains_in_log("KerberosInit failure:") if __name__ == "__main__": From 2b76d0c6a992631b7727c5b051c98d4e077f4803 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 8 Jun 2022 12:26:35 +0300 Subject: [PATCH 006/123] Add new integration test for kerberized Kafka; remove old kinit code from HDFSCommon --- src/Access/KerberosInit.cpp | 1 - src/Storages/HDFS/HDFSCommon.cpp | 38 +---------------- src/Storages/HDFS/HDFSCommon.h | 2 - .../test_storage_kerberized_kafka/test.py | 41 ++++++++++++++++++- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 724aa223c2b..1fa550a4ec1 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -13,7 +13,6 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con krb5_error_code ret; - // todo: use deftype const char *deftype = nullptr; int flags = 0; diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 35319ce1d83..a3554bafeff 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -77,44 +77,8 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration } } -String HDFSBuilderWrapper::getKinitCmd() -{ - - if (hadoop_kerberos_keytab.empty() || hadoop_kerberos_principal.empty()) - { - throw Exception("Not enough parameters to run kinit", - ErrorCodes::NO_ELEMENTS_IN_CONFIG); - } - - WriteBufferFromOwnString ss; - - String cache_name = hadoop_security_kerberos_ticket_cache_path.empty() ? - String() : - (String(" -c \"") + hadoop_security_kerberos_ticket_cache_path + "\""); - - // command to run looks like - // kinit -R -t /keytab_dir/clickhouse.keytab -k somebody@TEST.CLICKHOUSE.TECH || .. - ss << hadoop_kerberos_kinit_command << cache_name << - " -R -t \"" << hadoop_kerberos_keytab << "\" -k " << hadoop_kerberos_principal << - "|| " << hadoop_kerberos_kinit_command << cache_name << " -t \"" << - hadoop_kerberos_keytab << "\" -k " << hadoop_kerberos_principal; - return ss.str(); -} - void HDFSBuilderWrapper::runKinit() -{ /* - String cmd = getKinitCmd(); - LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "running kinit: {}", cmd); - - std::unique_lock lck(kinit_mtx); - - auto command = ShellCommand::execute(cmd); - auto status = command->tryWait(); - if (status) - { - throw Exception("kinit failure: " + cmd, ErrorCodes::BAD_ARGUMENTS); - } - */ +{ LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: running KerberosInit"); std::unique_lock lck(kinit_mtx); KerberosInit k_init; diff --git a/src/Storages/HDFS/HDFSCommon.h b/src/Storages/HDFS/HDFSCommon.h index 0523849abe5..773661b0745 100644 --- a/src/Storages/HDFS/HDFSCommon.h +++ b/src/Storages/HDFS/HDFSCommon.h @@ -69,8 +69,6 @@ public: private: void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const String & prefix, bool isUser = false); - String getKinitCmd(); - void runKinit(); // hdfs builder relies on an external config data storage diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index 98d39dd5d4f..091307a2b01 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -122,12 +122,51 @@ def test_kafka_json_as_string(kafka_cluster): {"t": 124, "e": {"x": "test"} } {"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"} """ - logging.debug("ADQM: logs: %s", instance.grep_in_log("ADQM")) assert TSV(result) == TSV(expected) assert instance.contains_in_log( "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows" ) +def test_kafka_json_as_string_request_new_ticket_after_expiration(kafka_cluster): + # Ticket should be expired after the wait time + # On run of SELECT query new ticket should be requested and SELECT query should run fine. + + kafka_produce( + kafka_cluster, + "kafka_json_as_string", + [ + '{"t": 123, "e": {"x": "woof"} }', + "", + '{"t": 124, "e": {"x": "test"} }', + '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}', + ], + ) + + instance.query( + """ + CREATE TABLE test.kafka (field String) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kerberized_kafka1:19092', + kafka_topic_list = 'kafka_json_as_string', + kafka_commit_on_select = 1, + kafka_group_name = 'kafka_json_as_string', + kafka_format = 'JSONAsString', + kafka_flush_interval_ms=1000; + """ + ) + + time.sleep(45) # wait for ticket expiration + + result = instance.query("SELECT * FROM test.kafka;") + expected = """\ +{"t": 123, "e": {"x": "woof"} } +{"t": 124, "e": {"x": "test"} } +{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"} +""" + assert TSV(result) == TSV(expected) + assert instance.contains_in_log( + "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows" + ) def test_kafka_json_as_string_no_kdc(kafka_cluster): # When the test is run alone (not preceded by any other kerberized kafka test), From 3cfea6e76f88e26b24cf4e6a371f033216dfa37f Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 8 Jun 2022 17:57:45 +0300 Subject: [PATCH 007/123] Cleanup code in KerberosInit, HDFSCommon and StorageKafka; update English and Russian documentation. --- .../table-engines/integrations/hdfs.md | 4 +- .../table-engines/integrations/kafka.md | 2 +- .../table-engines/integrations/hdfs.md | 3 +- .../table-engines/integrations/kafka.md | 2 +- src/Access/KerberosInit.cpp | 188 ++++++++---------- src/Access/KerberosInit.h | 19 +- src/Access/examples/kerberos_init.cpp | 2 + src/Storages/HDFS/HDFSCommon.cpp | 11 +- src/Storages/HDFS/HDFSCommon.h | 3 - src/Storages/Kafka/StorageKafka.cpp | 4 +- 10 files changed, 104 insertions(+), 134 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/hdfs.md b/docs/en/engines/table-engines/integrations/hdfs.md index 503bd779abf..146d1fcb3a6 100644 --- a/docs/en/engines/table-engines/integrations/hdfs.md +++ b/docs/en/engines/table-engines/integrations/hdfs.md @@ -186,7 +186,6 @@ Similar to GraphiteMergeTree, the HDFS engine supports extended configuration us | - | - | |hadoop\_kerberos\_keytab | "" | |hadoop\_kerberos\_principal | "" | -|hadoop\_kerberos\_kinit\_command | kinit | |libhdfs3\_conf | "" | ### Limitations {#limitations} @@ -200,8 +199,7 @@ Note that due to libhdfs3 limitations only old-fashioned approach is supported, datanode communications are not secured by SASL (`HADOOP_SECURE_DN_USER` is a reliable indicator of such security approach). Use `tests/integration/test_storage_kerberized_hdfs/hdfs_configs/bootstrap.sh` for reference. -If `hadoop_kerberos_keytab`, `hadoop_kerberos_principal` or `hadoop_kerberos_kinit_command` is specified, `kinit` will be invoked. `hadoop_kerberos_keytab` and `hadoop_kerberos_principal` are mandatory in this case. `kinit` tool and krb5 configuration files are required. - +If `hadoop_kerberos_keytab`, `hadoop_kerberos_principal` or `hadoop_security_kerberos_ticket_cache_path` are specified, Kerberos authentication will be used. `hadoop_kerberos_keytab` and `hadoop_kerberos_principal` are mandatory in this case. ## HDFS Namenode HA support {#namenode-ha} libhdfs3 support HDFS namenode HA. diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index a9d13194a59..94fcbab51ad 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -168,7 +168,7 @@ For a list of possible configuration options, see the [librdkafka configuration ### Kerberos support {#kafka-kerberos-support} To deal with Kerberos-aware Kafka, add `security_protocol` child element with `sasl_plaintext` value. It is enough if Kerberos ticket-granting ticket is obtained and cached by OS facilities. -ClickHouse is able to maintain Kerberos credentials using a keytab file. Consider `sasl_kerberos_service_name`, `sasl_kerberos_keytab`, `sasl_kerberos_principal` and `sasl.kerberos.kinit.cmd` child elements. +ClickHouse is able to maintain Kerberos credentials using a keytab file. Consider `sasl_kerberos_service_name`, `sasl_kerberos_keytab` and `sasl_kerberos_principal` child elements. Example: diff --git a/docs/ru/engines/table-engines/integrations/hdfs.md b/docs/ru/engines/table-engines/integrations/hdfs.md index 0857359e987..84f31c0afcc 100644 --- a/docs/ru/engines/table-engines/integrations/hdfs.md +++ b/docs/ru/engines/table-engines/integrations/hdfs.md @@ -183,7 +183,6 @@ CREATE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9 | - | - | |hadoop\_kerberos\_keytab | "" | |hadoop\_kerberos\_principal | "" | -|hadoop\_kerberos\_kinit\_command | kinit | ### Ограничения {#limitations} * `hadoop_security_kerberos_ticket_cache_path` и `libhdfs3_conf` могут быть определены только на глобальном, а не на пользовательском уровне @@ -196,7 +195,7 @@ CREATE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9 коммуникация с узлами данных не защищена SASL (`HADOOP_SECURE_DN_USER` надежный показатель такого подхода к безопасности). Используйте `tests/integration/test_storage_kerberized_hdfs/hdfs_configs/bootstrap.sh` для примера настроек. -Если `hadoop_kerberos_keytab`, `hadoop_kerberos_principal` или `hadoop_kerberos_kinit_command` указаны в настройках, `kinit` будет вызван. `hadoop_kerberos_keytab` и `hadoop_kerberos_principal` обязательны в этом случае. Необходимо также будет установить `kinit` и файлы конфигурации krb5. +Если `hadoop_kerberos_keytab`, `hadoop_kerberos_principal` или `hadoop_security_kerberos_ticket_cache_path` указаны в настройках, будет использоваться аутентификация с помощью Kerberos. `hadoop_kerberos_keytab` и `hadoop_kerberos_principal` обязательны в этом случае. ## Виртуальные столбцы {#virtual-columns} diff --git a/docs/ru/engines/table-engines/integrations/kafka.md b/docs/ru/engines/table-engines/integrations/kafka.md index b24a096015d..b51a0113302 100644 --- a/docs/ru/engines/table-engines/integrations/kafka.md +++ b/docs/ru/engines/table-engines/integrations/kafka.md @@ -167,7 +167,7 @@ Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format ### Поддержка Kerberos {#kafka-kerberos-support} Чтобы начать работу с Kafka с поддержкой Kerberos, добавьте дочерний элемент `security_protocol` со значением `sasl_plaintext`. Этого будет достаточно, если получен тикет на получение тикета (ticket-granting ticket) Kerberos и он кэшируется средствами ОС. -ClickHouse может поддерживать учетные данные Kerberos с помощью файла keytab. Рассмотрим дочерние элементы `sasl_kerberos_service_name`, `sasl_kerberos_keytab`, `sasl_kerberos_principal` и `sasl.kerberos.kinit.cmd`. +ClickHouse может поддерживать учетные данные Kerberos с помощью файла keytab. Рассмотрим дочерние элементы `sasl_kerberos_service_name`, `sasl_kerberos_keytab` и `sasl_kerberos_principal`. Пример: diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 1fa550a4ec1..1130447048b 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -5,11 +5,17 @@ #include #include +using namespace DB; + +std::mutex KerberosInit::kinit_mtx; + int KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) { - auto adqm_log = &Poco::Logger::get("ADQM"); - LOG_DEBUG(adqm_log,"KerberosInit: begin"); - //LOG_DEBUG(adqm_log,"KerberosInit: do nothing"); return 0; + // Using mutex to prevent cache file corruptions + std::unique_lock lck(kinit_mtx); + + auto log = &Poco::Logger::get("ADQM"); + LOG_DEBUG(log,"Trying to authenticate to Kerberos v5"); krb5_error_code ret; @@ -17,182 +23,152 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con int flags = 0; if (!std::filesystem::exists(keytab_file)) - throw DB::Exception(0, "Error keytab file does not exist"); + throw Exception("Error keytab file does not exist", ErrorCodes::KERBEROS_ERROR); - memset(&k5d, 0, sizeof(k5d)); - k5 = &k5d; - // begin - ret = krb5_init_context(&k5->ctx); + memset(&k5, 0, sizeof(k5)); + ret = krb5_init_context(&k5.ctx); if (ret) - throw DB::Exception(0, "Error while initializing Kerberos 5 library"); + throw Exception("Error while initializing Kerberos 5 library", ErrorCodes::KERBEROS_ERROR); if (!cache_name.empty()) { - ret = krb5_cc_resolve(k5->ctx, cache_name.c_str(), &k5->out_cc); - // todo: analyze return code - LOG_DEBUG(adqm_log,"Resolved cache"); + ret = krb5_cc_resolve(k5.ctx, cache_name.c_str(), &k5.out_cc); + if (ret) + throw Exception("Error in resolving cache", ErrorCodes::KERBEROS_ERROR); + LOG_DEBUG(log,"Resolved cache"); } else { // Resolve the default ccache and get its type and default principal (if it is initialized). - ret = krb5_cc_default(k5->ctx, &defcache); + ret = krb5_cc_default(k5.ctx, &defcache); if (ret) - throw DB::Exception(0, "Error while getting default ccache"); - LOG_DEBUG(adqm_log,"Resolved default cache"); - deftype = krb5_cc_get_type(k5->ctx, defcache); - if (krb5_cc_get_principal(k5->ctx, defcache, &defcache_princ) != 0) + throw Exception("Error while getting default ccache", ErrorCodes::KERBEROS_ERROR); + LOG_DEBUG(log,"Resolved default cache"); + deftype = krb5_cc_get_type(k5.ctx, defcache); + if (krb5_cc_get_principal(k5.ctx, defcache, &defcache_princ) != 0) defcache_princ = nullptr; } // Use the specified principal name. - ret = krb5_parse_name_flags(k5->ctx, principal.c_str(), flags, &k5->me); + ret = krb5_parse_name_flags(k5.ctx, principal.c_str(), flags, &k5.me); if (ret) - throw DB::Exception(0, "Error when parsing principal name " + principal); + throw Exception("Error when parsing principal name " + principal, ErrorCodes::KERBEROS_ERROR); // Cache related commands - if (k5->out_cc == nullptr && krb5_cc_support_switch(k5->ctx, deftype)) + if (k5.out_cc == nullptr && krb5_cc_support_switch(k5.ctx, deftype)) { // Use an existing cache for the client principal if we can. - ret = krb5_cc_cache_match(k5->ctx, k5->me, &k5->out_cc); + ret = krb5_cc_cache_match(k5.ctx, k5.me, &k5.out_cc); if (ret && ret != KRB5_CC_NOTFOUND) - throw DB::Exception(0, "Error while searching for ccache for " + principal); + throw Exception("Error while searching for cache for " + principal, ErrorCodes::KERBEROS_ERROR); if (!ret) { - LOG_DEBUG(adqm_log,"Using default cache: {}", krb5_cc_get_name(k5->ctx, k5->out_cc)); - k5->switch_to_cache = 1; + LOG_DEBUG(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); + k5.switch_to_cache = 1; } else if (defcache_princ != nullptr) { // Create a new cache to avoid overwriting the initialized default cache. - ret = krb5_cc_new_unique(k5->ctx, deftype, nullptr, &k5->out_cc); + ret = krb5_cc_new_unique(k5.ctx, deftype, nullptr, &k5.out_cc); if (ret) - throw DB::Exception(0, "Error while generating new ccache"); - LOG_DEBUG(adqm_log,"Using default cache: {}", krb5_cc_get_name(k5->ctx, k5->out_cc)); - k5->switch_to_cache = 1; + throw Exception("Error while generating new cache", ErrorCodes::KERBEROS_ERROR); + LOG_DEBUG(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); + k5.switch_to_cache = 1; } } // Use the default cache if we haven't picked one yet. - if (k5->out_cc == nullptr) + if (k5.out_cc == nullptr) { - k5->out_cc = defcache; + k5.out_cc = defcache; defcache = nullptr; - LOG_DEBUG(adqm_log,"Using default cache: {}", krb5_cc_get_name(k5->ctx, k5->out_cc)); + LOG_DEBUG(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); } - ret = krb5_unparse_name(k5->ctx, k5->me, &k5->name); + ret = krb5_unparse_name(k5.ctx, k5.me, &k5.name); if (ret) - throw DB::Exception(0, "Error when unparsing name"); + throw Exception("Error when unparsing name", ErrorCodes::KERBEROS_ERROR); + LOG_DEBUG(log,"Using principal: {}", k5.name); - LOG_DEBUG(adqm_log,"KerberosInit: Using principal: {}", k5->name); - - // init: memset(&my_creds, 0, sizeof(my_creds)); - - ret = krb5_get_init_creds_opt_alloc(k5->ctx, &options); + ret = krb5_get_init_creds_opt_alloc(k5.ctx, &options); if (ret) - throw DB::Exception(0, "Error in options allocation"); + throw Exception("Error in options allocation", ErrorCodes::KERBEROS_ERROR); - // todo -/* -#ifndef _WIN32 - if (strncmp(opts->keytab_name, "KDB:", 4) == 0) { - ret = kinit_kdb_init(&k5->ctx, k5->me->realm.data); - errctx = k5->ctx; - if (ret) { - com_err(progname, ret, - _("while setting up KDB keytab for realm %s"), - k5->me->realm.data); - goto cleanup; - } - } -#endif -*/ // Resolve keytab - ret = krb5_kt_resolve(k5->ctx, keytab_file.c_str(), &keytab); + ret = krb5_kt_resolve(k5.ctx, keytab_file.c_str(), &keytab); if (ret) - throw DB::Exception(0, "Error resolving keytab "+keytab_file); + throw Exception("Error in resolving keytab "+keytab_file, ErrorCodes::KERBEROS_ERROR); + LOG_DEBUG(log,"Using keytab: {}", keytab_file); - LOG_DEBUG(adqm_log,"KerberosInit: Using keytab: {}", keytab_file); - - if (k5->in_cc) + if (k5.in_cc) { - ret = krb5_get_init_creds_opt_set_in_ccache(k5->ctx, options, k5->in_cc); + ret = krb5_get_init_creds_opt_set_in_ccache(k5.ctx, options, k5.in_cc); if (ret) - throw DB::Exception(0, "Error in setting input credential cache"); + throw Exception("Error in setting input credential cache", ErrorCodes::KERBEROS_ERROR); } - ret = krb5_get_init_creds_opt_set_out_ccache(k5->ctx, options, k5->out_cc); + ret = krb5_get_init_creds_opt_set_out_ccache(k5.ctx, options, k5.out_cc); if (ret) - throw DB::Exception(0, "Error in setting output credential cache"); + throw Exception("Error in setting output credential cache", ErrorCodes::KERBEROS_ERROR); - // action: init or renew - LOG_DEBUG(adqm_log,"Trying to renew credentials"); - ret = krb5_get_renewed_creds(k5->ctx, &my_creds, k5->me, k5->out_cc, nullptr); + // Action: init or renew + LOG_DEBUG(log,"Trying to renew credentials"); + ret = krb5_get_renewed_creds(k5.ctx, &my_creds, k5.me, k5.out_cc, nullptr); if (ret) { - LOG_DEBUG(adqm_log,"Renew failed, making init credentials"); - ret = krb5_get_init_creds_keytab(k5->ctx, &my_creds, k5->me, keytab, 0, nullptr, options); + LOG_DEBUG(log,"Renew failed, trying to get initial credentials"); + ret = krb5_get_init_creds_keytab(k5.ctx, &my_creds, k5.me, keytab, 0, nullptr, options); if (ret) - throw DB::Exception(0, "Error in init"); + throw Exception("Error in getting initial credentials", ErrorCodes::KERBEROS_ERROR); else - LOG_DEBUG(adqm_log,"Getting initial credentials"); + LOG_DEBUG(log,"Got initial credentials"); } else { - LOG_DEBUG(adqm_log,"Successfull reviewal"); - ret = krb5_cc_initialize(k5->ctx, k5->out_cc, k5->me); + LOG_DEBUG(log,"Successfull renewal"); + ret = krb5_cc_initialize(k5.ctx, k5.out_cc, k5.me); if (ret) - throw DB::Exception(0, "Error when initializing cache"); - LOG_DEBUG(adqm_log,"Initialized cache"); - ret = krb5_cc_store_cred(k5->ctx, k5->out_cc, &my_creds); + throw Exception("Error when initializing cache", ErrorCodes::KERBEROS_ERROR); + LOG_DEBUG(log,"Initialized cache"); + ret = krb5_cc_store_cred(k5.ctx, k5.out_cc, &my_creds); if (ret) - LOG_DEBUG(adqm_log,"Error while storing credentials"); - LOG_DEBUG(adqm_log,"Stored credentials"); + LOG_DEBUG(log,"Error while storing credentials"); + LOG_DEBUG(log,"Stored credentials"); } - if (k5->switch_to_cache) { - ret = krb5_cc_switch(k5->ctx, k5->out_cc); + if (k5.switch_to_cache) { + ret = krb5_cc_switch(k5.ctx, k5.out_cc); if (ret) - throw DB::Exception(0, "Error while switching to new ccache"); + throw Exception("Error while switching to new ccache", ErrorCodes::KERBEROS_ERROR); } - LOG_DEBUG(adqm_log,"Authenticated to Kerberos v5"); - LOG_DEBUG(adqm_log,"KerberosInit: end"); + LOG_DEBUG(log,"Authenticated to Kerberos v5"); return 0; } KerberosInit::~KerberosInit() { - if (k5->ctx) + std::unique_lock lck(kinit_mtx); + if (k5.ctx) { - //begin. cleanup: if (defcache) - krb5_cc_close(k5->ctx, defcache); - krb5_free_principal(k5->ctx, defcache_princ); + krb5_cc_close(k5.ctx, defcache); + krb5_free_principal(k5.ctx, defcache_princ); - // init. cleanup: - //todo: - /* - #ifndef _WIN32 - kinit_kdb_fini(); - #endif - */ if (options) - krb5_get_init_creds_opt_free(k5->ctx, options); - if (my_creds.client == k5->me) + krb5_get_init_creds_opt_free(k5.ctx, options); + if (my_creds.client == k5.me) my_creds.client = nullptr; - krb5_free_cred_contents(k5->ctx, &my_creds); + krb5_free_cred_contents(k5.ctx, &my_creds); if (keytab) - krb5_kt_close(k5->ctx, keytab); + krb5_kt_close(k5.ctx, keytab); - // end. cleanup: - krb5_free_unparsed_name(k5->ctx, k5->name); - krb5_free_principal(k5->ctx, k5->me); - if (k5->in_cc != nullptr) - krb5_cc_close(k5->ctx, k5->in_cc); - if (k5->out_cc != nullptr) - krb5_cc_close(k5->ctx, k5->out_cc); - krb5_free_context(k5->ctx); + krb5_free_unparsed_name(k5.ctx, k5.name); + krb5_free_principal(k5.ctx, k5.me); + if (k5.in_cc != nullptr) + krb5_cc_close(k5.ctx, k5.in_cc); + if (k5.out_cc != nullptr) + krb5_cc_close(k5.ctx, k5.out_cc); + krb5_free_context(k5.ctx); } - memset(k5, 0, sizeof(*k5)); } diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index 07f619aaa0b..48401cbac1b 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -4,10 +4,16 @@ #include -//#include -//#include "k5-platform.h" #include -//#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int KERBEROS_ERROR; +} +} struct k5_data { @@ -24,13 +30,14 @@ public: int init(const String & keytab_file, const String & principal, const String & cache_name = ""); ~KerberosInit(); private: - struct k5_data * k5 = nullptr; - //struct k5_data k5; - struct k5_data k5d; + //struct k5_data * k5 = nullptr; + struct k5_data k5; + //struct k5_data k5d; krb5_ccache defcache = nullptr; krb5_get_init_creds_opt *options = nullptr; krb5_creds my_creds; krb5_keytab keytab = nullptr; krb5_principal defcache_princ = nullptr; + static std::mutex kinit_mtx; }; diff --git a/src/Access/examples/kerberos_init.cpp b/src/Access/examples/kerberos_init.cpp index 554d57fb8b2..49923e81fd7 100644 --- a/src/Access/examples/kerberos_init.cpp +++ b/src/Access/examples/kerberos_init.cpp @@ -13,6 +13,7 @@ int main(int argc, char ** argv) if (argc < 3) { + std::cout << "kerberos_init obtains and caches an initial ticket-granting ticket for principal." << "\n\n"; std::cout << "Usage:" << "\n" << " kerberos_init keytab principal [cache]" << "\n"; return 0; } @@ -31,6 +32,7 @@ int main(int argc, char ** argv) k_init.init(argv[1], argv[2], cache_name); } catch (const Exception & e) { std::cout << "KerberosInit failure: " << getExceptionMessage(e, false) << "\n"; + return -1; } std::cout << "Done" << "\n"; return 0; diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index a3554bafeff..e3592a7e53b 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -52,12 +52,6 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration hdfsBuilderSetPrincipal(hdfs_builder, hadoop_kerberos_principal.c_str()); continue; } - else if (key == "hadoop_kerberos_kinit_command") - { - need_kinit = true; - hadoop_kerberos_kinit_command = config.getString(key_path); - continue; - } else if (key == "hadoop_security_kerberos_ticket_cache_path") { if (isUser) @@ -80,13 +74,12 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration void HDFSBuilderWrapper::runKinit() { LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: running KerberosInit"); - std::unique_lock lck(kinit_mtx); KerberosInit k_init; try { k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal,hadoop_security_kerberos_ticket_cache_path); } catch (const DB::Exception & e) { - throw Exception("KerberosInit failure: "+ DB::getExceptionMessage(e, false), ErrorCodes::BAD_ARGUMENTS); + throw Exception("KerberosInit failure: "+ getExceptionMessage(e, false), ErrorCodes::KERBEROS_ERROR); } LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: finished KerberosInit"); } @@ -168,8 +161,6 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A return builder; } -std::mutex HDFSBuilderWrapper::kinit_mtx; - HDFSFSPtr createHDFSFS(hdfsBuilder * builder) { HDFSFSPtr fs(hdfsBuilderConnect(builder)); diff --git a/src/Storages/HDFS/HDFSCommon.h b/src/Storages/HDFS/HDFSCommon.h index 773661b0745..cacb986a091 100644 --- a/src/Storages/HDFS/HDFSCommon.h +++ b/src/Storages/HDFS/HDFSCommon.h @@ -9,7 +9,6 @@ #include #include -#include #include #include @@ -80,10 +79,8 @@ private: hdfsBuilder * hdfs_builder; String hadoop_kerberos_keytab; String hadoop_kerberos_principal; - String hadoop_kerberos_kinit_command = "kinit"; String hadoop_security_kerberos_ticket_cache_path; - static std::mutex kinit_mtx; std::vector> config_stor; bool need_kinit{false}; }; diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index f6850f02511..d8018aebbcc 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -528,8 +528,8 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) try { k_init.init(keytab,principal); - } catch (const DB::Exception & e) { - LOG_ERROR(log, "ADQM: KerberosInit failure: {}", DB::getExceptionMessage(e, false)); + } catch (const Exception & e) { + LOG_ERROR(log, "ADQM: KerberosInit failure: {}", getExceptionMessage(e, false)); } LOG_DEBUG(log, "ADQM: finished KerberosInit"); conf.set("sasl.kerberos.kinit.cmd",""); From d1d6d874322a012fc5f16975d668dfd0dd554cad Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Thu, 9 Jun 2022 11:51:15 +0300 Subject: [PATCH 008/123] Cleanup code in KerberosInit --- src/Access/KerberosInit.cpp | 10 +++++----- src/Storages/HDFS/HDFSCommon.cpp | 4 ++-- src/Storages/Kafka/StorageKafka.cpp | 8 +++----- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 1130447048b..549520e63ea 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -14,7 +14,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Using mutex to prevent cache file corruptions std::unique_lock lck(kinit_mtx); - auto log = &Poco::Logger::get("ADQM"); + auto log = &Poco::Logger::get("KerberosInit"); LOG_DEBUG(log,"Trying to authenticate to Kerberos v5"); krb5_error_code ret; @@ -39,10 +39,10 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con } else { - // Resolve the default ccache and get its type and default principal (if it is initialized). + // Resolve the default cache and get its type and default principal (if it is initialized). ret = krb5_cc_default(k5.ctx, &defcache); if (ret) - throw Exception("Error while getting default ccache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error while getting default cache", ErrorCodes::KERBEROS_ERROR); LOG_DEBUG(log,"Resolved default cache"); deftype = krb5_cc_get_type(k5.ctx, defcache); if (krb5_cc_get_principal(k5.ctx, defcache, &defcache_princ) != 0) @@ -116,7 +116,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con ret = krb5_get_renewed_creds(k5.ctx, &my_creds, k5.me, k5.out_cc, nullptr); if (ret) { - LOG_DEBUG(log,"Renew failed, trying to get initial credentials"); + LOG_DEBUG(log,"Renew failed ({}). Trying to get initial credentials", ret); ret = krb5_get_init_creds_keytab(k5.ctx, &my_creds, k5.me, keytab, 0, nullptr, options); if (ret) throw Exception("Error in getting initial credentials", ErrorCodes::KERBEROS_ERROR); @@ -139,7 +139,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con if (k5.switch_to_cache) { ret = krb5_cc_switch(k5.ctx, k5.out_cc); if (ret) - throw Exception("Error while switching to new ccache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error while switching to new cache", ErrorCodes::KERBEROS_ERROR); } LOG_DEBUG(log,"Authenticated to Kerberos v5"); diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index e3592a7e53b..160c2279d17 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -73,7 +73,7 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration void HDFSBuilderWrapper::runKinit() { - LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: running KerberosInit"); + LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Running KerberosInit"); KerberosInit k_init; try { @@ -81,7 +81,7 @@ void HDFSBuilderWrapper::runKinit() } catch (const DB::Exception & e) { throw Exception("KerberosInit failure: "+ getExceptionMessage(e, false), ErrorCodes::KERBEROS_ERROR); } - LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "ADQM: finished KerberosInit"); + LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Finished KerberosInit"); } HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::AbstractConfiguration & config) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index d8018aebbcc..b473b9fee56 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -519,19 +519,17 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) if (conf.has_property("sasl.kerberos.keytab") && conf.has_property("sasl.kerberos.principal")) { - LOG_DEBUG(log, "ADQM: preparing KerberosInit"); String keytab = conf.get("sasl.kerberos.keytab"); String principal = conf.get("sasl.kerberos.principal"); - LOG_DEBUG(log, "ADQM: keytab: {}, principal: {}", keytab, principal); - LOG_DEBUG(log, "ADQM: running KerberosInit"); + LOG_DEBUG(log, "Running KerberosInit"); KerberosInit k_init; try { k_init.init(keytab,principal); } catch (const Exception & e) { - LOG_ERROR(log, "ADQM: KerberosInit failure: {}", getExceptionMessage(e, false)); + LOG_ERROR(log, "KerberosInit failure: {}", getExceptionMessage(e, false)); } - LOG_DEBUG(log, "ADQM: finished KerberosInit"); + LOG_DEBUG(log, "Finished KerberosInit"); conf.set("sasl.kerberos.kinit.cmd",""); conf.set("sasl.kerberos.min.time.before.relogin","0"); } From 4c560584c700cdffbd66beab89a7e06e7546f646 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Fri, 10 Jun 2022 12:38:39 +0300 Subject: [PATCH 009/123] Code cleanup in KerberosInit and kafka integration tests --- src/Access/KerberosInit.h | 6 ++---- tests/integration/test_storage_kerberized_kafka/test.py | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index 48401cbac1b..f5269793c97 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -20,7 +20,7 @@ struct k5_data krb5_context ctx; krb5_ccache in_cc, out_cc; krb5_principal me; - char *name; + char * name; krb5_boolean switch_to_cache; }; @@ -30,11 +30,9 @@ public: int init(const String & keytab_file, const String & principal, const String & cache_name = ""); ~KerberosInit(); private: - //struct k5_data * k5 = nullptr; struct k5_data k5; - //struct k5_data k5d; krb5_ccache defcache = nullptr; - krb5_get_init_creds_opt *options = nullptr; + krb5_get_init_creds_opt * options = nullptr; krb5_creds my_creds; krb5_keytab keytab = nullptr; krb5_principal defcache_princ = nullptr; diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index 091307a2b01..7856361deda 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -127,6 +127,7 @@ def test_kafka_json_as_string(kafka_cluster): "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows" ) + def test_kafka_json_as_string_request_new_ticket_after_expiration(kafka_cluster): # Ticket should be expired after the wait time # On run of SELECT query new ticket should be requested and SELECT query should run fine. @@ -155,7 +156,7 @@ def test_kafka_json_as_string_request_new_ticket_after_expiration(kafka_cluster) """ ) - time.sleep(45) # wait for ticket expiration + time.sleep(45) # wait for ticket expiration result = instance.query("SELECT * FROM test.kafka;") expected = """\ @@ -168,6 +169,7 @@ def test_kafka_json_as_string_request_new_ticket_after_expiration(kafka_cluster) "Parsing of message (topic: kafka_json_as_string, partition: 0, offset: 1) return no rows" ) + def test_kafka_json_as_string_no_kdc(kafka_cluster): # When the test is run alone (not preceded by any other kerberized kafka test), # we need a ticket to @@ -222,7 +224,6 @@ def test_kafka_json_as_string_no_kdc(kafka_cluster): assert TSV(result) == TSV(expected) assert instance.contains_in_log("StorageKafka (kafka_no_kdc): Nothing to commit") assert instance.contains_in_log("Ticket expired") - #~ assert instance.contains_in_log("Kerberos ticket refresh failed") assert instance.contains_in_log("KerberosInit failure:") From c149c916ec98e9d6a89a9d69a8d0ddb30b9cf6c9 Mon Sep 17 00:00:00 2001 From: zvonand Date: Wed, 15 Jun 2022 11:49:55 +0500 Subject: [PATCH 010/123] initial setup --- .gitmodules | 3 + contrib/CMakeLists.txt | 1 + contrib/base58 | 1 + contrib/base58-cmake/CMakeLists.txt | 22 +++ src/Functions/CMakeLists.txt | 4 + src/Functions/FunctionBase58Conversion.h | 179 ++++++++++++++++++++++ src/Functions/FunctionsBase58.cpp | 23 +++ src/Functions/config_functions.h.in | 1 + src/Functions/configure_config.cmake | 3 + src/Functions/registerFunctionsString.cpp | 12 ++ src/configure_config.cmake | 3 + 11 files changed, 252 insertions(+) create mode 160000 contrib/base58 create mode 100644 contrib/base58-cmake/CMakeLists.txt create mode 100644 src/Functions/FunctionBase58Conversion.h create mode 100644 src/Functions/FunctionsBase58.cpp diff --git a/.gitmodules b/.gitmodules index 55fd684fddb..568dab1eb26 100644 --- a/.gitmodules +++ b/.gitmodules @@ -268,3 +268,6 @@ [submodule "contrib/hashidsxx"] path = contrib/hashidsxx url = https://github.com/schoentoon/hashidsxx.git +[submodule "contrib/base58"] + path = contrib/base58 + url = https://github.com/Kronuz/base-x.git diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 943e0e0ebc1..a356ade7eb8 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -153,6 +153,7 @@ endif() add_contrib (sqlite-cmake sqlite-amalgamation) add_contrib (s2geometry-cmake s2geometry) +add_contrib (base58-cmake base58) # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear diff --git a/contrib/base58 b/contrib/base58 new file mode 160000 index 00000000000..a85f98fb4ed --- /dev/null +++ b/contrib/base58 @@ -0,0 +1 @@ +Subproject commit a85f98fb4ed52c2f4029a4b6ac1ef0bafdfc56f5 diff --git a/contrib/base58-cmake/CMakeLists.txt b/contrib/base58-cmake/CMakeLists.txt new file mode 100644 index 00000000000..26783e0177d --- /dev/null +++ b/contrib/base58-cmake/CMakeLists.txt @@ -0,0 +1,22 @@ +set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/base58") + +set (SRCS + ${LIBRARY_DIR}/base_x.hh + ${LIBRARY_DIR}/uinteger_t.hh + ) + +add_library(_base58 ${SRCS}) + +target_include_directories(_base58 SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}) + +if (XCODE OR XCODE_VERSION) + # https://gitlab.kitware.com/cmake/cmake/issues/17457 + # Some native build systems may not like targets that have only object files, so consider adding at least one real source file + # This applies to Xcode. + if (NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/dummy.c") + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/dummy.c" "") + endif () + target_sources(_base58 PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/dummy.c") +endif () + +add_library(ch_contrib::base58 ALIAS _base58) \ No newline at end of file diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 60386908f01..bf72795aae0 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -72,6 +72,10 @@ if (TARGET ch_contrib::llvm) target_link_libraries(clickhouse_functions PRIVATE ch_contrib::llvm) endif () +if (TARGET ch_contrib::base58) + target_link_libraries(clickhouse_functions PRIVATE ch_contrib::base58) +endif() + if (TARGET ch_contrib::base64) target_link_libraries(clickhouse_functions PRIVATE ch_contrib::base64) endif() diff --git a/src/Functions/FunctionBase58Conversion.h b/src/Functions/FunctionBase58Conversion.h new file mode 100644 index 00000000000..fd1bc81842f --- /dev/null +++ b/src/Functions/FunctionBase58Conversion.h @@ -0,0 +1,179 @@ +#pragma once +#include "config_functions.h" + +#if USE_BASE58 +# include +# include +# include +# include +# include +# include +# include +# include +# include + + +namespace DB +{ +using namespace GatherUtils; + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int INCORRECT_DATA; + extern const int BAD_ARGUMENTS; +} + +struct Base58Encode +{ + static constexpr auto name = "base58Encode"; + static size_t getBufferSize(size_t string_length, size_t string_count) + { + return ((string_length - string_count) / 3 + string_count) * 4 + string_count; + } + + void process(ColumnString source, ColumnString result, std::string alphabet) + { + + } +}; + +struct Base58Decode +{ + static constexpr auto name = "base58Decode"; + + static size_t getBufferSize(size_t string_length, size_t string_count) + { + return ((string_length - string_count) / 4 + string_count) * 3 + string_count; + } +}; + +struct TryBase58Decode +{ + static constexpr auto name = "tryBase58Decode"; + + static size_t getBufferSize(size_t string_length, size_t string_count) + { + return Base58Decode::getBufferSize(string_length, string_count); + } +}; + +template +class FunctionBase58Conversion : public IFunction +{ +public: + static constexpr auto name = Func::name; + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + String getName() const override + { + return Func::name; + } + + bool isVariadic() const override { return true; } + + size_t getNumberOfArguments() const override { return 0; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + bool useDefaultImplementationForConstants() const override { return true; } + + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() != 1 || arguments.size() != 2) + throw Exception( + "Wrong number of arguments for function " + getName() + ": " + arguments.size() + " provided, 1 or 2 expected.", + ErrorCodes::BAD_ARGUMENTS); + + if (!isString(arguments[0].type)) + throw Exception( + "Illegal type " + arguments[0].type->getName() + " of 1st argument of function " + getName() + ". Must be String.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (!isString(arguments[1].type)) + throw Exception( + "Illegal type " + arguments[1].type->getName() + " of 2nd argument of function " + getName() + ". Must be String.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const ColumnPtr column_string = arguments[0].column; + const ColumnString * input = checkAndGetColumn(column_string.get()); + if (!input) + throw Exception( + "Illegal column " + arguments[0].column->getName() + " of first argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + + std::string alphabet = "bitcoin"; + + if (arguments.size() == 2) + { + const auto * alphabet_column = checkAndGetColumn(arguments[1].column.get()); + + if (!alphabet_column) + throw Exception("Second argument for function " + getName() + " must be constant String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (alphabet = alphabet_column->getValue(); alphabet != "bitcoin" && alphabet != "ripple" && alphabet != "flickr" && alphabet != "gmp") + throw Exception("Second argument for function " + getName() + " must be 'bitcoin', 'ripple', 'flickr' or 'gmp'", ErrorCodes::ILLEGAL_COLUMN); + + } + + auto dst_column = ColumnString::create(); + auto & dst_data = dst_column->getChars(); + auto & dst_offsets = dst_column->getOffsets(); + + size_t reserve = Func::getBufferSize(input->getChars().size(), input->size()); + dst_data.resize(reserve); + dst_offsets.resize(input_rows_count); + + const ColumnString::Offsets & src_offsets = input->getOffsets(); + + const auto * source = input->getChars().data(); + auto * dst = dst_data.data(); + auto * dst_pos = dst; + + size_t src_offset_prev = 0; + + for (size_t row = 0; row < input_rows_count; ++row) + { + size_t srclen = src_offsets[row] - src_offset_prev - 1; + size_t outlen = 0; + + if constexpr (std::is_same_v) + { + Base58:: + } + else if constexpr (std::is_same_v) + { + } + else + { + } + + source += srclen + 1; + dst_pos += outlen; + *dst_pos = '\0'; + dst_pos += 1; + + dst_offsets[row] = dst_pos - dst; + src_offset_prev = src_offsets[row]; + } + + dst_data.resize(dst_pos - dst); + + return dst_column; + } +}; +} + +#endif diff --git a/src/Functions/FunctionsBase58.cpp b/src/Functions/FunctionsBase58.cpp new file mode 100644 index 00000000000..efd3f42c6da --- /dev/null +++ b/src/Functions/FunctionsBase58.cpp @@ -0,0 +1,23 @@ +#include +#if USE_BASE58 +#include +#include + +namespace DB +{ +void registerFunctionBase58Encode(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +void registerFunctionBase58Decode(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +void registerFunctionTryBase58Decode(FunctionFactory & factory) +{ + factory.registerFunction>(); +} +} +#endif diff --git a/src/Functions/config_functions.h.in b/src/Functions/config_functions.h.in index a693611f975..001712d5cef 100644 --- a/src/Functions/config_functions.h.in +++ b/src/Functions/config_functions.h.in @@ -2,6 +2,7 @@ // .h autogenerated by cmake! +#cmakedefine01 USE_BASE58 #cmakedefine01 USE_BASE64 #cmakedefine01 USE_SIMDJSON #cmakedefine01 USE_RAPIDJSON diff --git a/src/Functions/configure_config.cmake b/src/Functions/configure_config.cmake index 7615a2eeeaf..776996d7e17 100644 --- a/src/Functions/configure_config.cmake +++ b/src/Functions/configure_config.cmake @@ -1,6 +1,9 @@ if (TARGET ch_contrib::fastops) set(USE_FASTOPS 1) endif() +if (TARGET ch_contrib::base58) + set(USE_BASE58 1) +endif() if (TARGET ch_contrib::base64) set(USE_BASE64 1) endif() diff --git a/src/Functions/registerFunctionsString.cpp b/src/Functions/registerFunctionsString.cpp index f86043c6959..2e2975a459c 100644 --- a/src/Functions/registerFunctionsString.cpp +++ b/src/Functions/registerFunctionsString.cpp @@ -49,6 +49,12 @@ void registerFunctionBase64Decode(FunctionFactory &); void registerFunctionTryBase64Decode(FunctionFactory &); #endif +#if USE_BASE58 +void registerFunctionBase58Encode(FunctionFactory &); +void registerFunctionBase58Decode(FunctionFactory &); +void registerFunctionTryBase58Decode(FunctionFactory &); +#endif + #if USE_NLP void registerFunctionStem(FunctionFactory &); void registerFunctionSynonyms(FunctionFactory &); @@ -105,6 +111,12 @@ void registerFunctionsString(FunctionFactory & factory) registerFunctionTryBase64Decode(factory); #endif +#if USE_BASE58 + registerFunctionBase58Encode(factory); + registerFunctionBase58Decode(factory); + registerFunctionTryBase58Decode(factory); +#endif + #if USE_NLP registerFunctionStem(factory); registerFunctionSynonyms(factory); diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 519307ba28a..fc2a858e75a 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -55,6 +55,9 @@ endif() if (TARGET ch_contrib::base64) set(USE_BASE64 1) endif() +if (TARGET ch_contrib::base58) + set(USE_BASE58 1) +endif() if (TARGET ch_contrib::yaml_cpp) set(USE_YAML_CPP 1) endif() From 9bf6b9d491fa5e8bba23a61fcca0a004b6c66451 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 15 Jun 2022 11:37:02 +0300 Subject: [PATCH 011/123] Add kinit presence handling in StorageKafka; Cleanup code in HDFSCommon --- src/Storages/HDFS/HDFSCommon.cpp | 4 +++- src/Storages/Kafka/StorageKafka.cpp | 12 +++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 160c2279d17..3ade1b1cfdf 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -78,7 +78,9 @@ void HDFSBuilderWrapper::runKinit() try { k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal,hadoop_security_kerberos_ticket_cache_path); - } catch (const DB::Exception & e) { + } + catch (const DB::Exception & e) + { throw Exception("KerberosInit failure: "+ getExceptionMessage(e, false), ErrorCodes::KERBEROS_ERROR); } LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Finished KerberosInit"); diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index b473b9fee56..39b8adfbec4 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -517,6 +517,12 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) if (config.has(config_prefix)) loadFromConfig(conf, config, config_prefix); + if (conf.has_property("sasl.kerberos.kinit.cmd")) + LOG_WARNING(log, "kinit executable is not allowed."); + + conf.set("sasl.kerberos.kinit.cmd",""); + conf.set("sasl.kerberos.min.time.before.relogin","0"); + if (conf.has_property("sasl.kerberos.keytab") && conf.has_property("sasl.kerberos.principal")) { String keytab = conf.get("sasl.kerberos.keytab"); @@ -526,12 +532,12 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) try { k_init.init(keytab,principal); - } catch (const Exception & e) { + } + catch (const Exception & e) + { LOG_ERROR(log, "KerberosInit failure: {}", getExceptionMessage(e, false)); } LOG_DEBUG(log, "Finished KerberosInit"); - conf.set("sasl.kerberos.kinit.cmd",""); - conf.set("sasl.kerberos.min.time.before.relogin","0"); } // Update consumer topic-specific configuration From dd5b0ee0656c6b10f25920f8557c2ef9f2b08260 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 15 Jun 2022 17:02:53 +0300 Subject: [PATCH 012/123] Add kerberosInit() function to call KeberosInit --- src/Access/KerberosInit.cpp | 36 +++++++++++++++++++++++---- src/Access/KerberosInit.h | 25 +------------------ src/Access/examples/kerberos_init.cpp | 3 +-- src/Storages/HDFS/HDFSCommon.cpp | 3 +-- src/Storages/Kafka/StorageKafka.cpp | 3 +-- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 549520e63ea..95633dea077 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -7,13 +7,31 @@ using namespace DB; -std::mutex KerberosInit::kinit_mtx; +struct k5_data +{ + krb5_context ctx; + krb5_ccache in_cc, out_cc; + krb5_principal me; + char * name; + krb5_boolean switch_to_cache; +}; + +class KerberosInit +{ +public: + int init(const String & keytab_file, const String & principal, const String & cache_name = ""); + ~KerberosInit(); +private: + struct k5_data k5; + krb5_ccache defcache = nullptr; + krb5_get_init_creds_opt * options = nullptr; + krb5_creds my_creds; + krb5_keytab keytab = nullptr; + krb5_principal defcache_princ = nullptr; +}; int KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) { - // Using mutex to prevent cache file corruptions - std::unique_lock lck(kinit_mtx); - auto log = &Poco::Logger::get("KerberosInit"); LOG_DEBUG(log,"Trying to authenticate to Kerberos v5"); @@ -148,7 +166,6 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con KerberosInit::~KerberosInit() { - std::unique_lock lck(kinit_mtx); if (k5.ctx) { if (defcache) @@ -172,3 +189,12 @@ KerberosInit::~KerberosInit() krb5_free_context(k5.ctx); } } + +int kerberosInit(const String & keytab_file, const String & principal, const String & cache_name) +{ + // Using mutex to prevent cache file corruptions + static std::mutex kinit_mtx; + std::unique_lock lck(kinit_mtx); + KerberosInit k_init; + return k_init.init(keytab_file, principal, cache_name); +} diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index f5269793c97..94910661056 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -15,27 +15,4 @@ namespace ErrorCodes } } -struct k5_data -{ - krb5_context ctx; - krb5_ccache in_cc, out_cc; - krb5_principal me; - char * name; - krb5_boolean switch_to_cache; -}; - -class KerberosInit -{ -public: - int init(const String & keytab_file, const String & principal, const String & cache_name = ""); - ~KerberosInit(); -private: - struct k5_data k5; - krb5_ccache defcache = nullptr; - krb5_get_init_creds_opt * options = nullptr; - krb5_creds my_creds; - krb5_keytab keytab = nullptr; - krb5_principal defcache_princ = nullptr; - static std::mutex kinit_mtx; -}; - +int kerberosInit(const String & keytab_file, const String & principal, const String & cache_name = ""); diff --git a/src/Access/examples/kerberos_init.cpp b/src/Access/examples/kerberos_init.cpp index 49923e81fd7..9a42a9e00e5 100644 --- a/src/Access/examples/kerberos_init.cpp +++ b/src/Access/examples/kerberos_init.cpp @@ -26,10 +26,9 @@ int main(int argc, char ** argv) Poco::Logger::root().setChannel(app_channel); Poco::Logger::root().setLevel("trace"); - KerberosInit k_init; try { - k_init.init(argv[1], argv[2], cache_name); + kerberosInit(argv[1], argv[2], cache_name); } catch (const Exception & e) { std::cout << "KerberosInit failure: " << getExceptionMessage(e, false) << "\n"; return -1; diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 3ade1b1cfdf..9d3cb2353b6 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -74,10 +74,9 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration void HDFSBuilderWrapper::runKinit() { LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Running KerberosInit"); - KerberosInit k_init; try { - k_init.init(hadoop_kerberos_keytab,hadoop_kerberos_principal,hadoop_security_kerberos_ticket_cache_path); + kerberosInit(hadoop_kerberos_keytab,hadoop_kerberos_principal,hadoop_security_kerberos_ticket_cache_path); } catch (const DB::Exception & e) { diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 39b8adfbec4..789df36fcf0 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -528,10 +528,9 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) String keytab = conf.get("sasl.kerberos.keytab"); String principal = conf.get("sasl.kerberos.principal"); LOG_DEBUG(log, "Running KerberosInit"); - KerberosInit k_init; try { - k_init.init(keytab,principal); + kerberosInit(keytab,principal); } catch (const Exception & e) { From 1c26424371adc53f8eefd8d0e3cc604954be089e Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 15 Jun 2022 19:35:21 +0300 Subject: [PATCH 013/123] Change message in StorageKafka; Code style correction --- src/Access/KerberosInit.cpp | 3 ++- src/Storages/Kafka/StorageKafka.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 95633dea077..6c4927dd4fa 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -154,7 +154,8 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con LOG_DEBUG(log,"Stored credentials"); } - if (k5.switch_to_cache) { + if (k5.switch_to_cache) + { ret = krb5_cc_switch(k5.ctx, k5.out_cc); if (ret) throw Exception("Error while switching to new cache", ErrorCodes::KERBEROS_ERROR); diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 789df36fcf0..b4072cd6c0e 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -518,7 +518,7 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) loadFromConfig(conf, config, config_prefix); if (conf.has_property("sasl.kerberos.kinit.cmd")) - LOG_WARNING(log, "kinit executable is not allowed."); + LOG_WARNING(log, "sasl.kerberos.kinit.cmd configuration parameter is ignored."); conf.set("sasl.kerberos.kinit.cmd",""); conf.set("sasl.kerberos.min.time.before.relogin","0"); From 89a659e738a6a7236300f918b508ad0273745304 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 15 Jun 2022 20:08:16 +0300 Subject: [PATCH 014/123] Move krb header files from KerberosInit.h to KerberosInit.cpp --- src/Access/KerberosInit.cpp | 2 ++ src/Access/KerberosInit.h | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 6c4927dd4fa..4fcb280f033 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include using namespace DB; diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index 94910661056..60f959ab4ba 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -4,9 +4,6 @@ #include -#include -#include - namespace DB { namespace ErrorCodes From 344fbe8de4038ec6f93d67eb3455eda1ad82ce87 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 15 Jun 2022 20:26:42 +0300 Subject: [PATCH 015/123] Fix code style --- src/Access/KerberosInit.cpp | 10 +++++++++- src/Access/KerberosInit.h | 8 -------- src/Access/examples/kerberos_init.cpp | 6 ++++-- src/Storages/HDFS/HDFSCommon.cpp | 1 + 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 4fcb280f033..304bb69d424 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -9,6 +9,14 @@ using namespace DB; +namespace DB +{ +namespace ErrorCodes +{ + extern const int KERBEROS_ERROR; +} +} + struct k5_data { krb5_context ctx; @@ -145,7 +153,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con } else { - LOG_DEBUG(log,"Successfull renewal"); + LOG_DEBUG(log,"Successful renewal"); ret = krb5_cc_initialize(k5.ctx, k5.out_cc, k5.me); if (ret) throw Exception("Error when initializing cache", ErrorCodes::KERBEROS_ERROR); diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index 60f959ab4ba..cfbbfc2e7d8 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -4,12 +4,4 @@ #include -namespace DB -{ -namespace ErrorCodes -{ - extern const int KERBEROS_ERROR; -} -} - int kerberosInit(const String & keytab_file, const String & principal, const String & cache_name = ""); diff --git a/src/Access/examples/kerberos_init.cpp b/src/Access/examples/kerberos_init.cpp index 9a42a9e00e5..4868182fba2 100644 --- a/src/Access/examples/kerberos_init.cpp +++ b/src/Access/examples/kerberos_init.cpp @@ -29,8 +29,10 @@ int main(int argc, char ** argv) try { kerberosInit(argv[1], argv[2], cache_name); - } catch (const Exception & e) { - std::cout << "KerberosInit failure: " << getExceptionMessage(e, false) << "\n"; + } + catch (const Exception & e) + { + std::cout << "KerberosInit failure: " << getExceptionMessage(e, false) << "\n"; return -1; } std::cout << "Done" << "\n"; diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 9d3cb2353b6..8134aaf4981 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -21,6 +21,7 @@ namespace ErrorCodes extern const int NETWORK_ERROR; extern const int EXCESSIVE_ELEMENT_IN_CONFIG; extern const int NO_ELEMENTS_IN_CONFIG; + extern const int KERBEROS_ERROR; } const String HDFSBuilderWrapper::CONFIG_PREFIX = "hdfs"; From d93fd3bd2dcdfec6d1eba790508e393d91858999 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Thu, 16 Jun 2022 09:30:40 +0000 Subject: [PATCH 016/123] Add complilation support for case when krb5 is not used --- src/Access/KerberosInit.cpp | 2 ++ src/Access/KerberosInit.h | 4 ++++ src/Storages/HDFS/HDFSCommon.cpp | 5 ++++- src/Storages/Kafka/StorageKafka.cpp | 6 ++++-- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 304bb69d424..8d4e2339bbf 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -4,6 +4,7 @@ #include #include #include +#if USE_KRB5 #include #include @@ -209,3 +210,4 @@ int kerberosInit(const String & keytab_file, const String & principal, const Str KerberosInit k_init; return k_init.init(keytab_file, principal, cache_name); } +#endif // USE_KRB5 diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index cfbbfc2e7d8..4731baf706f 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -4,4 +4,8 @@ #include +#if USE_KRB5 + int kerberosInit(const String & keytab_file, const String & principal, const String & cache_name = ""); + +#endif // USE_KRB5 diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 8134aaf4981..bee8ac21acb 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -10,8 +10,9 @@ #include #include #include +#if USE_KRB5 #include - +#endif // USE_KRB5 namespace DB { @@ -74,6 +75,7 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration void HDFSBuilderWrapper::runKinit() { + #if USE_KRB5 LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Running KerberosInit"); try { @@ -84,6 +86,7 @@ void HDFSBuilderWrapper::runKinit() throw Exception("KerberosInit failure: "+ getExceptionMessage(e, false), ErrorCodes::KERBEROS_ERROR); } LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Finished KerberosInit"); + #endif // USE_KRB5 } HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::AbstractConfiguration & config) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index b4072cd6c0e..a24630fa3b1 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -43,9 +43,9 @@ #include #include - +#if USE_KRB5 #include - +#endif // USE_KRB5 namespace CurrentMetrics { @@ -517,6 +517,7 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) if (config.has(config_prefix)) loadFromConfig(conf, config, config_prefix); + #if USE_KRB5 if (conf.has_property("sasl.kerberos.kinit.cmd")) LOG_WARNING(log, "sasl.kerberos.kinit.cmd configuration parameter is ignored."); @@ -538,6 +539,7 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) } LOG_DEBUG(log, "Finished KerberosInit"); } + #endif // USE_KRB5 // Update consumer topic-specific configuration for (const auto & topic : topics) From a800158438d939555b8f8cb472c9d3ffc51523e5 Mon Sep 17 00:00:00 2001 From: zvonand Date: Thu, 16 Jun 2022 15:11:41 +0500 Subject: [PATCH 017/123] wip upload --- src/Functions/FunctionBase58Conversion.h | 102 ++++++++++++----------- 1 file changed, 53 insertions(+), 49 deletions(-) diff --git a/src/Functions/FunctionBase58Conversion.h b/src/Functions/FunctionBase58Conversion.h index fd1bc81842f..9aacbc4c9e7 100644 --- a/src/Functions/FunctionBase58Conversion.h +++ b/src/Functions/FunctionBase58Conversion.h @@ -8,14 +8,12 @@ # include # include # include -# include # include # include namespace DB { -using namespace GatherUtils; namespace ErrorCodes { @@ -28,14 +26,58 @@ namespace ErrorCodes struct Base58Encode { static constexpr auto name = "base58Encode"; - static size_t getBufferSize(size_t string_length, size_t string_count) - { - return ((string_length - string_count) / 3 + string_count) * 4 + string_count; - } - void process(ColumnString source, ColumnString result, std::string alphabet) + static void process(const ColumnString * input, ColumnString * dst_column, std::string& alphabet, size_t input_rows_count) { + auto & dst_data = dst_column->getChars(); + auto & dst_offsets = dst_column->getOffsets(); + size_t current_allocated_size = input->getChars().size(); + + dst_data.resize(current_allocated_size); + dst_offsets.resize(input_rows_count); + + const ColumnString::Offsets & src_offsets = input->getOffsets(); + + const auto * source = input->getChars().raw_data(); + auto * dst = dst_data.data(); + auto * dst_pos = dst; + + size_t src_offset_prev = 0; + size_t processed_size = 0; + + const auto& encoder = (alphabet == "bitcoin") ? Base58::bitcoin() : + ((alphabet == "flickr") ? Base58::flickr() : + ((alphabet == "ripple") ? Base58::ripple() : Base58::base58())); + + for (size_t row = 0; row < input_rows_count; ++row) + { + size_t srclen = src_offsets[row] - src_offset_prev - 1; + /// Why we didn't simply operate on char* here? + /// We don't know the size of the result string beforehand (it's not byte-to-byte encoding), + /// so we may need to do many resizes (the worst case -- we'll do it for each row) + /// Using std::string allows to do exponential resizes and one final resize after whole operation is complete + std::string encoded; + encoder.encode(encoded, source, srclen); + size_t outlen = encoded.size(); + + if (processed_size + outlen >= current_allocated_size) + { + current_allocated_size += current_allocated_size; + dst_data.resize(current_allocated_size); + } + + source += srclen + 1; + dst_pos += outlen; + *dst_pos = '\0'; + dst_pos += 1; + + dst_offsets[row] = dst_pos - dst; + src_offset_prev = src_offsets[row]; + processed_size += outlen; + } + + dst_data.resize(dst_pos - dst); } }; @@ -123,53 +165,15 @@ public: if (!alphabet_column) throw Exception("Second argument for function " + getName() + " must be constant String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (alphabet = alphabet_column->getValue(); alphabet != "bitcoin" && alphabet != "ripple" && alphabet != "flickr" && alphabet != "gmp") - throw Exception("Second argument for function " + getName() + " must be 'bitcoin', 'ripple', 'flickr' or 'gmp'", ErrorCodes::ILLEGAL_COLUMN); + if (alphabet = alphabet_column->getValue(); + alphabet != "bitcoin" && alphabet != "ripple" && alphabet != "flickr" && alphabet != "gmp") + throw Exception("Second argument for function " + getName() + " must be 'bitcoin', 'ripple', 'gmp' or 'flickr'", ErrorCodes::ILLEGAL_COLUMN); } auto dst_column = ColumnString::create(); - auto & dst_data = dst_column->getChars(); - auto & dst_offsets = dst_column->getOffsets(); - size_t reserve = Func::getBufferSize(input->getChars().size(), input->size()); - dst_data.resize(reserve); - dst_offsets.resize(input_rows_count); - - const ColumnString::Offsets & src_offsets = input->getOffsets(); - - const auto * source = input->getChars().data(); - auto * dst = dst_data.data(); - auto * dst_pos = dst; - - size_t src_offset_prev = 0; - - for (size_t row = 0; row < input_rows_count; ++row) - { - size_t srclen = src_offsets[row] - src_offset_prev - 1; - size_t outlen = 0; - - if constexpr (std::is_same_v) - { - Base58:: - } - else if constexpr (std::is_same_v) - { - } - else - { - } - - source += srclen + 1; - dst_pos += outlen; - *dst_pos = '\0'; - dst_pos += 1; - - dst_offsets[row] = dst_pos - dst; - src_offset_prev = src_offsets[row]; - } - - dst_data.resize(dst_pos - dst); + Func::process(column_string, dst_column, alphabet, input_rows_count); return dst_column; } From 6e28275569d9ede757ab490f110e578524775458 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Thu, 16 Jun 2022 14:21:04 +0300 Subject: [PATCH 018/123] Add warnings about using krb5 parameters --- src/Storages/HDFS/HDFSCommon.cpp | 19 ++++++++++++++++--- src/Storages/HDFS/HDFSCommon.h | 9 +++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index bee8ac21acb..87b2d02ada0 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -20,9 +20,10 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int NETWORK_ERROR; + #if USE_KRB5 extern const int EXCESSIVE_ELEMENT_IN_CONFIG; - extern const int NO_ELEMENTS_IN_CONFIG; extern const int KERBEROS_ERROR; + #endif // USE_KRB5 } const String HDFSBuilderWrapper::CONFIG_PREFIX = "hdfs"; @@ -43,19 +44,28 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration String key_name; if (key == "hadoop_kerberos_keytab") { + #if USE_KRB5 need_kinit = true; hadoop_kerberos_keytab = config.getString(key_path); + #else // USE_KRB5 + LOG_WARNING(&Poco::Logger::get("HDFSClient"), "hadoop_kerberos_keytab parameter is ignored because ClickHouse was built without support of krb5 library."); + #endif // USE_KRB5 continue; } else if (key == "hadoop_kerberos_principal") { + #if USE_KRB5 need_kinit = true; hadoop_kerberos_principal = config.getString(key_path); hdfsBuilderSetPrincipal(hdfs_builder, hadoop_kerberos_principal.c_str()); + #else // USE_KRB5 + LOG_WARNING(&Poco::Logger::get("HDFSClient"), "hadoop_kerberos_principal parameter is ignored because ClickHouse was built without support of krb5 library."); + #endif // USE_KRB5 continue; } else if (key == "hadoop_security_kerberos_ticket_cache_path") { + #if USE_KRB5 if (isUser) { throw Exception("hadoop.security.kerberos.ticket.cache.path cannot be set per user", @@ -64,6 +74,9 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration hadoop_security_kerberos_ticket_cache_path = config.getString(key_path); // standard param - pass further + #else // USE_KRB5 + LOG_WARNING(&Poco::Logger::get("HDFSClient"), "hadoop.security.kerberos.ticket.cache.path parameter is ignored because ClickHouse was built without support of krb5 library."); + #endif // USE_KRB5 } key_name = boost::replace_all_copy(key, "_", "."); @@ -73,9 +86,9 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration } } +#if USE_KRB5 void HDFSBuilderWrapper::runKinit() { - #if USE_KRB5 LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Running KerberosInit"); try { @@ -86,8 +99,8 @@ void HDFSBuilderWrapper::runKinit() throw Exception("KerberosInit failure: "+ getExceptionMessage(e, false), ErrorCodes::KERBEROS_ERROR); } LOG_DEBUG(&Poco::Logger::get("HDFSClient"), "Finished KerberosInit"); - #endif // USE_KRB5 } +#endif // USE_KRB5 HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::AbstractConfiguration & config) { diff --git a/src/Storages/HDFS/HDFSCommon.h b/src/Storages/HDFS/HDFSCommon.h index cacb986a091..9eb2dfd3e46 100644 --- a/src/Storages/HDFS/HDFSCommon.h +++ b/src/Storages/HDFS/HDFSCommon.h @@ -68,8 +68,6 @@ public: private: void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const String & prefix, bool isUser = false); - void runKinit(); - // hdfs builder relies on an external config data storage std::pair& keep(const String & k, const String & v) { @@ -77,12 +75,15 @@ private: } hdfsBuilder * hdfs_builder; + std::vector> config_stor; + + #if USE_KRB5 + void runKinit(); String hadoop_kerberos_keytab; String hadoop_kerberos_principal; String hadoop_security_kerberos_ticket_cache_path; - - std::vector> config_stor; bool need_kinit{false}; + #endif // USE_KRB5 }; using HDFSFSPtr = std::unique_ptr, detail::HDFSFsDeleter>; From 0ab6bfd5d805b358cb61ae053909527a9253e47b Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Thu, 16 Jun 2022 14:25:37 +0300 Subject: [PATCH 019/123] Add warning about using krb5 parameter in StorageKafka.cpp --- src/Storages/Kafka/StorageKafka.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index a24630fa3b1..e65ac27b096 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -539,6 +539,9 @@ void StorageKafka::updateConfiguration(cppkafka::Configuration & conf) } LOG_DEBUG(log, "Finished KerberosInit"); } + #else // USE_KRB5 + if (conf.has_property("sasl.kerberos.keytab") || conf.has_property("sasl.kerberos.principal")) + LOG_WARNING(log, "Kerberos-related parameters are ignored because ClickHouse was built without support of krb5 library."); #endif // USE_KRB5 // Update consumer topic-specific configuration From ed3fe84b63d14cb05898cf12eac8024ae3921110 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Thu, 16 Jun 2022 14:45:27 +0300 Subject: [PATCH 020/123] Fix runKinit() is called only for USE_KRB5 --- src/Storages/HDFS/HDFSCommon.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index 87b2d02ada0..3ac9f50231d 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -171,10 +171,12 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A } } + #if USE_KRB5 if (builder.need_kinit) { builder.runKinit(); } + #endif // USE_KRB5 return builder; } From c1b2b669ab8a8750be5c79a622dc863f0fdffb12 Mon Sep 17 00:00:00 2001 From: zvonand Date: Fri, 17 Jun 2022 01:52:45 +0500 Subject: [PATCH 021/123] remove wrong code --- src/Functions/FunctionBase58Conversion.h | 78 +++++++++++++++++------ src/Functions/FunctionsBase58.cpp | 5 -- src/Functions/registerFunctionsString.cpp | 2 - 3 files changed, 58 insertions(+), 27 deletions(-) diff --git a/src/Functions/FunctionBase58Conversion.h b/src/Functions/FunctionBase58Conversion.h index 9aacbc4c9e7..97dc2a4f40c 100644 --- a/src/Functions/FunctionBase58Conversion.h +++ b/src/Functions/FunctionBase58Conversion.h @@ -27,7 +27,7 @@ struct Base58Encode { static constexpr auto name = "base58Encode"; - static void process(const ColumnString * input, ColumnString * dst_column, std::string& alphabet, size_t input_rows_count) + static void process(const ColumnString * input, ColumnString::MutablePtr& dst_column, std::string& alphabet, size_t input_rows_count) { auto & dst_data = dst_column->getChars(); auto & dst_offsets = dst_column->getOffsets(); @@ -48,15 +48,16 @@ struct Base58Encode const auto& encoder = (alphabet == "bitcoin") ? Base58::bitcoin() : ((alphabet == "flickr") ? Base58::flickr() : - ((alphabet == "ripple") ? Base58::ripple() : Base58::base58())); + ((alphabet == "ripple") ? Base58::ripple() : + Base58::base58())); for (size_t row = 0; row < input_rows_count; ++row) { size_t srclen = src_offsets[row] - src_offset_prev - 1; - /// Why we didn't simply operate on char* here? + /// Why we didn't use char* here? /// We don't know the size of the result string beforehand (it's not byte-to-byte encoding), /// so we may need to do many resizes (the worst case -- we'll do it for each row) - /// Using std::string allows to do exponential resizes and one final resize after whole operation is complete + /// This way we do exponential resizes and one final resize after whole operation is complete std::string encoded; encoder.encode(encoded, source, srclen); size_t outlen = encoded.size(); @@ -66,11 +67,10 @@ struct Base58Encode current_allocated_size += current_allocated_size; dst_data.resize(current_allocated_size); } + std::strcpy(reinterpret_cast(dst_pos), encoded.c_str()); source += srclen + 1; - dst_pos += outlen; - *dst_pos = '\0'; - dst_pos += 1; + dst_pos += outlen + 1; dst_offsets[row] = dst_pos - dst; src_offset_prev = src_offsets[row]; @@ -85,19 +85,57 @@ struct Base58Decode { static constexpr auto name = "base58Decode"; - static size_t getBufferSize(size_t string_length, size_t string_count) + static void process(const ColumnString * input, ColumnString::MutablePtr& dst_column, std::string& alphabet, size_t input_rows_count) { - return ((string_length - string_count) / 4 + string_count) * 3 + string_count; - } -}; + auto & dst_data = dst_column->getChars(); + auto & dst_offsets = dst_column->getOffsets(); -struct TryBase58Decode -{ - static constexpr auto name = "tryBase58Decode"; + size_t current_allocated_size = input->getChars().size(); - static size_t getBufferSize(size_t string_length, size_t string_count) - { - return Base58Decode::getBufferSize(string_length, string_count); + dst_data.resize(current_allocated_size); + dst_offsets.resize(input_rows_count); + + const ColumnString::Offsets & src_offsets = input->getOffsets(); + + const auto * source = input->getChars().raw_data(); + auto * dst = dst_data.data(); + auto * dst_pos = dst; + + size_t src_offset_prev = 0; + size_t processed_size = 0; + + const auto& decoder = (alphabet == "bitcoin") ? Base58::bitcoin() : + ((alphabet == "flickr") ? Base58::flickr() : + ((alphabet == "ripple") ? Base58::ripple() : + Base58::base58())); + + for (size_t row = 0; row < input_rows_count; ++row) + { + size_t srclen = src_offsets[row] - src_offset_prev - 1; + /// Why we didn't use char* here? + /// We don't know the size of the result string beforehand (it's not byte-to-byte encoding), + /// so we may need to do many resizes (the worst case -- we'll do it for each row) + /// This way we do exponential resizes and one final resize after whole operation is complete + std::string decoded; + decoder.decode(decoded, source, srclen); + size_t outlen = decoded.size(); + + if (processed_size + outlen >= current_allocated_size) + { + current_allocated_size += current_allocated_size; + dst_data.resize(current_allocated_size); + } + std::strcpy(reinterpret_cast(dst_pos), decoded.c_str()); + + source += srclen + 1; + dst_pos += outlen + 1; + + dst_offsets[row] = dst_pos - dst; + src_offset_prev = src_offsets[row]; + processed_size += outlen; + } + + dst_data.resize(dst_pos - dst); } }; @@ -129,9 +167,9 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.size() != 1 || arguments.size() != 2) + if (arguments.size() != 1 && arguments.size() != 2) throw Exception( - "Wrong number of arguments for function " + getName() + ": " + arguments.size() + " provided, 1 or 2 expected.", + "Wrong number of arguments for function " + getName() + ": 1 or 2 expected.", ErrorCodes::BAD_ARGUMENTS); if (!isString(arguments[0].type)) @@ -173,7 +211,7 @@ public: auto dst_column = ColumnString::create(); - Func::process(column_string, dst_column, alphabet, input_rows_count); + Func::process(input, dst_column, alphabet, input_rows_count); return dst_column; } diff --git a/src/Functions/FunctionsBase58.cpp b/src/Functions/FunctionsBase58.cpp index efd3f42c6da..3ccb4d790ce 100644 --- a/src/Functions/FunctionsBase58.cpp +++ b/src/Functions/FunctionsBase58.cpp @@ -14,10 +14,5 @@ void registerFunctionBase58Decode(FunctionFactory & factory) { factory.registerFunction>(); } - -void registerFunctionTryBase58Decode(FunctionFactory & factory) -{ - factory.registerFunction>(); -} } #endif diff --git a/src/Functions/registerFunctionsString.cpp b/src/Functions/registerFunctionsString.cpp index 2e2975a459c..43035ef51e7 100644 --- a/src/Functions/registerFunctionsString.cpp +++ b/src/Functions/registerFunctionsString.cpp @@ -52,7 +52,6 @@ void registerFunctionTryBase64Decode(FunctionFactory &); #if USE_BASE58 void registerFunctionBase58Encode(FunctionFactory &); void registerFunctionBase58Decode(FunctionFactory &); -void registerFunctionTryBase58Decode(FunctionFactory &); #endif #if USE_NLP @@ -114,7 +113,6 @@ void registerFunctionsString(FunctionFactory & factory) #if USE_BASE58 registerFunctionBase58Encode(factory); registerFunctionBase58Decode(factory); - registerFunctionTryBase58Decode(factory); #endif #if USE_NLP From f987f461e5da22921c753f5e5261c2cecf9f49ea Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Fri, 17 Jun 2022 15:00:32 +0500 Subject: [PATCH 022/123] fix style -- rm unused ErrorCode --- src/Functions/FunctionBase58Conversion.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/FunctionBase58Conversion.h b/src/Functions/FunctionBase58Conversion.h index 97dc2a4f40c..87a1821b2de 100644 --- a/src/Functions/FunctionBase58Conversion.h +++ b/src/Functions/FunctionBase58Conversion.h @@ -19,7 +19,6 @@ namespace ErrorCodes { extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int INCORRECT_DATA; extern const int BAD_ARGUMENTS; } From f4b3af091d0a24bb3f1ac8dec26f7f22ad3e8623 Mon Sep 17 00:00:00 2001 From: zvonand Date: Fri, 17 Jun 2022 23:48:14 +0500 Subject: [PATCH 023/123] fix zero byte --- src/Functions/FunctionBase58Conversion.h | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/Functions/FunctionBase58Conversion.h b/src/Functions/FunctionBase58Conversion.h index 87a1821b2de..a0431ca47df 100644 --- a/src/Functions/FunctionBase58Conversion.h +++ b/src/Functions/FunctionBase58Conversion.h @@ -58,7 +58,8 @@ struct Base58Encode /// so we may need to do many resizes (the worst case -- we'll do it for each row) /// This way we do exponential resizes and one final resize after whole operation is complete std::string encoded; - encoder.encode(encoded, source, srclen); + if (srclen) + encoder.encode(encoded, source, srclen); size_t outlen = encoded.size(); if (processed_size + outlen >= current_allocated_size) @@ -66,10 +67,13 @@ struct Base58Encode current_allocated_size += current_allocated_size; dst_data.resize(current_allocated_size); } - std::strcpy(reinterpret_cast(dst_pos), encoded.c_str()); + if (srclen) + std::strcpy(reinterpret_cast(dst_pos), encoded.c_str()); source += srclen + 1; - dst_pos += outlen + 1; + dst_pos += outlen; + *dst_pos = '\0'; + dst_pos += 1; dst_offsets[row] = dst_pos - dst; src_offset_prev = src_offsets[row]; @@ -127,7 +131,9 @@ struct Base58Decode std::strcpy(reinterpret_cast(dst_pos), decoded.c_str()); source += srclen + 1; - dst_pos += outlen + 1; + dst_pos += outlen; + *dst_pos = '\0'; + dst_pos += 1; dst_offsets[row] = dst_pos - dst; src_offset_prev = src_offsets[row]; @@ -176,7 +182,7 @@ public: "Illegal type " + arguments[0].type->getName() + " of 1st argument of function " + getName() + ". Must be String.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (!isString(arguments[1].type)) + if (arguments.size() == 2 && !isString(arguments[1].type)) throw Exception( "Illegal type " + arguments[1].type->getName() + " of 2nd argument of function " + getName() + ". Must be String.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); From 832fd6e0a938c7a50247a6b233b7ee681dccaf14 Mon Sep 17 00:00:00 2001 From: zvonand Date: Sun, 19 Jun 2022 23:10:28 +0500 Subject: [PATCH 024/123] Added tests + minor updates --- .gitmodules | 4 +- contrib/CMakeLists.txt | 2 +- .../CMakeLists.txt | 10 +- contrib/base-x/.gitignore | 4 + contrib/base-x/.travis.yml | 36 + contrib/base-x/LICENSE | 21 + contrib/base-x/README.md | 97 + contrib/base-x/base_x.hh | 614 ++++ contrib/base-x/tests/test.cc | 30 + contrib/base-x/tests/testcases/tests.cc | 359 +++ contrib/base-x/uinteger_t.hh | 2546 +++++++++++++++++ contrib/base58 | 1 - src/Functions/CMakeLists.txt | 4 +- src/Functions/FunctionBase58Conversion.h | 46 +- src/Functions/configure_config.cmake | 2 +- .../0_stateless/02337_base58.reference | 48 + tests/queries/0_stateless/02337_base58.sql | 17 + 17 files changed, 3809 insertions(+), 32 deletions(-) rename contrib/{base58-cmake => base-x-cmake}/CMakeLists.txt (65%) create mode 100644 contrib/base-x/.gitignore create mode 100755 contrib/base-x/.travis.yml create mode 100644 contrib/base-x/LICENSE create mode 100644 contrib/base-x/README.md create mode 100644 contrib/base-x/base_x.hh create mode 100644 contrib/base-x/tests/test.cc create mode 100644 contrib/base-x/tests/testcases/tests.cc create mode 100644 contrib/base-x/uinteger_t.hh delete mode 160000 contrib/base58 create mode 100644 tests/queries/0_stateless/02337_base58.reference create mode 100644 tests/queries/0_stateless/02337_base58.sql diff --git a/.gitmodules b/.gitmodules index 568dab1eb26..e1960d2144b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -268,6 +268,6 @@ [submodule "contrib/hashidsxx"] path = contrib/hashidsxx url = https://github.com/schoentoon/hashidsxx.git -[submodule "contrib/base58"] - path = contrib/base58 +[submodule "contrib/base-x"] + path = contrib/base-x url = https://github.com/Kronuz/base-x.git diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index a356ade7eb8..2ade6c139f6 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -153,7 +153,7 @@ endif() add_contrib (sqlite-cmake sqlite-amalgamation) add_contrib (s2geometry-cmake s2geometry) -add_contrib (base58-cmake base58) +add_contrib (base-x-cmake base-x) # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear diff --git a/contrib/base58-cmake/CMakeLists.txt b/contrib/base-x-cmake/CMakeLists.txt similarity index 65% rename from contrib/base58-cmake/CMakeLists.txt rename to contrib/base-x-cmake/CMakeLists.txt index 26783e0177d..48cb54d307f 100644 --- a/contrib/base58-cmake/CMakeLists.txt +++ b/contrib/base-x-cmake/CMakeLists.txt @@ -1,13 +1,13 @@ -set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/base58") +set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/base-x") set (SRCS ${LIBRARY_DIR}/base_x.hh ${LIBRARY_DIR}/uinteger_t.hh ) -add_library(_base58 ${SRCS}) +add_library(_base-x ${SRCS}) -target_include_directories(_base58 SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}) +target_include_directories(_base-x SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}) if (XCODE OR XCODE_VERSION) # https://gitlab.kitware.com/cmake/cmake/issues/17457 @@ -16,7 +16,7 @@ if (XCODE OR XCODE_VERSION) if (NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/dummy.c") file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/dummy.c" "") endif () - target_sources(_base58 PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/dummy.c") + target_sources(_base-x PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/dummy.c") endif () -add_library(ch_contrib::base58 ALIAS _base58) \ No newline at end of file +add_library(ch_contrib::base-x ALIAS _base-x) \ No newline at end of file diff --git a/contrib/base-x/.gitignore b/contrib/base-x/.gitignore new file mode 100644 index 00000000000..b63b40c8b71 --- /dev/null +++ b/contrib/base-x/.gitignore @@ -0,0 +1,4 @@ +.DS_Store +test +*.o +*.dSYM \ No newline at end of file diff --git a/contrib/base-x/.travis.yml b/contrib/base-x/.travis.yml new file mode 100755 index 00000000000..f55132e614f --- /dev/null +++ b/contrib/base-x/.travis.yml @@ -0,0 +1,36 @@ +sudo: false + +language: cpp + +compiler: + - clang + - gcc + +addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-precise-3.8 + packages: + - g++-6 + - clang-3.8 + +install: + - if [ "$CXX" = "g++" ]; then export CXX="g++-6"; fi + - if [ "$CXX" == "clang++" ]; then export CXX="clang++-3.8"; fi + - sudo apt-get install -qq git cmake + +before_script: + # not much better than git submodules, but there was never a need/want for the repo in this repo + - cd .. + - git clone https://github.com/google/googletest.git + - cd googletest + - git reset --hard d62d6c6556d96dda924382547c54a4b3afedb22c + - cmake CMakeLists.txt + - make + + - cd ../base-x/tests + - make + +script: + - make run diff --git a/contrib/base-x/LICENSE b/contrib/base-x/LICENSE new file mode 100644 index 00000000000..f7b3408abac --- /dev/null +++ b/contrib/base-x/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 German Mendez Bravo (Kronuz) @ german dot mb at gmail.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/contrib/base-x/README.md b/contrib/base-x/README.md new file mode 100644 index 00000000000..5dc4a068043 --- /dev/null +++ b/contrib/base-x/README.md @@ -0,0 +1,97 @@ +# base-x [![License][license-img]][license-url] [![GitHub Stars][stars-img]][stars-url] [![GitHub Forks][forks-img]][forks-url] [![GitHub Watchers][watchers-img]][watchers-url] [![Tweet][tweet-img]][tweet-url] + +[![Build Status](https://travis-ci.org/Kronuz/base-x.svg?branch=master)](https://travis-ci.org/Kronuz/base-x) + + +### BaseX encoder / decoder for C++ + +This is a fast base encoder / decoder of any given alphabet. + + +#### Example + +``` cpp +// example.cc +// g++ -std=c++14 -o example example.cc + +#include +#include "base_x.hh" + +int main() { + auto encoded = Base58::base58().encode("Hello world!"); + + std::cout << encoded << std::endl; + // => 1LDlk6QWOejX6rPrJ + + return 0; +} +``` + + +#### Compilation + +* g++ and clang++ are supported. +* C++14 is required. + + +### Alphabets + +See below for a list of commonly recognized alphabets, and their respective base. + +Base | Factory | Alphabet +-----|---------------------|------------- + 2 | base2::base2() | `01` + 2 | base8::base8() | `01234567` + 11 | bas11::bas11() | `0123456789a` + 16 | base16::base16() | `0123456789abcdef` + 32 | base32::base32() | `0123456789ABCDEFGHJKMNPQRSTVWXYZ` + 36 | base36::base36() | `0123456789abcdefghijklmnopqrstuvwxyz` + 58 | base58::base58() | `123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz` + 58 | base58::bitcoin() | `123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz` + 58 | base58::gmp() | `0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv` + 58 | base58::ripple() | `rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz` + 58 | base58::flickr() | `123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ` + 62 | base62::base62() | `0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz` + 62 | base62::inverted() | `0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ` + 64 | base64::base64() | `ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/` + 64 | base64::urlsafe() | `ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_` + 66 | base66::base66() | `ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.!~` + + +### How it works + +It encodes octet arrays by doing long divisions on all significant digits in the +array, creating a representation of that number in the new base. + +**If you need standard hex encoding, or base64 encoding, this module is NOT +appropriate.** + + +## Author +[**German Mendez Bravo (Kronuz)**](https://kronuz.io/) + +[![Follow on GitHub][github-follow-img]][github-follow-url] +[![Follow on Twitter][twitter-follow-img]][twitter-follow-url] + + +## License + +MIT License. See [LICENSE](LICENSE) for details. + +Copyright (c) 2017 German Mendez Bravo (Kronuz) @ german dot mb at gmail.com + + +[license-url]: https://github.com/Kronuz/base-x/blob/master/LICENSE +[license-img]: https://img.shields.io/github/license/Kronuz/base-x.svg +[stars-url]: https://github.com/Kronuz/base-x/stargazers +[stars-img]: https://img.shields.io/github/stars/Kronuz/base-x.svg?style=social&label=Stars +[forks-url]: https://github.com/Kronuz/base-x/network/members +[forks-img]: https://img.shields.io/github/forks/Kronuz/base-x.svg?style=social&label=Forks +[watchers-url]: https://github.com/Kronuz/base-x/watchers +[watchers-img]: https://img.shields.io/github/watchers/Kronuz/base-x.svg?style=social&label=Watchers +[tweet-img]: https://img.shields.io/twitter/url/https/github.com/Kronuz/base-x.svg?style=social +[tweet-url]: https://twitter.com/intent/tweet?text=Base-X+encoding%2Fdecoding+for+modern+C%2B%2B+by+%40germbravo:&url=https%3A%2F%2Fgithub.com%2FKronuz%2Fbase-x +[github-follow-url]: https://github.com/Kronuz +[github-follow-img]: https://img.shields.io/github/followers/Kronuz.svg?style=social&label=Follow +[twitter-follow-url]: https://twitter.com/intent/follow?screen_name=germbravo +[twitter-follow-img]: https://img.shields.io/twitter/follow/germbravo.svg?style=social&label=Follow diff --git a/contrib/base-x/base_x.hh b/contrib/base-x/base_x.hh new file mode 100644 index 00000000000..fdc06fead2f --- /dev/null +++ b/contrib/base-x/base_x.hh @@ -0,0 +1,614 @@ +/* +base_x.hh +BaseX encoder / decoder for C++ + +Copyright (c) 2017 German Mendez Bravo (Kronuz) @ german dot mb at gmail.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#ifndef __BASE_X__H_ +#define __BASE_X__H_ + +#include // for std::find_if, std::reverse +#include // for std::invalid_argument +#include // for std::string +#include // for std::enable_if_t + +#include "uinteger_t.hh" + + +class BaseX { + char _chr[256]; + int _ord[256]; + + const int size; + const int alphabet_base; + const unsigned base_size; + const unsigned alphabet_base_bits; + const unsigned block_size; + const uinteger_t::digit alphabet_base_mask; + const unsigned padding_size; + const char padding; + const int flags; + + constexpr char chr(unsigned char ord) const { + return _chr[ord]; + } + + constexpr int ord(unsigned char chr) const { + return _ord[chr]; + } + +public: + static constexpr int ignore_case = (1 << 0); + static constexpr int with_checksum = (1 << 1); + static constexpr int with_check = (1 << 2); + static constexpr int block_padding = (1 << 3); + + template + constexpr BaseX(int flgs, const char (&alphabet)[alphabet_size1], const char (&extended)[extended_size1], const char (&padding_string)[padding_size1], const char (&translate)[translate_size1]) : + _chr(), + _ord(), + size(alphabet_size1 - 1 + extended_size1 - 1), + alphabet_base(alphabet_size1 - 1), + base_size(uinteger_t::base_size(alphabet_base)), + alphabet_base_bits(uinteger_t::base_bits(alphabet_base)), + block_size((flgs & BaseX::block_padding) ? alphabet_base_bits : 0), + alphabet_base_mask(alphabet_base - 1), + padding_size(padding_size1 - 1), + padding(padding_size ? padding_string[0] : '\0'), + flags(flgs) + { + for (int c = 0; c < 256; ++c) { + _chr[c] = 0; + _ord[c] = alphabet_base; + } + for (int cp = 0; cp < alphabet_base; ++cp) { + auto ch = alphabet[cp]; + _chr[cp] = ch; + ASSERT(_ord[(unsigned char)ch] == alphabet_base); // Duplicate character in the alphabet + _ord[(unsigned char)ch] = cp; + if (flags & BaseX::ignore_case) { + if (ch >= 'A' && ch <='Z') { + _ord[(unsigned char)ch - 'A' + 'a'] = cp; + } else if (ch >= 'a' && ch <='z') { + _ord[(unsigned char)ch - 'a' + 'A'] = cp; + } + } + } + for (std::size_t i = 0; i < extended_size1 - 1; ++i) { + auto ch = extended[i]; + auto cp = alphabet_base + i; + _chr[cp] = ch; + ASSERT(_ord[(unsigned char)ch] == alphabet_base); // Duplicate character in the extended alphabet + _ord[(unsigned char)ch] = cp; + if (flags & BaseX::ignore_case) { + if (ch >= 'A' && ch <='Z') { + _ord[(unsigned char)ch - 'A' + 'a'] = cp; + } else if (ch >= 'a' && ch <='z') { + _ord[(unsigned char)ch - 'a' + 'A'] = cp; + } + } + } + int cp = -1; + for (std::size_t i = 0; i < translate_size1 - 1; ++i) { + auto ch = translate[i]; + auto ncp = _ord[(unsigned char)ch]; + if (ncp >= alphabet_base) { + ASSERT(_ord[(unsigned char)ch] == alphabet_base); // Invalid translation character + _ord[(unsigned char)ch] = cp; + if (flags & BaseX::ignore_case) { + if (ch >= 'A' && ch <='Z') { + _ord[(unsigned char)ch - 'A' + 'a'] = cp; + } else if (ch >= 'a' && ch <='z') { + _ord[(unsigned char)ch - 'a' + 'A'] = cp; + } + } + } else { + cp = ncp; + } + } + } + + // Get string representation of value + template ::value>> + void encode(Result& result, const uinteger_t& input) const { + std::size_t bp = 0; + uinteger_t quotient; + if (block_size) { + bp = ((input.bits() + 7) & 0xf8) % block_size; + bp = bp ? (block_size - bp) % block_size : 0; + if (bp) { + quotient = input << bp; + } + } + const uinteger_t& num = bp ? quotient : input; + auto num_sz = num.size(); + if (num_sz) { + int sum = 0; + result.reserve(num_sz * base_size); + if (alphabet_base_bits) { + std::size_t shift = 0; + auto ptr = reinterpret_cast(num.data()); + uinteger_t::digit v = *ptr++; + v <<= uinteger_t::half_digit_bits; + for (auto i = num_sz * 2 - 1; i; --i) { + v >>= uinteger_t::half_digit_bits; + v |= (static_cast(*ptr++) << uinteger_t::half_digit_bits); + do { + auto d = static_cast((v >> shift) & alphabet_base_mask); + result.push_back(chr(d)); + shift += alphabet_base_bits; + sum += d; + } while (shift <= uinteger_t::half_digit_bits); + shift -= uinteger_t::half_digit_bits; + } + v >>= (shift + uinteger_t::half_digit_bits); + while (v) { + auto d = static_cast(v & alphabet_base_mask); + result.push_back(chr(d)); + v >>= alphabet_base_bits; + sum += d; + } + auto s = chr(0); + auto rit_f = std::find_if(result.rbegin(), result.rend(), [s](const char& c) { return c != s; }); + result.resize(result.rend() - rit_f); // shrink + } else { + uinteger_t uint_base = alphabet_base; + if (!bp) { + quotient = num; + } + do { + auto r = quotient.divmod(uint_base); + auto d = static_cast(r.second); + result.push_back(chr(d)); + quotient = std::move(r.first); + sum += d; + } while (quotient); + } + std::reverse(result.begin(), result.end()); + if (padding_size) { + Result p; + p.resize((padding_size - (result.size() % padding_size)) % padding_size, padding); + result.append(p); + } + if (flags & BaseX::with_check) { + auto chk = static_cast(num % size); + result.push_back(chr(chk)); + sum += chk; + } + if (flags & BaseX::with_checksum) { + auto sz = result.size(); + sz = (sz + sz / size) % size; + sum += sz; + sum = (size - sum % size) % size; + result.push_back(chr(sum)); + } + } else { + result.push_back(chr(0)); + } + } + + template ::value>> + Result encode(const uinteger_t& num) const { + Result result; + encode(result, num); + return result; + } + + template ::value>> + void encode(Result& result, const unsigned char* decoded, std::size_t decoded_size) const { + encode(result, uinteger_t(decoded, decoded_size, 256)); + } + + template ::value>> + Result encode(const unsigned char* decoded, std::size_t decoded_size) const { + Result result; + encode(result, uinteger_t(decoded, decoded_size, 256)); + return result; + } + + template ::value>> + void encode(Result& result, const char* decoded, std::size_t decoded_size) const { + encode(result, uinteger_t(decoded, decoded_size, 256)); + } + + template ::value>> + Result encode(const char* decoded, std::size_t decoded_size) const { + Result result; + encode(result, uinteger_t(decoded, decoded_size, 256)); + return result; + } + + template ::value>> + void encode(Result& result, T (&s)[N]) const { + encode(result, s, N - 1); + } + + template ::value>> + Result encode(T (&s)[N]) const { + Result result; + encode(result, s, N - 1); + return result; + } + + template ::value>> + void encode(Result& result, const std::string& binary) const { + return encode(result, binary.data(), binary.size()); + } + + template ::value>> + Result encode(const std::string& binary) const { + Result result; + encode(result, binary.data(), binary.size()); + return result; + } + + void decode(uinteger_t& result, const char* encoded, std::size_t encoded_size) const { + result = 0; + int sum = 0; + int sumsz = 0; + int direction = 1; + + auto sz = encoded_size; + if (flags & BaseX::with_checksum) --sz; + if (flags & BaseX::with_check) --sz; + + int bp = 0; + + if (alphabet_base_bits) { + for (; sz; --sz, encoded += direction) { + auto c = *encoded; + if (c == padding) break; + auto d = ord(static_cast(c)); + if (d < 0) continue; // ignored character + if (d >= alphabet_base) { + throw std::invalid_argument("Error: Invalid character: '" + std::string(1, c) + "' at " + std::to_string(encoded_size - sz)); + } + sum += d; + ++sumsz; + result = (result << alphabet_base_bits) | d; + bp += block_size; + } + } else { + uinteger_t uint_base = alphabet_base; + for (; sz; --sz, encoded += direction) { + auto c = *encoded; + if (c == padding) break; + auto d = ord(static_cast(c)); + if (d < 0) continue; // ignored character + if (d >= alphabet_base) { + throw std::invalid_argument("Error: Invalid character: '" + std::string(1, c) + "' at " + std::to_string(encoded_size - sz)); + } + sum += d; + ++sumsz; + result = (result * uint_base) + d; + bp += block_size; + } + } + + for (; sz && *encoded == padding; --sz, ++encoded); + + result >>= (bp & 7); + + if (flags & BaseX::with_check) { + auto c = *encoded; + auto d = ord(static_cast(c)); + if (d < 0 || d >= size) { + throw std::invalid_argument("Error: Invalid character: '" + std::string(1, c) + "' at " + std::to_string(encoded_size - sz)); + } + auto chk = static_cast(result % size); + if (d != chk) { + throw std::invalid_argument("Error: Invalid check"); + } + sum += chk; + ++sumsz; + ++encoded; + } + + if (flags & BaseX::with_checksum) { + auto c = *encoded; + auto d = ord(static_cast(c)); + if (d < 0 || d >= size) { + throw std::invalid_argument("Error: Invalid character: '" + std::string(1, c) + "' at " + std::to_string(encoded_size - sz)); + } + sum += d; + sum += (sumsz + sumsz / size) % size; + if (sum % size) { + throw std::invalid_argument("Error: Invalid checksum"); + } + } + } + + template ::value>> + void decode(Result& result, const char* encoded, std::size_t encoded_size) const { + uinteger_t num; + decode(num, encoded, encoded_size); + result = num.template str(256); + } + + template ::value or std::is_integral::value>> + Result decode(const char* encoded, std::size_t encoded_size) const { + Result result; + decode(result, encoded, encoded_size); + return result; + } + + template ::value or std::is_integral::value>> + void decode(Result& result, T (&s)[N]) const { + decode(result, s, N - 1); + } + + template ::value or std::is_integral::value>> + Result decode(T (&s)[N]) const { + Result result; + decode(result, s, N - 1); + return result; + } + + template ::value or std::is_integral::value>> + void decode(Result& result, const std::string& encoded) const { + decode(result, encoded.data(), encoded.size()); + } + + template ::value or std::is_integral::value>> + Result decode(const std::string& encoded) const { + Result result; + decode(result, encoded.data(), encoded.size()); + return result; + } + + bool is_valid(const char* encoded, std::size_t encoded_size) const { + int sum = 0; + int sumsz = 0; + if (flags & BaseX::with_checksum) --sumsz; + for (; encoded_size; --encoded_size, ++encoded) { + auto d = ord(static_cast(*encoded)); + if (d < 0) continue; // ignored character + if (d >= alphabet_base) { + return false; + } + sum += d; + ++sumsz; + } + if (flags & BaseX::with_checksum) { + sum += (sumsz + sumsz / size) % size; + if (sum % size) { + return false; + } + } + return true; + } + + template + bool is_valid(T (&s)[N]) const { + return is_valid(s, N - 1); + } + + bool is_valid(const std::string& encoded) const { + return is_valid(encoded.data(), encoded.size()); + } +}; + +// base2 +struct Base2 { + static const BaseX& base2() { + static constexpr BaseX encoder(0, "01", "", "", ""); + return encoder; + } + static const BaseX& base2chk() { + static constexpr BaseX encoder(BaseX::with_checksum, "01", "", "", ""); + return encoder; + } +}; + +// base8 +struct Base8 { + static const BaseX& base8() { + static constexpr BaseX encoder(0, "01234567", "", "", ""); + return encoder; + } + static const BaseX& base8chk() { + static constexpr BaseX encoder(BaseX::with_checksum, "01234567", "", "", ""); + return encoder; + } +}; + +// base11 +struct Base11 { + static const BaseX& base11() { + static constexpr BaseX encoder(BaseX::ignore_case, "0123456789a", "", "", ""); + return encoder; + } + static const BaseX& base11chk() { + static constexpr BaseX encoder(BaseX::ignore_case | BaseX::with_checksum, "0123456789a", "", "", ""); + return encoder; + } +}; + +// base16 +struct Base16 { + static const BaseX& base16() { + static constexpr BaseX encoder(BaseX::ignore_case, "0123456789abcdef", "", "", ""); + return encoder; + } + static const BaseX& base16chk() { + static constexpr BaseX encoder(BaseX::ignore_case | BaseX::with_checksum, "0123456789abcdef", "", "", ""); + return encoder; + } + static const BaseX& rfc4648() { + static constexpr BaseX encoder(0, "0123456789ABCDEF", "", "", ""); + return encoder; + } +}; + +// base32 +struct Base32 { + static const BaseX& base32() { + static constexpr BaseX encoder(BaseX::ignore_case, "0123456789abcdefghijklmnopqrstuv", "", "", ""); + return encoder; + } + static const BaseX& base32chk() { + static constexpr BaseX encoder(BaseX::ignore_case | BaseX::with_checksum, "0123456789abcdefghijklmnopqrstuv", "", "", ""); + return encoder; + } + static const BaseX& crockford() { + static constexpr BaseX encoder(BaseX::ignore_case, "0123456789ABCDEFGHJKMNPQRSTVWXYZ", "", "", "-0O1IL"); + return encoder; + } + static const BaseX& crockfordchk() { + static constexpr BaseX encoder(BaseX::ignore_case | BaseX::with_check, "0123456789ABCDEFGHJKMNPQRSTVWXYZ", "*~$=U", "", "-0O1IL"); + return encoder; + } + static const BaseX& rfc4648() { + static constexpr BaseX encoder(BaseX::block_padding, "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567", "", "========", "\n\r"); + return encoder; + } + static const BaseX& rfc4648hex() { + static constexpr BaseX encoder(BaseX::block_padding, "0123456789ABCDEFGHIJKLMNOPQRSTUV", "", "========", "\n\r"); + return encoder; + } +}; + +// base36 +struct Base36 { + static const BaseX& base36() { + static constexpr BaseX encoder(BaseX::ignore_case, "0123456789abcdefghijklmnopqrstuvwxyz", "", "", ""); + return encoder; + } + static const BaseX& base36chk() { + static constexpr BaseX encoder(BaseX::ignore_case | BaseX::with_checksum, "0123456789abcdefghijklmnopqrstuvwxyz", "", "", ""); + return encoder; + } +}; + +// base58 +struct Base58 { + static const BaseX& base58() { + static constexpr BaseX encoder(0, "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv", "", "", ""); + return encoder; + } + static const BaseX& base58chk() { + static constexpr BaseX encoder(BaseX::with_checksum, "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv", "", "", ""); + return encoder; + } + static const BaseX& bitcoin() { + static constexpr BaseX encoder(0, "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", "", "", ""); + return encoder; + } + static const BaseX& bitcoinchk() { + static constexpr BaseX encoder(BaseX::with_checksum, "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", "", "", ""); + return encoder; + } + static const BaseX& ripple() { + static constexpr BaseX encoder(0, "rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz", "", "", ""); + return encoder; + } + static const BaseX& ripplechk() { + static constexpr BaseX encoder(BaseX::with_checksum, "rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz", "", "", ""); + return encoder; + } + static const BaseX& flickr() { + static constexpr BaseX encoder(0, "123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ", "", "", ""); + return encoder; + } + static const BaseX& flickrchk() { + static constexpr BaseX encoder(BaseX::with_checksum, "123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ", "", "", ""); + return encoder; + } +}; + +// base59 +struct Base59 { + static const BaseX& base59() { + static constexpr BaseX encoder(0, "23456789abcdefghijklmnopqrstuvwxyzABCDEFGHJKLMNOPQRSTUVWXYZ", "", "", "l1IO0"); + return encoder; + } + static const BaseX& base59chk() { + static constexpr BaseX encoder(BaseX::with_checksum, "23456789abcdefghijklmnopqrstuvwxyzABCDEFGHJKLMNOPQRSTUVWXYZ", "", "", "l1IO0"); + return encoder; + } + static const BaseX& dubaluchk() { + static constexpr BaseX encoder(BaseX::with_checksum, "zy9MalDxwpKLdvW2AtmscgbYUq6jhP7E53TiXenZRkVCrouBH4GSQf8FNJO", "", "", "-l1IO0"); + return encoder; + } +}; + +// base62 +struct Base62 { + static const BaseX& base62() { + static constexpr BaseX encoder(0, "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", "", "", ""); + return encoder; + } + static const BaseX& base62chk() { + static constexpr BaseX encoder(BaseX::with_checksum, "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", "", "", ""); + return encoder; + } + static const BaseX& inverted() { + static constexpr BaseX encoder(0, "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ", "", "", ""); + return encoder; + } + static const BaseX& invertedchk() { + static constexpr BaseX encoder(BaseX::with_checksum, "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ", "", "", ""); + return encoder; + } +}; + +// base64 +struct Base64 { + static const BaseX& base64() { + static constexpr BaseX encoder(0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", "", "", ""); + return encoder; + } + static const BaseX& base64chk() { + static constexpr BaseX encoder(BaseX::with_checksum, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", "", "", ""); + return encoder; + } + static const BaseX& url() { + static constexpr BaseX encoder(0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", "", "", ""); + return encoder; + } + static const BaseX& urlchk() { + static constexpr BaseX encoder(BaseX::with_checksum, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", "", "", ""); + return encoder; + } + static const BaseX& rfc4648() { + static constexpr BaseX encoder(BaseX::block_padding, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", "", "====", "\n\r"); + return encoder; + } + static const BaseX& rfc4648url() { + static constexpr BaseX encoder(BaseX::block_padding, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", "", "====", "\n\r"); + return encoder; + } +}; + +// base66 +struct Base66 { + static const BaseX& base66() { + static constexpr BaseX encoder(0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.!~", "", "", ""); + return encoder; + } + static const BaseX& base66chk() { + static constexpr BaseX encoder(BaseX::with_checksum, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.!~", "", "", ""); + return encoder; + } +}; + +#endif diff --git a/contrib/base-x/tests/test.cc b/contrib/base-x/tests/test.cc new file mode 100644 index 00000000000..d47d211173e --- /dev/null +++ b/contrib/base-x/tests/test.cc @@ -0,0 +1,30 @@ +/* +The MIT License (MIT) + +Copyright (c) 2017 German Mendez Bravo (Kronuz) @ german dot mb at gmail.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include + +int main(int argc, char * argv[]){ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/contrib/base-x/tests/testcases/tests.cc b/contrib/base-x/tests/testcases/tests.cc new file mode 100644 index 00000000000..c5bebfc8288 --- /dev/null +++ b/contrib/base-x/tests/testcases/tests.cc @@ -0,0 +1,359 @@ +/* +The MIT License (MIT) + +Copyright (c) 2017 German Mendez Bravo (Kronuz) @ german dot mb at gmail.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include + +#include "base_x.hh" + + +static constexpr BaseX test_base2(0, "01", "", "", ""); +static constexpr BaseX test_base16(0, "0123456789abcdef", "", "", ""); +static constexpr BaseX test_base58(0, "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", "", "", ""); + + +TEST(UUID, Encode) { + EXPECT_EQ(Base62::base62().encode("\330\105\140\310\23\117\21\346\241\342\64\66\73\322\155\256"), "6a630O1jrtMjCrQDyG3D3O"); + EXPECT_EQ(Base58::bitcoin().encode("\330\105\140\310\23\117\21\346\241\342\64\66\73\322\155\256"), "ThxCy1Ek2q6UhWQhj9CK1o"); + EXPECT_EQ(Base58::base58().encode("\330\105\140\310\23\117\21\346\241\342\64\66\73\322\155\256"), "QetBu0Dh1m5ReTNeg8BI0k"); +} + +TEST(BaseX, checksums) { + EXPECT_EQ(Base64::base64().encode("Hello world!"), "SGVsbG8gd29ybGQh"); + EXPECT_EQ(Base64::base64chk().encode("Hello world!"), "SGVsbG8gd29ybGQhG"); + + EXPECT_EQ(Base64::base64().decode("SGVsbG8gd29ybGQh"), "Hello world!"); + EXPECT_EQ(Base64::base64chk().decode("SGVsbG8gd29ybGQhG"), "Hello world!"); + + EXPECT_EQ(Base62::base62().encode("Hello world!"), "T8dgcjRGuYUueWht"); + EXPECT_EQ(Base62::base62chk().encode("Hello world!"), "T8dgcjRGuYUueWhtE"); + + EXPECT_EQ(Base62::base62().decode("T8dgcjRGuYUueWht"), "Hello world!"); + EXPECT_EQ(Base62::base62chk().decode("T8dgcjRGuYUueWhtE"), "Hello world!"); + + EXPECT_EQ(Base62::base62chk().is_valid("T8dgcjRGuYUueWhtE"), true); + EXPECT_EQ(Base62::base62chk().is_valid("Some random text!"), false); +} + +TEST(base16, Encoder) { + EXPECT_EQ(Base16::base16().encode("A"), "41"); + EXPECT_EQ(Base16::base16().encode("AB"), "4142"); + EXPECT_EQ(Base16::base16().encode("ABC"), "414243"); + EXPECT_EQ(Base16::base16().encode("ABCD"), "41424344"); + EXPECT_EQ(Base16::base16().encode("ABCDE"), "4142434445"); + EXPECT_EQ(Base16::base16().encode("ABCDEF"), "414243444546"); + + EXPECT_EQ(Base16::rfc4648().encode("A"), "41"); + EXPECT_EQ(Base16::rfc4648().encode("AB"), "4142"); + EXPECT_EQ(Base16::rfc4648().encode("ABC"), "414243"); + EXPECT_EQ(Base16::rfc4648().encode("ABCD"), "41424344"); + EXPECT_EQ(Base16::rfc4648().encode("ABCDE"), "4142434445"); + EXPECT_EQ(Base16::rfc4648().encode("ABCDEF"), "414243444546"); +} + +TEST(base16, Decoder) { + EXPECT_EQ(Base16::base16().decode("41"), "A"); + EXPECT_EQ(Base16::base16().decode("4142"), "AB"); + EXPECT_EQ(Base16::base16().decode("414243"), "ABC"); + EXPECT_EQ(Base16::base16().decode("41424344"), "ABCD"); + EXPECT_EQ(Base16::base16().decode("4142434445"), "ABCDE"); + EXPECT_EQ(Base16::base16().decode("414243444546"), "ABCDEF"); + + EXPECT_EQ(Base16::rfc4648().decode("41"), "A"); + EXPECT_EQ(Base16::rfc4648().decode("4142"), "AB"); + EXPECT_EQ(Base16::rfc4648().decode("414243"), "ABC"); + EXPECT_EQ(Base16::rfc4648().decode("41424344"), "ABCD"); + EXPECT_EQ(Base16::rfc4648().decode("4142434445"), "ABCDE"); + EXPECT_EQ(Base16::rfc4648().decode("414243444546"), "ABCDEF"); +} + +TEST(base32, Encoder) { + // Note base64() encoding is NOT the same as the standard (rfc4648) + EXPECT_EQ(Base32::base32().encode("A"), "21"); + EXPECT_EQ(Base32::base32().encode("AB"), "ga2"); + EXPECT_EQ(Base32::base32().encode("ABC"), "42gi3"); + EXPECT_EQ(Base32::base32().encode("ABCD"), "10k4gq4"); + EXPECT_EQ(Base32::base32().encode("ABCDE"), "85146h25"); + EXPECT_EQ(Base32::base32().encode("ABCDEF"), "21891k8ha6"); + EXPECT_EQ(Base32::base32().encode("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), "21891k8ha68t44iiib9h6ksjqga5956l2lapblgmaq"); + + EXPECT_EQ(Base32::rfc4648().encode("A"), "IE======"); + EXPECT_EQ(Base32::rfc4648().encode("AB"), "IFBA===="); + EXPECT_EQ(Base32::rfc4648().encode("ABC"), "IFBEG==="); + EXPECT_EQ(Base32::rfc4648().encode("ABCD"), "IFBEGRA="); + EXPECT_EQ(Base32::rfc4648().encode("ABCDE"), "IFBEGRCF"); + EXPECT_EQ(Base32::rfc4648().encode("ABCDEF"), "IFBEGRCFIY======"); + EXPECT_EQ(Base32::rfc4648().encode("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), "IFBEGRCFIZDUQSKKJNGE2TSPKBIVEU2UKVLFOWCZLI======"); + + EXPECT_EQ(Base32::crockford().encode(519571), "FVCK"); + EXPECT_EQ(Base32::crockfordchk().encode(1234), "16JD"); + EXPECT_EQ(Base32::crockfordchk().encode("Hello World"), "28CNP6RVS0AXQQ4V348"); +} + +TEST(base32, Decoder) { + // Note base64() encoding is NOT the same as the standard (rfc4648) + EXPECT_EQ(Base32::base32().decode("21"), "A"); + EXPECT_EQ(Base32::base32().decode("ga2"), "AB"); + EXPECT_EQ(Base32::base32().decode("42gi3"), "ABC"); + EXPECT_EQ(Base32::base32().decode("10k4gq4"), "ABCD"); + EXPECT_EQ(Base32::base32().decode("85146h25"), "ABCDE"); + EXPECT_EQ(Base32::base32().decode("21891k8ha6"), "ABCDEF"); + EXPECT_EQ(Base32::base32().decode("21891k8ha68t44iiib9h6ksjqga5956l2lapblgmaq"), "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + + EXPECT_EQ(Base32::rfc4648().decode("IE======"), "A"); + EXPECT_EQ(Base32::rfc4648().decode("IFBA===="), "AB"); + EXPECT_EQ(Base32::rfc4648().decode("IFBEG==="), "ABC"); + EXPECT_EQ(Base32::rfc4648().decode("IFBEGRA="), "ABCD"); + EXPECT_EQ(Base32::rfc4648().decode("IFBEGRCF"), "ABCDE"); + EXPECT_EQ(Base32::rfc4648().decode("IFBEGRCFIY======"), "ABCDEF"); + EXPECT_EQ(Base32::rfc4648().decode("IFBEGRCFIZDUQSKKJNGE2TSPKBIVEU2UKVLFOWCZLI======"), "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + + EXPECT_EQ(Base32::crockford().decode("FVCK"), 519571); + EXPECT_EQ(Base32::crockfordchk().is_valid("16JD"), true); + EXPECT_EQ(Base32::crockfordchk().decode("16JD"), 1234); + + EXPECT_EQ(Base32::crockfordchk().decode("2-8cn-p6r-vso-axq-q4v-348"), "Hello World"); +} + +TEST(base58, Encoder) { + EXPECT_EQ(Base58::base58().decode("1TFvCj"), 987654321); + EXPECT_EQ(Base58::base58().encode(987654321), "1TFvCj"); + EXPECT_EQ(Base58::base58().encode("Hello world!"), "1LDlk6QWOejX6rPrJ"); + EXPECT_EQ(Base58::bitcoin().encode("Hello world!"), "2NEpo7TZRhna7vSvL"); +} + +TEST(base62, Encoder) { + EXPECT_EQ(Base62::base62().decode("14q60P"), 987654321); + EXPECT_EQ(Base62::base62().encode(987654321), "14q60P"); + EXPECT_EQ(Base62::base62().encode("Hello world!"), "T8dgcjRGuYUueWht"); + EXPECT_EQ(Base62::inverted().encode("Hello world!"), "t8DGCJrgUyuUEwHT"); +} + +TEST(base64, Encoder) { + // Note Base64 encoding is NOT the same as the standard (rfc4648) + EXPECT_EQ(Base64::base64().encode("A"), "BB"); + EXPECT_EQ(Base64::base64().encode("AB"), "EFC"); + EXPECT_EQ(Base64::base64().encode("ABC"), "QUJD"); + EXPECT_EQ(Base64::base64().encode("ABCD"), "BBQkNE"); + EXPECT_EQ(Base64::base64().encode("ABCDE"), "EFCQ0RF"); + EXPECT_EQ(Base64::base64().encode("ABCDEF"), "QUJDREVG"); + EXPECT_EQ(Base64::base64().encode("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), "EFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFla"); + + EXPECT_EQ(Base64::rfc4648().encode("A"), "QQ=="); + EXPECT_EQ(Base64::rfc4648().encode("AB"), "QUI="); + EXPECT_EQ(Base64::rfc4648().encode("ABC"), "QUJD"); + EXPECT_EQ(Base64::rfc4648().encode("ABCD"), "QUJDRA=="); + EXPECT_EQ(Base64::rfc4648().encode("ABCDE"), "QUJDREU="); + EXPECT_EQ(Base64::rfc4648().encode("ABCDEF"), "QUJDREVG"); + EXPECT_EQ(Base64::rfc4648().encode("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo="); +} + +TEST(base64, Decoder) { + // Note Base64 encoding is NOT the same as the standard (rfc4648) + EXPECT_EQ(Base64::base64().decode("BB"), "A"); + EXPECT_EQ(Base64::base64().decode("EFC"), "AB"); + EXPECT_EQ(Base64::base64().decode("QUJD"), "ABC"); + EXPECT_EQ(Base64::base64().decode("BBQkNE"), "ABCD"); + EXPECT_EQ(Base64::base64().decode("EFCQ0RF"), "ABCDE"); + EXPECT_EQ(Base64::base64().decode("QUJDREVG"), "ABCDEF"); + EXPECT_EQ(Base64::base64().decode("EFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFla"), "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + + EXPECT_EQ(Base64::rfc4648().decode("QQ=="), "A"); + EXPECT_EQ(Base64::rfc4648().decode("QUI="), "AB"); + EXPECT_EQ(Base64::rfc4648().decode("QUJD"), "ABC"); + EXPECT_EQ(Base64::rfc4648().decode("QUJDRA=="), "ABCD"); + EXPECT_EQ(Base64::rfc4648().decode("QUJDREU="), "ABCDE"); + EXPECT_EQ(Base64::rfc4648().decode("QUJDREVG"), "ABCDEF"); + EXPECT_EQ(Base64::rfc4648().decode("QUJDREVG\nR0hJSktM\nTU5PUFFS\nU1RVVldY\nWVo="), "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); +} + +TEST(base58, ShouldEncodeAndDecodeIntegers) { + auto data = 987654321; + + auto gmpEncoded = Base58::base58().encode(data); + auto bitcoinEncoded = Base58::bitcoin().encode(data); + auto rippleEncoded = Base58::ripple().encode(data); + auto flickrEncoded = Base58::flickr().encode(data); + + EXPECT_EQ(gmpEncoded, "1TFvCj"); + EXPECT_EQ(bitcoinEncoded, "2WGzDn"); + EXPECT_EQ(rippleEncoded, "pWGzD8"); + EXPECT_EQ(flickrEncoded, "2vgZdM"); + + auto gmpDecoded = Base58::base58().decode(gmpEncoded); + auto bitcoinDecoded = Base58::bitcoin().decode(bitcoinEncoded); + auto rippleDecoded = Base58::ripple().decode(rippleEncoded); + auto flickrDecoded = Base58::flickr().decode(flickrEncoded); + + EXPECT_EQ(gmpDecoded, data); + EXPECT_EQ(bitcoinDecoded, data); + EXPECT_EQ(rippleDecoded, data); + EXPECT_EQ(flickrDecoded, data); + + auto encoded = Base58::base58().encode(data); + auto decoded = Base58::base58().decode(encoded); + + EXPECT_EQ(decoded, data); +} + +TEST(base58, LongText) { + auto data = "Lorem ipsum dolor consectetur."; + + auto gmpEncoded = Base58::base58().encode(data); + auto bitcoinEncoded = Base58::bitcoin().encode(data); + auto rippleEncoded = Base58::ripple().encode(data); + auto flickrEncoded = Base58::flickr().encode(data); + + EXPECT_EQ(gmpEncoded, "FIHZQEpJ739QdqChX1PkgTBqP1FaDgJWQiGvY92YA"); + EXPECT_EQ(bitcoinEncoded, "GKJcTFtL84ATguDka2SojWCuS2GdEjLZTmHzbA3bB"); + EXPECT_EQ(rippleEncoded, "GKJcTEtL3hwTguDk2pSojWUuSpGdNjLZTmHzbwsbB"); + EXPECT_EQ(flickrEncoded, "gjiBsfTk84asFUdKz2rNJvcUr2gCeJkysLhZAa3Ab"); + + auto gmpDecoded = Base58::base58().decode(gmpEncoded); + auto bitcoinDecoded = Base58::bitcoin().decode(bitcoinEncoded); + auto rippleDecoded = Base58::ripple().decode(rippleEncoded); + auto flickrDecoded = Base58::flickr().decode(flickrEncoded); + + EXPECT_EQ(gmpDecoded, data); + EXPECT_EQ(bitcoinDecoded, data); + EXPECT_EQ(rippleDecoded, data); + EXPECT_EQ(flickrDecoded, data); +} + +TEST(base58, Tests) { + EXPECT_EQ(test_base2.encode(uinteger_t("000f", 16)), "1111"); + // EXPECT_EQ(test_base2.encode(uinteger_t("00ff", 16)), "011111111"); // ->> + EXPECT_EQ(test_base2.encode(uinteger_t("00ff", 16)), "11111111"); + EXPECT_EQ(test_base2.encode(uinteger_t("0fff", 16)), "111111111111"); + EXPECT_EQ(test_base2.encode(uinteger_t("ff00ff00", 16)), "11111111000000001111111100000000"); + // EXPECT_EQ(test_base16.encode(uinteger_t("0000000f", 16)), "000f"); // ->> + EXPECT_EQ(test_base16.encode(uinteger_t("0000000f", 16)), "f"); + // EXPECT_EQ(test_base16.encode(uinteger_t("000fff", 16)), "0fff"); // ->> + EXPECT_EQ(test_base16.encode(uinteger_t("000fff", 16)), "fff"); + EXPECT_EQ(test_base16.encode(uinteger_t("ffff", 16)), "ffff"); + // EXPECT_EQ(test_base58.encode(uinteger_t("", 16)), ""); // ->> + EXPECT_EQ(test_base58.encode(uinteger_t("", 16)), "1"); + EXPECT_EQ(test_base58.encode(uinteger_t("61", 16)), "2g"); + EXPECT_EQ(test_base58.encode(uinteger_t("626262", 16)), "a3gV"); + EXPECT_EQ(test_base58.encode(uinteger_t("636363", 16)), "aPEr"); + EXPECT_EQ(test_base58.encode(uinteger_t("73696d706c792061206c6f6e6720737472696e67", 16)), "2cFupjhnEsSn59qHXstmK2ffpLv2"); + // EXPECT_EQ(test_base58.encode(uinteger_t("00eb15231dfceb60925886b67d065299925915aeb172c06647", 16)), "1NS17iag9jJgTHD1VXjvLCEnZuQ3rJDE9L"); // ->> + EXPECT_EQ(test_base58.encode(uinteger_t("00eb15231dfceb60925886b67d065299925915aeb172c06647", 16)), "NS17iag9jJgTHD1VXjvLCEnZuQ3rJDE9L"); + EXPECT_EQ(test_base58.encode(uinteger_t("516b6fcd0f", 16)), "ABnLTmg"); + EXPECT_EQ(test_base58.encode(uinteger_t("bf4f89001e670274dd", 16)), "3SEo3LWLoPntC"); + EXPECT_EQ(test_base58.encode(uinteger_t("572e4794", 16)), "3EFU7m"); + EXPECT_EQ(test_base58.encode(uinteger_t("ecac89cad93923c02321", 16)), "EJDM8drfXA6uyA"); + EXPECT_EQ(test_base58.encode(uinteger_t("10c8511e", 16)), "Rt5zm"); + // EXPECT_EQ(test_base58.encode(uinteger_t("00000000000000000000", 16)), "1111111111"); // ->> + EXPECT_EQ(test_base58.encode(uinteger_t("00000000000000000000", 16)), "1"); + EXPECT_EQ(test_base58.encode(uinteger_t("801184cd2cdd640ca42cfc3a091c51d549b2f016d454b2774019c2b2d2e08529fd206ec97e", 16)), "5Hx15HFGyep2CfPxsJKe2fXJsCVn5DEiyoeGGF6JZjGbTRnqfiD"); + // EXPECT_EQ(test_base58.encode(uinteger_t("003c176e659bea0f29a3e9bf7880c112b1b31b4dc826268187", 16)), "16UjcYNBG9GTK4uq2f7yYEbuifqCzoLMGS"); // ->> + EXPECT_EQ(test_base58.encode(uinteger_t("003c176e659bea0f29a3e9bf7880c112b1b31b4dc826268187", 16)), "6UjcYNBG9GTK4uq2f7yYEbuifqCzoLMGS"); + EXPECT_EQ(test_base58.encode(uinteger_t("ffffffffffffffffffff", 16)), "FPBt6CHo3fovdL"); + EXPECT_EQ(test_base58.encode(uinteger_t("ffffffffffffffffffffffffff", 16)), "NKioeUVktgzXLJ1B3t"); + EXPECT_EQ(test_base58.encode(uinteger_t("ffffffffffffffffffffffffffffffff", 16)), "YcVfxkQb6JRzqk5kF2tNLv"); + EXPECT_EQ(test_base2.encode(uinteger_t("fb6f9ac3", 16)), "11111011011011111001101011000011"); + EXPECT_EQ(test_base2.encode(uinteger_t("179eea7a", 16)), "10111100111101110101001111010"); + EXPECT_EQ(test_base2.encode(uinteger_t("6db825db", 16)), "1101101101110000010010111011011"); + EXPECT_EQ(test_base2.encode(uinteger_t("93976aa7", 16)), "10010011100101110110101010100111"); + EXPECT_EQ(test_base58.encode(uinteger_t("ef41b9ce7e830af7", 16)), "h26E62FyLQN"); + EXPECT_EQ(test_base58.encode(uinteger_t("606cbc791036d2e9", 16)), "H8Sa62HVULG"); + EXPECT_EQ(test_base58.encode(uinteger_t("bdcb0ea69c2c8ec8", 16)), "YkESUPpnfoD"); + EXPECT_EQ(test_base58.encode(uinteger_t("1a2358ba67fb71d5", 16)), "5NaBN89ajtQ"); + EXPECT_EQ(test_base58.encode(uinteger_t("e6173f0f4d5fb5d7", 16)), "fVAoezT1ZkS"); + EXPECT_EQ(test_base58.encode(uinteger_t("91c81cbfdd58bbd2", 16)), "RPGNSU3bqTX"); + EXPECT_EQ(test_base58.encode(uinteger_t("329e0bf0e388dbfe", 16)), "9U41ZkwwysT"); + EXPECT_EQ(test_base58.encode(uinteger_t("30b10393210fa65b", 16)), "99NMW3WHjjY"); + EXPECT_EQ(test_base58.encode(uinteger_t("ab3bdd18e3623654", 16)), "VeBbqBb4rCT"); + EXPECT_EQ(test_base58.encode(uinteger_t("fe29d1751ec4af8a", 16)), "jWhmYLN9dUm"); + EXPECT_EQ(test_base58.encode(uinteger_t("c1273ab5488769807d", 16)), "3Tbh4kL3WKW6g"); + EXPECT_EQ(test_base58.encode(uinteger_t("6c7907904de934f852", 16)), "2P5jNYhfpTJxy"); + EXPECT_EQ(test_base58.encode(uinteger_t("05f0be055db47a0dc9", 16)), "5PN768Kr5oEp"); + EXPECT_EQ(test_base58.encode(uinteger_t("3511e6206829b35b12", 16)), "gBREojGaJ6DF"); + EXPECT_EQ(test_base58.encode(uinteger_t("d1c7c2ddc4a459d503", 16)), "3fsekq5Esq2KC"); + EXPECT_EQ(test_base58.encode(uinteger_t("1f88efd17ab073e9a1", 16)), "QHJbmW9ZY7jn"); + EXPECT_EQ(test_base58.encode(uinteger_t("0f45dadf4e64c5d5c2", 16)), "CGyVUMmCKLRf"); + EXPECT_EQ(test_base58.encode(uinteger_t("de1e5c5f718bb7fafa", 16)), "3pyy8U7w3KUa5"); + EXPECT_EQ(test_base58.encode(uinteger_t("123190b93e9a49a46c", 16)), "ES3DeFrG1zbd"); + EXPECT_EQ(test_base58.encode(uinteger_t("8bee94a543e7242e5a", 16)), "2nJnuWyLpGf6y"); + EXPECT_EQ(test_base58.encode(uinteger_t("9fd5f2285362f5cfd834", 16)), "9yqFhqeewcW3pF"); + EXPECT_EQ(test_base58.encode(uinteger_t("6987bac63ad23828bb31", 16)), "6vskE5Y1LhS3U4"); + EXPECT_EQ(test_base58.encode(uinteger_t("19d4a0f9d459cc2a08b0", 16)), "2TAsHPuaLhh5Aw"); + EXPECT_EQ(test_base58.encode(uinteger_t("a1e47ffdbea5a807ab26", 16)), "A6XzPgSUJDf1W5"); + EXPECT_EQ(test_base58.encode(uinteger_t("35c231e5b3a86a9b83db", 16)), "42B8reRwPAAoAa"); + EXPECT_EQ(test_base58.encode(uinteger_t("b2351012a48b8347c351", 16)), "B1hPyomGx4Vhqa"); + EXPECT_EQ(test_base58.encode(uinteger_t("71d402694dd9517ea653", 16)), "7Pv2SyAQx2Upu8"); + EXPECT_EQ(test_base58.encode(uinteger_t("55227c0ec7955c2bd6e8", 16)), "5nR64BkskyjHMq"); + EXPECT_EQ(test_base58.encode(uinteger_t("17b3d8ee7907c1be34df", 16)), "2LEg7TxosoxTGS"); + EXPECT_EQ(test_base58.encode(uinteger_t("7e7bba7b68bb8e95827f", 16)), "879o2ATGnmYyAW"); + EXPECT_EQ(test_base58.encode(uinteger_t("db9c13f5ba7654b01407fb", 16)), "wTYfxjDVbiks874"); + EXPECT_EQ(test_base58.encode(uinteger_t("6186449d20f5fd1e6c4393", 16)), "RBeiWhzZNL6VtMG"); + EXPECT_EQ(test_base58.encode(uinteger_t("5248751cebf4ad1c1a83c3", 16)), "MQSVNnc8ehFCqtW"); + EXPECT_EQ(test_base58.encode(uinteger_t("32090ef18cd479fc376a74", 16)), "DQdu351ExDaeYeX"); + EXPECT_EQ(test_base58.encode(uinteger_t("7cfa5d6ed1e467d986c426", 16)), "XzW67T5qfEnFcaZ"); + EXPECT_EQ(test_base58.encode(uinteger_t("9d8707723c7ede51103b6d", 16)), "g4eTCg6QJnB1UU4"); + EXPECT_EQ(test_base58.encode(uinteger_t("6f4d1e392d6a9b4ed8b223", 16)), "Ubo7kZY5aDpAJp2"); + EXPECT_EQ(test_base58.encode(uinteger_t("38057d98797cd39f80a0c9", 16)), "EtjQ2feamJvuqse"); + EXPECT_EQ(test_base58.encode(uinteger_t("de7e59903177e20880e915", 16)), "xB2N7yRBnDYEoT2"); + EXPECT_EQ(test_base58.encode(uinteger_t("b2ea24a28bc4a60b5c4b8d", 16)), "mNFMpJ2P3TGYqhv"); + EXPECT_EQ(test_base58.encode(uinteger_t("cf84938958589b6ffba6114d", 16)), "4v8ZbsGh2ePz5sipt"); + EXPECT_EQ(test_base58.encode(uinteger_t("dee13be7b8d8a08c94a3c02a", 16)), "5CwmE9jQqwtHkTF45"); + EXPECT_EQ(test_base58.encode(uinteger_t("14cb9c6b3f8cd2e02710f569", 16)), "Pm85JHVAAdeUdxtp"); + EXPECT_EQ(test_base58.encode(uinteger_t("ca3f2d558266bdcc44c79cb5", 16)), "4pMwomBAQHuUnoLUC"); + EXPECT_EQ(test_base58.encode(uinteger_t("c031215be44cbad745f38982", 16)), "4dMeTrcxiVw9RWvj3"); + EXPECT_EQ(test_base58.encode(uinteger_t("1435ab1dbc403111946270a5", 16)), "P7wX3sCWNrbqhBEC"); + EXPECT_EQ(test_base58.encode(uinteger_t("d8c6e4d775e7a66a0d0f9f41", 16)), "56GLoRDGWGuGJJwPN"); + EXPECT_EQ(test_base58.encode(uinteger_t("dcee35e74f0fd74176fce2f4", 16)), "5Ap1zyuYiJJFwWcMR"); + EXPECT_EQ(test_base58.encode(uinteger_t("bfcc0ca4b4855d1cf8993fc0", 16)), "4cvafQW4PEhARKv9D"); + EXPECT_EQ(test_base58.encode(uinteger_t("e02a3ac25ece7b54584b670a", 16)), "5EMM28xkpxZ1kkVUM"); + EXPECT_EQ(test_base58.encode(uinteger_t("fe4d938fc3719f064cabb4bfff", 16)), "NBXKkbHwrAsiWTLAk6"); + EXPECT_EQ(test_base58.encode(uinteger_t("9289cb4f6b15c57e6086b87ea5", 16)), "DCvDpjEXEbHjZqskKv"); + EXPECT_EQ(test_base58.encode(uinteger_t("fc266f35626b3612bfe978537b", 16)), "N186PVoBWrNre35BGE"); + EXPECT_EQ(test_base58.encode(uinteger_t("33ff08c06d92502bf258c07166", 16)), "5LC4SoW6jmTtbkbePw"); + EXPECT_EQ(test_base58.encode(uinteger_t("6a81cac1f3666bc59dc67b1c3c", 16)), "9sXgUySUzwiqDU5WHy"); + EXPECT_EQ(test_base58.encode(uinteger_t("9dfb8e7e744c544c0f323ea729", 16)), "EACsmGmkgcwsrPFzLg"); + EXPECT_EQ(test_base58.encode(uinteger_t("1e7a1e284f70838b38442b682b", 16)), "3YEVk9bE7rw5qExMkv"); + EXPECT_EQ(test_base58.encode(uinteger_t("2a862ad57901a8235f5dc74eaf", 16)), "4YS259nuTLfeXa5Wuc"); + EXPECT_EQ(test_base58.encode(uinteger_t("74c82096baef21f9d3089e5462", 16)), "AjAcKEhUfrqm8smvM7"); + EXPECT_EQ(test_base58.encode(uinteger_t("7a3edbc23d7b600263920261cc", 16)), "BBZXyRgey5S5DDZkcK"); + EXPECT_EQ(test_base58.encode(uinteger_t("20435664c357d25a9c8df751cf4f", 16)), "CrwNL6Fbv4pbRx1zd9g"); + EXPECT_EQ(test_base58.encode(uinteger_t("51a7aa87cf5cb1c12d045ec3422d", 16)), "X27NHGgKXmGzzQvDtpC"); + EXPECT_EQ(test_base58.encode(uinteger_t("344d2e116aa26f1062a2cb6ebbef", 16)), "LEDLDvL1Hg4qt1efVXt"); + EXPECT_EQ(test_base58.encode(uinteger_t("6941add7be4c0b5c7163e4928f8e", 16)), "fhMyN6gwoxE3uYraVzV"); + EXPECT_EQ(test_base58.encode(uinteger_t("10938fcbb7c4ab991649734a14bf", 16)), "76TPrSDxzGQfSzMu974"); + EXPECT_EQ(test_base58.encode(uinteger_t("eafe04d944ba504e9af9117b07de", 16)), "2VPgov563ryfe4L2Bj6M"); + EXPECT_EQ(test_base58.encode(uinteger_t("58d0aeed4d35da20b6f052127edf", 16)), "ZenZhXF9YwP8nQvNtNz"); + EXPECT_EQ(test_base58.encode(uinteger_t("d734984e2f5aecf25f7a3e353f8a", 16)), "2N7n3jFsTdyN49Faoq6h"); + EXPECT_EQ(test_base58.encode(uinteger_t("57d873fdb405b7daf4bafa62068a", 16)), "ZJ7NwoP4wHvwyZg3Wjs"); + EXPECT_EQ(test_base58.encode(uinteger_t("bda4ec7b40d0d65ca95dec4c4d3b", 16)), "2CijxjsNyvqTwPCfDcpA"); + EXPECT_EQ(test_base58.encode(uinteger_t("826c4abdceb1b91f0d4ad665f86d2e", 16)), "4edfvuDQu9KzVxLuXHfMo"); + EXPECT_EQ(test_base58.encode(uinteger_t("e7ecb35d07e65b960cb10574a4f51a", 16)), "7VLRYdB4cToipp2J2p3v9"); + EXPECT_EQ(test_base58.encode(uinteger_t("4f2d72ead87b31d6869fba39eac6dc", 16)), "3DUjqJRcfdWhpsrLrGcQs"); + EXPECT_EQ(test_base58.encode(uinteger_t("8b4f5788d60030950d5dfbf94c585d", 16)), "4u44JSRH5jP5X39YhPsmE"); + EXPECT_EQ(test_base58.encode(uinteger_t("ee4c0a0025d1a74ace9fe349355cc5", 16)), "7fgACjABRQUGUEpN6VBBA"); + EXPECT_EQ(test_base58.encode(uinteger_t("58ac05b9a0b4b66083ff1d489b8d84", 16)), "3UtJPyTwGXapcxHx8Rom5"); + EXPECT_EQ(test_base58.encode(uinteger_t("1aa35c05e1132e8e049aafaef035d8", 16)), "kE2eSU7gM2619pT82iGP"); + EXPECT_EQ(test_base58.encode(uinteger_t("771b0c28608484562a292e5d5d2b30", 16)), "4LGYeWhyfrjUByibUqdVR"); + EXPECT_EQ(test_base58.encode(uinteger_t("78ff9a0e56f9e88dc1cd654b40d019", 16)), "4PLggs66qAdbmZgkaPihe"); + EXPECT_EQ(test_base58.encode(uinteger_t("6d691bdd736346aa5a0a95b373b2ab", 16)), "44Y6qTgSvRMkdqpQ5ufkN"); +} diff --git a/contrib/base-x/uinteger_t.hh b/contrib/base-x/uinteger_t.hh new file mode 100644 index 00000000000..901460f75c4 --- /dev/null +++ b/contrib/base-x/uinteger_t.hh @@ -0,0 +1,2546 @@ +/* +uinteger_t.hh +An arbitrary precision unsigned integer type for C++ + +Copyright (c) 2017 German Mendez Bravo (Kronuz) @ german dot mb at gmail.com +Copyright (c) 2013 - 2017 Jason Lee @ calccrypto at gmail.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +With much help from Auston Sterling + +Thanks to Stefan Deigmüller for finding +a bug in operator*. + +Thanks to François Dessenne for convincing me +to do a general rewrite of this class. + +Germán Mández Bravo (Kronuz) converted Jason Lee's uint128_t +to header-only and extended to arbitrary bit length. +*/ + +#ifndef __uint_t__ +#define __uint_t__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ASSERT assert + +// Compatibility inlines +#ifndef __has_builtin // Optional of course +#define __has_builtin(x) 0 // Compatibility with non-clang compilers +#endif + +#if defined _MSC_VER +# define HAVE___ADDCARRY_U64 +# define HAVE___SUBBORROW_U64 +# define HAVE___ADDCARRY_U32 +# define HAVE___SUBBORROW_U32 +# define HAVE___ADDCARRY_U16 +# define HAVE___SUBBORROW_U16 +# define HAVE___UMUL128 +# define HAVE___UMUL64 +# define HAVE___UMUL32 +# include +#endif + +#if (defined(__clang__) && __has_builtin(__builtin_clzll)) || (defined(__GNUC__ ) && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 3))) +# define HAVE____BUILTIN_CLZLL +#endif +#if (defined(__clang__) && __has_builtin(__builtin_clzl)) || (defined(__GNUC__ ) && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 3))) +# define HAVE____BUILTIN_CLZL +#endif +#if (defined(__clang__) && __has_builtin(__builtin_clz)) || (defined(__GNUC__ ) && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 3))) +# define HAVE____BUILTIN_CLZ +#endif +#if (defined(__clang__) && __has_builtin(__builtin_addcll)) +# define HAVE____BUILTIN_ADDCLL +#endif +#if (defined(__clang__) && __has_builtin(__builtin_addcl)) +# define HAVE____BUILTIN_ADDCL +#endif +#if (defined(__clang__) && __has_builtin(__builtin_addc)) +# define HAVE____BUILTIN_ADDC +#endif +#if (defined(__clang__) && __has_builtin(__builtin_subcll)) +# define HAVE____BUILTIN_SUBCLL +#endif +#if (defined(__clang__) && __has_builtin(__builtin_subcl)) +# define HAVE____BUILTIN_SUBCL +#endif +#if (defined(__clang__) && __has_builtin(__builtin_subc)) +# define HAVE____BUILTIN_SUBC +#endif + +#if defined __SIZEOF_INT128__ +#define HAVE____INT128_T +#endif + + +#ifndef DIGIT_T +#define DIGIT_T std::uint64_t +#endif + +#ifndef HALF_DIGIT_T +#define HALF_DIGIT_T std::uint32_t +#endif + +class uinteger_t; + +namespace std { // This is probably not a good idea + // Give uinteger_t type traits + template <> struct is_arithmetic : std::true_type {}; + template <> struct is_integral : std::true_type {}; + template <> struct is_unsigned : std::true_type {}; +} + +class uinteger_t { +public: + using digit = DIGIT_T; + using half_digit = HALF_DIGIT_T; + + static constexpr std::size_t digit_octets = sizeof(digit); // number of octets per digit + static constexpr std::size_t digit_bits = digit_octets * 8; // number of bits per digit + static constexpr std::size_t half_digit_octets = sizeof(half_digit); // number of octets per half_digit + static constexpr std::size_t half_digit_bits = half_digit_octets * 8; // number of bits per half_digit + + using container = std::vector; + + template + struct is_result { + static const bool value = false; + }; + + template + struct is_result> { + static const bool value = true; + }; + + template + struct is_result> { + static const bool value = true; + }; + +private: + static_assert(digit_octets == half_digit_octets * 2, "half_digit must be exactly half the size of digit"); + + static constexpr std::size_t karatsuba_cutoff = 1024 / digit_bits; + static constexpr double growth_factor = 1.5; + + std::size_t _begin; + std::size_t _end; + container _value_instance; + container& _value; + bool _carry; + +public: + // Window to vector (uses _begin and _end) + + void reserve(std::size_t sz) { + _value.reserve(sz + _begin); + } + + std::size_t grow(std::size_t n) { + // expands the vector using a growth factor + // and returns the new capacity. + auto cc = _value.capacity(); + if (n >= cc) { + cc = n * growth_factor; + _value.reserve(cc); + } + return cc; + } + + void resize(std::size_t sz) { + grow(sz + _begin); + _value.resize(sz + _begin); + } + + void resize(std::size_t sz, const digit& c) { + grow(sz + _begin); + _value.resize(sz + _begin, c); + } + + void clear() { + _value.clear(); + _begin = 0; + _end = 0; + _carry = false; + } + + digit* data() noexcept { + return _value.data() + _begin; + } + + const digit* data() const noexcept { + return _value.data() + _begin; + } + + std::size_t size() const noexcept { + return _end ? _end - _begin : _value.size() - _begin; + } + + void prepend(std::size_t sz, const digit& c) { + // Efficiently prepend by growing backwards by growth factor + auto min = std::min(_begin, sz); + if (min) { + // If there is some space before `_begin`, we try using it first: + _begin -= min; + std::fill_n(_value.begin() + _begin, min, c); + sz -= min; + } + if (sz) { + ASSERT(_begin == 0); // _begin should be 0 in here + // If there's still more room needed, we grow the vector: + // Ex.: grow using prepend(3, y) + // sz = 3 + // _begin = 0 (B) + // _end = 1 (E) + // initially (capacity == 12): + // |xxxxxxxxxx- | + // B E + // after reclaiming space after `_end` (same capacity == 12): + // |xxxxxxxxxx | + // B + // _end = 0 + // csz = 10 + // grow returns the new capacity (22) + // isz = 12 (22 - 10) + // _begin = 9 (12 - 3) + // after (capacity == (12 + 3) * 1.5 == 22): + // |---------yyyxxxxxxxxxx| + // B + if (_end) { + // reclaim space after `_end` + _value.resize(_end); + _end = 0; + } + auto csz = _value.size(); + auto isz = grow(csz + sz) - csz; + _value.insert(_value.begin(), isz, c); + _begin = isz - sz; + } + } + + void prepend(const digit& c) { + prepend(1, c); + } + + void prepend(const uinteger_t& num) { + prepend(num.size(), 0); + std::copy(num.begin(), num.end(), begin()); + } + + void append(std::size_t sz, const digit& c) { + // Efficiently append by growing by growth factor + if (_end) { + // reclaim space after `_end` + _value.resize(_end); + _end = 0; + } + auto nsz = _value.size() + sz; + grow(nsz); + _value.resize(nsz, c); + } + + void append(const digit& c) { + append(1, c); + } + + void append(const uinteger_t& num) { + auto sz = num.size(); + append(sz, 0); + std::copy(num.begin(), num.end(), end() - sz); + } + + container::iterator begin() noexcept { + return _value.begin() + _begin; + } + + container::const_iterator begin() const noexcept { + return _value.cbegin() + _begin; + } + + container::iterator end() noexcept { + return _end ? _value.begin() + _end : _value.end(); + } + + container::const_iterator end() const noexcept { + return _end ? _value.cbegin() + _end : _value.cend(); + } + + container::reverse_iterator rbegin() noexcept { + return _end ? container::reverse_iterator(_value.begin() + _end) : _value.rbegin(); + } + + container::const_reverse_iterator rbegin() const noexcept { + return _end ? container::const_reverse_iterator(_value.cbegin() + _end) : _value.crbegin(); + } + + container::reverse_iterator rend() noexcept { + return container::reverse_iterator(_value.begin() + _begin); + } + + container::const_reverse_iterator rend() const noexcept { + return container::const_reverse_iterator(_value.cbegin() + _begin); + } + + container::reference front() { + return *begin(); + } + + container::const_reference front() const { + return *begin(); + } + + container::reference back() { + return *rbegin(); + } + + container::const_reference back() const { + return *rbegin(); + } + +private: + // Optimized primitives for operations + + static digit _bits(digit x) { + #if defined HAVE____BUILTIN_CLZLL + if (digit_octets == sizeof(unsigned long long)) { + return x ? digit_bits - __builtin_clzll(x) : 1; + } + #endif + #if defined HAVE____BUILTIN_CLZL + if (digit_octets == sizeof(unsigned long)) { + return x ? digit_bits - __builtin_clzl(x) : 1; + } + #endif + #if defined HAVE____BUILTIN_CLZ + if (digit_octets == sizeof(unsigned)) { + return x ? digit_bits - __builtin_clz(x) : 1; + } + #endif + { + digit c = x ? 0 : 1; + while (x) { + x >>= 1; + ++c; + } + return c; + } + } + + static digit _mult(digit x, digit y, digit* lo) { + #if defined HAVE___UMUL128 + if (digit_bits == 64) { + digit h; + digit l = _umul128(x, y, &h); // _umul128(x, y, *hi) -> lo + return h; + } + #endif + #if defined HAVE___UMUL64 + if (digit_bits == 32) { + digit h; + digit l = _umul64(x, y, &h); // _umul64(x, y, *hi) -> lo + return h; + } + #endif + #if defined HAVE___UMUL32 + if (digit_bits == 16) { + digit h; + digit l = _umul32(x, y, &h); // _umul32(x, y, *hi) -> lo + return h; + } + #endif + #if defined HAVE____INT128_T + if (digit_bits == 64) { + auto r = static_cast<__uint128_t>(x) * static_cast<__uint128_t>(y); + *lo = r; + return r >> digit_bits; + } + #endif + if (digit_bits == 64) { + digit x0 = x & 0xffffffffUL; + digit x1 = x >> 32; + digit y0 = y & 0xffffffffUL; + digit y1 = y >> 32; + + digit u = (x0 * y0); + digit v = (x1 * y0) + (u >> 32); + digit w = (x0 * y1) + (v & 0xffffffffUL); + + *lo = (w << 32) + (u & 0xffffffffUL); // low + return (x1 * y1) + (v >> 32) + (w >> 32); // high + } if (digit_bits == 32) { + auto r = static_cast(x) * static_cast(y); + *lo = r; + return r >> 32; + } if (digit_bits == 16) { + auto r = static_cast(x) * static_cast(y); + *lo = r; + return r >> 16; + } if (digit_bits == 8) { + auto r = static_cast(x) * static_cast(y); + *lo = r; + return r >> 8; + } + } + + static digit _multadd(digit x, digit y, digit a, digit c, digit* lo) { + #if defined HAVE___UMUL128 && defined HAVE___ADDCARRY_U64 + if (digit_bits == 64) { + digit h; + digit l = _umul128(x, y, &h); // _umul128(x, y, *hi) -> lo + return h + _addcarry_u64(c, l, a, lo); // _addcarry_u64(carryin, x, y, *sum) -> carryout + } + #endif + #if defined HAVE___UMUL64 && defined HAVE___ADDCARRY_U32 + if (digit_bits == 32) { + digit h; + digit l = _umul64(x, y, &h); // _umul64(x, y, *hi) -> lo + return h + _addcarry_u32(c, l, a, lo); // _addcarry_u32(carryin, x, y, *sum) -> carryout + } + #endif + #if defined HAVE___UMUL32 && defined HAVE___ADDCARRY_U16 + if (digit_bits == 16) { + digit h; + digit l = _umul32(x, y, &h); // _umul32(x, y, *hi) -> lo + return h + _addcarry_u16(c, l, a, lo); // _addcarry_u16(carryin, x, y, *sum) -> carryout + } + #endif + #if defined HAVE____INT128_T + if (digit_bits == 64) { + auto r = static_cast<__uint128_t>(x) * static_cast<__uint128_t>(y) + static_cast<__uint128_t>(a) + static_cast<__uint128_t>(c); + *lo = r; + return r >> digit_bits; + } + #endif + if (digit_bits == 64) { + digit x0 = x & 0xffffffffUL; + digit x1 = x >> 32; + digit y0 = y & 0xffffffffUL; + digit y1 = y >> 32; + + digit u = (x0 * y0) + (a & 0xffffffffUL) + (c & 0xffffffffUL); + digit v = (x1 * y0) + (u >> 32) + (a >> 32) + (c >> 32); + digit w = (x0 * y1) + (v & 0xffffffffUL); + + *lo = (w << 32) + (u & 0xffffffffUL); // low + return (x1 * y1) + (v >> 32) + (w >> 32); // high + } + if (digit_bits == 32) { + auto r = static_cast(x) * static_cast(y) + static_cast(a) + static_cast(c); + *lo = r; + return r >> 32; + } + if (digit_bits == 16) { + auto r = static_cast(x) * static_cast(y) + static_cast(a) + static_cast(c); + *lo = r; + return r >> 16; + } + if (digit_bits == 8) { + auto r = static_cast(x) * static_cast(y) + static_cast(a) + static_cast(c); + *lo = r; + return r >> 8; + } + } + + static digit _divmod(digit x_hi, digit x_lo, digit y, digit* result) { + #if defined HAVE____INT128_T + if (digit_bits == 64) { + auto x = static_cast<__uint128_t>(x_hi) << digit_bits | static_cast<__uint128_t>(x_lo); + digit q = x / y; + digit r = x % y; + + *result = q; + return r; + } + #endif + if (digit_bits == 64) { + // quotient + digit q = x_lo << 1; + + // remainder + digit r = x_hi; + + digit carry = x_lo >> 63; + int i; + + for (i = 0; i < 64; i++) { + auto tmp = r >> 63; + r <<= 1; + r |= carry; + carry = tmp; + + if (carry == 0) { + if (r >= y) { + carry = 1; + } else { + tmp = q >> 63; + q <<= 1; + q |= carry; + carry = tmp; + continue; + } + } + + r -= y; + r -= (1 - carry); + carry = 1; + tmp = q >> 63; + q <<= 1; + q |= carry; + carry = tmp; + } + + *result = q; + return r; + } + if (digit_bits == 32) { + auto x = static_cast(x_hi) << 32 | static_cast(x_lo); + digit q = x / y; + digit r = x % y; + + *result = q; + return r; + } + if (digit_bits == 16) { + auto x = static_cast(x_hi) << 16 | static_cast(x_lo); + digit q = x / y; + digit r = x % y; + + *result = q; + return r; + } + if (digit_bits == 8) { + auto x = static_cast(x_hi) << 8 | static_cast(x_lo); + digit q = x / y; + digit r = x % y; + + *result = q; + return r; + } + } + + static digit _addcarry(digit x, digit y, digit c, digit* result) { + #if defined HAVE___ADDCARRY_U64 + if (digit_bits == 64) { + return _addcarry_u64(c, x, y, result); // _addcarry_u64(carryin, x, y, *sum) -> carryout + } + #endif + #if defined HAVE___ADDCARRY_U32 + if (digit_bits == 32) { + return _addcarry_u32(c, x, y, result); // _addcarry_u32(carryin, x, y, *sum) -> carryout + } + #endif + #if defined HAVE___ADDCARRY_U16 + if (digit_bits == 16) { + return _addcarry_u16(c, x, y, result); // _addcarry_u16(carryin, x, y, *sum) -> carryout + } + #endif + #if defined HAVE____BUILTIN_ADDCLL + if (digit_octets == sizeof(unsigned long long)) { + unsigned long long carryout; + *result = __builtin_addcll(x, y, c, &carryout); // __builtin_addcll(x, y, carryin, *carryout) -> sum + return carryout; + } + #endif + #if defined HAVE____BUILTIN_ADDCL + if (digit_octets == sizeof(unsigned long)) { + unsigned long carryout; + *result = __builtin_addcl(x, y, c, &carryout); // __builtin_addcl(x, y, carryin, *carryout) -> sum + return carryout; + } + #endif + #if defined HAVE____BUILTIN_ADDC + if (digit_octets == sizeof(unsigned)) { + unsigned carryout; + *result = __builtin_addc(x, y, c, &carryout); // __builtin_addc(x, y, carryin, *carryout) -> sum + return carryout; + } + #endif + #if defined HAVE____INT128_T + if (digit_bits == 64) { + auto r = static_cast<__uint128_t>(x) + static_cast<__uint128_t>(y) + static_cast<__uint128_t>(c); + *result = r; + return static_cast(r >> digit_bits); + } + #endif + if (digit_bits == 64) { + digit x0 = x & 0xffffffffUL; + digit x1 = x >> 32; + digit y0 = y & 0xffffffffUL; + digit y1 = y >> 32; + + auto u = x0 + y0 + c; + auto v = x1 + y1 + static_cast(u >> 32); + *result = (v << 32) + (u & 0xffffffffUL); + return static_cast(v >> 32); + } + if (digit_bits == 32) { + auto r = static_cast(x) + static_cast(y) + static_cast(c); + *result = r; + return static_cast(r >> 32); + } + if (digit_bits == 16) { + auto r = static_cast(x) + static_cast(y) + static_cast(c); + *result = r; + return static_cast(r >> 16); + } + if (digit_bits == 8) { + auto r = static_cast(x) + static_cast(y) + static_cast(c); + *result = r; + return static_cast(r >> 8); + } + } + + static digit _subborrow(digit x, digit y, digit c, digit* result) { + #if defined HAVE___SUBBORROW_U64 + if (digit_bits == 64) { + return _subborrow_u64(c, x, y, result); // _subborrow_u64(carryin, x, y, *sum) -> carryout + } + #endif + #if defined HAVE___SUBBORROW_U32 + if (digit_bits == 64) { + return _subborrow_u32(c, x, y, result); // _subborrow_u32(carryin, x, y, *sum) -> carryout + } + #endif + #if defined HAVE___SUBBORROW_U16 + if (digit_bits == 64) { + return _subborrow_u16(c, x, y, result); // _subborrow_u16(carryin, x, y, *sum) -> carryout + } + #endif + #if defined HAVE____BUILTIN_SUBCLL + if (digit_octets == sizeof(unsigned long long)) { + unsigned long long carryout; + *result = __builtin_subcll(x, y, c, &carryout); // __builtin_subcll(x, y, carryin, *carryout) -> sum + return carryout; + } + #endif + #if defined HAVE____BUILTIN_SUBCL + if (digit_octets == sizeof(unsigned long)) { + unsigned long carryout; + *result = __builtin_subcl(x, y, c, &carryout); // __builtin_subcl(x, y, carryin, *carryout) -> sum + return carryout; + } + #endif + #if defined HAVE____BUILTIN_SUBC + if (digit_octets == sizeof(unsigned)) { + unsigned carryout; + *result = __builtin_subc(x, y, c, &carryout); // __builtin_subc(x, y, carryin, *carryout) -> sum + return carryout; + } + #endif + #if defined HAVE____INT128_T + if (digit_bits == 64) { + auto r = static_cast<__uint128_t>(x) - static_cast<__uint128_t>(y) - static_cast<__uint128_t>(c); + *result = r; + return static_cast(r >> 64); + } + #endif + if (digit_bits == 64) { + digit x0 = x & 0xffffffffUL; + digit x1 = x >> 32; + digit y0 = y & 0xffffffffUL; + digit y1 = y >> 32; + + auto u = x0 - y0 - c; + auto v = x1 - y1 - static_cast(u >> 32); + *result = (v << 32) + (u & 0xffffffffUL); + return static_cast(v >> 32); + } + if (digit_bits == 32) { + auto r = static_cast(x) - static_cast(y) - static_cast(c); + *result = r; + return static_cast(r >> 32); + } + if (digit_bits == 16) { + auto r = static_cast(x) - static_cast(y) - static_cast(c); + *result = r; + return static_cast(r >> 16); + } + if (digit_bits == 8) { + auto r = static_cast(x) - static_cast(y) - static_cast(c); + *result = r; + return static_cast(r >> 8); + } + } + + // Helper functions + + void trim(digit mask = 0) { + auto rit = rbegin(); + auto rit_e = rend(); + + // Masks the last value of internal vector + mask &= (digit_bits - 1); + if (mask && rit != rit_e) { + *rit &= (static_cast(1) << mask) - 1; + } + + // Removes all unused zeros from the internal vector + auto rit_f = std::find_if(rit, rit_e, [](const digit& c) { return c; }); + resize(rit_e - rit_f); // shrink + } + + static constexpr char chr(int ord) { + constexpr const char _[256] = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', + 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + }; + return _[ord]; + } + + static constexpr int ord(int chr) { + constexpr const int _[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + + -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, -1, + + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + }; + return _[chr]; + } + +public: + static constexpr unsigned base_bits(int base) { + constexpr const unsigned _[256] = { + 0, 1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, + }; + return _[base - 1]; + } + + static constexpr unsigned base_size(int base) { + constexpr const unsigned _[256] = { + 0, 64, 41, 32, 28, 25, 23, 22, 21, 20, 19, 18, 18, 17, 17, 16, + 16, 16, 16, 15, 15, 15, 15, 14, 14, 14, 14, 14, 14, 14, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12, 12, 12, 12, 11, 11, 11, 11, 11, 11, 11, 11, + + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 8, + }; + return _[base - 1]; + } + + static const uinteger_t uint_0() { + static uinteger_t uint_0(0); + return uint_0; + } + + static const uinteger_t uint_1() { + static uinteger_t uint_1(1); + return uint_1; + } + +private: + // Public Implementation +#ifdef UINT_T_PUBLIC_IMPLEMENTATION +public: +#endif + static uinteger_t& bitwise_and(uinteger_t& lhs, const uinteger_t& rhs) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + if (lhs_sz > rhs_sz) { + lhs.resize(rhs_sz); // shrink + } + + auto lhs_it = lhs.begin(); + auto lhs_it_e = lhs.end(); + + auto rhs_it = rhs.begin(); + + for (; lhs_it != lhs_it_e; ++lhs_it, ++rhs_it) { + *lhs_it &= *rhs_it; + } + + // Finish up + lhs.trim(); + return lhs; + } + + static uinteger_t& bitwise_and(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + auto result_sz = std::max(lhs_sz, rhs_sz); + result.resize(result_sz); + + // not using `end()` because resize of `result.resize()` could have + // resized `lhs` or `rhs` if `result` is also either `rhs` or `lhs`. + auto lhs_it = lhs.begin(); + auto lhs_it_e = lhs_it + lhs_sz; + + auto rhs_it = rhs.begin(); + auto rhs_it_e = rhs_it + rhs_sz; + + auto it = result.begin(); + + if (lhs_sz < rhs_sz) { + for (; lhs_it != lhs_it_e; ++lhs_it, ++rhs_it, ++it) { + *it = *lhs_it & *rhs_it; + } + for (; rhs_it != rhs_it_e; ++rhs_it, ++it) { + *it = 0; + } + } else { + for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it, ++it) { + *it = *lhs_it & *rhs_it; + } + for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { + *it = 0; + } + } + + // Finish up + result.trim(); + return result; + } + + static uinteger_t bitwise_and(const uinteger_t& lhs, const uinteger_t& rhs) { + uinteger_t result; + bitwise_and(result, lhs, rhs); + return result; + } + + static uinteger_t& bitwise_or(uinteger_t& lhs, const uinteger_t& rhs) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + if (lhs_sz < rhs_sz) { + lhs.resize(rhs_sz, 0); // grow + } + + auto lhs_it = lhs.begin(); + + auto rhs_it = rhs.begin(); + auto rhs_it_e = rhs.end(); + + for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it) { + *lhs_it |= *rhs_it; + } + + // Finish up + lhs.trim(); + return lhs; + } + + static uinteger_t& bitwise_or(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + auto result_sz = std::max(lhs_sz, rhs_sz); + result.resize(result_sz); + + // not using `end()` because resize of `result.resize()` could have + // resized `lhs` or `rhs` if `result` is also either `rhs` or `lhs`. + auto lhs_it = lhs.begin(); + auto lhs_it_e = lhs_it + lhs_sz; + + auto rhs_it = rhs.begin(); + auto rhs_it_e = rhs_it + rhs_sz; + + auto it = result.begin(); + + if (lhs_sz < rhs_sz) { + for (; lhs_it != lhs_it_e; ++lhs_it, ++rhs_it, ++it) { + *it = *lhs_it | *rhs_it; + } + for (; rhs_it != rhs_it_e; ++rhs_it, ++it) { + *it = *rhs_it; + } + } else { + for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it, ++it) { + *it = *lhs_it | *rhs_it; + } + for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { + *it = *lhs_it; + } + } + + // Finish up + result.trim(); + return result; + } + static uinteger_t bitwise_or(const uinteger_t& lhs, const uinteger_t& rhs) { + uinteger_t result; + bitwise_or(result, lhs, rhs); + return result; + } + + static uinteger_t& bitwise_xor(uinteger_t& lhs, const uinteger_t& rhs) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + if (lhs_sz < rhs_sz) { + lhs.resize(rhs_sz, 0); // grow + } + + auto lhs_it = lhs.begin(); + + auto rhs_it = rhs.begin(); + auto rhs_it_e = rhs.end(); + + for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it) { + *lhs_it ^= *rhs_it; + } + + // Finish up + lhs.trim(); + return lhs; + } + + static uinteger_t& bitwise_xor(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + auto result_sz = std::max(lhs_sz, rhs_sz); + result.resize(result_sz); + + // not using `end()` because resize of `result.resize()` could have + // resized `lhs` or `rhs` if `result` is also either `rhs` or `lhs`. + auto lhs_it = lhs.begin(); + auto lhs_it_e = lhs_it + lhs_sz; + + auto rhs_it = rhs.begin(); + auto rhs_it_e = rhs_it + rhs_sz; + + auto it = result.begin(); + + if (lhs_sz < rhs_sz) { + for (; lhs_it != lhs_it_e; ++lhs_it, ++rhs_it, ++it) { + *it = *lhs_it ^ *rhs_it; + } + for (; rhs_it != rhs_it_e; ++rhs_it, ++it) { + *it = *rhs_it; + } + } else { + for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it, ++it) { + *it = *lhs_it ^ *rhs_it; + } + for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { + *it = *lhs_it; + } + } + + // Finish up + result.trim(); + return result; + } + + static uinteger_t bitwise_xor(const uinteger_t& lhs, const uinteger_t& rhs) { + uinteger_t result; + bitwise_xor(result, lhs, rhs); + return result; + } + + static uinteger_t& bitwise_inv(uinteger_t& lhs) { + auto lhs_sz = lhs.size(); + + auto b = lhs.bits(); + + if (!lhs_sz) { + lhs.append(0); + } + + // not using `end()` because resize of `result.resize()` could have + // resized `lhs` if `result` is also `lhs`. + auto lhs_it = lhs.begin(); + auto lhs_it_e = lhs_it + lhs_sz; + + for (; lhs_it != lhs_it_e; ++lhs_it) { + *lhs_it = ~*lhs_it; + } + + // Finish up + lhs.trim(b ? b : 1); + return lhs; + } + + static uinteger_t& bitwise_inv(uinteger_t& result, const uinteger_t& lhs) { + auto lhs_sz = lhs.size(); + + auto b = lhs.bits(); + + auto result_sz = lhs_sz ? lhs_sz : 1; + result.resize(result_sz); + + // not using `end()` because resize of `result.resize()` could have + // resized `lhs` if `result` is also `lhs`. + auto lhs_it = lhs.begin(); + auto lhs_it_e = lhs_it + lhs_sz; + + auto it = result.begin(); + auto it_e = it + result_sz; + + for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { + *it = ~*lhs_it; + } + for (; it != it_e; ++it) { + *it = ~static_cast(0); + } + + // Finish up + result.trim(b ? b : 1); + return result; + } + + static uinteger_t bitwise_inv(const uinteger_t& lhs) { + uinteger_t result; + bitwise_inv(result, lhs); + return result; + } + + static uinteger_t& bitwise_lshift(uinteger_t& lhs, const uinteger_t& rhs) { + if (!rhs) { + return lhs; + } + + uinteger_t shifts_q; + uinteger_t shifts_r; + auto _digit_bits = digit_bits; + auto uint_digit_bits = uinteger_t(_digit_bits); + divmod(shifts_q, shifts_r, rhs, uint_digit_bits); + std::size_t shifts = static_cast(shifts_q); + std::size_t shift = static_cast(shifts_r); + + if (shifts) { + lhs.prepend(shifts, 0); + } + if (shift) { + digit shifted = 0; + auto lhs_it = lhs.begin() + shifts; + auto lhs_it_e = lhs.end(); + for (; lhs_it != lhs_it_e; ++lhs_it) { + auto v = (*lhs_it << shift) | shifted; + shifted = *lhs_it >> (_digit_bits - shift); + *lhs_it = v; + } + if (shifted) { + lhs.append(shifted); + } + } + + // Finish up + lhs.trim(); + return lhs; + } + + static uinteger_t& bitwise_lshift(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { + if (&result._value == &lhs._value) { + bitwise_lshift(result, rhs); + return result; + } + if (!rhs) { + result = lhs; + return result; + } + + auto lhs_sz = lhs.size(); + + uinteger_t shifts_q; + uinteger_t shifts_r; + auto _digit_bits = digit_bits; + auto uint_digit_bits = uinteger_t(_digit_bits); + divmod(shifts_q, shifts_r, rhs, uint_digit_bits); + std::size_t shifts = static_cast(shifts_q); + std::size_t shift = static_cast(shifts_r); + + auto result_sz = lhs_sz + shifts; + result.grow(result_sz + 1); + result.resize(shifts, 0); + result.resize(result_sz); + + // not using `end()` because resize of `result.resize()` could have + // resized `lhs` if `result` is also `lhs`. + auto lhs_it = lhs.begin(); + auto lhs_it_e = lhs_it + lhs_sz; + + auto it = result.begin() + shifts; + + if (shift) { + digit shifted = 0; + for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { + auto v = (*lhs_it << shift) | shifted; + shifted = *lhs_it >> (_digit_bits - shift); + *it = v; + } + if (shifted) { + result.append(shifted); + } + } else { + for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { + *it = *lhs_it; + } + } + + // Finish up + result.trim(); + return result; + } + + static uinteger_t bitwise_lshift(const uinteger_t& lhs, const uinteger_t& rhs) { + uinteger_t result; + bitwise_lshift(result, lhs, rhs); + return result; + } + + static uinteger_t& bitwise_rshift(uinteger_t& lhs, const uinteger_t& rhs) { + if (!rhs) { + return lhs; + } + + auto lhs_sz = lhs.size(); + + auto _digit_bits = digit_bits; + if (compare(rhs, uinteger_t(lhs_sz * _digit_bits)) >= 0) { + lhs = uint_0(); + return lhs; + } + + uinteger_t shifts_q; + uinteger_t shifts_r; + auto uint_digit_bits = uinteger_t(_digit_bits); + divmod(shifts_q, shifts_r, rhs, uint_digit_bits); + std::size_t shifts = static_cast(shifts_q); + std::size_t shift = static_cast(shifts_r); + + if (shifts) { + lhs._begin += shifts; + } + if (shift) { + digit shifted = 0; + auto lhs_rit = lhs.rbegin(); + auto lhs_rit_e = lhs.rend(); + for (; lhs_rit != lhs_rit_e; ++lhs_rit) { + auto v = (*lhs_rit >> shift) | shifted; + shifted = *lhs_rit << (_digit_bits - shift); + *lhs_rit = v; + } + lhs.trim(); + } + + return lhs; + } + + static uinteger_t& bitwise_rshift(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { + if (&result._value == &lhs._value) { + bitwise_lshift(result, rhs); + return result; + } + if (!rhs) { + result = lhs; + return result; + } + + auto lhs_sz = lhs.size(); + + auto _digit_bits = digit_bits; + if (compare(rhs, uinteger_t(lhs_sz * _digit_bits)) >= 0) { + result = uint_0(); + return result; + } + + uinteger_t shifts_q; + uinteger_t shifts_r; + auto uint_digit_bits = uinteger_t(_digit_bits); + divmod(shifts_q, shifts_r, rhs, uint_digit_bits); + std::size_t shifts = static_cast(shifts_q); + std::size_t shift = static_cast(shifts_r); + + auto result_sz = lhs_sz - shifts; + result.resize(result_sz); + + // not using `end()` because resize of `result.resize()` could have + // resized `lhs` if `result` is also `lhs`. + auto lhs_rit = lhs.rbegin(); + auto lhs_rit_e = lhs_rit + lhs_sz - shifts; + + auto rit = result.rbegin(); + auto rit_e = rit + result_sz; + + if (shift) { + digit shifted = 0; + for (; lhs_rit != lhs_rit_e; ++lhs_rit, ++rit) { + ASSERT(rit != rit_e); (void)(rit_e); + auto v = (*lhs_rit >> shift) | shifted; + shifted = *lhs_rit << (_digit_bits - shift); + *rit = v; + } + } else { + for (; lhs_rit != lhs_rit_e; ++lhs_rit, ++rit) { + ASSERT(rit != rit_e); (void)(rit_e); + *rit = *lhs_rit; + } + } + + // Finish up + result.trim(); + return result; + } + + static uinteger_t bitwise_rshift(const uinteger_t& lhs, const uinteger_t& rhs) { + uinteger_t result; + bitwise_rshift(result, lhs, rhs); + return result; + } + + static int compare(const uinteger_t& lhs, const uinteger_t& rhs) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + if (lhs_sz > rhs_sz) return 1; + if (lhs_sz < rhs_sz) return -1; + + auto lhs_rit = lhs.rbegin(); + auto lhs_rit_e = lhs.rend(); + + auto rhs_rit = rhs.rbegin(); + + for (; lhs_rit != lhs_rit_e && *lhs_rit == *rhs_rit; ++lhs_rit, ++rhs_rit); + + if (lhs_rit != lhs_rit_e) { + if (*lhs_rit > *rhs_rit) return 1; + if (*lhs_rit < *rhs_rit) return -1; + } + + return 0; + } + + static uinteger_t& long_add(uinteger_t& lhs, const uinteger_t& rhs) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + if (lhs_sz < rhs_sz) { + lhs.reserve(rhs_sz + 1); + lhs.resize(rhs_sz, 0); // grow + } + + // not using `end()` because resize of `lhs.resize()` could have + // resized `lhs`. + auto lhs_it = lhs.begin(); + auto lhs_it_e = lhs_it + lhs_sz; + + auto rhs_it = rhs.begin(); + auto rhs_it_e = rhs_it + rhs_sz; + + digit carry = 0; + if (lhs_sz < rhs_sz) { + for (; lhs_it != lhs_it_e; ++rhs_it, ++lhs_it) { + carry = _addcarry(*lhs_it, *rhs_it, carry, &*lhs_it); + } + for (; carry && rhs_it != rhs_it_e; ++rhs_it, ++lhs_it) { + carry = _addcarry(0, *rhs_it, carry, &*lhs_it); + } + for (; rhs_it != rhs_it_e; ++rhs_it, ++lhs_it) { + *lhs_it = *rhs_it; + } + } else { + for (; rhs_it != rhs_it_e; ++rhs_it, ++lhs_it) { + carry = _addcarry(*lhs_it, *rhs_it, carry, &*lhs_it); + } + for (; carry && lhs_it != lhs_it_e; ++lhs_it) { + carry = _addcarry(*lhs_it, 0, carry, &*lhs_it); + } + } + + if (carry) { + lhs.append(1); + } + + lhs._carry = false; + + // Finish up + lhs.trim(); + return lhs; + } + + static uinteger_t& long_add(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + auto result_sz = std::max(lhs_sz, rhs_sz); + result.reserve(result_sz + 1); + result.resize(result_sz, 0); + + // not using `end()` because resize of `result.resize()` could have + // resized `lhs` or `rhs` if `result` is also either `rhs` or `lhs`. + auto lhs_it = lhs.begin(); + auto lhs_it_e = lhs_it + lhs_sz; + + auto rhs_it = rhs.begin(); + auto rhs_it_e = rhs_it + rhs_sz; + + auto it = result.begin(); + + digit carry = 0; + if (lhs_sz < rhs_sz) { + for (; lhs_it != lhs_it_e; ++lhs_it, ++rhs_it, ++it) { + carry = _addcarry(*lhs_it, *rhs_it, carry, &*it); + } + for (; carry && rhs_it != rhs_it_e; ++rhs_it, ++it) { + carry = _addcarry(0, *rhs_it, carry, &*it); + } + for (; rhs_it != rhs_it_e; ++rhs_it, ++it) { + *it = *rhs_it; + } + } else { + for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it, ++it) { + carry = _addcarry(*lhs_it, *rhs_it, carry, &*it); + } + for (; carry && lhs_it != lhs_it_e; ++lhs_it, ++it) { + carry = _addcarry(*lhs_it, 0, carry, &*it); + } + for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { + *it = *lhs_it; + } + } + + if (carry) { + result.append(1); + } + result._carry = false; + + // Finish up + result.trim(); + return result; + } + + static uinteger_t& add(uinteger_t& lhs, const uinteger_t& rhs) { + // First try saving some calculations: + if (!rhs) { + return lhs; + } + if (!lhs) { + lhs = rhs; + return lhs; + } + + return long_add(lhs, rhs); + } + + static uinteger_t& add(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { + // First try saving some calculations: + if (!rhs) { + result = lhs; + return result; + } + if (!lhs) { + result = rhs; + return result; + } + + return long_add(result, lhs, rhs); + } + + static uinteger_t add(const uinteger_t& lhs, const uinteger_t& rhs) { + uinteger_t result; + add(result, lhs, rhs); + return result; + } + + static uinteger_t& long_sub(uinteger_t& lhs, const uinteger_t& rhs) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + if (lhs_sz < rhs_sz) { + lhs.resize(rhs_sz, 0); // grow + } + + // not using `end()` because resize of `lhs.resize()` could have + // resized `lhs`. + auto lhs_it = lhs.begin(); + auto lhs_it_e = lhs_it + lhs_sz; + + auto rhs_it = rhs.begin(); + auto rhs_it_e = rhs_it + rhs_sz; + + digit borrow = 0; + if (lhs_sz < rhs_sz) { + for (; lhs_it != lhs_it_e; ++lhs_it, ++rhs_it) { + borrow = _subborrow(*lhs_it, *rhs_it, borrow, &*lhs_it); + } + for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it) { + borrow = _subborrow(0, *rhs_it, borrow, &*lhs_it); + } + } else { + for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it) { + borrow = _subborrow(*lhs_it, *rhs_it, borrow, &*lhs_it); + } + for (; borrow && lhs_it != lhs_it_e; ++lhs_it) { + borrow = _subborrow(*lhs_it, 0, borrow, &*lhs_it); + } + } + + lhs._carry = borrow; + + // Finish up + lhs.trim(); + return lhs; + } + + static uinteger_t& long_sub(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + auto result_sz = std::max(lhs_sz, rhs_sz); + result.resize(result_sz, 0); + + // not using `end()` because resize of `result.resize()` could have + // resized `lhs` or `rhs` if `result` is also either `rhs` or `lhs`. + auto lhs_it = lhs.begin(); + auto lhs_it_e = lhs_it + lhs_sz; + + auto rhs_it = rhs.begin(); + auto rhs_it_e = rhs_it + rhs_sz; + + auto it = result.begin(); + + digit borrow = 0; + if (lhs_sz < rhs_sz) { + for (; lhs_it != lhs_it_e; ++lhs_it, ++rhs_it, ++it) { + borrow = _subborrow(*lhs_it, *rhs_it, borrow, &*it); + } + for (; rhs_it != rhs_it_e; ++rhs_it, ++it) { + borrow = _subborrow(0, *rhs_it, borrow, &*it); + } + } else { + for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it, ++it) { + borrow = _subborrow(*lhs_it, *rhs_it, borrow, &*it); + } + for (; borrow && lhs_it != lhs_it_e; ++lhs_it, ++it) { + borrow = _subborrow(*lhs_it, 0, borrow, &*it); + } + for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { + *it = *lhs_it; + } + } + + result._carry = borrow; + + // Finish up + result.trim(); + return result; + } + + static uinteger_t& sub(uinteger_t& lhs, const uinteger_t& rhs) { + // First try saving some calculations: + if (!rhs) { + return lhs; + } + + return long_sub(lhs, rhs); + } + + static uinteger_t& sub(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { + // First try saving some calculations: + if (!rhs) { + result = lhs; + return result; + } + + return long_sub(result, lhs, rhs); + } + + static uinteger_t sub(const uinteger_t& lhs, const uinteger_t& rhs) { + uinteger_t result; + sub(result, lhs, rhs); + return result; + } + + // Single word long multiplication + // Fastests, but ONLY for single sized rhs + static uinteger_t& single_mult(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + ASSERT(rhs_sz == 1); (void)(rhs_sz); + auto n = rhs.front(); + + uinteger_t tmp; + tmp.resize(lhs_sz + 1, 0); + + auto it_lhs = lhs.begin(); + auto it_lhs_e = lhs.end(); + + auto it_result = tmp.begin(); + + digit carry = 0; + for (; it_lhs != it_lhs_e; ++it_lhs, ++it_result) { + carry = _multadd(*it_lhs, n, 0, carry, &*it_result); + } + if (carry) { + *it_result = carry; + } + + result = std::move(tmp); + + // Finish up + result.trim(); + return result; + } + + static uinteger_t& long_mult(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + if (lhs_sz > rhs_sz) { + // rhs should be the largest: + return long_mult(result, rhs, lhs); + } + + if (lhs_sz == 1) { + return single_mult(result, rhs, lhs); + } + + uinteger_t tmp; + tmp.resize(lhs_sz + rhs_sz, 0); + + auto it_lhs = lhs.begin(); + auto it_lhs_e = lhs.end(); + + auto it_rhs = rhs.begin(); + auto it_rhs_e = rhs.end(); + + auto it_result = tmp.begin(); + auto it_result_s = it_result; + auto it_result_l = it_result; + + for (; it_lhs != it_lhs_e; ++it_lhs, ++it_result) { + if (auto lhs_it_val = *it_lhs) { + auto _it_rhs = it_rhs; + auto _it_result = it_result; + digit carry = 0; + for (; _it_rhs != it_rhs_e; ++_it_rhs, ++_it_result) { + carry = _multadd(*_it_rhs, lhs_it_val, *_it_result, carry, &*_it_result); + } + if (carry) { + *_it_result++ = carry; + } + if (it_result_l < _it_result) { + it_result_l = _it_result; + } + } + } + + tmp.resize(it_result_l - it_result_s); // shrink + + result = std::move(tmp); + + // Finish up + result.trim(); + return result; + } + + // A helper for Karatsuba multiplication to split a number in two, at n. + static std::pair karatsuba_mult_split(const uinteger_t& num, std::size_t n) { + const uinteger_t a(num, num._begin, num._begin + n); + const uinteger_t b(num, num._begin + n, num._end); + return std::make_pair(std::move(a), std::move(b)); + } + + // If rhs has at least twice the digits of lhs, and lhs is big enough that + // Karatsuba would pay off *if* the inputs had balanced sizes. + // View rhs as a sequence of slices, each with lhs.size() digits, + // and multiply the slices by lhs, one at a time. + static uinteger_t& karatsuba_lopsided_mult(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs, std::size_t cutoff) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + ASSERT(lhs_sz > cutoff); + ASSERT(2 * lhs_sz <= rhs_sz); + + auto rhs_begin = rhs._begin; + std::size_t shift = 0; + + uinteger_t r; + while (rhs_sz > 0) { + // Multiply the next slice of rhs by lhs and add into result: + auto slice_size = std::min(lhs_sz, rhs_sz); + const uinteger_t rhs_slice(rhs, rhs_begin, rhs_begin + slice_size); + uinteger_t p; + karatsuba_mult(p, lhs, rhs_slice, cutoff); + uinteger_t rs(r, shift, 0); + add(rs, rs, p); + shift += slice_size; + rhs_sz -= slice_size; + rhs_begin += slice_size; + } + + result = std::move(r); + return result; + } + + // Karatsuba multiplication + static uinteger_t& karatsuba_mult(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs, std::size_t cutoff = 1) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + if (lhs_sz > rhs_sz) { + // rhs should be the largest: + return karatsuba_mult(result, rhs, lhs, cutoff); + } + + if (lhs_sz <= cutoff) { + return long_mult(result, lhs, rhs); + } + + // If a is too small compared to b, splitting on b gives a degenerate case + // in which Karatsuba may be (even much) less efficient than long multiplication. + if (2 * lhs_sz <= rhs_sz) { + return karatsuba_lopsided_mult(result, lhs, rhs, cutoff); + } + + // Karatsuba: + // + // A B + // x C D + // --------------------- + // AD BD + // AC BC + // --------------------- + // AC AD + BC BD + // + // AD + BC = + // AC + AD + BC + BD - AC - BD + // (A + B) (C + D) - AC - BD + + // Calculate the split point near the middle of the largest (rhs). + auto shift = rhs_sz >> 1; + + // Split to get A and B: + const auto lhs_pair = karatsuba_mult_split(lhs, shift); + const auto& A = lhs_pair.second; // hi + const auto& B = lhs_pair.first; // lo + + // Split to get C and D: + const auto rhs_pair = karatsuba_mult_split(rhs, shift); + const auto& C = rhs_pair.second; // hi + const auto& D = rhs_pair.first; // lo + + // Get the pieces: + uinteger_t AC; + karatsuba_mult(AC, A, C, cutoff); + + uinteger_t BD; + karatsuba_mult(BD, B, D, cutoff); + uinteger_t AD_BC, AB, CD; + karatsuba_mult(AD_BC, A + B, C + D, cutoff); + AD_BC -= AC; + AD_BC -= BD; + + // Join the pieces, AC and BD (can't overlap) into BD: + BD.reserve(shift * 2 + AC.size()); + BD.resize(shift * 2, 0); + BD.append(AC); + + // And add AD_BC to the middle: (AC BD) + ( AD + BC ): + uinteger_t BDs(BD, shift, 0); + add(BDs, BDs, AD_BC); + + result = std::move(BD); + + // Finish up + result.trim(); + return result; + } + + static uinteger_t& mult(uinteger_t& lhs, const uinteger_t& rhs) { + // Hard to see how this could have a further optimized implementation. + return mult(lhs, lhs, rhs); + } + + static uinteger_t& mult(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { + // First try saving some calculations: + if (!lhs || !rhs) { + result = uint_0(); + return result; + } + if (compare(lhs, uint_1()) == 0) { + result = rhs; + return result; + } + if (compare(rhs, uint_1()) == 0) { + result = lhs; + return result; + } + + return karatsuba_mult(result, lhs, rhs, karatsuba_cutoff); + } + + static uinteger_t mult(const uinteger_t& lhs, const uinteger_t& rhs) { + uinteger_t result; + mult(result, lhs, rhs); + return result; + } + + // Single word long division + // Fastests, but ONLY for single sized rhs + static std::pair, std::reference_wrapper> single_divmod(uinteger_t& quotient, uinteger_t& remainder, const uinteger_t& lhs, const uinteger_t& rhs) { + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + + ASSERT(rhs_sz == 1); (void)(rhs_sz); + auto n = rhs.front(); + + auto rit_lhs = lhs.rbegin(); + auto rit_lhs_e = lhs.rend(); + + auto q = uint_0(); + q.resize(lhs_sz, 0); + auto rit_q = q.rbegin(); + + digit r = 0; + for (; rit_lhs != rit_lhs_e; ++rit_lhs, ++rit_q) { + r = _divmod(r, *rit_lhs, n, &*rit_q); + } + + q.trim(); + + quotient = std::move(q); + remainder = r; + return std::make_pair(std::ref(quotient), std::ref(remainder)); + } + + // Implementation of Knuth's Algorithm D + static std::pair, std::reference_wrapper> knuth_divmod(uinteger_t& quotient, uinteger_t& remainder, const uinteger_t& lhs, const uinteger_t& rhs) { + uinteger_t v(lhs); + uinteger_t w(rhs); + + auto v_size = v.size(); + auto w_size = w.size(); + ASSERT(v_size >= w_size && w_size >= 2); + + // D1. normalize: shift rhs left so that its top digit is >= 63 bits. + // shift lhs left by the same amount. Results go into w and v. + auto d = uinteger_t(digit_bits - _bits(w.back())); + v <<= d; + w <<= d; + + if (*v.rbegin() >= *w.rbegin()) { + v.append(0); + } + v_size = v.size(); + v.append(0); + + // Now *v.rbegin() < *w.rbegin() so quotient has at most + // (and usually exactly) k = v.size() - w.size() digits. + auto k = v_size - w_size; + auto q = uint_0(); + q.resize(k + 1, 0); + + auto rit_q = q.rend() - (k + 1); + + auto it_v_b = v.begin(); + auto it_v_k = it_v_b + k; + + auto it_w = w.begin(); + auto it_w_e = w.end(); + + auto rit_w = w.rbegin(); + auto wm1 = *rit_w++; + auto wm2 = *rit_w; + + // D2. inner loop: divide v[k+0..k+n] by w[0..n] + for (; it_v_k >= it_v_b; --it_v_k, ++rit_q) { + // D3. Compute estimate quotient digit q; may overestimate by 1 (rare) + digit _q; + auto _r = _divmod(*(it_v_k + w_size), *(it_v_k + w_size - 1), wm1, &_q); + digit mullo = 0; + auto mulhi = _mult(_q, wm2, &mullo); + auto rlo = *(it_v_k + w_size - 2); + while (mulhi > _r || (mulhi == _r && mullo > rlo)) { + --_q; + if (_addcarry(_r, wm1, 0, &_r)) { + break; + } + mulhi = _mult(_q, wm2, &mullo); + } + + // D4. Multiply and subtract _q * w0[0:size_w] from vk[0:size_w+1] + auto _it_v = it_v_k; + auto _it_w = it_w; + mulhi = 0; + digit carry = 0; + for (; _it_w != it_w_e; ++_it_v, ++_it_w) { + mullo = 0; + mulhi = _multadd(*_it_w, _q, 0, mulhi, &mullo); + carry = _subborrow(*_it_v, mullo, carry, &*_it_v); + } + carry = _subborrow(*_it_v, 0, carry, &*_it_v); + + if (carry) { + // D6. Add w back if q was too large (this branch taken rarely) + --_q; + + _it_v = it_v_k; + _it_w = it_w; + carry = 0; + for (; _it_w != it_w_e; ++_it_v, ++_it_w) { + carry = _addcarry(*_it_v, *_it_w, carry, &*_it_v); + } + carry = _addcarry(*_it_v, 0, carry, &*_it_v); + } + + /* store quotient digit */ + *rit_q = _q; + } + + // D8. unnormalize: unshift remainder. + v.resize(w_size); + v >>= d; + + q.trim(); + v.trim(); + + quotient = std::move(q); + remainder = std::move(v); + return std::make_pair(std::ref(quotient), std::ref(remainder)); + } + + static std::pair, std::reference_wrapper> divmod(uinteger_t& quotient, uinteger_t& remainder, const uinteger_t& lhs, const uinteger_t& rhs) { + // First try saving some calculations: + if (!rhs) { + throw std::domain_error("Error: division or modulus by 0"); + } + auto lhs_sz = lhs.size(); + auto rhs_sz = rhs.size(); + if (lhs_sz == 1 && rhs_sz == 1) { + // Fast division and modulo for single value + auto a = *lhs.begin(); + auto b = *rhs.begin(); + quotient = a / b; + remainder = a % b; + return std::make_pair(std::ref(quotient), std::ref(remainder)); + } + if (compare(rhs, uint_1()) == 0) { + quotient = lhs; + remainder = uint_0(); + return std::make_pair(std::ref(quotient), std::ref(remainder)); + } + auto compared = compare(lhs, rhs); + if (compared == 0) { + quotient = uint_1(); + remainder = uint_0(); + return std::make_pair(std::ref(quotient), std::ref(remainder)); + } + if (!lhs || compared < 0) { + quotient = uint_0(); + remainder = lhs; + return std::make_pair(std::ref(quotient), std::ref(remainder)); + } + if (rhs_sz == 1) { + return single_divmod(quotient, remainder, lhs, rhs); + } + + return knuth_divmod(quotient, remainder, lhs, rhs); + } + + static std::pair divmod(const uinteger_t& lhs, const uinteger_t& rhs) { + uinteger_t quotient; + uinteger_t remainder; + divmod(quotient, remainder, lhs, rhs); + return std::make_pair(std::move(quotient), std::move(remainder)); + } + +private: + // Constructors + + template ::value and not std::is_same>::value>> + void _uint_t(const T& value) { + append(static_cast(value)); + } + + template ::value and not std::is_same>::value>> + void _uint_t(const T& value, Args... args) { + _uint_t(args...); + append(static_cast(value)); + } + + // This constructor creates a window view of the _value + uinteger_t(const uinteger_t& o, std::size_t begin, std::size_t end) : + _begin(begin), + _end(end), + _value(o._value), + _carry(o._carry) { } + +public: + uinteger_t() : + _begin(0), + _end(0), + _value(_value_instance), + _carry(false) { } + + uinteger_t(const uinteger_t& o) : + _begin(0), + _end(0), + _value_instance(o.begin(), o.end()), + _value(_value_instance), + _carry(o._carry) { } + + uinteger_t(uinteger_t&& o) : + _begin(std::move(o._begin)), + _end(std::move(o._end)), + _value_instance(std::move(o._value_instance)), + _value(_value_instance), + _carry(std::move(o._carry)) { } + + template ::value and not std::is_same>::value>> + uinteger_t(const T& value) : + _begin(0), + _end(0), + _value(_value_instance), + _carry(false) { + if (value) { + append(static_cast(value)); + } + } + + template ::value and not std::is_same>::value>> + uinteger_t(const T& value, Args... args) : + _begin(0), + _end(0), + _value(_value_instance), + _carry(false) { + _uint_t(args...); + append(static_cast(value)); + trim(); + } + + template ::value and not std::is_same>::value>> + uinteger_t(std::initializer_list list) : + _begin(0), + _end(0), + _value(_value_instance), + _carry(false) { + reserve(list.size()); + for (const auto& value : list) { + append(static_cast(value)); + } + trim(); + } + + template + explicit uinteger_t(T (&s)[N], int base=10) : + uinteger_t(s, N - 1, base) { } + + explicit uinteger_t(const unsigned char* bytes, std::size_t sz, int base) : + uinteger_t(strtouint(bytes, sz, base)) { } + + explicit uinteger_t(const char* bytes, std::size_t sz, int base) : + uinteger_t(strtouint(bytes, sz, base)) { } + + template + explicit uinteger_t(const std::vector& bytes, int base=10) : + uinteger_t(bytes.data(), bytes.size(), base) { } + + explicit uinteger_t(const std::string& bytes, int base=10) : + uinteger_t(bytes.data(), bytes.size(), base) { } + + // Assignment Operator + uinteger_t& operator=(const uinteger_t& o) { + _begin = 0; + _end = 0; + _value = container(o.begin(), o.end()); + _carry = o._carry; + return *this; + } + uinteger_t& operator=(uinteger_t&& o) { + _begin = std::move(o._begin); + _end = std::move(o._end); + _value_instance = std::move(o._value_instance); + _carry = std::move(o._carry); + return *this; + } + + // Typecast Operators + explicit operator bool() const { + return static_cast(size()); + } + explicit operator unsigned char() const { + return static_cast(size() ? front() : 0); + } + explicit operator unsigned short() const { + return static_cast(size() ? front() : 0); + } + explicit operator unsigned int() const { + return static_cast(size() ? front() : 0); + } + explicit operator unsigned long() const { + return static_cast(size() ? front() : 0); + } + explicit operator unsigned long long() const { + return static_cast(size() ? front() : 0); + } + explicit operator char() const { + return static_cast(size() ? front() : 0); + } + explicit operator short() const { + return static_cast(size() ? front() : 0); + } + explicit operator int() const { + return static_cast(size() ? front() : 0); + } + explicit operator long() const { + return static_cast(size() ? front() : 0); + } + explicit operator long long() const { + return static_cast(size() ? front() : 0); + } + + // Bitwise Operators + uinteger_t operator&(const uinteger_t& rhs) const { + return bitwise_and(*this, rhs); + } + + uinteger_t& operator&=(const uinteger_t& rhs) { + return bitwise_and(*this, rhs); + } + + uinteger_t operator|(const uinteger_t& rhs) const { + return bitwise_or(*this, rhs); + } + + uinteger_t& operator|=(const uinteger_t& rhs) { + return bitwise_or(*this, rhs); + } + + uinteger_t operator^(const uinteger_t& rhs) const { + return bitwise_xor(*this, rhs); + } + + uinteger_t& operator^=(const uinteger_t& rhs) { + return bitwise_xor(*this, rhs); + } + + uinteger_t operator~() const { + return bitwise_inv(*this); + } + + uinteger_t inv() { + return bitwise_inv(*this); + } + + // Bit Shift Operators + uinteger_t operator<<(const uinteger_t& rhs) const { + return bitwise_lshift(*this, rhs); + } + + uinteger_t& operator<<=(const uinteger_t& rhs) { + return bitwise_lshift(*this, rhs); + } + + uinteger_t operator>>(const uinteger_t& rhs) const { + return bitwise_rshift(*this, rhs); + } + + uinteger_t& operator>>=(const uinteger_t& rhs) { + return bitwise_rshift(*this, rhs); + } + + // Logical Operators + bool operator!() const { + return !static_cast(*this); + } + + bool operator&&(const uinteger_t& rhs) const { + return static_cast(*this) && rhs; + } + + bool operator||(const uinteger_t& rhs) const { + return static_cast(*this) || rhs; + } + + // Comparison Operators + bool operator==(const uinteger_t& rhs) const { + return compare(*this, rhs) == 0; + } + + bool operator!=(const uinteger_t& rhs) const { + return compare(*this, rhs) != 0; + } + + bool operator>(const uinteger_t& rhs) const { + return compare(*this, rhs) > 0; + } + + bool operator<(const uinteger_t& rhs) const { + return compare(*this, rhs) < 0; + } + + bool operator>=(const uinteger_t& rhs) const { + return compare(*this, rhs) >= 0; + } + + bool operator<=(const uinteger_t& rhs) const { + return compare(*this, rhs) <= 0; + } + + // Arithmetic Operators + uinteger_t operator+(const uinteger_t& rhs) const { + return add(*this, rhs); + } + + uinteger_t& operator+=(const uinteger_t& rhs) { + return add(*this, rhs); + } + + uinteger_t operator-(const uinteger_t& rhs) const { + return sub(*this, rhs); + } + + uinteger_t& operator-=(const uinteger_t& rhs) { + return sub(*this, rhs); + } + + uinteger_t operator*(const uinteger_t& rhs) const { + return mult(*this, rhs); + } + + uinteger_t& operator*=(const uinteger_t& rhs) { + return mult(*this, rhs); + } + + std::pair divmod(const uinteger_t& rhs) const { + return divmod(*this, rhs); + } + + uinteger_t operator/(const uinteger_t& rhs) const { + return divmod(*this, rhs).first; + } + + uinteger_t& operator/=(const uinteger_t& rhs) { + uinteger_t quotient; + uinteger_t remainder; + divmod(quotient, remainder, *this, rhs); + *this = std::move(quotient); + return *this; + } + + uinteger_t operator%(const uinteger_t& rhs) const { + return divmod(*this, rhs).second; + } + + uinteger_t& operator%=(const uinteger_t& rhs) { + uinteger_t quotient; + uinteger_t remainder; + divmod(quotient, remainder, *this, rhs); + *this = std::move(remainder); + return *this; + } + + // Increment Operator + uinteger_t& operator++() { + return *this += uint_1(); + } + uinteger_t operator++(int) { + uinteger_t temp(*this); + ++*this; + return temp; + } + + // Decrement Operator + uinteger_t& operator--() { + return *this -= uint_1(); + } + uinteger_t operator--(int) { + uinteger_t temp(*this); + --*this; + return temp; + } + + // Nothing done since promotion doesn't work here + uinteger_t operator+() const { + return *this; + } + + // two's complement + uinteger_t operator-() const { + return uint_0() - *this; + } + + // Get private value at index + const digit& value(std::size_t idx) const { + static const digit zero = 0; + return idx < size() ? *(begin() + idx) : zero; + } + + // Get value of bit N + bool operator[](std::size_t n) const { + auto nd = n / digit_bits; + auto nm = n % digit_bits; + return nd < size() ? (*(begin() + nd) >> nm) & 1 : 0; + } + + // Get bitsize of value + std::size_t bits() const { + auto sz = size(); + if (sz) { + return _bits(back()) + (sz - 1) * digit_bits; + } + return 0; + } + + // Get string representation of value + template ::value>> + Result str(int alphabet_base = 10) const { + auto num_sz = size(); + if (alphabet_base >= 2 && alphabet_base <= 36) { + Result result; + if (num_sz) { + auto alphabet_base_bits = base_bits(alphabet_base); + result.reserve(num_sz * base_size(alphabet_base)); + if (alphabet_base_bits) { + digit alphabet_base_mask = alphabet_base - 1; + std::size_t shift = 0; + auto ptr = reinterpret_cast(data()); + digit v = *ptr++; + v <<= half_digit_bits; + for (auto i = num_sz * 2 - 1; i; --i) { + v >>= half_digit_bits; + v |= (static_cast(*ptr++) << half_digit_bits); + do { + auto d = static_cast((v >> shift) & alphabet_base_mask); + result.push_back(chr(d)); + shift += alphabet_base_bits; + } while (shift <= half_digit_bits); + shift -= half_digit_bits; + } + v >>= (shift + half_digit_bits); + while (v) { + auto d = static_cast(v & alphabet_base_mask); + result.push_back(chr(d)); + v >>= alphabet_base_bits; + } + auto s = chr(0); + auto rit_f = std::find_if(result.rbegin(), result.rend(), [s](const char& c) { return c != s; }); + result.resize(result.rend() - rit_f); // shrink + } else { + uinteger_t uint_base = alphabet_base; + uinteger_t quotient = *this; + do { + auto r = quotient.divmod(uint_base); + auto d = static_cast(r.second); + result.push_back(chr(d)); + quotient = std::move(r.first); + } while (quotient); + } + std::reverse(result.begin(), result.end()); + } else { + result.push_back(chr(0)); + } + return result; + } else if (alphabet_base == 256) { + if (num_sz) { + auto ptr = reinterpret_cast(data()); + Result result(ptr, ptr + num_sz * digit_octets); + auto rit_f = std::find_if(result.rbegin(), result.rend(), [](const char& c) { return c; }); + result.resize(result.rend() - rit_f); // shrink + std::reverse(result.begin(), result.end()); + return result; + } else { + Result result; + result.push_back('\x00'); + return result; + } + } else { + throw std::invalid_argument("Base must be in the range [2, 36]"); + } + } + + static uinteger_t strtouint(const void* encoded, std::size_t encoded_size, int alphabet_base) { + const char* data = (const char *)encoded; + uinteger_t result; + + if (alphabet_base >= 2 && alphabet_base <= 36) { + uinteger_t alphabet_base_bits = base_bits(alphabet_base); + uinteger_t uint_base = alphabet_base; + if (alphabet_base_bits) { + for (; encoded_size; --encoded_size, ++data) { + auto d = ord(static_cast(*data)); + if (d < 0) { + throw std::invalid_argument("Error: Not a digit in base " + std::to_string(alphabet_base) + ": '" + std::string(1, *data) + "' at " + std::to_string(encoded_size)); + } + result = (result << alphabet_base_bits) | d; + } + } else { + for (; encoded_size; --encoded_size, ++data) { + auto d = ord(static_cast(*data)); + if (d < 0) { + throw std::invalid_argument("Error: Not a digit in base " + std::to_string(alphabet_base) + ": '" + std::string(1, *data) + "' at " + std::to_string(encoded_size)); + } + result = (result * uint_base) + d; + } + } + } else if (encoded_size && alphabet_base == 256) { + auto value_size = encoded_size / digit_octets; + auto value_padding = encoded_size % digit_octets; + if (value_padding) { + value_padding = digit_octets - value_padding; + ++value_size; + } + result.resize(value_size); // grow (no initialization) + *result.begin() = 0; // initialize value + auto ptr = reinterpret_cast(result.data()); + std::copy(data, data + encoded_size, ptr + value_padding); + std::reverse(ptr, ptr + value_size * digit_octets); + } else { + throw std::invalid_argument("Error: Cannot convert from base " + std::to_string(alphabet_base)); + } + + return result; + } + + template ::value>> + Result bin() const { + return str(2); + } + + template ::value>> + Result oct() const { + return str(8); + } + + template ::value>> + Result hex() const { + return str(16); + } + + template ::value>> + Result raw() const { + return str(256); + } +}; + +namespace std { // This is probably not a good idea + // Make it work with std::string() + inline std::string to_string(uinteger_t& num) { + return num.str(); + } + inline const std::string to_string(const uinteger_t& num) { + return num.str(); + } +} + +// lhs type T as first arguemnt +// If the output is not a bool, casts to type T + +// Bitwise Operators +template ::value and not std::is_same>::value>> +uinteger_t operator&(const T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) & rhs; +} + +template ::value and not std::is_same>::value>> +T& operator&=(T& lhs, const uinteger_t& rhs) { + return lhs = static_cast(rhs & lhs); +} + +template ::value and not std::is_same>::value>> +uinteger_t operator|(const T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) | rhs; +} + +template ::value and not std::is_same>::value>> +T& operator|=(T& lhs, const uinteger_t& rhs) { + return lhs = static_cast(rhs | lhs); +} + +template ::value and not std::is_same>::value>> +uinteger_t operator^(const T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) ^ rhs; +} + +template ::value and not std::is_same>::value>> +T& operator^=(T& lhs, const uinteger_t& rhs) { + return lhs = static_cast(rhs ^ lhs); +} + +// Bitshift operators +template ::value and not std::is_same>::value>> +inline uinteger_t operator<<(T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) << rhs; +} + +template ::value and not std::is_same>::value>> +T& operator<<=(T& lhs, const uinteger_t& rhs) { + return lhs = static_cast(lhs << rhs); +} + +template ::value and not std::is_same>::value>> +inline uinteger_t operator>>(T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) >> rhs; +} + +template ::value and not std::is_same>::value>> +T& operator>>=(T& lhs, const uinteger_t& rhs) { + return lhs = static_cast(lhs >> rhs); +} + +// Comparison Operators +template ::value and not std::is_same>::value>> +bool operator==(const T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) == rhs; +} + +template ::value and not std::is_same>::value>> +bool operator!=(const T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) != rhs; +} + +template ::value and not std::is_same>::value>> +bool operator>(const T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) > rhs; +} + +template ::value and not std::is_same>::value>> +bool operator<(const T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) < rhs; +} + +template ::value and not std::is_same>::value>> +bool operator>=(const T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) >= rhs; +} + +template ::value and not std::is_same>::value>> +bool operator<=(const T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) <= rhs; +} + +// Arithmetic Operators +template ::value and not std::is_same>::value>> +uinteger_t operator+(const T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) + rhs; +} + +template ::value and not std::is_same>::value>> +T& operator+=(T& lhs, const uinteger_t& rhs) { + return lhs = static_cast(rhs + lhs); +} + +template ::value and not std::is_same>::value>> +uinteger_t operator-(const T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) - rhs; +} + +template ::value and not std::is_same>::value>> +T& operator-=(T& lhs, const uinteger_t& rhs) { + return lhs = static_cast(lhs - rhs); +} + +template ::value and not std::is_same>::value>> +uinteger_t operator*(const T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) * rhs; +} + +template ::value and not std::is_same>::value>> +T& operator*=(T& lhs, const uinteger_t& rhs) { + return lhs = static_cast(rhs * lhs); +} + +template ::value and not std::is_same>::value>> +uinteger_t operator/(const T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) / rhs; +} + +template ::value and not std::is_same>::value>> +T& operator/=(T& lhs, const uinteger_t& rhs) { + return lhs = static_cast(lhs / rhs); +} + +template ::value and not std::is_same>::value>> +uinteger_t operator%(const T& lhs, const uinteger_t& rhs) { + return uinteger_t(lhs) % rhs; +} + +template ::value and not std::is_same>::value>> +T& operator%=(T& lhs, const uinteger_t& rhs) { + return lhs = static_cast(lhs % rhs); +} + +// IO Operator +inline std::ostream& operator<<(std::ostream& stream, const uinteger_t& rhs) { + if (stream.flags() & stream.oct) { + stream << rhs.str(8); + } else if (stream.flags() & stream.dec) { + stream << rhs.str(10); + } else if (stream.flags() & stream.hex) { + stream << rhs.str(16); + } + return stream; +} + +#endif diff --git a/contrib/base58 b/contrib/base58 deleted file mode 160000 index a85f98fb4ed..00000000000 --- a/contrib/base58 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a85f98fb4ed52c2f4029a4b6ac1ef0bafdfc56f5 diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index bf72795aae0..32f7952961c 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -72,8 +72,8 @@ if (TARGET ch_contrib::llvm) target_link_libraries(clickhouse_functions PRIVATE ch_contrib::llvm) endif () -if (TARGET ch_contrib::base58) - target_link_libraries(clickhouse_functions PRIVATE ch_contrib::base58) +if (TARGET ch_contrib::base-x) + target_link_libraries(clickhouse_functions PRIVATE ch_contrib::base-x) endif() if (TARGET ch_contrib::base64) diff --git a/src/Functions/FunctionBase58Conversion.h b/src/Functions/FunctionBase58Conversion.h index a0431ca47df..c0c54fd0ba9 100644 --- a/src/Functions/FunctionBase58Conversion.h +++ b/src/Functions/FunctionBase58Conversion.h @@ -26,19 +26,19 @@ struct Base58Encode { static constexpr auto name = "base58Encode"; - static void process(const ColumnString * input, ColumnString::MutablePtr& dst_column, std::string& alphabet, size_t input_rows_count) + static void process(const ColumnString& input, ColumnString::MutablePtr& dst_column, std::string& alphabet, size_t input_rows_count) { auto & dst_data = dst_column->getChars(); auto & dst_offsets = dst_column->getOffsets(); - size_t current_allocated_size = input->getChars().size(); + size_t current_allocated_size = input.getChars().size(); dst_data.resize(current_allocated_size); dst_offsets.resize(input_rows_count); - const ColumnString::Offsets & src_offsets = input->getOffsets(); + const ColumnString::Offsets & src_offsets = input.getOffsets(); - const auto * source = input->getChars().raw_data(); + const auto * source = input.getChars().raw_data(); auto * dst = dst_data.data(); auto * dst_pos = dst; @@ -48,8 +48,9 @@ struct Base58Encode const auto& encoder = (alphabet == "bitcoin") ? Base58::bitcoin() : ((alphabet == "flickr") ? Base58::flickr() : ((alphabet == "ripple") ? Base58::ripple() : - Base58::base58())); + Base58::base58())); //GMP + std::string encoded; for (size_t row = 0; row < input_rows_count; ++row) { size_t srclen = src_offsets[row] - src_offset_prev - 1; @@ -57,7 +58,7 @@ struct Base58Encode /// We don't know the size of the result string beforehand (it's not byte-to-byte encoding), /// so we may need to do many resizes (the worst case -- we'll do it for each row) /// This way we do exponential resizes and one final resize after whole operation is complete - std::string encoded; + encoded.clear(); if (srclen) encoder.encode(encoded, source, srclen); size_t outlen = encoded.size(); @@ -66,14 +67,15 @@ struct Base58Encode { current_allocated_size += current_allocated_size; dst_data.resize(current_allocated_size); + auto processed_offset = dst_pos - dst; + dst = dst_data.data(); + dst_pos = dst; + dst_pos += processed_offset; } - if (srclen) - std::strcpy(reinterpret_cast(dst_pos), encoded.c_str()); + std::memcpy(dst_pos, encoded.c_str(), ++outlen); source += srclen + 1; dst_pos += outlen; - *dst_pos = '\0'; - dst_pos += 1; dst_offsets[row] = dst_pos - dst; src_offset_prev = src_offsets[row]; @@ -88,19 +90,19 @@ struct Base58Decode { static constexpr auto name = "base58Decode"; - static void process(const ColumnString * input, ColumnString::MutablePtr& dst_column, std::string& alphabet, size_t input_rows_count) + static void process(const ColumnString& input, ColumnString::MutablePtr& dst_column, std::string& alphabet, size_t input_rows_count) { auto & dst_data = dst_column->getChars(); auto & dst_offsets = dst_column->getOffsets(); - size_t current_allocated_size = input->getChars().size(); + size_t current_allocated_size = input.getChars().size(); dst_data.resize(current_allocated_size); dst_offsets.resize(input_rows_count); - const ColumnString::Offsets & src_offsets = input->getOffsets(); + const ColumnString::Offsets & src_offsets = input.getOffsets(); - const auto * source = input->getChars().raw_data(); + const auto * source = input.getChars().raw_data(); auto * dst = dst_data.data(); auto * dst_pos = dst; @@ -112,6 +114,7 @@ struct Base58Decode ((alphabet == "ripple") ? Base58::ripple() : Base58::base58())); + std::string decoded; for (size_t row = 0; row < input_rows_count; ++row) { size_t srclen = src_offsets[row] - src_offset_prev - 1; @@ -119,21 +122,24 @@ struct Base58Decode /// We don't know the size of the result string beforehand (it's not byte-to-byte encoding), /// so we may need to do many resizes (the worst case -- we'll do it for each row) /// This way we do exponential resizes and one final resize after whole operation is complete - std::string decoded; - decoder.decode(decoded, source, srclen); + decoded.clear(); + if (srclen) + decoder.decode(decoded, source, srclen); size_t outlen = decoded.size(); if (processed_size + outlen >= current_allocated_size) { current_allocated_size += current_allocated_size; dst_data.resize(current_allocated_size); + auto processed_offset = dst_pos - dst; + dst = dst_data.data(); + dst_pos = dst; + dst_pos += processed_offset; } - std::strcpy(reinterpret_cast(dst_pos), decoded.c_str()); + std::memcpy(dst_pos, decoded.c_str(), ++outlen); source += srclen + 1; dst_pos += outlen; - *dst_pos = '\0'; - dst_pos += 1; dst_offsets[row] = dst_pos - dst; src_offset_prev = src_offsets[row]; @@ -216,7 +222,7 @@ public: auto dst_column = ColumnString::create(); - Func::process(input, dst_column, alphabet, input_rows_count); + Func::process(*input, dst_column, alphabet, input_rows_count); return dst_column; } diff --git a/src/Functions/configure_config.cmake b/src/Functions/configure_config.cmake index 776996d7e17..856d9a5682a 100644 --- a/src/Functions/configure_config.cmake +++ b/src/Functions/configure_config.cmake @@ -1,7 +1,7 @@ if (TARGET ch_contrib::fastops) set(USE_FASTOPS 1) endif() -if (TARGET ch_contrib::base58) +if (TARGET ch_contrib::base-x) set(USE_BASE58 1) endif() if (TARGET ch_contrib::base64) diff --git a/tests/queries/0_stateless/02337_base58.reference b/tests/queries/0_stateless/02337_base58.reference new file mode 100644 index 00000000000..718dfeb4a34 --- /dev/null +++ b/tests/queries/0_stateless/02337_base58.reference @@ -0,0 +1,48 @@ +32YCBjgZhV4AdCWHaCDNu + +f +fo +foo +foob +fooba +foobar +Hello world! + +f +fo +foo +foob +fooba +foobar +Hello world! + +f +fo +foo +foob +fooba +foobar +Hello world! + +f +fo +foo +foob +fooba +foobar +Hello world! + +2m +8o8 +bQbp +3csAg9 +CZJRhmz +t1Zv2yaZ + +f +fo +foo +foob +fooba +foobar +1 1 diff --git a/tests/queries/0_stateless/02337_base58.sql b/tests/queries/0_stateless/02337_base58.sql new file mode 100644 index 00000000000..b67993d2cc9 --- /dev/null +++ b/tests/queries/0_stateless/02337_base58.sql @@ -0,0 +1,17 @@ +-- Tags: no-fasttest + +SET send_logs_level = 'fatal'; + +SELECT base58Encode('Hold my beer...'); + +SELECT base58Decode(encoded, 'gmp') FROM (SELECT base58Encode(val, 'gmp') as encoded FROM (select arrayJoin(['', 'f', 'fo', 'foo', 'foob', 'fooba', 'foobar', 'Hello world!']) val)); +SELECT base58Decode(encoded, 'ripple') FROM (SELECT base58Encode(val, 'ripple') as encoded FROM (select arrayJoin(['', 'f', 'fo', 'foo', 'foob', 'fooba', 'foobar', 'Hello world!']) val)); +SELECT base58Decode(encoded, 'flickr') FROM (SELECT base58Encode(val, 'flickr') as encoded FROM (select arrayJoin(['', 'f', 'fo', 'foo', 'foob', 'fooba', 'foobar', 'Hello world!']) val)); +SELECT base58Decode(encoded, 'bitcoin') FROM (SELECT base58Encode(val, 'bitcoin') as encoded FROM (select arrayJoin(['', 'f', 'fo', 'foo', 'foob', 'fooba', 'foobar', 'Hello world!']) val)); + +SELECT base58Encode(val) FROM (select arrayJoin(['', 'f', 'fo', 'foo', 'foob', 'fooba', 'foobar']) val); +SELECT base58Decode(val) FROM (select arrayJoin(['', '2m', '8o8', 'bQbp', '3csAg9', 'CZJRhmz', 't1Zv2yaZ']) val); + +SELECT base58Decode(base58Encode('foo')) = 'foo', base58Encode(base58Decode('bQbp')) == 'bQbp'; + +SELECT base58Decode('Why_not?'); -- { serverError 1001 } \ No newline at end of file From 78d55d6f46ab3aee8cfdce1bb17e131dba54f6eb Mon Sep 17 00:00:00 2001 From: zvonand Date: Mon, 20 Jun 2022 19:30:54 +0500 Subject: [PATCH 025/123] small fixes --- .gitmodules | 2 +- src/Functions/FunctionBase58Conversion.h | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.gitmodules b/.gitmodules index e1960d2144b..a0ad0b69822 100644 --- a/.gitmodules +++ b/.gitmodules @@ -270,4 +270,4 @@ url = https://github.com/schoentoon/hashidsxx.git [submodule "contrib/base-x"] path = contrib/base-x - url = https://github.com/Kronuz/base-x.git + url = https://github.com/ClickHouse/base-x.git diff --git a/src/Functions/FunctionBase58Conversion.h b/src/Functions/FunctionBase58Conversion.h index c0c54fd0ba9..e472e997c32 100644 --- a/src/Functions/FunctionBase58Conversion.h +++ b/src/Functions/FunctionBase58Conversion.h @@ -26,12 +26,13 @@ struct Base58Encode { static constexpr auto name = "base58Encode"; - static void process(const ColumnString& input, ColumnString::MutablePtr& dst_column, std::string& alphabet, size_t input_rows_count) + static void process(const ColumnString & input, ColumnString::MutablePtr & dst_column, const std::string & alphabet, size_t input_rows_count) { auto & dst_data = dst_column->getChars(); auto & dst_offsets = dst_column->getOffsets(); - size_t current_allocated_size = input.getChars().size(); + /// Wikipedia states Base58 has efficiency of 73%, and we take 1.5 scale to avoid reallocation in most cases + size_t current_allocated_size = ceil(1.5 * input.getChars().size()); dst_data.resize(current_allocated_size); dst_offsets.resize(input_rows_count); @@ -90,11 +91,12 @@ struct Base58Decode { static constexpr auto name = "base58Decode"; - static void process(const ColumnString& input, ColumnString::MutablePtr& dst_column, std::string& alphabet, size_t input_rows_count) + static void process(const ColumnString & input, ColumnString::MutablePtr & dst_column, const std::string & alphabet, size_t input_rows_count) { auto & dst_data = dst_column->getChars(); auto & dst_offsets = dst_column->getOffsets(); + /// We allocate probably even more then needed to avoid many resizes size_t current_allocated_size = input.getChars().size(); dst_data.resize(current_allocated_size); @@ -202,7 +204,7 @@ public: const ColumnString * input = checkAndGetColumn(column_string.get()); if (!input) throw Exception( - "Illegal column " + arguments[0].column->getName() + " of first argument of function " + getName(), + "Illegal column " + arguments[0].column->getName() + " of first argument of function " + getName() + ", must be String", ErrorCodes::ILLEGAL_COLUMN); std::string alphabet = "bitcoin"; @@ -214,8 +216,8 @@ public: if (!alphabet_column) throw Exception("Second argument for function " + getName() + " must be constant String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (alphabet = alphabet_column->getValue(); - alphabet != "bitcoin" && alphabet != "ripple" && alphabet != "flickr" && alphabet != "gmp") + alphabet = alphabet_column->getValue(); + if (alphabet != "bitcoin" && alphabet != "ripple" && alphabet != "flickr" && alphabet != "gmp") throw Exception("Second argument for function " + getName() + " must be 'bitcoin', 'ripple', 'gmp' or 'flickr'", ErrorCodes::ILLEGAL_COLUMN); } From 9c6b2b9ba075d61a022f02e5b49cf59cfa03450d Mon Sep 17 00:00:00 2001 From: zvonand Date: Mon, 20 Jun 2022 20:10:28 +0500 Subject: [PATCH 026/123] added docs --- docs/en/development/contrib.md | 1 + .../functions/encoding-functions.md | 43 ++++++++++++++++++ docs/ru/development/contrib.md | 1 + .../functions/encoding-functions.md | 44 +++++++++++++++++++ 4 files changed, 89 insertions(+) diff --git a/docs/en/development/contrib.md b/docs/en/development/contrib.md index 7713c397e46..831923dc43a 100644 --- a/docs/en/development/contrib.md +++ b/docs/en/development/contrib.md @@ -18,6 +18,7 @@ The list of third-party libraries: | aws-c-common | [Apache](https://github.com/ClickHouse-Extras/aws-c-common/blob/736a82d1697c108b04a277e66438a7f4e19b6857/LICENSE) | | aws-c-event-stream | [Apache](https://github.com/ClickHouse-Extras/aws-c-event-stream/blob/3bc33662f9ccff4f4cbcf9509cc78c26e022fde0/LICENSE) | | aws-checksums | [Apache](https://github.com/ClickHouse-Extras/aws-checksums/blob/519d6d9093819b6cf89ffff589a27ef8f83d0f65/LICENSE) | +| base58 | [MIT](https://github.com/ClickHouse/base-x/blob/3e58874643c087f57e82b0ff03825c933fab945a/LICENSE) | | base64 | [BSD 2-clause](https://github.com/ClickHouse-Extras/Turbo-Base64/blob/af9b331f2b4f30b41c70f3a571ff904a8251c1d3/LICENSE) | | boost | [Boost](https://github.com/ClickHouse-Extras/boost/blob/9cf09dbfd55a5c6202dedbdf40781a51b02c2675/LICENSE_1_0.txt) | | boringssl | [BSD](https://github.com/ClickHouse-Extras/boringssl/blob/a6a2e2ab3e44d97ce98e51c558e989f211de7eb3/LICENSE) | diff --git a/docs/en/sql-reference/functions/encoding-functions.md b/docs/en/sql-reference/functions/encoding-functions.md index 4ee71267a09..42a6d75952c 100644 --- a/docs/en/sql-reference/functions/encoding-functions.md +++ b/docs/en/sql-reference/functions/encoding-functions.md @@ -440,3 +440,46 @@ Result: │ [0,1,2,3,4,5,6,7] │ └───────────────────┘ ``` + +## Base58Encode(plaintext[, alphabet_name]) +## Base58Decode(plaintext[, alphabet_name]) + +Accepts a String and encodes/decodes it using [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) encoding scheme using specified alphabet. + +**Syntax** + +```sql +base58Encode(decoded[, alphabet_name]) +base58Decode(encoded[, alphabet_name]) +``` + +**Arguments** + +- `decoded` — [String](../../sql-reference/data-types/string.md) column or constant. +- `encoded` — [String](../../sql-reference/data-types/string.md) column or constant. If the string is not a valid base58-encoded value, `1001 Exception` will be thrown. +- `alphabet_name` — String constant. Specifies alphabet used for encoding. Possible values: `gmp`, `bitcoin`, `ripple`, `flickr`. Default: `gmp`. + +**Returned value** + +- A string containing encoded/decoded value of 1st argument. + +Type: [String](../../sql-reference/data-types/string.md). + +**Example** + +Query: + +``` sql +SELECT base58Encode('encode', 'flickr'); +SELECT base58Decode('izCFiDUY', 'ripple'); +``` + +Result: +```text +┌─base58Encode('encode', 'flickr')─┐ +│ SvyTHb1D │ +└──────────────────────────────────┘ +┌─base58Decode('izCFiDUY', 'ripple')─┐ +│ decode │ +└────────────────────────────────────┘ +``` diff --git a/docs/ru/development/contrib.md b/docs/ru/development/contrib.md index 1b99ec97553..0f4d22e90ce 100644 --- a/docs/ru/development/contrib.md +++ b/docs/ru/development/contrib.md @@ -18,6 +18,7 @@ sidebar_label: "Используемые сторонние библиотеки | aws-c-common | [Apache](https://github.com/ClickHouse-Extras/aws-c-common/blob/736a82d1697c108b04a277e66438a7f4e19b6857/LICENSE) | | aws-c-event-stream | [Apache](https://github.com/ClickHouse-Extras/aws-c-event-stream/blob/3bc33662f9ccff4f4cbcf9509cc78c26e022fde0/LICENSE) | | aws-checksums | [Apache](https://github.com/ClickHouse-Extras/aws-checksums/blob/519d6d9093819b6cf89ffff589a27ef8f83d0f65/LICENSE) | +| base58 | [MIT](https://github.com/ClickHouse/base-x/blob/3e58874643c087f57e82b0ff03825c933fab945a/LICENSE) | | base64 | [BSD 2-clause](https://github.com/ClickHouse-Extras/Turbo-Base64/blob/af9b331f2b4f30b41c70f3a571ff904a8251c1d3/LICENSE) | | boost | [Boost](https://github.com/ClickHouse-Extras/boost/blob/9cf09dbfd55a5c6202dedbdf40781a51b02c2675/LICENSE_1_0.txt) | | boringssl | [BSD](https://github.com/ClickHouse-Extras/boringssl/blob/a6a2e2ab3e44d97ce98e51c558e989f211de7eb3/LICENSE) | diff --git a/docs/ru/sql-reference/functions/encoding-functions.md b/docs/ru/sql-reference/functions/encoding-functions.md index 65d2b0e6538..255985fcc92 100644 --- a/docs/ru/sql-reference/functions/encoding-functions.md +++ b/docs/ru/sql-reference/functions/encoding-functions.md @@ -404,3 +404,47 @@ SELECT bitPositionsToArray(toInt8(-1)) AS bit_positions; │ [0,1,2,3,4,5,6,7] │ └───────────────────┘ ``` + +## Base58Encode(plaintext[, alphabet_name]) +## Base58Decode(plaintext[, alphabet_name]) + +Принимает на вход строку или колонку строк и кодирует/раскодирует их с помощью схемы кодирования [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) с использованием указанного алфавита. + +**Синтаксис** + +```sql +base58Encode(decoded[, alphabet_name]) +base58Decode(encoded[, alphabet_name]) +``` + +**Аргументы** + +- `decoded` — Колонка или строка типа [String](../../sql-reference/data-types/string.md). +- `encoded` — Колонка или строка типа [String](../../sql-reference/data-types/string.md). Если входная строка не является корректным кодом для какой-либо другой строки, возникнет исключение `1001`. +- `alphabet_name` — Строковая константа. Указывает алфавит, для которого необходимо получить код. Может принимать одно из следующих значений: `gmp`, `bitcoin`, `ripple`, `flickr`. По умолчанию: `gmp`. + +**Возвращаемое значение** + +- Строка, содержащая раскодированный/закодированный первый аргумент. + +Тип: [String](../../sql-reference/data-types/string.md). + +**Пример:** + +Запрос: + +``` sql +SELECT base58Encode('encode', 'flickr'); +SELECT base58Decode('izCFiDUY', 'ripple'); +``` + +Результат: +```text +┌─base58Encode('encode', 'flickr')─┐ +│ SvyTHb1D │ +└──────────────────────────────────┘ +┌─base58Decode('izCFiDUY', 'ripple')─┐ +│ decode │ +└────────────────────────────────────┘ +``` + From d4e5686b997660b7943d0e6760b6f11baf91acf9 Mon Sep 17 00:00:00 2001 From: zvonand Date: Mon, 20 Jun 2022 20:13:09 +0500 Subject: [PATCH 027/123] minor: fix message for base64 --- src/Functions/FunctionBase64Conversion.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionBase64Conversion.h b/src/Functions/FunctionBase64Conversion.h index a1d6b966660..87a3309f7ef 100644 --- a/src/Functions/FunctionBase64Conversion.h +++ b/src/Functions/FunctionBase64Conversion.h @@ -85,7 +85,7 @@ public: { if (!WhichDataType(arguments[0].type).isString()) throw Exception( - "Illegal type " + arguments[0].type->getName() + " of 1 argument of function " + getName() + ". Must be String.", + "Illegal type " + arguments[0].type->getName() + " of 1st argument of function " + getName() + ". Must be String.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared(); @@ -98,7 +98,7 @@ public: if (!input) throw Exception( - "Illegal column " + arguments[0].column->getName() + " of first argument of function " + getName(), + "Illegal column " + arguments[0].column->getName() + " of first argument of function " + getName() + ", must be of type String", ErrorCodes::ILLEGAL_COLUMN); auto dst_column = ColumnString::create(); From 53d656d89f9f1dafd89c21d32e3f73ed741f72db Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 20 Jun 2022 17:35:24 +0000 Subject: [PATCH 028/123] Fix deadlock in OvercommitTracker logging --- src/Common/MemoryTracker.cpp | 7 +-- src/Common/OvercommitTracker.cpp | 31 ++++++++---- src/Interpreters/InternalTextLogsQueue.cpp | 1 - src/Interpreters/InternalTextLogsQueue.h | 56 ---------------------- 4 files changed, 23 insertions(+), 72 deletions(-) diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 51f4c83dc23..a9aef7d8465 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -338,11 +338,8 @@ void MemoryTracker::free(Int64 size) accounted_size += new_amount; } } - if (!OvercommitTrackerBlockerInThread::isBlocked()) - { - if (auto * overcommit_tracker_ptr = overcommit_tracker.load(std::memory_order_relaxed); overcommit_tracker_ptr) - overcommit_tracker_ptr->tryContinueQueryExecutionAfterFree(accounted_size); - } + if (auto * overcommit_tracker_ptr = overcommit_tracker.load(std::memory_order_relaxed)) + overcommit_tracker_ptr->tryContinueQueryExecutionAfterFree(accounted_size); if (auto * loaded_next = parent.load(std::memory_order_relaxed)) loaded_next->free(size); diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index 3cef72eb8b4..097b6637569 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -23,8 +23,16 @@ OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_) , allow_release(true) {} +#define LOG_DEBUG_SAFE(...) \ + { \ + OvercommitTrackerBlockerInThread blocker; \ + LOG_DEBUG(__VA_ARGS__); \ + } + OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) { + if (OvercommitTrackerBlockerInThread::isBlocked()) + return OvercommitResult::NONE; // NOTE: Do not change the order of locks // // global_mutex must be acquired before overcommit_m, because @@ -73,7 +81,7 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int }); auto wait_end_time = std::chrono::system_clock::now(); ProfileEvents::increment(ProfileEvents::MemoryOvercommitWaitTimeMicroseconds, (wait_end_time - wait_start_time) / 1us); - LOG_DEBUG(getLogger(), "Memory was{} freed within timeout", (timeout ? " not" : "")); + LOG_DEBUG_SAFE(getLogger(), "Memory was{} freed within timeout", (timeout ? " not" : "")) required_memory -= amount; Int64 still_need = required_per_thread[tracker]; // If enough memory is freed it will be 0 @@ -98,6 +106,9 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int void OvercommitTracker::tryContinueQueryExecutionAfterFree(Int64 amount) { + if (OvercommitTrackerBlockerInThread::isBlocked()) + return; + std::lock_guard guard(overcommit_m); if (cancellation_state != QueryCancellationState::NONE) { @@ -112,7 +123,7 @@ void OvercommitTracker::onQueryStop(MemoryTracker * tracker) std::unique_lock lk(overcommit_m); if (picked_tracker == tracker) { - LOG_DEBUG(getLogger(), "Picked query stopped"); + LOG_DEBUG_SAFE(getLogger(), "Picked query stopped") reset(); cv.notify_all(); @@ -140,7 +151,7 @@ void UserOvercommitTracker::pickQueryToExcludeImpl() // At this moment query list must be read only. // This is guaranteed by locking global_mutex in OvercommitTracker::needToStopQuery. auto & queries = user_process_list->queries; - LOG_DEBUG(logger, "Trying to choose query to stop from {} queries", queries.size()); + LOG_DEBUG_SAFE(logger, "Trying to choose query to stop from {} queries", queries.size()) for (auto const & query : queries) { if (query.second->isKilled()) @@ -151,15 +162,15 @@ void UserOvercommitTracker::pickQueryToExcludeImpl() continue; auto ratio = memory_tracker->getOvercommitRatio(); - LOG_DEBUG(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit); + LOG_DEBUG_SAFE(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit) if (ratio.soft_limit != 0 && current_ratio < ratio) { query_tracker = memory_tracker; current_ratio = ratio; } } - LOG_DEBUG(logger, "Selected to stop query with overcommit ratio {}/{}", - current_ratio.committed, current_ratio.soft_limit); + LOG_DEBUG_SAFE(logger, "Selected to stop query with overcommit ratio {}/{}", + current_ratio.committed, current_ratio.soft_limit) picked_tracker = query_tracker; } @@ -174,7 +185,7 @@ void GlobalOvercommitTracker::pickQueryToExcludeImpl() OvercommitRatio current_ratio{0, 0}; // At this moment query list must be read only. // This is guaranteed by locking global_mutex in OvercommitTracker::needToStopQuery. - LOG_DEBUG(logger, "Trying to choose query to stop from {} queries", process_list->size()); + LOG_DEBUG_SAFE(logger, "Trying to choose query to stop from {} queries", process_list->size()) for (auto const & query : process_list->processes) { if (query.isKilled()) @@ -190,15 +201,15 @@ void GlobalOvercommitTracker::pickQueryToExcludeImpl() if (!memory_tracker) continue; auto ratio = memory_tracker->getOvercommitRatio(user_soft_limit); - LOG_DEBUG(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit); + LOG_DEBUG_SAFE(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit) if (current_ratio < ratio) { query_tracker = memory_tracker; current_ratio = ratio; } } - LOG_DEBUG(logger, "Selected to stop query with overcommit ratio {}/{}", - current_ratio.committed, current_ratio.soft_limit); + LOG_DEBUG_SAFE(logger, "Selected to stop query with overcommit ratio {}/{}", + current_ratio.committed, current_ratio.soft_limit) picked_tracker = query_tracker; } diff --git a/src/Interpreters/InternalTextLogsQueue.cpp b/src/Interpreters/InternalTextLogsQueue.cpp index 6176e3cc865..2172a6f4261 100644 --- a/src/Interpreters/InternalTextLogsQueue.cpp +++ b/src/Interpreters/InternalTextLogsQueue.cpp @@ -38,7 +38,6 @@ MutableColumns InternalTextLogsQueue::getSampleColumns() void InternalTextLogsQueue::pushBlock(Block && log_block) { - OvercommitTrackerBlockerInThread blocker; static Block sample_block = getSampleBlock(); if (blocksHaveEqualStructure(sample_block, log_block)) diff --git a/src/Interpreters/InternalTextLogsQueue.h b/src/Interpreters/InternalTextLogsQueue.h index a7193a55178..53710fa3bd2 100644 --- a/src/Interpreters/InternalTextLogsQueue.h +++ b/src/Interpreters/InternalTextLogsQueue.h @@ -18,62 +18,6 @@ public: static Block getSampleBlock(); static MutableColumns getSampleColumns(); - template - bool push(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::push(std::forward(args)...); - } - - template - bool emplace(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::emplace(std::forward(args)...); - } - - template - bool pop(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::pop(std::forward(args)...); - } - - template - bool tryPush(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::tryPush(std::forward(args)...); - } - - template - bool tryEmplace(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::tryEmplace(std::forward(args)...); - } - - template - bool tryPop(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::tryPop(std::forward(args)...); - } - - template - void clear(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::clear(std::forward(args)...); - } - - template - void clearAndFinish(Args &&... args) - { - OvercommitTrackerBlockerInThread blocker; - return ConcurrentBoundedQueue::clearAndFinish(std::forward(args)...); - } - /// Is used to pass block from remote server to the client void pushBlock(Block && log_block); From ad2b9cc4e4dcb740f42976e0e0ceffbc8eaf45b6 Mon Sep 17 00:00:00 2001 From: zvonand Date: Mon, 20 Jun 2022 23:38:45 +0500 Subject: [PATCH 029/123] upd tests --- tests/queries/0_stateless/02337_base58.reference | 1 - tests/queries/0_stateless/02337_base58.sql | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02337_base58.reference b/tests/queries/0_stateless/02337_base58.reference index 718dfeb4a34..f4a2c95304b 100644 --- a/tests/queries/0_stateless/02337_base58.reference +++ b/tests/queries/0_stateless/02337_base58.reference @@ -45,4 +45,3 @@ foo foob fooba foobar -1 1 diff --git a/tests/queries/0_stateless/02337_base58.sql b/tests/queries/0_stateless/02337_base58.sql index b67993d2cc9..68dac97a20b 100644 --- a/tests/queries/0_stateless/02337_base58.sql +++ b/tests/queries/0_stateless/02337_base58.sql @@ -3,6 +3,8 @@ SET send_logs_level = 'fatal'; SELECT base58Encode('Hold my beer...'); +SELECT base58Encode('Hold my beer...', ''); -- { serverError 44 } +SELECT base58Encode('Hold my beer...', 'gmp', 'third'); -- { serverError 36 } SELECT base58Decode(encoded, 'gmp') FROM (SELECT base58Encode(val, 'gmp') as encoded FROM (select arrayJoin(['', 'f', 'fo', 'foo', 'foob', 'fooba', 'foobar', 'Hello world!']) val)); SELECT base58Decode(encoded, 'ripple') FROM (SELECT base58Encode(val, 'ripple') as encoded FROM (select arrayJoin(['', 'f', 'fo', 'foo', 'foob', 'fooba', 'foobar', 'Hello world!']) val)); @@ -12,6 +14,4 @@ SELECT base58Decode(encoded, 'bitcoin') FROM (SELECT base58Encode(val, 'bitcoin' SELECT base58Encode(val) FROM (select arrayJoin(['', 'f', 'fo', 'foo', 'foob', 'fooba', 'foobar']) val); SELECT base58Decode(val) FROM (select arrayJoin(['', '2m', '8o8', 'bQbp', '3csAg9', 'CZJRhmz', 't1Zv2yaZ']) val); -SELECT base58Decode(base58Encode('foo')) = 'foo', base58Encode(base58Decode('bQbp')) == 'bQbp'; - -SELECT base58Decode('Why_not?'); -- { serverError 1001 } \ No newline at end of file +SELECT base58Decode('Why_not?'); -- { serverError 1001 } From 22af00b757f14b30c3912c57ad7ccbe1ea25e6c9 Mon Sep 17 00:00:00 2001 From: zvonand Date: Mon, 20 Jun 2022 23:53:47 +0500 Subject: [PATCH 030/123] rename variable + fix handling of ENABLE_LIBRARIES --- contrib/base-x-cmake/CMakeLists.txt | 7 +++++++ src/Functions/FunctionBase58Conversion.h | 2 +- src/Functions/FunctionsBase58.cpp | 3 +-- src/Functions/config_functions.h.in | 2 +- src/Functions/configure_config.cmake | 2 +- src/Functions/registerFunctionsString.cpp | 4 ++-- src/configure_config.cmake | 4 ++-- 7 files changed, 15 insertions(+), 9 deletions(-) diff --git a/contrib/base-x-cmake/CMakeLists.txt b/contrib/base-x-cmake/CMakeLists.txt index 48cb54d307f..ab5696c9fb6 100644 --- a/contrib/base-x-cmake/CMakeLists.txt +++ b/contrib/base-x-cmake/CMakeLists.txt @@ -1,3 +1,10 @@ +option (ENABLE_BASEX "Enable base-x" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_BASEX) + message(STATUS "Not using base-x") + return() +endif() + set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/base-x") set (SRCS diff --git a/src/Functions/FunctionBase58Conversion.h b/src/Functions/FunctionBase58Conversion.h index e472e997c32..ed4667aa63b 100644 --- a/src/Functions/FunctionBase58Conversion.h +++ b/src/Functions/FunctionBase58Conversion.h @@ -1,7 +1,7 @@ #pragma once #include "config_functions.h" -#if USE_BASE58 +#if USE_BASEX # include # include # include diff --git a/src/Functions/FunctionsBase58.cpp b/src/Functions/FunctionsBase58.cpp index 3ccb4d790ce..dc325ff8cfc 100644 --- a/src/Functions/FunctionsBase58.cpp +++ b/src/Functions/FunctionsBase58.cpp @@ -1,7 +1,6 @@ #include -#if USE_BASE58 +#if USE_BASEX #include -#include namespace DB { diff --git a/src/Functions/config_functions.h.in b/src/Functions/config_functions.h.in index 001712d5cef..0bfea78922b 100644 --- a/src/Functions/config_functions.h.in +++ b/src/Functions/config_functions.h.in @@ -2,7 +2,7 @@ // .h autogenerated by cmake! -#cmakedefine01 USE_BASE58 +#cmakedefine01 USE_BASEX #cmakedefine01 USE_BASE64 #cmakedefine01 USE_SIMDJSON #cmakedefine01 USE_RAPIDJSON diff --git a/src/Functions/configure_config.cmake b/src/Functions/configure_config.cmake index 856d9a5682a..1038c09e53f 100644 --- a/src/Functions/configure_config.cmake +++ b/src/Functions/configure_config.cmake @@ -2,7 +2,7 @@ if (TARGET ch_contrib::fastops) set(USE_FASTOPS 1) endif() if (TARGET ch_contrib::base-x) - set(USE_BASE58 1) + set(USE_BASEX 1) endif() if (TARGET ch_contrib::base64) set(USE_BASE64 1) diff --git a/src/Functions/registerFunctionsString.cpp b/src/Functions/registerFunctionsString.cpp index 43035ef51e7..248b6391b4f 100644 --- a/src/Functions/registerFunctionsString.cpp +++ b/src/Functions/registerFunctionsString.cpp @@ -49,7 +49,7 @@ void registerFunctionBase64Decode(FunctionFactory &); void registerFunctionTryBase64Decode(FunctionFactory &); #endif -#if USE_BASE58 +#if USE_BASEX void registerFunctionBase58Encode(FunctionFactory &); void registerFunctionBase58Decode(FunctionFactory &); #endif @@ -110,7 +110,7 @@ void registerFunctionsString(FunctionFactory & factory) registerFunctionTryBase64Decode(factory); #endif -#if USE_BASE58 +#if USE_BASEX registerFunctionBase58Encode(factory); registerFunctionBase58Decode(factory); #endif diff --git a/src/configure_config.cmake b/src/configure_config.cmake index fc2a858e75a..45e45b505d4 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -55,8 +55,8 @@ endif() if (TARGET ch_contrib::base64) set(USE_BASE64 1) endif() -if (TARGET ch_contrib::base58) - set(USE_BASE58 1) +if (TARGET ch_contrib::base-x) + set(USE_BASEX 1) endif() if (TARGET ch_contrib::yaml_cpp) set(USE_YAML_CPP 1) From 9ca368ac20a35e6e49f3162f00f06a6206b9e6dd Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 20 Jun 2022 23:10:40 +0000 Subject: [PATCH 031/123] Use do while(false) in macro --- src/Common/OvercommitTracker.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index 097b6637569..1e36c99b41c 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -24,10 +24,10 @@ OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_) {} #define LOG_DEBUG_SAFE(...) \ - { \ + do { \ OvercommitTrackerBlockerInThread blocker; \ LOG_DEBUG(__VA_ARGS__); \ - } + } while (false) OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) { @@ -81,7 +81,7 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int }); auto wait_end_time = std::chrono::system_clock::now(); ProfileEvents::increment(ProfileEvents::MemoryOvercommitWaitTimeMicroseconds, (wait_end_time - wait_start_time) / 1us); - LOG_DEBUG_SAFE(getLogger(), "Memory was{} freed within timeout", (timeout ? " not" : "")) + LOG_DEBUG_SAFE(getLogger(), "Memory was{} freed within timeout", (timeout ? " not" : "")); required_memory -= amount; Int64 still_need = required_per_thread[tracker]; // If enough memory is freed it will be 0 @@ -123,7 +123,7 @@ void OvercommitTracker::onQueryStop(MemoryTracker * tracker) std::unique_lock lk(overcommit_m); if (picked_tracker == tracker) { - LOG_DEBUG_SAFE(getLogger(), "Picked query stopped") + LOG_DEBUG_SAFE(getLogger(), "Picked query stopped"); reset(); cv.notify_all(); @@ -151,7 +151,7 @@ void UserOvercommitTracker::pickQueryToExcludeImpl() // At this moment query list must be read only. // This is guaranteed by locking global_mutex in OvercommitTracker::needToStopQuery. auto & queries = user_process_list->queries; - LOG_DEBUG_SAFE(logger, "Trying to choose query to stop from {} queries", queries.size()) + LOG_DEBUG_SAFE(logger, "Trying to choose query to stop from {} queries", queries.size()); for (auto const & query : queries) { if (query.second->isKilled()) @@ -162,7 +162,7 @@ void UserOvercommitTracker::pickQueryToExcludeImpl() continue; auto ratio = memory_tracker->getOvercommitRatio(); - LOG_DEBUG_SAFE(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit) + LOG_DEBUG_SAFE(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit); if (ratio.soft_limit != 0 && current_ratio < ratio) { query_tracker = memory_tracker; @@ -170,7 +170,7 @@ void UserOvercommitTracker::pickQueryToExcludeImpl() } } LOG_DEBUG_SAFE(logger, "Selected to stop query with overcommit ratio {}/{}", - current_ratio.committed, current_ratio.soft_limit) + current_ratio.committed, current_ratio.soft_limit); picked_tracker = query_tracker; } @@ -185,7 +185,7 @@ void GlobalOvercommitTracker::pickQueryToExcludeImpl() OvercommitRatio current_ratio{0, 0}; // At this moment query list must be read only. // This is guaranteed by locking global_mutex in OvercommitTracker::needToStopQuery. - LOG_DEBUG_SAFE(logger, "Trying to choose query to stop from {} queries", process_list->size()) + LOG_DEBUG_SAFE(logger, "Trying to choose query to stop from {} queries", process_list->size()); for (auto const & query : process_list->processes) { if (query.isKilled()) @@ -201,7 +201,7 @@ void GlobalOvercommitTracker::pickQueryToExcludeImpl() if (!memory_tracker) continue; auto ratio = memory_tracker->getOvercommitRatio(user_soft_limit); - LOG_DEBUG_SAFE(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit) + LOG_DEBUG_SAFE(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit); if (current_ratio < ratio) { query_tracker = memory_tracker; @@ -209,7 +209,7 @@ void GlobalOvercommitTracker::pickQueryToExcludeImpl() } } LOG_DEBUG_SAFE(logger, "Selected to stop query with overcommit ratio {}/{}", - current_ratio.committed, current_ratio.soft_limit) + current_ratio.committed, current_ratio.soft_limit); picked_tracker = query_tracker; } From e10f079bd35c377170abb2a1b984e83f1ce7c0a3 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 21 Jun 2022 10:15:33 +0000 Subject: [PATCH 032/123] Get rid of allocations in OvercommitTracker --- src/Common/OvercommitTracker.cpp | 19 ++++++++++--------- src/Common/OvercommitTracker.h | 10 ++++++++-- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index 1e36c99b41c..6f688ca28ff 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -20,6 +20,8 @@ OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_) , global_mutex(global_mutex_) , freed_memory(0) , required_memory(0) + , next_id(0) + , id_to_release(0) , allow_release(true) {} @@ -42,6 +44,8 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int std::unique_lock global_lock(global_mutex); std::unique_lock lk(overcommit_m); + size_t id = next_id++; + auto max_wait_time = tracker->getOvercommitWaitingTime(); if (max_wait_time == ZERO_MICROSEC) @@ -73,23 +77,21 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int allow_release = true; required_memory += amount; - required_per_thread[tracker] = amount; auto wait_start_time = std::chrono::system_clock::now(); - bool timeout = !cv.wait_for(lk, max_wait_time, [this, tracker]() + bool timeout = !cv.wait_for(lk, max_wait_time, [this, id]() { - return required_per_thread[tracker] == 0 || cancellation_state == QueryCancellationState::NONE; + return id < id_to_release || cancellation_state == QueryCancellationState::NONE; }); auto wait_end_time = std::chrono::system_clock::now(); ProfileEvents::increment(ProfileEvents::MemoryOvercommitWaitTimeMicroseconds, (wait_end_time - wait_start_time) / 1us); LOG_DEBUG_SAFE(getLogger(), "Memory was{} freed within timeout", (timeout ? " not" : "")); required_memory -= amount; - Int64 still_need = required_per_thread[tracker]; // If enough memory is freed it will be 0 - required_per_thread.erase(tracker); + bool still_need = !(id < id_to_release); // True if thread wasn't released // If threads where not released since last call of this method, // we can release them now. - if (allow_release && required_memory <= freed_memory && still_need != 0) + if (allow_release && required_memory <= freed_memory && still_need) releaseThreads(); // All required amount of memory is free now and selected query to stop doesn't know about it. @@ -98,7 +100,7 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int reset(); if (timeout) return OvercommitResult::TIMEOUTED; - if (still_need != 0) + if (still_need) return OvercommitResult::NOT_ENOUGH_FREED; else return OvercommitResult::MEMORY_FREED; @@ -132,8 +134,7 @@ void OvercommitTracker::onQueryStop(MemoryTracker * tracker) void OvercommitTracker::releaseThreads() { - for (auto & required : required_per_thread) - required.second = 0; + id_to_release = next_id; freed_memory = 0; allow_release = false; // To avoid repeating call of this method in OvercommitTracker::needToStopQuery cv.notify_all(); diff --git a/src/Common/OvercommitTracker.h b/src/Common/OvercommitTracker.h index 80aaed68e37..6a03c679422 100644 --- a/src/Common/OvercommitTracker.h +++ b/src/Common/OvercommitTracker.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -104,6 +105,10 @@ private: picked_tracker = nullptr; cancellation_state = QueryCancellationState::NONE; freed_memory = 0; + + next_id = 0; + id_to_release = 0; + allow_release = true; } @@ -111,8 +116,6 @@ private: QueryCancellationState cancellation_state; - std::unordered_map required_per_thread; - // Global mutex which is used in ProcessList to synchronize // insertion and deletion of queries. // OvercommitTracker::pickQueryToExcludeImpl() implementations @@ -122,6 +125,9 @@ private: Int64 freed_memory; Int64 required_memory; + size_t next_id; // Id provided to the next thread to come in OvercommitTracker + size_t id_to_release; // We can release all threads with id smaller than this + bool allow_release; }; From ba0fcec993953fc129f4305f890f1cdca4e4b8e1 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 21 Jun 2022 12:35:47 +0200 Subject: [PATCH 033/123] add background cleanup of store/ subdirs --- src/Access/DiskAccessStorage.cpp | 10 +- src/Common/filesystemHelpers.cpp | 2 - src/Databases/DatabaseOrdinary.cpp | 6 +- src/IO/ReadHelpers.h | 2 + src/Interpreters/DatabaseCatalog.cpp | 149 +++++++++++++- src/Interpreters/DatabaseCatalog.h | 15 ++ src/Interpreters/InterpreterCreateQuery.cpp | 8 + tests/config/config.d/database_atomic.xml | 5 + .../configs/store_cleanup.xml | 11 ++ .../test.py | 183 +++++++++++++++++- 10 files changed, 371 insertions(+), 20 deletions(-) create mode 100644 tests/integration/test_broken_detached_part_clean_up/configs/store_cleanup.xml diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index a9b7a6a265b..231e325196d 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -153,15 +153,7 @@ namespace bool tryParseUUID(const String & str, UUID & id) { - try - { - id = parseFromString(str); - return true; - } - catch (...) - { - return false; - } + return tryParse(id, str); } } diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 7c10b24eb9c..32ae3219d9f 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -1,12 +1,10 @@ #include "filesystemHelpers.h" #if defined(OS_LINUX) -# include # include # include #endif #include -#include #include #include #include diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index ada9030905d..2e4ee785a94 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -172,9 +172,9 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables if (fs::exists(full_path.string() + detached_suffix)) { - /// FIXME: even if we don't load the table we can still mark the uuid of it as taken. - /// if (create_query->uuid != UUIDHelpers::Nil) - /// DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); + /// Even if we don't load the table we can still mark the uuid of it as taken. + if (create_query->uuid != UUIDHelpers::Nil) + DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); const std::string table_name = unescapeForFileName(file_name.substr(0, file_name.size() - 4)); LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name)); diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 8fe8e2aa23e..4cd07dddf25 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -1065,6 +1065,8 @@ inline bool tryReadText(is_integer auto & x, ReadBuffer & buf) return tryReadIntText(x, buf); } +inline bool tryReadText(UUID & x, ReadBuffer & buf) { return tryReadUUIDText(x, buf); } + inline void readText(is_floating_point auto & x, ReadBuffer & buf) { readFloatText(x, buf); } inline void readText(String & x, ReadBuffer & buf) { readEscapedString(x, buf); } diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index fa9444a7e66..623e4ed0845 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -16,9 +16,11 @@ #include #include #include -#include #include +#include +#include + #include "config_core.h" #if USE_MYSQL @@ -27,12 +29,11 @@ #endif #if USE_LIBPQXX -# include +# include # include +# include #endif -namespace fs = std::filesystem; - namespace CurrentMetrics { extern const Metric TablesToDropQueueSize; @@ -143,6 +144,9 @@ StoragePtr TemporaryTableHolder::getTable() const void DatabaseCatalog::initializeAndLoadTemporaryDatabase() { drop_delay_sec = getContext()->getConfigRef().getInt("database_atomic_delay_before_drop_table_sec", default_drop_delay_sec); + unused_dir_hide_timeout_sec = getContext()->getConfigRef().getInt("database_catalog_unused_dir_hide_timeout_sec", unused_dir_hide_timeout_sec); + unused_dir_rm_timeout_sec = getContext()->getConfigRef().getInt("database_catalog_unused_dir_rm_timeout_sec", unused_dir_rm_timeout_sec); + unused_dir_cleanup_period_sec = getContext()->getConfigRef().getInt("database_catalog_unused_dir_cleanup_period_sec", unused_dir_cleanup_period_sec); auto db_for_temporary_and_external_tables = std::make_shared(TEMPORARY_DATABASE, getContext()); attachDatabase(TEMPORARY_DATABASE, db_for_temporary_and_external_tables); @@ -150,6 +154,12 @@ void DatabaseCatalog::initializeAndLoadTemporaryDatabase() void DatabaseCatalog::loadDatabases() { + auto cleanup_task_holder = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this](){ this->cleanupStoreDirectoryTask(); }); + cleanup_task = std::make_unique(std::move(cleanup_task_holder)); + (*cleanup_task)->activate(); + /// Do not start task immediately on server startup, it's not urgent. + (*cleanup_task)->scheduleAfter(unused_dir_hide_timeout_sec * 1000); + auto task_holder = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this](){ this->dropTableDataTask(); }); drop_task = std::make_unique(std::move(task_holder)); (*drop_task)->activate(); @@ -166,6 +176,9 @@ void DatabaseCatalog::shutdownImpl() { TemporaryLiveViewCleaner::shutdown(); + if (cleanup_task) + (*cleanup_task)->deactivate(); + if (drop_task) (*drop_task)->deactivate(); @@ -372,6 +385,10 @@ DatabasePtr DatabaseCatalog::detachDatabase(ContextPtr local_context, const Stri if (drop) { + UUID db_uuid = db->getUUID(); + if (db_uuid != UUIDHelpers::Nil) + removeUUIDMappingFinally(db_uuid); + /// Delete the database. db->drop(local_context); @@ -560,6 +577,15 @@ void DatabaseCatalog::updateUUIDMapping(const UUID & uuid, DatabasePtr database, prev_table = std::move(table); } +bool DatabaseCatalog::hasUUIDMapping(const UUID & uuid) +{ + assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); + UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; + std::lock_guard lock{map_part.mutex}; + auto it = map_part.map.find(uuid); + return it != map_part.map.end(); +} + std::unique_ptr DatabaseCatalog::database_catalog; DatabaseCatalog::DatabaseCatalog(ContextMutablePtr global_context_) @@ -1072,6 +1098,121 @@ void DatabaseCatalog::updateLoadingDependencies(const StorageID & table_id, Tabl old_dependencies = std::move(new_dependencies); } +void DatabaseCatalog::cleanupStoreDirectoryTask() +{ + fs::path store_path = fs::path(getContext()->getPath()) / "store"; + size_t affected_dirs = 0; + for (const auto & prefix_dir : fs::directory_iterator{store_path}) + { + String prefix = prefix_dir.path().filename(); + bool expected_prefix_dir = prefix_dir.is_directory() && + prefix.size() == 3 && + isHexDigit(prefix[0]) && + isHexDigit(prefix[1]) && + isHexDigit(prefix[2]); + + if (!expected_prefix_dir) + { + LOG_WARNING(log, "Found invalid directory {}, will try to remove it", prefix_dir.path().string()); + maybeRemoveDirectory(prefix_dir.path()); + continue; + } + + for (const auto & uuid_dir : fs::directory_iterator{prefix_dir.path()}) + { + String uuid_str = uuid_dir.path().filename(); + UUID uuid; + bool parsed = tryParse(uuid, uuid_str); + + bool expected_dir = uuid_dir.is_directory() && + parsed && + uuid != UUIDHelpers::Nil && + uuid_str.starts_with(prefix); + + if (!expected_dir) + { + LOG_WARNING(log, "Found invalid directory {}, will try to remove it", uuid_dir.path().string()); + maybeRemoveDirectory(uuid_dir.path()); + continue; + } + + if (!hasUUIDMapping(uuid)) + { + /// We load uuids even for detached and permanently detached tables, + /// so it looks safe enough to remove directory if we don't have uuid mapping for it. + /// No table or database using this directory should concurrently appear, + /// because creation of new table would fail with "directory already exists". + affected_dirs += maybeRemoveDirectory(uuid_dir.path()); + } + } + } + + if (affected_dirs) + LOG_INFO(log, "Cleaned up {} directories from store/", affected_dirs); + + (*cleanup_task)->scheduleAfter(unused_dir_cleanup_period_sec * 1000); +} + +bool DatabaseCatalog::maybeRemoveDirectory(const fs::path & unused_dir) +{ + /// "Safe" automatic removal of some directory. + /// At first we do not remove anything and only revoke all access right. + /// And remove only if nobody noticed it after, for example, one month. + + struct stat st; + if (stat(unused_dir.string().c_str(), &st)) + { + LOG_ERROR(log, "Failed to stat {}, errno: {}", unused_dir.string(), errno); + return false; + } + + if (st.st_uid != geteuid()) + { + /// Directory is not owned by clickhouse, it's weird, let's ignore it (chmod will likely fail anyway). + LOG_WARNING(log, "Found directory {} with unexpected owner (uid={})", unused_dir.string(), st.st_uid); + return false; + } + + time_t max_modification_time = std::max(st.st_atime, std::max(st.st_mtime, st.st_ctime)); + time_t current_time = time(nullptr); + if (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) + { + if (current_time <= max_modification_time + unused_dir_hide_timeout_sec) + return false; + + LOG_INFO(log, "Removing access rights for unused directory {} (will remove it when timeout exceed)", unused_dir.string()); + + /// Explicitly update modification time just in case + + struct utimbuf tb; + tb.actime = current_time; + tb.modtime = current_time; + if (utime(unused_dir.string().c_str(), &tb) != 0) + LOG_ERROR(log, "Failed to utime {}, errno: {}", unused_dir.string(), errno); + + /// Remove all access right + if (chmod(unused_dir.string().c_str(), 0)) + LOG_ERROR(log, "Failed to chmod {}, errno: {}", unused_dir.string(), errno); + + return true; + } + else + { + if (current_time <= max_modification_time + unused_dir_rm_timeout_sec) + return false; + + LOG_INFO(log, "Removing unused directory {}", unused_dir.string()); + + /// We have to set these access rights to make recursive removal work + if (chmod(unused_dir.string().c_str(), S_IRWXU)) + LOG_ERROR(log, "Failed to chmod {}, errno: {}", unused_dir.string(), errno); + + fs::remove_all(unused_dir); + + return true; + } +} + DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock guards_lock_, const String & elem, const String & database_name) : map(map_), db_mutex(db_mutex_), guards_lock(std::move(guards_lock_)) diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 79ba4052038..9ca0bb276d6 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -20,7 +20,9 @@ #include #include #include +#include +namespace fs = std::filesystem; namespace DB { @@ -203,6 +205,8 @@ public: /// this method will throw an exception. void addUUIDMapping(const UUID & uuid); + bool hasUUIDMapping(const UUID & uuid); + static String getPathForUUID(const UUID & uuid); DatabaseAndTable tryGetByUUID(const UUID & uuid) const; @@ -261,6 +265,9 @@ private: void dropTableDataTask(); void dropTableFinally(const TableMarkedAsDropped & table); + void cleanupStoreDirectoryTask(); + bool maybeRemoveDirectory(const fs::path & unused_dir); + static constexpr size_t reschedule_time_ms = 100; static constexpr time_t drop_error_cooldown_sec = 5; @@ -298,6 +305,14 @@ private: static constexpr time_t default_drop_delay_sec = 8 * 60; time_t drop_delay_sec = default_drop_delay_sec; std::condition_variable wait_table_finally_dropped; + + std::unique_ptr cleanup_task; + static constexpr time_t default_unused_dir_hide_timeout_sec = 60 * 60; /// 1 hour + time_t unused_dir_hide_timeout_sec = default_unused_dir_hide_timeout_sec; + static constexpr time_t default_unused_dir_rm_timeout_sec = 30 * 24 * 60 * 60; /// 30 days + time_t unused_dir_rm_timeout_sec = default_unused_dir_rm_timeout_sec; + static constexpr time_t default_unused_dir_cleanup_period_sec = 24 * 60 * 60; /// 1 day + time_t unused_dir_cleanup_period_sec = default_unused_dir_cleanup_period_sec; }; } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 304cfa2f3f4..9e745ffcabe 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -255,6 +255,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) create.setDatabase(TABLE_WITH_UUID_NAME_PLACEHOLDER); bool need_write_metadata = !create.attach || !fs::exists(metadata_file_path); + bool need_lock_uuid = internal || !create.attach; if (need_write_metadata) { @@ -279,6 +280,13 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) /// We attach database before loading it's tables, so do not allow concurrent DDL queries auto db_guard = DatabaseCatalog::instance().getExclusiveDDLGuardForDatabase(database_name); + if (create.uuid != UUIDHelpers::Nil && need_lock_uuid) + { + /// Lock uuid, so we will known it's already in use. + /// We do it when attaching databases on server startup (internal) and on CREATE query (!create.attach); + DatabaseCatalog::instance().addUUIDMapping(create.uuid); + } + bool added = false; bool renamed = false; try diff --git a/tests/config/config.d/database_atomic.xml b/tests/config/config.d/database_atomic.xml index b3f51d51a79..a551e710ca3 100644 --- a/tests/config/config.d/database_atomic.xml +++ b/tests/config/config.d/database_atomic.xml @@ -1,3 +1,8 @@ 60 + + + 0 + 5 + 10 diff --git a/tests/integration/test_broken_detached_part_clean_up/configs/store_cleanup.xml b/tests/integration/test_broken_detached_part_clean_up/configs/store_cleanup.xml new file mode 100644 index 00000000000..3b0260dd07a --- /dev/null +++ b/tests/integration/test_broken_detached_part_clean_up/configs/store_cleanup.xml @@ -0,0 +1,11 @@ + + 0 + 15 + 1 + + + + testkeeper + + \ No newline at end of file diff --git a/tests/integration/test_broken_detached_part_clean_up/test.py b/tests/integration/test_broken_detached_part_clean_up/test.py index 3d9134bdc54..dd6744b4228 100644 --- a/tests/integration/test_broken_detached_part_clean_up/test.py +++ b/tests/integration/test_broken_detached_part_clean_up/test.py @@ -1,14 +1,15 @@ import pytest from helpers.cluster import ClickHouseCluster -from multiprocessing.dummy import Pool from helpers.corrupt_part_data_on_disk import corrupt_part_data_on_disk from helpers.corrupt_part_data_on_disk import break_part import time cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance("node1", stay_alive=True, with_zookeeper=True) +node1 = cluster.add_instance( + "node1", stay_alive=True, main_configs=["configs/store_cleanup.xml"] +) path_to_data = "/var/lib/clickhouse/" @@ -147,3 +148,181 @@ def test_remove_broken_detached_part_replicated_merge_tree(started_cluster): ) remove_broken_detached_part_impl("replicated_mt", node1, "broken") + + +def test_store_cleanup(started_cluster): + node1.query("CREATE DATABASE db UUID '10000000-1000-4000-8000-000000000001'") + node1.query( + "CREATE TABLE db.log UUID '10000000-1000-4000-8000-000000000002' ENGINE=Log AS SELECT 1" + ) + node1.query( + "CREATE TABLE db.mt UUID '10000000-1000-4000-8000-000000000003' ENGINE=MergeTree ORDER BY tuple() AS SELECT 1" + ) + node1.query( + "CREATE TABLE db.mem UUID '10000000-1000-4000-8000-000000000004' ENGINE=Memory AS SELECT 1" + ) + + node1.query("CREATE DATABASE db2 UUID '20000000-1000-4000-8000-000000000001'") + node1.query( + "CREATE TABLE db2.log UUID '20000000-1000-4000-8000-000000000002' ENGINE=Log AS SELECT 1" + ) + node1.query("DETACH DATABASE db2") + + node1.query("CREATE DATABASE db3 UUID '30000000-1000-4000-8000-000000000001'") + node1.query( + "CREATE TABLE db3.log UUID '30000000-1000-4000-8000-000000000002' ENGINE=Log AS SELECT 1" + ) + node1.query( + "CREATE TABLE db3.log2 UUID '30000000-1000-4000-8000-000000000003' ENGINE=Log AS SELECT 1" + ) + node1.query("DETACH TABLE db3.log") + node1.query("DETACH TABLE db3.log2 PERMANENTLY") + + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store"] + ) + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/100"] + ) + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/200"] + ) + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/300"] + ) + + node1.stop_clickhouse(kill=True) + # All dirs related to `db` will be removed + node1.exec_in_container(["rm", f"{path_to_data}/metadata/db.sql"]) + + node1.exec_in_container(["mkdir", f"{path_to_data}/store/kek"]) + node1.exec_in_container(["touch", f"{path_to_data}/store/12"]) + node1.exec_in_container(["mkdir", f"{path_to_data}/store/456"]) + node1.exec_in_container(["mkdir", f"{path_to_data}/store/456/testgarbage"]) + node1.exec_in_container( + ["mkdir", f"{path_to_data}/store/456/30000000-1000-4000-8000-000000000003"] + ) + node1.exec_in_container( + ["touch", f"{path_to_data}/store/456/45600000-1000-4000-8000-000000000003"] + ) + node1.exec_in_container( + ["mkdir", f"{path_to_data}/store/456/45600000-1000-4000-8000-000000000004"] + ) + + node1.start_clickhouse() + node1.query("DETACH DATABASE db2") + node1.query("DETACH TABLE db3.log") + + node1.wait_for_log_line("Removing access rights for unused directory") + time.sleep(1) + node1.wait_for_log_line("testgarbage") + node1.wait_for_log_line("directories from store") + + store = node1.exec_in_container(["ls", f"{path_to_data}/store"]) + assert "100" in store + assert "200" in store + assert "300" in store + assert "456" in store + assert "kek" in store + assert "12" in store + assert "d---------" in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store"] + ) + assert "d---------" in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/456"] + ) + + # Metadata is removed, so store/100 contains garbage + store100 = node1.exec_in_container(["ls", f"{path_to_data}/store/100"]) + assert "10000000-1000-4000-8000-000000000001" in store100 + assert "10000000-1000-4000-8000-000000000002" in store100 + assert "10000000-1000-4000-8000-000000000003" in store100 + assert "d---------" in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/100"] + ) + + # Database is detached, nothing to clean up + store200 = node1.exec_in_container(["ls", f"{path_to_data}/store/200"]) + assert "20000000-1000-4000-8000-000000000001" in store200 + assert "20000000-1000-4000-8000-000000000002" in store200 + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/200"] + ) + + # Tables are detached, nothing to clean up + store300 = node1.exec_in_container(["ls", f"{path_to_data}/store/300"]) + assert "30000000-1000-4000-8000-000000000001" in store300 + assert "30000000-1000-4000-8000-000000000002" in store300 + assert "30000000-1000-4000-8000-000000000003" in store300 + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/300"] + ) + + # Manually created garbage + store456 = node1.exec_in_container(["ls", f"{path_to_data}/store/456"]) + assert "30000000-1000-4000-8000-000000000003" in store456 + assert "45600000-1000-4000-8000-000000000003" in store456 + assert "45600000-1000-4000-8000-000000000004" in store456 + assert "testgarbage" in store456 + assert "----------" in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/456"] + ) + + node1.wait_for_log_line("Removing unused directory") + time.sleep(1) + node1.wait_for_log_line("directories from store") + + store = node1.exec_in_container(["ls", f"{path_to_data}/store"]) + assert "100" in store + assert "200" in store + assert "300" in store + assert "456" in store + assert "kek" not in store # changed + assert "12" not in store # changed + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store"] + ) # changed + + # Metadata is removed, so store/100 contains garbage + store100 = node1.exec_in_container(["ls", f"{path_to_data}/store/100"]) # changed + assert "10000000-1000-4000-8000-000000000001" not in store100 # changed + assert "10000000-1000-4000-8000-000000000002" not in store100 # changed + assert "10000000-1000-4000-8000-000000000003" not in store100 # changed + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/100"] + ) # changed + + # Database is detached, nothing to clean up + store200 = node1.exec_in_container(["ls", f"{path_to_data}/store/200"]) + assert "20000000-1000-4000-8000-000000000001" in store200 + assert "20000000-1000-4000-8000-000000000002" in store200 + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/200"] + ) + + # Tables are detached, nothing to clean up + store300 = node1.exec_in_container(["ls", f"{path_to_data}/store/300"]) + assert "30000000-1000-4000-8000-000000000001" in store300 + assert "30000000-1000-4000-8000-000000000002" in store300 + assert "30000000-1000-4000-8000-000000000003" in store300 + assert "d---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/300"] + ) + + # Manually created garbage + store456 = node1.exec_in_container(["ls", f"{path_to_data}/store/456"]) + assert "30000000-1000-4000-8000-000000000003" not in store456 # changed + assert "45600000-1000-4000-8000-000000000003" not in store456 # changed + assert "45600000-1000-4000-8000-000000000004" not in store456 # changed + assert "testgarbage" not in store456 # changed + assert "---------" not in node1.exec_in_container( + ["ls", "-l", f"{path_to_data}/store/456"] + ) # changed + + node1.query("ATTACH TABLE db3.log2") + node1.query("ATTACH DATABASE db2") + node1.query("ATTACH TABLE db3.log") + + assert "1\n" == node1.query("SELECT * FROM db3.log") + assert "1\n" == node1.query("SELECT * FROM db3.log2") + assert "1\n" == node1.query("SELECT * FROM db2.log") From 099055c1833224bf9212897a7a283a7cf529c803 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 21 Jun 2022 12:26:13 +0000 Subject: [PATCH 034/123] Explicitly forbid allocations in OvercommitTracker --- src/Common/OvercommitTracker.cpp | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index 6f688ca28ff..74235748345 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -25,14 +25,24 @@ OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_) , allow_release(true) {} -#define LOG_DEBUG_SAFE(...) \ - do { \ - OvercommitTrackerBlockerInThread blocker; \ - LOG_DEBUG(__VA_ARGS__); \ +#define LOG_DEBUG_SAFE(...) \ + do { \ + OvercommitTrackerBlockerInThread blocker; \ + try \ + { \ + ALLOW_ALLOCATIONS_IN_SCOPE; \ + LOG_DEBUG(__VA_ARGS__); \ + } \ + catch (std::bad_alloc const &) \ + { \ + fprintf(stderr, "Allocation failed during writing to log in OvercommitTracker\n"); \ + } \ } while (false) OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) { + DENY_ALLOCATIONS_IN_SCOPE; + if (OvercommitTrackerBlockerInThread::isBlocked()) return OvercommitResult::NONE; // NOTE: Do not change the order of locks @@ -108,6 +118,8 @@ OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int void OvercommitTracker::tryContinueQueryExecutionAfterFree(Int64 amount) { + DENY_ALLOCATIONS_IN_SCOPE; + if (OvercommitTrackerBlockerInThread::isBlocked()) return; @@ -122,6 +134,8 @@ void OvercommitTracker::tryContinueQueryExecutionAfterFree(Int64 amount) void OvercommitTracker::onQueryStop(MemoryTracker * tracker) { + DENY_ALLOCATIONS_IN_SCOPE; + std::unique_lock lk(overcommit_m); if (picked_tracker == tracker) { From 13e3d40db43bf0a257f608ddab344ee22d0fa3f1 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 21 Jun 2022 15:11:03 +0200 Subject: [PATCH 035/123] fix --- src/Interpreters/DatabaseCatalog.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 623e4ed0845..dc0d346be92 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -20,6 +20,7 @@ #include #include +#include #include "config_core.h" From 1d6eea6cfa8068284ab06f2c7d5f8dfc8e091783 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Tue, 21 Jun 2022 18:55:17 +0300 Subject: [PATCH 036/123] Replace LOG_DEBUG by LOG_TRACE in KerberosInit; Correct exception message; Prohibit making a copy of KerberosInit --- src/Access/KerberosInit.cpp | 40 ++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 8d4e2339bbf..e78830bdaa5 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #if USE_KRB5 #include #include @@ -27,7 +28,10 @@ struct k5_data krb5_boolean switch_to_cache; }; -class KerberosInit +/** + * This class implements programmatic implementation of kinit. + */ +class KerberosInit : boost::noncopyable { public: int init(const String & keytab_file, const String & principal, const String & cache_name = ""); @@ -44,7 +48,7 @@ private: int KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) { auto log = &Poco::Logger::get("KerberosInit"); - LOG_DEBUG(log,"Trying to authenticate to Kerberos v5"); + LOG_TRACE(log,"Trying to authenticate to Kerberos v5"); krb5_error_code ret; @@ -52,7 +56,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con int flags = 0; if (!std::filesystem::exists(keytab_file)) - throw Exception("Error keytab file does not exist", ErrorCodes::KERBEROS_ERROR); + throw Exception("Keytab file does not exist", ErrorCodes::KERBEROS_ERROR); memset(&k5, 0, sizeof(k5)); ret = krb5_init_context(&k5.ctx); @@ -64,7 +68,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con ret = krb5_cc_resolve(k5.ctx, cache_name.c_str(), &k5.out_cc); if (ret) throw Exception("Error in resolving cache", ErrorCodes::KERBEROS_ERROR); - LOG_DEBUG(log,"Resolved cache"); + LOG_TRACE(log,"Resolved cache"); } else { @@ -72,7 +76,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con ret = krb5_cc_default(k5.ctx, &defcache); if (ret) throw Exception("Error while getting default cache", ErrorCodes::KERBEROS_ERROR); - LOG_DEBUG(log,"Resolved default cache"); + LOG_TRACE(log,"Resolved default cache"); deftype = krb5_cc_get_type(k5.ctx, defcache); if (krb5_cc_get_principal(k5.ctx, defcache, &defcache_princ) != 0) defcache_princ = nullptr; @@ -92,7 +96,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con throw Exception("Error while searching for cache for " + principal, ErrorCodes::KERBEROS_ERROR); if (!ret) { - LOG_DEBUG(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); + LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); k5.switch_to_cache = 1; } else if (defcache_princ != nullptr) @@ -101,7 +105,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con ret = krb5_cc_new_unique(k5.ctx, deftype, nullptr, &k5.out_cc); if (ret) throw Exception("Error while generating new cache", ErrorCodes::KERBEROS_ERROR); - LOG_DEBUG(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); + LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); k5.switch_to_cache = 1; } } @@ -111,13 +115,13 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con { k5.out_cc = defcache; defcache = nullptr; - LOG_DEBUG(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); + LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); } ret = krb5_unparse_name(k5.ctx, k5.me, &k5.name); if (ret) throw Exception("Error when unparsing name", ErrorCodes::KERBEROS_ERROR); - LOG_DEBUG(log,"Using principal: {}", k5.name); + LOG_TRACE(log,"Using principal: {}", k5.name); memset(&my_creds, 0, sizeof(my_creds)); ret = krb5_get_init_creds_opt_alloc(k5.ctx, &options); @@ -128,7 +132,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con ret = krb5_kt_resolve(k5.ctx, keytab_file.c_str(), &keytab); if (ret) throw Exception("Error in resolving keytab "+keytab_file, ErrorCodes::KERBEROS_ERROR); - LOG_DEBUG(log,"Using keytab: {}", keytab_file); + LOG_TRACE(log,"Using keytab: {}", keytab_file); if (k5.in_cc) { @@ -141,28 +145,28 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con throw Exception("Error in setting output credential cache", ErrorCodes::KERBEROS_ERROR); // Action: init or renew - LOG_DEBUG(log,"Trying to renew credentials"); + LOG_TRACE(log,"Trying to renew credentials"); ret = krb5_get_renewed_creds(k5.ctx, &my_creds, k5.me, k5.out_cc, nullptr); if (ret) { - LOG_DEBUG(log,"Renew failed ({}). Trying to get initial credentials", ret); + LOG_TRACE(log,"Renew failed ({}). Trying to get initial credentials", ret); ret = krb5_get_init_creds_keytab(k5.ctx, &my_creds, k5.me, keytab, 0, nullptr, options); if (ret) throw Exception("Error in getting initial credentials", ErrorCodes::KERBEROS_ERROR); else - LOG_DEBUG(log,"Got initial credentials"); + LOG_TRACE(log,"Got initial credentials"); } else { - LOG_DEBUG(log,"Successful renewal"); + LOG_TRACE(log,"Successful renewal"); ret = krb5_cc_initialize(k5.ctx, k5.out_cc, k5.me); if (ret) throw Exception("Error when initializing cache", ErrorCodes::KERBEROS_ERROR); - LOG_DEBUG(log,"Initialized cache"); + LOG_TRACE(log,"Initialized cache"); ret = krb5_cc_store_cred(k5.ctx, k5.out_cc, &my_creds); if (ret) - LOG_DEBUG(log,"Error while storing credentials"); - LOG_DEBUG(log,"Stored credentials"); + LOG_TRACE(log,"Error while storing credentials"); + LOG_TRACE(log,"Stored credentials"); } if (k5.switch_to_cache) @@ -172,7 +176,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con throw Exception("Error while switching to new cache", ErrorCodes::KERBEROS_ERROR); } - LOG_DEBUG(log,"Authenticated to Kerberos v5"); + LOG_TRACE(log,"Authenticated to Kerberos v5"); return 0; } From 8288b9fe609485804c5c41f387d8ec3322e5c9be Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 21 Jun 2022 18:47:55 +0200 Subject: [PATCH 037/123] fixes --- src/Databases/DatabaseOrdinary.cpp | 15 ++++++++++++++- tests/queries/0_stateless/replication.lib | 4 ++-- tests/queries/shell_config.sh | 2 +- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 2e4ee785a94..ff5628ecf0f 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -174,7 +174,20 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables { /// Even if we don't load the table we can still mark the uuid of it as taken. if (create_query->uuid != UUIDHelpers::Nil) - DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); + { + /// A bit tricky way to distinguish ATTACH DATABASE and server startup. + if (getContext()->isServerCompletelyStarted()) + { + /// It's ATTACH DATABASE. UUID for permanently detached table must be already locked. + if (!DatabaseCatalog::instance().hasUUIDMapping(create_query->uuid)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create_query->uuid); + } + else + { + /// Server is starting up. Lock UUID used by permanently detached table. + DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); + } + } const std::string table_name = unescapeForFileName(file_name.substr(0, file_name.size() - 4)); LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name)); diff --git a/tests/queries/0_stateless/replication.lib b/tests/queries/0_stateless/replication.lib index 6bf3c35f344..fd32fa28ba0 100755 --- a/tests/queries/0_stateless/replication.lib +++ b/tests/queries/0_stateless/replication.lib @@ -45,8 +45,8 @@ function check_replication_consistency() while [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE current_database=currentDatabase() AND query LIKE '%$table_name_prefix%'") -ne 1 ]]; do sleep 0.5; num_tries=$((num_tries+1)) - if [ $num_tries -eq 100 ]; then - $CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE current_database=currentDatabase() AND query LIKE '%$table_name_prefix%' FORMAT Vertical" + if [ $num_tries -eq 200 ]; then + $CLICKHOUSE_CLIENT -q "SELECT * FROM system.processes WHERE current_database=currentDatabase() AND query LIKE '%$table_name_prefix%' FORMAT Vertical" break fi done diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index 8e9b9c2ac20..e3ab454f356 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -138,7 +138,7 @@ function wait_for_queries_to_finish() sleep 0.5; num_tries=$((num_tries+1)) if [ $num_tries -eq 20 ]; then - $CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE current_database=currentDatabase() AND query NOT LIKE '%system.processes%' FORMAT Vertical" + $CLICKHOUSE_CLIENT -q "SELECT * FROM system.processes WHERE current_database=currentDatabase() AND query NOT LIKE '%system.processes%' FORMAT Vertical" break fi done From dc1f5963265d08f027592ae96fd3976e4c112c6b Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 22 Jun 2022 00:50:16 +0200 Subject: [PATCH 038/123] fix --- src/Databases/DatabaseOrdinary.cpp | 25 +++++---- src/Databases/DatabaseOrdinary.h | 2 +- src/Databases/IDatabase.h | 2 +- src/Databases/TablesLoader.cpp | 2 +- src/Interpreters/DatabaseCatalog.cpp | 57 ++++++++++++++++----- src/Interpreters/DatabaseCatalog.h | 16 ++++++ src/Interpreters/InterpreterCreateQuery.cpp | 2 + 7 files changed, 81 insertions(+), 25 deletions(-) diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index ff5628ecf0f..e44f5e5e628 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -29,6 +29,12 @@ namespace fs = std::filesystem; namespace DB { + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768; namespace @@ -83,7 +89,7 @@ void DatabaseOrdinary::loadStoredObjects( */ ParsedTablesMetadata metadata; - loadTablesMetadata(local_context, metadata); + loadTablesMetadata(local_context, metadata, force_attach); size_t total_tables = metadata.parsed_tables.size() - metadata.total_dictionaries; @@ -151,12 +157,12 @@ void DatabaseOrdinary::loadStoredObjects( } } -void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTablesMetadata & metadata) +void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTablesMetadata & metadata, bool is_startup) { size_t prev_tables_count = metadata.parsed_tables.size(); size_t prev_total_dictionaries = metadata.total_dictionaries; - auto process_metadata = [&metadata, this](const String & file_name) + auto process_metadata = [&metadata, is_startup, this](const String & file_name) { fs::path path(getMetadataPath()); fs::path file_path(file_name); @@ -176,17 +182,16 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables if (create_query->uuid != UUIDHelpers::Nil) { /// A bit tricky way to distinguish ATTACH DATABASE and server startup. - if (getContext()->isServerCompletelyStarted()) - { - /// It's ATTACH DATABASE. UUID for permanently detached table must be already locked. - if (!DatabaseCatalog::instance().hasUUIDMapping(create_query->uuid)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create_query->uuid); - } - else + if (is_startup) { /// Server is starting up. Lock UUID used by permanently detached table. DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); } + else if (!DatabaseCatalog::instance().hasUUIDMapping(create_query->uuid)) + { + /// It's ATTACH DATABASE. UUID for permanently detached table must be already locked. + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create_query->uuid); + } } const std::string table_name = unescapeForFileName(file_name.substr(0, file_name.size() - 4)); diff --git a/src/Databases/DatabaseOrdinary.h b/src/Databases/DatabaseOrdinary.h index 982be2024ce..6e524ae18b0 100644 --- a/src/Databases/DatabaseOrdinary.h +++ b/src/Databases/DatabaseOrdinary.h @@ -25,7 +25,7 @@ public: bool supportsLoadingInTopologicalOrder() const override { return true; } - void loadTablesMetadata(ContextPtr context, ParsedTablesMetadata & metadata) override; + void loadTablesMetadata(ContextPtr context, ParsedTablesMetadata & metadata, bool is_startup) override; void loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast, bool force_restore) override; diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 38c85cf3d05..a81df0a389a 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -148,7 +148,7 @@ public: { } - virtual void loadTablesMetadata(ContextPtr /*local_context*/, ParsedTablesMetadata & /*metadata*/) + virtual void loadTablesMetadata(ContextPtr /*local_context*/, ParsedTablesMetadata & /*metadata*/, bool /*is_startup*/) { throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); } diff --git a/src/Databases/TablesLoader.cpp b/src/Databases/TablesLoader.cpp index 898376f8e0f..e973c9211be 100644 --- a/src/Databases/TablesLoader.cpp +++ b/src/Databases/TablesLoader.cpp @@ -93,7 +93,7 @@ void TablesLoader::loadTables() for (auto & database_name : databases_to_load) { databases[database_name]->beforeLoadingMetadata(global_context, force_restore, force_attach); - databases[database_name]->loadTablesMetadata(global_context, metadata); + databases[database_name]->loadTablesMetadata(global_context, metadata, force_attach); } LOG_INFO(log, "Parsed metadata of {} tables in {} databases in {} sec", diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index dc0d346be92..116cc8aa359 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -155,11 +155,15 @@ void DatabaseCatalog::initializeAndLoadTemporaryDatabase() void DatabaseCatalog::loadDatabases() { - auto cleanup_task_holder = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this](){ this->cleanupStoreDirectoryTask(); }); - cleanup_task = std::make_unique(std::move(cleanup_task_holder)); - (*cleanup_task)->activate(); - /// Do not start task immediately on server startup, it's not urgent. - (*cleanup_task)->scheduleAfter(unused_dir_hide_timeout_sec * 1000); + if (Context::getGlobalContextInstance()->getApplicationType() == Context::ApplicationType::SERVER) + { + auto cleanup_task_holder + = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this]() { this->cleanupStoreDirectoryTask(); }); + cleanup_task = std::make_unique(std::move(cleanup_task_holder)); + (*cleanup_task)->activate(); + /// Do not start task immediately on server startup, it's not urgent. + (*cleanup_task)->scheduleAfter(unused_dir_hide_timeout_sec * 1000); + } auto task_holder = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this](){ this->dropTableDataTask(); }); drop_task = std::make_unique(std::move(task_holder)); @@ -387,8 +391,6 @@ DatabasePtr DatabaseCatalog::detachDatabase(ContextPtr local_context, const Stri if (drop) { UUID db_uuid = db->getUUID(); - if (db_uuid != UUIDHelpers::Nil) - removeUUIDMappingFinally(db_uuid); /// Delete the database. db->drop(local_context); @@ -399,6 +401,9 @@ DatabasePtr DatabaseCatalog::detachDatabase(ContextPtr local_context, const Stri fs::remove(database_metadata_dir); fs::path database_metadata_file = fs::path(getContext()->getPath()) / "metadata" / (escapeForFileName(database_name) + ".sql"); fs::remove(database_metadata_file); + + if (db_uuid != UUIDHelpers::Nil) + removeUUIDMappingFinally(db_uuid); } return db; @@ -583,8 +588,7 @@ bool DatabaseCatalog::hasUUIDMapping(const UUID & uuid) assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; std::lock_guard lock{map_part.mutex}; - auto it = map_part.map.find(uuid); - return it != map_part.map.end(); + return map_part.map.contains(uuid); } std::unique_ptr DatabaseCatalog::database_catalog; @@ -1115,7 +1119,7 @@ void DatabaseCatalog::cleanupStoreDirectoryTask() if (!expected_prefix_dir) { LOG_WARNING(log, "Found invalid directory {}, will try to remove it", prefix_dir.path().string()); - maybeRemoveDirectory(prefix_dir.path()); + affected_dirs += maybeRemoveDirectory(prefix_dir.path()); continue; } @@ -1133,11 +1137,12 @@ void DatabaseCatalog::cleanupStoreDirectoryTask() if (!expected_dir) { LOG_WARNING(log, "Found invalid directory {}, will try to remove it", uuid_dir.path().string()); - maybeRemoveDirectory(uuid_dir.path()); + affected_dirs += maybeRemoveDirectory(uuid_dir.path()); continue; } - if (!hasUUIDMapping(uuid)) + /// Order is important + if (!isProtectedUUIDDir(uuid) && !hasUUIDMapping(uuid)) { /// We load uuids even for detached and permanently detached tables, /// so it looks safe enough to remove directory if we don't have uuid mapping for it. @@ -1214,6 +1219,34 @@ bool DatabaseCatalog::maybeRemoveDirectory(const fs::path & unused_dir) } } +void DatabaseCatalog::addProtectedUUIDDir(const UUID & uuid) +{ + if (uuid == UUIDHelpers::Nil) + return; + std::lock_guard lock{protected_uuid_dirs_mutex}; + bool inserted = protected_uuid_dirs.insert(uuid).second; + if (inserted) + return; + + throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Mapping for table with UUID={} already exists. It happened due to UUID collision, " + "most likely because some not random UUIDs were manually specified in CREATE queries.", toString(uuid)); +} + +void DatabaseCatalog::removeProtectedUUIDDir(const UUID & uuid) +{ + if (uuid == UUIDHelpers::Nil) + return; + std::lock_guard lock{protected_uuid_dirs_mutex}; + chassert(protected_uuid_dirs.contains(uuid)); + protected_uuid_dirs.erase(uuid); +} + +bool DatabaseCatalog::isProtectedUUIDDir(const UUID & uuid) +{ + std::lock_guard lock{protected_uuid_dirs_mutex}; + return protected_uuid_dirs.contains(uuid); +} + DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock guards_lock_, const String & elem, const String & database_name) : map(map_), db_mutex(db_mutex_), guards_lock(std::move(guards_lock_)) diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 9ca0bb276d6..7dde0643dd9 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -207,6 +207,10 @@ public: bool hasUUIDMapping(const UUID & uuid); + void addProtectedUUIDDir(const UUID & uuid); + void removeProtectedUUIDDir(const UUID & uuid); + bool isProtectedUUIDDir(const UUID & uuid); + static String getPathForUUID(const UUID & uuid); DatabaseAndTable tryGetByUUID(const UUID & uuid) const; @@ -301,6 +305,9 @@ private: std::unordered_set tables_marked_dropped_ids; mutable std::mutex tables_marked_dropped_mutex; + std::unordered_set protected_uuid_dirs; + mutable std::mutex protected_uuid_dirs_mutex; + std::unique_ptr drop_task; static constexpr time_t default_drop_delay_sec = 8 * 60; time_t drop_delay_sec = default_drop_delay_sec; @@ -315,4 +322,13 @@ private: time_t unused_dir_cleanup_period_sec = default_unused_dir_cleanup_period_sec; }; + +class UUIDDirectoryProtector : private boost::noncopyable +{ + UUID uuid; +public: + UUIDDirectoryProtector(UUID uuid_) : uuid(uuid_) { DatabaseCatalog::instance().addProtectedUUIDDir(uuid); } + ~UUIDDirectoryProtector() { DatabaseCatalog::instance().removeProtectedUUIDDir(uuid); } +}; + } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 9e745ffcabe..20ebf96e617 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -249,6 +249,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) "Enable allow_experimental_database_materialized_postgresql to use it.", ErrorCodes::UNKNOWN_DATABASE_ENGINE); } + UUIDDirectoryProtector uuid_lock{create.uuid}; DatabasePtr database = DatabaseFactory::get(create, metadata_path / "", getContext()); if (create.uuid != UUIDHelpers::Nil) @@ -1269,6 +1270,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, database->checkDetachedTableNotInUse(create.uuid); } + UUIDDirectoryProtector uuid_lock{create.uuid}; StoragePtr res; /// NOTE: CREATE query may be rewritten by Storage creator or table function if (create.as_table_function) From f2811995887893e800bea3d0fe87cbd89d187d27 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 22 Jun 2022 11:28:00 +0300 Subject: [PATCH 039/123] Fix code style in KerberosInit; Add anonymous namespace; Add comment about using kerberos_init --- src/Access/KerberosInit.cpp | 9 ++++++--- src/Access/examples/kerberos_init.cpp | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index e78830bdaa5..435a8126b05 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -19,7 +19,9 @@ namespace ErrorCodes } } -struct k5_data +namespace +{ +struct K5Data { krb5_context ctx; krb5_ccache in_cc, out_cc; @@ -37,17 +39,18 @@ public: int init(const String & keytab_file, const String & principal, const String & cache_name = ""); ~KerberosInit(); private: - struct k5_data k5; + struct K5Data k5; krb5_ccache defcache = nullptr; krb5_get_init_creds_opt * options = nullptr; krb5_creds my_creds; krb5_keytab keytab = nullptr; krb5_principal defcache_princ = nullptr; }; +} int KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) { - auto log = &Poco::Logger::get("KerberosInit"); + auto * log = &Poco::Logger::get("KerberosInit"); LOG_TRACE(log,"Trying to authenticate to Kerberos v5"); krb5_error_code ret; diff --git a/src/Access/examples/kerberos_init.cpp b/src/Access/examples/kerberos_init.cpp index 4868182fba2..5dbe92a5b57 100644 --- a/src/Access/examples/kerberos_init.cpp +++ b/src/Access/examples/kerberos_init.cpp @@ -5,6 +5,13 @@ #include #include +/** The example demonstrates using of kerberosInit function to obtain and cache Kerberos ticket-granting ticket. + * The first argument specifies keytab file. The second argument specifies principal name. + * The third argument is optional. It specifies credentials cache location. + * After successful run of kerberos_init it is useful to call klist command to list cached Kerberos tickets. + * It is also useful to run kdestroy to destroy Kerberos tickets if needed. + */ + using namespace DB; int main(int argc, char ** argv) From a7ff956c83fc9c955d8d3b17a18e1a270db3b8f4 Mon Sep 17 00:00:00 2001 From: zvonand Date: Wed, 22 Jun 2022 14:56:37 +0500 Subject: [PATCH 040/123] updated docs --- .../functions/string-functions.md | 42 ++++++++++++++++++ .../functions/encoding-functions.md | 44 ------------------- .../functions/string-functions.md | 42 ++++++++++++++++++ 3 files changed, 84 insertions(+), 44 deletions(-) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 66e9aa98e67..7c6ae903acf 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -494,6 +494,48 @@ If the ‘s’ string is non-empty and does not contain the ‘c’ character at Returns the string ‘s’ that was converted from the encoding in ‘from’ to the encoding in ‘to’. +## Base58Encode(plaintext[, alphabet_name]), Base58Decode(encoded_text[, alphabet_name]) + +Accepts a String and encodes/decodes it using [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) encoding scheme using specified alphabet. + +**Syntax** + +```sql +base58Encode(decoded[, alphabet_name]) +base58Decode(encoded[, alphabet_name]) +``` + +**Arguments** + +- `decoded` — [String](../../sql-reference/data-types/string.md) column or constant. +- `encoded` — [String](../../sql-reference/data-types/string.md) column or constant. If the string is not a valid base58-encoded value, an exception is thrown. +- `alphabet_name` — String constant. Specifies alphabet used for encoding. Possible values: `gmp`, `bitcoin`, `ripple`, `flickr`. Default: `gmp`. + +**Returned value** + +- A string containing encoded/decoded value of 1st argument. + +Type: [String](../../sql-reference/data-types/string.md). + +**Example** + +Query: + +``` sql +SELECT base58Encode('encode', 'flickr'); +SELECT base58Decode('izCFiDUY', 'ripple'); +``` + +Result: +```text +┌─base58Encode('encode', 'flickr')─┐ +│ SvyTHb1D │ +└──────────────────────────────────┘ +┌─base58Decode('izCFiDUY', 'ripple')─┐ +│ decode │ +└────────────────────────────────────┘ +``` + ## base64Encode(s) Encodes ‘s’ string into base64 diff --git a/docs/ru/sql-reference/functions/encoding-functions.md b/docs/ru/sql-reference/functions/encoding-functions.md index 255985fcc92..65d2b0e6538 100644 --- a/docs/ru/sql-reference/functions/encoding-functions.md +++ b/docs/ru/sql-reference/functions/encoding-functions.md @@ -404,47 +404,3 @@ SELECT bitPositionsToArray(toInt8(-1)) AS bit_positions; │ [0,1,2,3,4,5,6,7] │ └───────────────────┘ ``` - -## Base58Encode(plaintext[, alphabet_name]) -## Base58Decode(plaintext[, alphabet_name]) - -Принимает на вход строку или колонку строк и кодирует/раскодирует их с помощью схемы кодирования [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) с использованием указанного алфавита. - -**Синтаксис** - -```sql -base58Encode(decoded[, alphabet_name]) -base58Decode(encoded[, alphabet_name]) -``` - -**Аргументы** - -- `decoded` — Колонка или строка типа [String](../../sql-reference/data-types/string.md). -- `encoded` — Колонка или строка типа [String](../../sql-reference/data-types/string.md). Если входная строка не является корректным кодом для какой-либо другой строки, возникнет исключение `1001`. -- `alphabet_name` — Строковая константа. Указывает алфавит, для которого необходимо получить код. Может принимать одно из следующих значений: `gmp`, `bitcoin`, `ripple`, `flickr`. По умолчанию: `gmp`. - -**Возвращаемое значение** - -- Строка, содержащая раскодированный/закодированный первый аргумент. - -Тип: [String](../../sql-reference/data-types/string.md). - -**Пример:** - -Запрос: - -``` sql -SELECT base58Encode('encode', 'flickr'); -SELECT base58Decode('izCFiDUY', 'ripple'); -``` - -Результат: -```text -┌─base58Encode('encode', 'flickr')─┐ -│ SvyTHb1D │ -└──────────────────────────────────┘ -┌─base58Decode('izCFiDUY', 'ripple')─┐ -│ decode │ -└────────────────────────────────────┘ -``` - diff --git a/docs/ru/sql-reference/functions/string-functions.md b/docs/ru/sql-reference/functions/string-functions.md index 6cd4cea403e..e85a97e0099 100644 --- a/docs/ru/sql-reference/functions/string-functions.md +++ b/docs/ru/sql-reference/functions/string-functions.md @@ -490,6 +490,48 @@ SELECT concat(key1, key2), sum(value) FROM key_val GROUP BY (key1, key2); Возвращает сконвертированную из кодировки from в кодировку to строку s. +## Base58Encode(plaintext[, alphabet_name]), Base58Decode(plaintext[, alphabet_name]) {#base58} + +Принимает на вход строку или колонку строк и кодирует/раскодирует их с помощью схемы кодирования [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) с использованием указанного алфавита. + +**Синтаксис** + +```sql +base58Encode(decoded[, alphabet_name]) +base58Decode(encoded[, alphabet_name]) +``` + +**Аргументы** + +- `decoded` — Колонка или строка типа [String](../../sql-reference/data-types/string.md). +- `encoded` — Колонка или строка типа [String](../../sql-reference/data-types/string.md). Если входная строка не является корректным кодом для какой-либо другой строки, возникнет исключение `1001`. +- `alphabet_name` — Строковая константа. Указывает алфавит, для которого необходимо получить код. Может принимать одно из следующих значений: `gmp`, `bitcoin`, `ripple`, `flickr`. По умолчанию: `gmp`. + +**Возвращаемое значение** + +- Строка, содержащая раскодированный/закодированный первый аргумент. + +Тип: [String](../../sql-reference/data-types/string.md). + +**Пример:** + +Запрос: + +``` sql +SELECT base58Encode('encode', 'flickr'); +SELECT base58Decode('izCFiDUY', 'ripple'); +``` + +Результат: +```text +┌─base58Encode('encode', 'flickr')─┐ +│ SvyTHb1D │ +└──────────────────────────────────┘ +┌─base58Decode('izCFiDUY', 'ripple')─┐ +│ decode │ +└────────────────────────────────────┘ +``` + ## base64Encode(s) {#base64encode} Производит кодирование строки s в base64-представление. From 7ae7b1e421d26ccdd491fb9b8379eb0df7610afe Mon Sep 17 00:00:00 2001 From: zvonand Date: Wed, 22 Jun 2022 14:58:39 +0500 Subject: [PATCH 041/123] fix wrong docs location --- .../functions/encoding-functions.md | 43 ------------------- 1 file changed, 43 deletions(-) diff --git a/docs/en/sql-reference/functions/encoding-functions.md b/docs/en/sql-reference/functions/encoding-functions.md index 42a6d75952c..4ee71267a09 100644 --- a/docs/en/sql-reference/functions/encoding-functions.md +++ b/docs/en/sql-reference/functions/encoding-functions.md @@ -440,46 +440,3 @@ Result: │ [0,1,2,3,4,5,6,7] │ └───────────────────┘ ``` - -## Base58Encode(plaintext[, alphabet_name]) -## Base58Decode(plaintext[, alphabet_name]) - -Accepts a String and encodes/decodes it using [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) encoding scheme using specified alphabet. - -**Syntax** - -```sql -base58Encode(decoded[, alphabet_name]) -base58Decode(encoded[, alphabet_name]) -``` - -**Arguments** - -- `decoded` — [String](../../sql-reference/data-types/string.md) column or constant. -- `encoded` — [String](../../sql-reference/data-types/string.md) column or constant. If the string is not a valid base58-encoded value, `1001 Exception` will be thrown. -- `alphabet_name` — String constant. Specifies alphabet used for encoding. Possible values: `gmp`, `bitcoin`, `ripple`, `flickr`. Default: `gmp`. - -**Returned value** - -- A string containing encoded/decoded value of 1st argument. - -Type: [String](../../sql-reference/data-types/string.md). - -**Example** - -Query: - -``` sql -SELECT base58Encode('encode', 'flickr'); -SELECT base58Decode('izCFiDUY', 'ripple'); -``` - -Result: -```text -┌─base58Encode('encode', 'flickr')─┐ -│ SvyTHb1D │ -└──────────────────────────────────┘ -┌─base58Decode('izCFiDUY', 'ripple')─┐ -│ decode │ -└────────────────────────────────────┘ -``` From 7bd65c8c24b444d23514758a27269b614560a96c Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Wed, 22 Jun 2022 16:31:48 +0300 Subject: [PATCH 042/123] Add comments to KerberosInit; Remove input cache and flags from KerberosInit --- src/Access/KerberosInit.cpp | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 435a8126b05..d3be559cba7 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -24,7 +24,7 @@ namespace struct K5Data { krb5_context ctx; - krb5_ccache in_cc, out_cc; + krb5_ccache out_cc; krb5_principal me; char * name; krb5_boolean switch_to_cache; @@ -42,6 +42,7 @@ private: struct K5Data k5; krb5_ccache defcache = nullptr; krb5_get_init_creds_opt * options = nullptr; + // Credentials structure including ticket, session key, and lifetime info. krb5_creds my_creds; krb5_keytab keytab = nullptr; krb5_principal defcache_princ = nullptr; @@ -56,7 +57,6 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con krb5_error_code ret; const char *deftype = nullptr; - int flags = 0; if (!std::filesystem::exists(keytab_file)) throw Exception("Keytab file does not exist", ErrorCodes::KERBEROS_ERROR); @@ -86,7 +86,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con } // Use the specified principal name. - ret = krb5_parse_name_flags(k5.ctx, principal.c_str(), flags, &k5.me); + ret = krb5_parse_name_flags(k5.ctx, principal.c_str(), 0, &k5.me); if (ret) throw Exception("Error when parsing principal name " + principal, ErrorCodes::KERBEROS_ERROR); @@ -126,7 +126,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con throw Exception("Error when unparsing name", ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Using principal: {}", k5.name); - memset(&my_creds, 0, sizeof(my_creds)); + // Allocate a new initial credential options structure. ret = krb5_get_init_creds_opt_alloc(k5.ctx, &options); if (ret) throw Exception("Error in options allocation", ErrorCodes::KERBEROS_ERROR); @@ -137,22 +137,20 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con throw Exception("Error in resolving keytab "+keytab_file, ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Using keytab: {}", keytab_file); - if (k5.in_cc) - { - ret = krb5_get_init_creds_opt_set_in_ccache(k5.ctx, options, k5.in_cc); - if (ret) - throw Exception("Error in setting input credential cache", ErrorCodes::KERBEROS_ERROR); - } + // Set an output credential cache in initial credential options. ret = krb5_get_init_creds_opt_set_out_ccache(k5.ctx, options, k5.out_cc); if (ret) throw Exception("Error in setting output credential cache", ErrorCodes::KERBEROS_ERROR); // Action: init or renew LOG_TRACE(log,"Trying to renew credentials"); + memset(&my_creds, 0, sizeof(my_creds)); + // Get renewed credential from KDC using an existing credential from output cache. ret = krb5_get_renewed_creds(k5.ctx, &my_creds, k5.me, k5.out_cc, nullptr); if (ret) { LOG_TRACE(log,"Renew failed ({}). Trying to get initial credentials", ret); + // Request KDC for an initial credentials using keytab. ret = krb5_get_init_creds_keytab(k5.ctx, &my_creds, k5.me, keytab, 0, nullptr, options); if (ret) throw Exception("Error in getting initial credentials", ErrorCodes::KERBEROS_ERROR); @@ -162,10 +160,12 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con else { LOG_TRACE(log,"Successful renewal"); + // Initialize a credential cache. Destroy any existing contents of cache and initialize it for the default principal. ret = krb5_cc_initialize(k5.ctx, k5.out_cc, k5.me); if (ret) throw Exception("Error when initializing cache", ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Initialized cache"); + // Store credentials in a credential cache. ret = krb5_cc_store_cred(k5.ctx, k5.out_cc, &my_creds); if (ret) LOG_TRACE(log,"Error while storing credentials"); @@ -174,6 +174,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con if (k5.switch_to_cache) { + // Make a credential cache the primary cache for its collection. ret = krb5_cc_switch(k5.ctx, k5.out_cc); if (ret) throw Exception("Error while switching to new cache", ErrorCodes::KERBEROS_ERROR); @@ -201,8 +202,6 @@ KerberosInit::~KerberosInit() krb5_free_unparsed_name(k5.ctx, k5.name); krb5_free_principal(k5.ctx, k5.me); - if (k5.in_cc != nullptr) - krb5_cc_close(k5.ctx, k5.in_cc); if (k5.out_cc != nullptr) krb5_cc_close(k5.ctx, k5.out_cc); krb5_free_context(k5.ctx); From 3544fbe5c49b13f81e5a1870b75b3950ab5f2647 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 22 Jun 2022 14:05:20 +0000 Subject: [PATCH 043/123] cleanup --- src/Common/OvercommitTracker.cpp | 25 +++++++++++++------------ src/Common/OvercommitTracker.h | 1 - 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp index 74235748345..4faed833428 100644 --- a/src/Common/OvercommitTracker.cpp +++ b/src/Common/OvercommitTracker.cpp @@ -25,18 +25,19 @@ OvercommitTracker::OvercommitTracker(std::mutex & global_mutex_) , allow_release(true) {} -#define LOG_DEBUG_SAFE(...) \ - do { \ - OvercommitTrackerBlockerInThread blocker; \ - try \ - { \ - ALLOW_ALLOCATIONS_IN_SCOPE; \ - LOG_DEBUG(__VA_ARGS__); \ - } \ - catch (std::bad_alloc const &) \ - { \ - fprintf(stderr, "Allocation failed during writing to log in OvercommitTracker\n"); \ - } \ +#define LOG_DEBUG_SAFE(...) \ + do { \ + OvercommitTrackerBlockerInThread blocker; \ + try \ + { \ + ALLOW_ALLOCATIONS_IN_SCOPE; \ + LOG_DEBUG(__VA_ARGS__); \ + } \ + catch (...) \ + { \ + if (fprintf(stderr, "Allocation failed during writing to log in OvercommitTracker\n") != -1) \ + ; \ + } \ } while (false) OvercommitResult OvercommitTracker::needToStopQuery(MemoryTracker * tracker, Int64 amount) diff --git a/src/Common/OvercommitTracker.h b/src/Common/OvercommitTracker.h index 6a03c679422..6f6788493a9 100644 --- a/src/Common/OvercommitTracker.h +++ b/src/Common/OvercommitTracker.h @@ -7,7 +7,6 @@ #include #include #include -#include #include #include From d553b30f808d2adb0ec58242769d1ac3e79d6128 Mon Sep 17 00:00:00 2001 From: Laurie Li Date: Wed, 22 Jun 2022 23:28:25 +0800 Subject: [PATCH 044/123] Update replication.md of zh docs docs/zh/engines/table-engines/mergetree-family/replication.md Signed-off-by: Laurie Li --- .../mergetree-family/replication.md | 100 ++++++++++++++++-- 1 file changed, 89 insertions(+), 11 deletions(-) diff --git a/docs/zh/engines/table-engines/mergetree-family/replication.md b/docs/zh/engines/table-engines/mergetree-family/replication.md index c3be3a382cb..568484c6256 100644 --- a/docs/zh/engines/table-engines/mergetree-family/replication.md +++ b/docs/zh/engines/table-engines/mergetree-family/replication.md @@ -18,11 +18,19 @@ 而 `CREATE`，`DROP`，`ATTACH`，`DETACH` 和 `RENAME` 语句只会在单个服务器上执行，不会被复制。 -- `The CREATE TABLE` 在运行此语句的服务器上创建一个新的可复制表。如果此表已存在其他服务器上，则给该表添加新副本。 -- `The DROP TABLE` 删除运行此查询的服务器上的副本。 -- `The RENAME` 重命名一个副本。换句话说，可复制表不同的副本可以有不同的名称。 +- `CREATE TABLE` 在运行此语句的服务器上创建一个新的可复制表。如果此表已存在其他服务器上，则给该表添加新副本。 +- `DROP TABLE` 删除运行此查询的服务器上的副本。 +- `RENAME` 重命名一个副本。换句话说，可复制表不同的副本可以有不同的名称。 -要使用副本，需在配置文件中设置 ZooKeeper 集群的地址。例如： +ClickHouse 使用 [Apache ZooKeeper](https://zookeeper.apache.org) 存储副本的元信息。请使用 ZooKeeper 3.4.5 或更高版本。 + +要使用副本，需在 [Zookeeper](../../../operations/server-configuration-parameters/settings.md#server-settings_zookeeper) 服务器的配置部分设置相应参数。 + +:::warning +不要忽视安全设置。 ClickHouse 支持 ZooKeeper 安全子系统中的 `digest` [ACL 方案](https://zookeeper.apache.org/doc/current/zookeeperProgrammers.html#sc_ZooKeeperAccessControl) 。 +::: + +Zookeeper 集群地址的设置样例如下： ``` xml @@ -41,7 +49,41 @@ ``` -需要 ZooKeeper 3.4.5 或更高版本。 +通过以引擎参数的形式提供 ZooKeeper 集群的名称和路径，ClickHouse 同样支持将副本的元信息存储在备用 ZooKeeper 集群上。也就是说，支持将不同数据表的元数据存储在不同的 ZooKeeper 集群上。 + +设置备用 ZooKeeper 集群地址的样例如下： + +``` xml + + + + example_2_1 + 2181 + + + example_2_2 + 2181 + + + example_2_3 + 2181 + + + + + example_3_1 + 2181 + + + +``` + +为了将数据表的元数据存储到备用 ZooKeeper 集群而非默认 ZooKeeper 集群，我们可以通过如下 SQL 的方式创建使用 +ReplicatedMergeTree 引擎的数据表： + +``` +CREATE TABLE table_name ( ... ) ENGINE = ReplicatedMergeTree('zookeeper_name_configured_in_auxiliary_zookeepers:path', 'replica_name') ... +``` 你可以配置任何现有的 ZooKeeper 集群，系统会使用里面的目录来存取元数据（该目录在创建可复制表时指定）。 @@ -53,7 +95,9 @@ 对于非常大的集群，你可以把不同的 ZooKeeper 集群用于不同的分片。然而，即使 Yandex.Metrica 集群（大约300台服务器）也证明还不需要这么做。 -复制是多主异步。 `INSERT` 语句（以及 `ALTER` ）可以发给任意可用的服务器。数据会先插入到执行该语句的服务器上，然后被复制到其他服务器。由于它是异步的，在其他副本上最近插入的数据会有一些延迟。如果部分副本不可用，则数据在其可用时再写入。副本可用的情况下，则延迟时长是通过网络传输压缩数据块所需的时间。 +复制是多主异步。 `INSERT` 语句（以及 `ALTER` ）可以发给任意可用的服务器。数据会先插入到执行该语句的服务器上，然后被复制到其他服务器。由于它是异步的，在其他副本上最近插入的数据会有一些延迟。如果部分副本不可用，则数据在其可用时再写入。副本可用的情况下，则延迟时长是通过网络传输压缩数据块所需的时间。为复制表执行后台任务的线程数量，可以通过 [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) 进行设置。 + +`ReplicatedMergeTree` 引擎采用一个独立的线程池进行复制拉取。线程池的大小通过 [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) 进行限定，它可以在重启服务器时进行调整。 默认情况下，INSERT 语句仅等待一个副本写入成功后返回。如果数据只成功写入一个副本后该副本所在的服务器不再存在，则存储的数据会丢失。要启用数据写入多个副本才确认返回，使用 `insert_quorum` 选项。 @@ -75,6 +119,7 @@ - `zoo_path` — ZooKeeper 中该表的路径。 - `replica_name` — ZooKeeper 中的该表的副本名称。 +- `other_parameters` — 关于引擎的一系列参数，这个引擎即是用来创建复制的引擎，例如，`ReplacingMergeTree` 。 示例: @@ -90,7 +135,9 @@ ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) ``` -已弃用的建表语法示例： +
+ +已弃用的建表语法示例： ``` sql CREATE TABLE table_name @@ -101,17 +148,19 @@ CREATE TABLE table_name ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192) ``` +
+ 如上例所示，这些参数可以包含宏替换的占位符，即大括号的部分。它们会被替换为配置文件里 ‘macros’ 那部分配置的值。示例： ``` xml 05 02 - example05-02-1.yandex.ru + example05-02-1 ``` -«ZooKeeper 中该表的路径»对每个可复制表都要是唯一的。不同分片上的表要有不同的路径。 +ZooKeeper 中该表的路径对每个可复制表都要是唯一的。不同分片上的表要有不同的路径。 这种情况下，路径包含下面这些部分： `/clickhouse/tables/` 是公共前缀，我们推荐使用这个。 @@ -119,7 +168,11 @@ CREATE TABLE table_name `{layer}-{shard}` 是分片标识部分。在此示例中，由于 Yandex.Metrica 集群使用了两级分片，所以它是由两部分组成的。但对于大多数情况来说，你只需保留 {shard} 占位符即可，它会替换展开为分片标识。 `table_name` 是该表在 ZooKeeper 中的名称。使其与 ClickHouse 中的表名相同比较好。 这里它被明确定义，跟 ClickHouse 表名不一样，它并不会被 RENAME 语句修改。 -*HINT*：你可以在前面添加一个数据库名称 `table_name` 也是 例如。 `db_name.table_name` +*HINT*：你也可以在 `table_name` 前面添加一个数据库名称。例如： `db_name.table_name` 。 + +两个内置的占位符 `{database}` 和 `{table}` 也可使用，它们可以展开成数据表名称和数据库名称（只有当这些宏指令在 `macros` 部分已经定义时才可以）。因此 ZooKeeper 路径可以指定为 `'/clickhouse/tables/{layer}-{shard}/{database}/{table}'` 。 + +当使用这些内置占位符时，请当心数据表重命名。 ZooKeeper 中的路径无法变更，而当数据表被重命名时，宏命令将展开为另一个路径，数据表将指向一个 ZooKeeper 上并不存在的路径，并因此转变为只读模式。 副本名称用于标识同一个表分片的不同副本。你可以使用服务器名称，如上例所示。同个分片中不同副本的副本名称要唯一。 @@ -127,6 +180,31 @@ CREATE TABLE table_name 使用大型集群时，我们建议使用宏替换，因为它可以降低出错的可能性。 +你可以在服务器的配置文件中指定 `Replicated` 数据表引擎的默认参数。例如： + +```xml +/clickhouse/tables/{shard}/{database}/{table} +{replica} +``` + +这样，你可以在建表时省略参数： + +``` sql +CREATE TABLE table_name ( + x UInt32 +) ENGINE = ReplicatedMergeTree +ORDER BY x; +``` + +它等价于： + +``` sql +CREATE TABLE table_name ( + x UInt32 +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/table_name', '{replica}') +ORDER BY x; +``` + 在每个副本服务器上运行 `CREATE TABLE` 查询。将创建新的复制表，或给现有表添加新副本。 如果其他副本上已包含了某些数据，在表上添加新副本，则在运行语句后，数据会从其他副本复制到新副本。换句话说，新副本会与其他副本同步。 @@ -164,7 +242,7 @@ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data 1. 在服务器上安装 ClickHouse。在包含分片标识符和副本的配置文件中正确定义宏配置，如果有用到的话， 2. 如果服务器上有非复制表则必须手动复制，可以从副本服务器上（在 `/var/lib/clickhouse/data/db_name/table_name/` 目录中）复制它们的数据。 3. 从副本服务器上中复制位于 `/var/lib/clickhouse/metadata/` 中的表定义信息。如果在表定义信息中显式指定了分片或副本标识符，请更正它以使其对应于该副本。（另外，启动服务器，然后会在 `/var/lib/clickhouse/metadata/` 中的.sql文件中生成所有的 `ATTACH TABLE` 语句。） - 4.要开始恢复，ZooKeeper 中创建节点 `/path_to_table/replica_name/flags/force_restore_data`，节点内容不限，或运行命令来恢复所有复制的表：`sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data` +4. 要开始恢复，ZooKeeper 中创建节点 `/path_to_table/replica_name/flags/force_restore_data`，节点内容不限，或运行命令来恢复所有复制的表：`sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data` 然后启动服务器（如果它已运行则重启）。数据会从副本中下载。 From ed8341025b49b31a674a27e58b1837a50bb0ba59 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 22 Jun 2022 18:31:42 +0200 Subject: [PATCH 045/123] make code less bad --- src/Databases/DatabaseAtomic.cpp | 7 +- src/Databases/DatabaseOnDisk.cpp | 12 ++ src/Databases/DatabaseOrdinary.cpp | 32 ++-- src/Interpreters/DatabaseCatalog.cpp | 97 ++++++++---- src/Interpreters/DatabaseCatalog.h | 30 ++-- src/Interpreters/InterpreterCreateQuery.cpp | 156 ++++++++++--------- src/Storages/System/attachSystemTablesImpl.h | 1 + 7 files changed, 188 insertions(+), 147 deletions(-) diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index ea887c84111..a4fa1fa267b 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -292,7 +292,6 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora { DetachedTables not_in_use; auto table_data_path = getTableDataPath(query); - bool locked_uuid = false; try { std::unique_lock lock{mutex}; @@ -302,9 +301,7 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora /// Do some checks before renaming file from .tmp to .sql not_in_use = cleanupDetachedTables(); assertDetachedTableNotInUse(query.uuid); - /// We will get en exception if some table with the same UUID exists (even if it's detached table or table from another database) - DatabaseCatalog::instance().addUUIDMapping(query.uuid); - locked_uuid = true; + chassert(DatabaseCatalog::instance().hasUUIDMapping(query.uuid)); auto txn = query_context->getZooKeeperMetadataTransaction(); if (txn && !query_context->isInternalSubquery()) @@ -321,8 +318,6 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora catch (...) { fs::remove(table_metadata_tmp_path); - if (locked_uuid) - DatabaseCatalog::instance().removeUUIDMappingFinally(query.uuid); throw; } if (table->storesDataOnDisk()) diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 9484da8ec2d..64bc9a4a094 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -395,6 +395,18 @@ void DatabaseOnDisk::renameTable( if (auto * target_db = dynamic_cast(&to_database)) target_db->checkMetadataFilenameAvailability(to_table_name); + /// This place is actually quite dangerous. Since data directory is moved to store/ + /// DatabaseCatalog may try to clean it up as unused. We add UUID mapping to avoid this. + /// However, we may fail after data directory move, but before metadata file creation in the destination db. + /// In this case nothing will protect data directory (except 30-days timeout). + /// But this situation (when table in Ordinary database is partially renamed) require manual intervention anyway. + if (from_ordinary_to_atomic) + { + DatabaseCatalog::instance().addUUIDMapping(create.uuid); + if (table->storesDataOnDisk()) + LOG_INFO(log, "Moving table from {} to {}", table_data_relative_path, to_database.getTableDataPath(create)); + } + /// Notify the table that it is renamed. It will move data to new path (if it stores data on disk) and update StorageID table->rename(to_database.getTableDataPath(create), StorageID(create)); } diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index e44f5e5e628..0bdabf7ef79 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -176,24 +176,24 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables auto * create_query = ast->as(); create_query->setDatabase(database_name); + /// Even if we don't load the table we can still mark the uuid of it as taken. + if (create_query->uuid != UUIDHelpers::Nil) + { + /// A bit tricky way to distinguish ATTACH DATABASE and server startup (actually it's "force_attach" flag). + if (is_startup) + { + /// Server is starting up. Lock UUID used by permanently detached table. + DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); + } + else if (!DatabaseCatalog::instance().hasUUIDMapping(create_query->uuid)) + { + /// It's ATTACH DATABASE. UUID for permanently detached table must be already locked. + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create_query->uuid); + } + } + if (fs::exists(full_path.string() + detached_suffix)) { - /// Even if we don't load the table we can still mark the uuid of it as taken. - if (create_query->uuid != UUIDHelpers::Nil) - { - /// A bit tricky way to distinguish ATTACH DATABASE and server startup. - if (is_startup) - { - /// Server is starting up. Lock UUID used by permanently detached table. - DatabaseCatalog::instance().addUUIDMapping(create_query->uuid); - } - else if (!DatabaseCatalog::instance().hasUUIDMapping(create_query->uuid)) - { - /// It's ATTACH DATABASE. UUID for permanently detached table must be already locked. - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create_query->uuid); - } - } - const std::string table_name = unescapeForFileName(file_name.substr(0, file_name.size() - 4)); LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name)); return; diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 116cc8aa359..159003138c2 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -79,6 +80,7 @@ TemporaryTableHolder::TemporaryTableHolder(ContextPtr context_, const TemporaryT } auto table_id = StorageID(DatabaseCatalog::TEMPORARY_DATABASE, global_name, id); auto table = creator(table_id); + DatabaseCatalog::instance().addUUIDMapping(id); temporary_tables->createTable(getContext(), global_name, table, original_create); table->startup(); } @@ -207,19 +209,25 @@ void DatabaseCatalog::shutdownImpl() tables_marked_dropped.clear(); std::lock_guard lock(databases_mutex); + for (const auto & db : databases) + { + UUID db_uuid = db.second->getUUID(); + if (db_uuid != UUIDHelpers::Nil) + removeUUIDMapping(db_uuid); + } assert(std::find_if(uuid_map.begin(), uuid_map.end(), [](const auto & elem) { /// Ensure that all UUID mappings are empty (i.e. all mappings contain nullptr instead of a pointer to storage) const auto & not_empty_mapping = [] (const auto & mapping) { + auto & db = mapping.second.first; auto & table = mapping.second.second; - return table; + return db || table; }; auto it = std::find_if(elem.map.begin(), elem.map.end(), not_empty_mapping); return it != elem.map.end(); }) == uuid_map.end()); databases.clear(); - db_uuid_map.clear(); view_dependencies.clear(); } @@ -349,9 +357,10 @@ void DatabaseCatalog::attachDatabase(const String & database_name, const Databas std::lock_guard lock{databases_mutex}; assertDatabaseDoesntExistUnlocked(database_name); databases.emplace(database_name, database); + NOEXCEPT_SCOPE; UUID db_uuid = database->getUUID(); if (db_uuid != UUIDHelpers::Nil) - db_uuid_map.emplace(db_uuid, database); + addUUIDMapping(db_uuid, database, nullptr); } @@ -365,7 +374,9 @@ DatabasePtr DatabaseCatalog::detachDatabase(ContextPtr local_context, const Stri std::lock_guard lock{databases_mutex}; assertDatabaseExistsUnlocked(database_name); db = databases.find(database_name)->second; - db_uuid_map.erase(db->getUUID()); + UUID db_uuid = db->getUUID(); + if (db_uuid != UUIDHelpers::Nil) + removeUUIDMapping(db_uuid); databases.erase(database_name); } @@ -450,21 +461,19 @@ DatabasePtr DatabaseCatalog::tryGetDatabase(const String & database_name) const DatabasePtr DatabaseCatalog::getDatabase(const UUID & uuid) const { - std::lock_guard lock{databases_mutex}; - auto it = db_uuid_map.find(uuid); - if (it == db_uuid_map.end()) + auto db_and_table = tryGetByUUID(uuid); + if (!db_and_table.first || db_and_table.second) throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database UUID {} does not exist", toString(uuid)); - return it->second; + return db_and_table.first; } DatabasePtr DatabaseCatalog::tryGetDatabase(const UUID & uuid) const { assert(uuid != UUIDHelpers::Nil); - std::lock_guard lock{databases_mutex}; - auto it = db_uuid_map.find(uuid); - if (it == db_uuid_map.end()) + auto db_and_table = tryGetByUUID(uuid); + if (!db_and_table.first || db_and_table.second) return {}; - return it->second; + return db_and_table.first; } bool DatabaseCatalog::isDatabaseExist(const String & database_name) const @@ -519,18 +528,22 @@ void DatabaseCatalog::addUUIDMapping(const UUID & uuid) void DatabaseCatalog::addUUIDMapping(const UUID & uuid, const DatabasePtr & database, const StoragePtr & table) { assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); - assert((database && table) || (!database && !table)); + assert(database || !table); UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; std::lock_guard lock{map_part.mutex}; auto [it, inserted] = map_part.map.try_emplace(uuid, database, table); if (inserted) + { + /// Mapping must be locked before actually inserting something + chassert((!database && !table)); return; + } auto & prev_database = it->second.first; auto & prev_table = it->second.second; - assert((prev_database && prev_table) || (!prev_database && !prev_table)); + assert(prev_database || !prev_table); - if (!prev_table && table) + if (!prev_database && database) { /// It's empty mapping, it was created to "lock" UUID and prevent collision. Just update it. prev_database = database; @@ -538,8 +551,8 @@ void DatabaseCatalog::addUUIDMapping(const UUID & uuid, const DatabasePtr & data return; } - /// We are trying to replace existing mapping (prev_table != nullptr), it's logical error - if (table) + /// We are trying to replace existing mapping (prev_database != nullptr), it's logical error + if (database || table) throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} already exists", toString(uuid)); /// Normally this should never happen, but it's possible when the same UUIDs are explicitly specified in different CREATE queries, /// so it's not LOGICAL_ERROR @@ -732,6 +745,8 @@ DatabaseAndTable DatabaseCatalog::tryGetDatabaseAndTable(const StorageID & table void DatabaseCatalog::loadMarkedAsDroppedTables() { + assert(!cleanup_task); + /// /clickhouse_root/metadata_dropped/ contains files with metadata of tables, /// which where marked as dropped by Atomic databases. /// Data directories of such tables still exists in store/ @@ -811,6 +826,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr time_t drop_time; if (table) { + chassert(hasUUIDMapping(table_id.uuid)); drop_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); table->is_dropped = true; } @@ -1142,7 +1158,7 @@ void DatabaseCatalog::cleanupStoreDirectoryTask() } /// Order is important - if (!isProtectedUUIDDir(uuid) && !hasUUIDMapping(uuid)) + if (!hasUUIDMapping(uuid)) { /// We load uuids even for detached and permanently detached tables, /// so it looks safe enough to remove directory if we don't have uuid mapping for it. @@ -1219,32 +1235,45 @@ bool DatabaseCatalog::maybeRemoveDirectory(const fs::path & unused_dir) } } -void DatabaseCatalog::addProtectedUUIDDir(const UUID & uuid) +static void maybeUnlockUUID(UUID uuid) { if (uuid == UUIDHelpers::Nil) return; - std::lock_guard lock{protected_uuid_dirs_mutex}; - bool inserted = protected_uuid_dirs.insert(uuid).second; - if (inserted) - return; - throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Mapping for table with UUID={} already exists. It happened due to UUID collision, " - "most likely because some not random UUIDs were manually specified in CREATE queries.", toString(uuid)); + chassert(DatabaseCatalog::instance().hasUUIDMapping(uuid)); + auto db_and_table = DatabaseCatalog::instance().tryGetByUUID(uuid); + if (!db_and_table.first && !db_and_table.second) + { + DatabaseCatalog::instance().removeUUIDMappingFinally(uuid); + return; + } + chassert(db_and_table.first || !db_and_table.second); } -void DatabaseCatalog::removeProtectedUUIDDir(const UUID & uuid) +TemporaryLockForUUIDDirectory::TemporaryLockForUUIDDirectory(UUID uuid_) + : uuid(uuid_) { - if (uuid == UUIDHelpers::Nil) - return; - std::lock_guard lock{protected_uuid_dirs_mutex}; - chassert(protected_uuid_dirs.contains(uuid)); - protected_uuid_dirs.erase(uuid); + if (uuid != UUIDHelpers::Nil) + DatabaseCatalog::instance().addUUIDMapping(uuid); } -bool DatabaseCatalog::isProtectedUUIDDir(const UUID & uuid) +TemporaryLockForUUIDDirectory::~TemporaryLockForUUIDDirectory() { - std::lock_guard lock{protected_uuid_dirs_mutex}; - return protected_uuid_dirs.contains(uuid); + maybeUnlockUUID(uuid); +} + +TemporaryLockForUUIDDirectory::TemporaryLockForUUIDDirectory(TemporaryLockForUUIDDirectory && rhs) noexcept + : uuid(rhs.uuid) +{ + rhs.uuid = UUIDHelpers::Nil; +} + +TemporaryLockForUUIDDirectory & TemporaryLockForUUIDDirectory::operator = (TemporaryLockForUUIDDirectory && rhs) noexcept +{ + maybeUnlockUUID(uuid); + uuid = rhs.uuid; + rhs.uuid = UUIDHelpers::Nil; + return *this; } diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 7dde0643dd9..d82a0594c9a 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -207,10 +207,6 @@ public: bool hasUUIDMapping(const UUID & uuid); - void addProtectedUUIDDir(const UUID & uuid); - void removeProtectedUUIDDir(const UUID & uuid); - bool isProtectedUUIDDir(const UUID & uuid); - static String getPathForUUID(const UUID & uuid); DatabaseAndTable tryGetByUUID(const UUID & uuid) const; @@ -275,14 +271,11 @@ private: static constexpr size_t reschedule_time_ms = 100; static constexpr time_t drop_error_cooldown_sec = 5; - using UUIDToDatabaseMap = std::unordered_map; - mutable std::mutex databases_mutex; ViewDependencies view_dependencies; Databases databases; - UUIDToDatabaseMap db_uuid_map; UUIDToStorageMap uuid_map; DependenciesInfos loading_dependencies; @@ -305,9 +298,6 @@ private: std::unordered_set tables_marked_dropped_ids; mutable std::mutex tables_marked_dropped_mutex; - std::unordered_set protected_uuid_dirs; - mutable std::mutex protected_uuid_dirs_mutex; - std::unique_ptr drop_task; static constexpr time_t default_drop_delay_sec = 8 * 60; time_t drop_delay_sec = default_drop_delay_sec; @@ -322,13 +312,23 @@ private: time_t unused_dir_cleanup_period_sec = default_unused_dir_cleanup_period_sec; }; - -class UUIDDirectoryProtector : private boost::noncopyable +/// This class is useful when creating a table or database. +/// Usually we create IStorage/IDatabase object first and then add it to IDatabase/DatabaseCatalog. +/// But such object may start using a directory in store/ since its creation. +/// To avoid race with cleanupStoreDirectoryTask() we have to mark UUID as used first. +/// Then we can either add DatabasePtr/StoragePtr to the created UUID mapping +/// or remove the lock if creation failed. +/// See also addUUIDMapping(...) +class TemporaryLockForUUIDDirectory : private boost::noncopyable { - UUID uuid; + UUID uuid = UUIDHelpers::Nil; public: - UUIDDirectoryProtector(UUID uuid_) : uuid(uuid_) { DatabaseCatalog::instance().addProtectedUUIDDir(uuid); } - ~UUIDDirectoryProtector() { DatabaseCatalog::instance().removeProtectedUUIDDir(uuid); } + TemporaryLockForUUIDDirectory() = default; + TemporaryLockForUUIDDirectory(UUID uuid_); + ~TemporaryLockForUUIDDirectory(); + + TemporaryLockForUUIDDirectory(TemporaryLockForUUIDDirectory && rhs) noexcept; + TemporaryLockForUUIDDirectory & operator = (TemporaryLockForUUIDDirectory && rhs) noexcept; }; } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 20ebf96e617..00597e0e3cd 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -249,15 +249,22 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) "Enable allow_experimental_database_materialized_postgresql to use it.", ErrorCodes::UNKNOWN_DATABASE_ENGINE); } - UUIDDirectoryProtector uuid_lock{create.uuid}; + bool need_write_metadata = !create.attach || !fs::exists(metadata_file_path); + bool need_lock_uuid = internal || need_write_metadata; + + /// Lock uuid, so we will known it's already in use. + /// We do it when attaching databases on server startup (internal) and on CREATE query (!create.attach); + TemporaryLockForUUIDDirectory uuid_lock; + if (need_lock_uuid) + uuid_lock = TemporaryLockForUUIDDirectory{create.uuid}; + else if (create.uuid != UUIDHelpers::Nil && !DatabaseCatalog::instance().hasUUIDMapping(create.uuid)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create.uuid); + DatabasePtr database = DatabaseFactory::get(create, metadata_path / "", getContext()); if (create.uuid != UUIDHelpers::Nil) create.setDatabase(TABLE_WITH_UUID_NAME_PLACEHOLDER); - bool need_write_metadata = !create.attach || !fs::exists(metadata_file_path); - bool need_lock_uuid = internal || !create.attach; - if (need_write_metadata) { create.attach = true; @@ -281,13 +288,6 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) /// We attach database before loading it's tables, so do not allow concurrent DDL queries auto db_guard = DatabaseCatalog::instance().getExclusiveDDLGuardForDatabase(database_name); - if (create.uuid != UUIDHelpers::Nil && need_lock_uuid) - { - /// Lock uuid, so we will known it's already in use. - /// We do it when attaching databases on server startup (internal) and on CREATE query (!create.attach); - DatabaseCatalog::instance().addUUIDMapping(create.uuid); - } - bool added = false; bool renamed = false; try @@ -1172,70 +1172,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, const InterpreterCreateQuery::TableProperties & properties) { - std::unique_ptr guard; - - String data_path; - DatabasePtr database; - - bool need_add_to_database = !create.temporary; - if (need_add_to_database) - { - /** If the request specifies IF NOT EXISTS, we allow concurrent CREATE queries (which do nothing). - * If table doesn't exist, one thread is creating table, while others wait in DDLGuard. - */ - guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); - - database = DatabaseCatalog::instance().getDatabase(create.getDatabase()); - assertOrSetUUID(create, database); - - String storage_name = create.is_dictionary ? "Dictionary" : "Table"; - auto storage_already_exists_error_code = create.is_dictionary ? ErrorCodes::DICTIONARY_ALREADY_EXISTS : ErrorCodes::TABLE_ALREADY_EXISTS; - - /// Table can be created before or it can be created concurrently in another thread, while we were waiting in DDLGuard. - if (database->isTableExist(create.getTable(), getContext())) - { - /// TODO Check structure of table - if (create.if_not_exists) - return false; - else if (create.replace_view) - { - /// when executing CREATE OR REPLACE VIEW, drop current existing view - auto drop_ast = std::make_shared(); - drop_ast->setDatabase(create.getDatabase()); - drop_ast->setTable(create.getTable()); - drop_ast->no_ddl_lock = true; - - auto drop_context = Context::createCopy(context); - InterpreterDropQuery interpreter(drop_ast, drop_context); - interpreter.execute(); - } - else - throw Exception(storage_already_exists_error_code, - "{} {}.{} already exists", storage_name, backQuoteIfNeed(create.getDatabase()), backQuoteIfNeed(create.getTable())); - } - else if (!create.attach) - { - /// Checking that table may exists in detached/detached permanently state - try - { - database->checkMetadataFilenameAvailability(create.getTable()); - } - catch (const Exception &) - { - if (create.if_not_exists) - return false; - throw; - } - } - - - data_path = database->getTableDataPath(create); - - if (!create.attach && !data_path.empty() && fs::exists(fs::path{getContext()->getPath()} / data_path)) - throw Exception(storage_already_exists_error_code, - "Directory for {} data {} already exists", Poco::toLower(storage_name), String(data_path)); - } - else + if (create.temporary) { if (create.if_not_exists && getContext()->tryResolveStorageID({"", create.getTable()}, Context::ResolveExternal)) return false; @@ -1246,6 +1183,65 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, return true; } + std::unique_ptr guard; + + String data_path; + DatabasePtr database; + + /** If the request specifies IF NOT EXISTS, we allow concurrent CREATE queries (which do nothing). + * If table doesn't exist, one thread is creating table, while others wait in DDLGuard. + */ + guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); + + database = DatabaseCatalog::instance().getDatabase(create.getDatabase()); + assertOrSetUUID(create, database); + + String storage_name = create.is_dictionary ? "Dictionary" : "Table"; + auto storage_already_exists_error_code = create.is_dictionary ? ErrorCodes::DICTIONARY_ALREADY_EXISTS : ErrorCodes::TABLE_ALREADY_EXISTS; + + /// Table can be created before or it can be created concurrently in another thread, while we were waiting in DDLGuard. + if (database->isTableExist(create.getTable(), getContext())) + { + /// TODO Check structure of table + if (create.if_not_exists) + return false; + else if (create.replace_view) + { + /// when executing CREATE OR REPLACE VIEW, drop current existing view + auto drop_ast = std::make_shared(); + drop_ast->setDatabase(create.getDatabase()); + drop_ast->setTable(create.getTable()); + drop_ast->no_ddl_lock = true; + + auto drop_context = Context::createCopy(context); + InterpreterDropQuery interpreter(drop_ast, drop_context); + interpreter.execute(); + } + else + throw Exception(storage_already_exists_error_code, + "{} {}.{} already exists", storage_name, backQuoteIfNeed(create.getDatabase()), backQuoteIfNeed(create.getTable())); + } + else if (!create.attach) + { + /// Checking that table may exists in detached/detached permanently state + try + { + database->checkMetadataFilenameAvailability(create.getTable()); + } + catch (const Exception &) + { + if (create.if_not_exists) + return false; + throw; + } + } + + data_path = database->getTableDataPath(create); + + if (!create.attach && !data_path.empty() && fs::exists(fs::path{getContext()->getPath()} / data_path)) + throw Exception(storage_already_exists_error_code, + "Directory for {} data {} already exists", Poco::toLower(storage_name), String(data_path)); + bool from_path = create.attach_from_path.has_value(); String actual_data_path = data_path; if (from_path) @@ -1270,7 +1266,15 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, database->checkDetachedTableNotInUse(create.uuid); } - UUIDDirectoryProtector uuid_lock{create.uuid}; + /// We should lock UUID on CREATE query (because for ATTACH it must be already locked previously). + /// But ATTACH without create.attach_short_syntax flag works like CREATE actually, that's why we check it. + bool need_lock_uuid = !create.attach_short_syntax; + TemporaryLockForUUIDDirectory uuid_lock; + if (need_lock_uuid) + uuid_lock = TemporaryLockForUUIDDirectory{create.uuid}; + else if (create.uuid != UUIDHelpers::Nil && !DatabaseCatalog::instance().hasUUIDMapping(create.uuid)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create.uuid); + StoragePtr res; /// NOTE: CREATE query may be rewritten by Storage creator or table function if (create.as_table_function) diff --git a/src/Storages/System/attachSystemTablesImpl.h b/src/Storages/System/attachSystemTablesImpl.h index fcc1ab43a64..a1fae985d92 100644 --- a/src/Storages/System/attachSystemTablesImpl.h +++ b/src/Storages/System/attachSystemTablesImpl.h @@ -22,6 +22,7 @@ void attach(ContextPtr context, IDatabase & system_database, const String & tabl /// NOTE: UUIDs are not persistent, but it's ok since no data are stored on disk for these storages /// and path is actually not used auto table_id = StorageID(DatabaseCatalog::SYSTEM_DATABASE, table_name, UUIDHelpers::generateV4()); + DatabaseCatalog::instance().addUUIDMapping(table_id.uuid); String path = "store/" + DatabaseCatalog::getPathForUUID(table_id.uuid); system_database.attachTable(context, table_name, std::make_shared(table_id, std::forward(args)...), path); } From ffab75baa7ec4242ef1d689095a3f64d6c0d4da3 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 22 Jun 2022 23:33:20 +0200 Subject: [PATCH 046/123] Checkout full repositories for performance tests --- .github/workflows/backport_branches.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index b93c1b61ffd..e88a121b5d3 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -143,6 +143,8 @@ jobs: sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - name: Check out repository code uses: actions/checkout@v2 + with: + fetch-depth: 0 # For a proper version and performance artifacts - name: Build run: | git -C "$GITHUB_WORKSPACE" submodule sync --recursive @@ -188,6 +190,8 @@ jobs: sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - name: Check out repository code uses: actions/checkout@v2 + with: + fetch-depth: 0 # For a proper version and performance artifacts - name: Build run: | git -C "$GITHUB_WORKSPACE" submodule sync --recursive From 3ab58fd14f20522de8cc3b1079e9388adaf6b477 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 20 Jun 2022 17:16:01 +0200 Subject: [PATCH 047/123] Add docusaurus header to generated changelogs --- utils/changelog/changelog.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/utils/changelog/changelog.py b/utils/changelog/changelog.py index c20c6cfd072..4e6bc7f4a6a 100755 --- a/utils/changelog/changelog.py +++ b/utils/changelog/changelog.py @@ -14,6 +14,7 @@ from typing import Dict, List, Optional, TextIO from fuzzywuzzy.fuzz import ratio # type: ignore from github import Github +from github.GithubException import UnknownObjectException from github.NamedUser import NamedUser from github.Issue import Issue from github.PullRequest import PullRequest @@ -66,7 +67,10 @@ class Description: r"\1[#\2](https://github.com/ClickHouse/ClickHouse/issues/\2)", entry, ) - user_name = self.user.name if self.user.name else self.user.login + try: + user_name = self.user.name if self.user.name else self.user.login + except UnknownObjectException: + user_name = self.user.login return ( f"* {entry} [#{self.number}]({self.html_url}) " f"([{user_name}]({self.user.html_url}))." @@ -306,7 +310,11 @@ def generate_description(item: PullRequest, repo: Repository) -> Optional[Descri def write_changelog(fd: TextIO, descriptions: Dict[str, List[Description]]): - fd.write(f"### ClickHouse release {TO_REF} FIXME as compared to {FROM_REF}\n\n") + year = date.today().year + fd.write( + f"---\nsidebar_position: 1\nsidebar_label: {year}\n---\n\n# {year} Changelog\n\n" + f"### ClickHouse release {TO_REF} FIXME as compared to {FROM_REF}\n\n" + ) seen_categories = [] # type: List[str] for category in categories_preferred_order: From 61d3e94d06668340e9fb06ae1aceb14a026172a3 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 20 Jun 2022 17:16:29 +0200 Subject: [PATCH 048/123] Regenerate v22.* changelogs --- docs/changelogs/v22.1.1.2542-prestable.md | 7 +++++++ docs/changelogs/v22.1.2.2-stable.md | 7 +++++++ docs/changelogs/v22.1.3.7-stable.md | 7 +++++++ docs/changelogs/v22.1.4.30-stable.md | 7 +++++++ docs/changelogs/v22.2.1.2139-prestable.md | 7 +++++++ docs/changelogs/v22.2.2.1-stable.md | 7 +++++++ docs/changelogs/v22.2.3.5-stable.md | 7 +++++++ docs/changelogs/v22.3.1.1262-prestable.md | 9 ++++++++- docs/changelogs/v22.3.2.2-lts.md | 7 +++++++ docs/changelogs/v22.3.3.44-lts.md | 7 +++++++ docs/changelogs/v22.3.4.20-lts.md | 7 +++++++ docs/changelogs/v22.3.5.5-lts.md | 7 +++++++ docs/changelogs/v22.3.6.5-lts.md | 7 +++++++ docs/changelogs/v22.4.1.2305-prestable.md | 7 +++++++ docs/changelogs/v22.4.2.1-stable.md | 7 +++++++ docs/changelogs/v22.4.3.3-stable.md | 7 +++++++ docs/changelogs/v22.4.4.7-stable.md | 7 +++++++ docs/changelogs/v22.4.5.9-stable.md | 7 +++++++ docs/changelogs/v22.5.1.2079-stable.md | 7 +++++++ docs/changelogs/v22.6.1.1985-stable.md | 7 +++++++ 20 files changed, 141 insertions(+), 1 deletion(-) diff --git a/docs/changelogs/v22.1.1.2542-prestable.md b/docs/changelogs/v22.1.1.2542-prestable.md index c9071422949..cbe8e781d3c 100644 --- a/docs/changelogs/v22.1.1.2542-prestable.md +++ b/docs/changelogs/v22.1.1.2542-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.1.1.2542-prestable FIXME as compared to v21.12.1.9017-prestable #### Backward Incompatible Change diff --git a/docs/changelogs/v22.1.2.2-stable.md b/docs/changelogs/v22.1.2.2-stable.md index 450c640bc5e..de2dc6e3cee 100644 --- a/docs/changelogs/v22.1.2.2-stable.md +++ b/docs/changelogs/v22.1.2.2-stable.md @@ -1,2 +1,9 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.1.2.2-stable FIXME as compared to v22.1.1.2542-prestable diff --git a/docs/changelogs/v22.1.3.7-stable.md b/docs/changelogs/v22.1.3.7-stable.md index f8bb8821031..72c7a9cc5da 100644 --- a/docs/changelogs/v22.1.3.7-stable.md +++ b/docs/changelogs/v22.1.3.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.1.3.7-stable FIXME as compared to v22.1.2.2-stable #### Improvement diff --git a/docs/changelogs/v22.1.4.30-stable.md b/docs/changelogs/v22.1.4.30-stable.md index 1ea56131481..7fa295cdcb0 100644 --- a/docs/changelogs/v22.1.4.30-stable.md +++ b/docs/changelogs/v22.1.4.30-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.1.4.30-stable FIXME as compared to v22.1.3.7-stable #### Build/Testing/Packaging Improvement diff --git a/docs/changelogs/v22.2.1.2139-prestable.md b/docs/changelogs/v22.2.1.2139-prestable.md index 26ac24a7778..f93a3a690c4 100644 --- a/docs/changelogs/v22.2.1.2139-prestable.md +++ b/docs/changelogs/v22.2.1.2139-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.2.1.2139-prestable FIXME as compared to v22.1.1.2542-prestable #### New Feature diff --git a/docs/changelogs/v22.2.2.1-stable.md b/docs/changelogs/v22.2.2.1-stable.md index c9158290753..0ea5be33c94 100644 --- a/docs/changelogs/v22.2.2.1-stable.md +++ b/docs/changelogs/v22.2.2.1-stable.md @@ -1,2 +1,9 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.2.2.1-stable FIXME as compared to v22.2.1.2139-prestable diff --git a/docs/changelogs/v22.2.3.5-stable.md b/docs/changelogs/v22.2.3.5-stable.md index 90c0d22d570..24b535a7f91 100644 --- a/docs/changelogs/v22.2.3.5-stable.md +++ b/docs/changelogs/v22.2.3.5-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.2.3.5-stable FIXME as compared to v22.2.2.1-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v22.3.1.1262-prestable.md b/docs/changelogs/v22.3.1.1262-prestable.md index 03e81bd1808..d2f976f4517 100644 --- a/docs/changelogs/v22.3.1.1262-prestable.md +++ b/docs/changelogs/v22.3.1.1262-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.1.1262-prestable FIXME as compared to v22.2.1.2139-prestable #### Backward Incompatible Change @@ -65,7 +72,7 @@ * Add setting to lower column case when reading parquet/ORC file. [#35145](https://github.com/ClickHouse/ClickHouse/pull/35145) ([shuchaome](https://github.com/shuchaome)). * Do not retry non-rertiable errors. Closes [#35161](https://github.com/ClickHouse/ClickHouse/issues/35161). [#35172](https://github.com/ClickHouse/ClickHouse/pull/35172) ([Kseniia Sumarokova](https://github.com/kssenii)). * Added disk_name to system.part_log. [#35178](https://github.com/ClickHouse/ClickHouse/pull/35178) ([Artyom Yurkov](https://github.com/Varinara)). -* Currently,ClickHouse validates hosts defined under for URL and Remote Table functions. This PR extends the RemoteHostFilter to Mysql and PostgreSQL table functions. [#35191](https://github.com/ClickHouse/ClickHouse/pull/35191) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Currently, ClickHouse validates hosts defined under for URL and Remote Table functions. This PR extends the RemoteHostFilter to Mysql and PostgreSQL table functions. [#35191](https://github.com/ClickHouse/ClickHouse/pull/35191) ([Heena Bansal](https://github.com/HeenaBansal2009)). * Sometimes it is not enough for us to distinguish queries hierachy only by is_initial_query in system.query_log and system.processes. So distributed_depth is needed. [#35207](https://github.com/ClickHouse/ClickHouse/pull/35207) ([李扬](https://github.com/taiyang-li)). * Support test mode for clickhouse-local. [#35264](https://github.com/ClickHouse/ClickHouse/pull/35264) ([Kseniia Sumarokova](https://github.com/kssenii)). * Return const for function getMacro if not in distributed query. Close [#34727](https://github.com/ClickHouse/ClickHouse/issues/34727). [#35289](https://github.com/ClickHouse/ClickHouse/pull/35289) ([李扬](https://github.com/taiyang-li)). diff --git a/docs/changelogs/v22.3.2.2-lts.md b/docs/changelogs/v22.3.2.2-lts.md index ef45265c7bd..b755db300c8 100644 --- a/docs/changelogs/v22.3.2.2-lts.md +++ b/docs/changelogs/v22.3.2.2-lts.md @@ -1,2 +1,9 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.2.2-lts FIXME as compared to v22.3.1.1262-prestable diff --git a/docs/changelogs/v22.3.3.44-lts.md b/docs/changelogs/v22.3.3.44-lts.md index 4214512c533..4fa98940290 100644 --- a/docs/changelogs/v22.3.3.44-lts.md +++ b/docs/changelogs/v22.3.3.44-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.3.44-lts FIXME as compared to v22.3.2.2-lts #### Bug Fix diff --git a/docs/changelogs/v22.3.4.20-lts.md b/docs/changelogs/v22.3.4.20-lts.md index 4bb4f1bf0f4..d820adbdbec 100644 --- a/docs/changelogs/v22.3.4.20-lts.md +++ b/docs/changelogs/v22.3.4.20-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.4.20-lts FIXME as compared to v22.3.3.44-lts #### Build/Testing/Packaging Improvement diff --git a/docs/changelogs/v22.3.5.5-lts.md b/docs/changelogs/v22.3.5.5-lts.md index d1c42807f41..7deff1be416 100644 --- a/docs/changelogs/v22.3.5.5-lts.md +++ b/docs/changelogs/v22.3.5.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.5.5-lts FIXME as compared to v22.3.4.20-lts #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v22.3.6.5-lts.md b/docs/changelogs/v22.3.6.5-lts.md index 16cf390c703..9d86d9e786c 100644 --- a/docs/changelogs/v22.3.6.5-lts.md +++ b/docs/changelogs/v22.3.6.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.3.6.5-lts FIXME as compared to v22.3.5.5-lts #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v22.4.1.2305-prestable.md b/docs/changelogs/v22.4.1.2305-prestable.md index eb1ed6decd3..5c6948251b3 100644 --- a/docs/changelogs/v22.4.1.2305-prestable.md +++ b/docs/changelogs/v22.4.1.2305-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.4.1.2305-prestable FIXME as compared to v22.3.1.1262-prestable #### Backward Incompatible Change diff --git a/docs/changelogs/v22.4.2.1-stable.md b/docs/changelogs/v22.4.2.1-stable.md index cd7ee75997c..fb77d3fee9b 100644 --- a/docs/changelogs/v22.4.2.1-stable.md +++ b/docs/changelogs/v22.4.2.1-stable.md @@ -1,2 +1,9 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.4.2.1-stable FIXME as compared to v22.4.1.2305-prestable diff --git a/docs/changelogs/v22.4.3.3-stable.md b/docs/changelogs/v22.4.3.3-stable.md index 5ab7872f880..4baa63672ab 100644 --- a/docs/changelogs/v22.4.3.3-stable.md +++ b/docs/changelogs/v22.4.3.3-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.4.3.3-stable FIXME as compared to v22.4.2.1-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v22.4.4.7-stable.md b/docs/changelogs/v22.4.4.7-stable.md index 9004ce2f6bc..525c0de1341 100644 --- a/docs/changelogs/v22.4.4.7-stable.md +++ b/docs/changelogs/v22.4.4.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.4.4.7-stable FIXME as compared to v22.4.3.3-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v22.4.5.9-stable.md b/docs/changelogs/v22.4.5.9-stable.md index ab43bf3eade..b5fffa56494 100644 --- a/docs/changelogs/v22.4.5.9-stable.md +++ b/docs/changelogs/v22.4.5.9-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.4.5.9-stable FIXME as compared to v22.4.4.7-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v22.5.1.2079-stable.md b/docs/changelogs/v22.5.1.2079-stable.md index aab8266c115..7614d678a2a 100644 --- a/docs/changelogs/v22.5.1.2079-stable.md +++ b/docs/changelogs/v22.5.1.2079-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.5.1.2079-stable FIXME as compared to v22.4.1.2305-prestable #### Backward Incompatible Change diff --git a/docs/changelogs/v22.6.1.1985-stable.md b/docs/changelogs/v22.6.1.1985-stable.md index 583ffddf279..9c7ecc1dae3 100644 --- a/docs/changelogs/v22.6.1.1985-stable.md +++ b/docs/changelogs/v22.6.1.1985-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v22.6.1.1985-stable FIXME as compared to v22.5.1.2079-stable #### Backward Incompatible Change From f06c533eaf6d573aff18b1f4bfc053b5352b8a46 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 20 Jun 2022 17:17:07 +0200 Subject: [PATCH 049/123] Add changelog for fresh v22.3.7.28-lts --- docs/changelogs/v22.3.7.28-lts.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 docs/changelogs/v22.3.7.28-lts.md diff --git a/docs/changelogs/v22.3.7.28-lts.md b/docs/changelogs/v22.3.7.28-lts.md new file mode 100644 index 00000000000..ecfd0ce2cf6 --- /dev/null +++ b/docs/changelogs/v22.3.7.28-lts.md @@ -0,0 +1,24 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + +### ClickHouse release v22.3.7.28-lts FIXME as compared to v22.3.6.5-lts + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#37715](https://github.com/ClickHouse/ClickHouse/issues/37715): Fix unexpected errors with a clash of constant strings in aggregate function, prewhere and join. Close [#36891](https://github.com/ClickHouse/ClickHouse/issues/36891). [#37336](https://github.com/ClickHouse/ClickHouse/pull/37336) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#37511](https://github.com/ClickHouse/ClickHouse/issues/37511): Fix logical error in normalizeUTF8 functions. Closes [#37298](https://github.com/ClickHouse/ClickHouse/issues/37298). [#37443](https://github.com/ClickHouse/ClickHouse/pull/37443) ([Maksim Kita](https://github.com/kitaisreal)). +* Backported in [#37843](https://github.com/ClickHouse/ClickHouse/issues/37843): Fix segmentation fault in `show create table` from mysql database when it is configured with named collections. Closes [#37683](https://github.com/ClickHouse/ClickHouse/issues/37683). [#37690](https://github.com/ClickHouse/ClickHouse/pull/37690) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backported in [#37940](https://github.com/ClickHouse/ClickHouse/issues/37940): Fix setting cast_ipv4_ipv6_default_on_conversion_error for internal cast function. Closes [#35156](https://github.com/ClickHouse/ClickHouse/issues/35156). [#37761](https://github.com/ClickHouse/ClickHouse/pull/37761) ([Maksim Kita](https://github.com/kitaisreal)). + +#### Bug Fix (user-visible misbehaviour in official stable or prestable release) + +* Backported in [#37926](https://github.com/ClickHouse/ClickHouse/issues/37926): Fix check asof join key nullability, close [#35565](https://github.com/ClickHouse/ClickHouse/issues/35565). [#35674](https://github.com/ClickHouse/ClickHouse/pull/35674) ([Vladimir C](https://github.com/vdimir)). +* Backported in [#37172](https://github.com/ClickHouse/ClickHouse/issues/37172): Fix bug in indexes of not presented columns in -WithNames formats that led to error `INCORRECT_NUMBER_OF_COLUMNS ` when the number of columns is more than 256. Closes [#35793](https://github.com/ClickHouse/ClickHouse/issues/35793). [#35803](https://github.com/ClickHouse/ClickHouse/pull/35803) ([Kruglov Pavel](https://github.com/Avogar)). +* Backported in [#37457](https://github.com/ClickHouse/ClickHouse/issues/37457): Server might fail to start if it cannot resolve hostname of external ClickHouse dictionary. It's fixed. Fixes [#36451](https://github.com/ClickHouse/ClickHouse/issues/36451). [#36463](https://github.com/ClickHouse/ClickHouse/pull/36463) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Backported in [#36925](https://github.com/ClickHouse/ClickHouse/issues/36925): Fix bug in clickhouse-keeper which can lead to corrupted compressed log files in case of small load and restarts. [#36910](https://github.com/ClickHouse/ClickHouse/pull/36910) ([alesapin](https://github.com/alesapin)). +* Backported in [#37364](https://github.com/ClickHouse/ClickHouse/issues/37364): Fixed problem with infs in `quantileTDigest`. Fixes [#32107](https://github.com/ClickHouse/ClickHouse/issues/32107). [#37021](https://github.com/ClickHouse/ClickHouse/pull/37021) ([Vladimir Chebotarev](https://github.com/excitoon)). + From 779b83262f770ba448852f9fcebfadbd3eda225e Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 22 Jun 2022 19:52:43 +0200 Subject: [PATCH 050/123] Add `not for changelog` category with titles as entries --- utils/changelog/changelog.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/utils/changelog/changelog.py b/utils/changelog/changelog.py index 4e6bc7f4a6a..e1797fb5dd3 100755 --- a/utils/changelog/changelog.py +++ b/utils/changelog/changelog.py @@ -284,7 +284,15 @@ def generate_description(item: PullRequest, repo: Repository) -> Optional[Descri # Filter out the PR categories that are not for changelog. if re.match( - r"(?i)doc|((non|in|not|un)[-\s]*significant)|(not[ ]*for[ ]*changelog)", + r"(?i)((non|in|not|un)[-\s]*significant)|(not[ ]*for[ ]*changelog)", + category, + ): + category = "NOT FOR CHANGELOG / INSIGNIFICANT" + return Description(item.number, item.user, item.html_url, item.title, category) + + # Filter out documentations changelog + if re.match( + r"(?i)doc", category, ): return None From 81becebfe3fbbfe9ae071739ac2f22147d63473d Mon Sep 17 00:00:00 2001 From: Laurie Li Date: Thu, 23 Jun 2022 09:29:46 +0800 Subject: [PATCH 051/123] Add "See Also" Signed-off-by: Laurie Li --- .../table-engines/mergetree-family/replication.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/zh/engines/table-engines/mergetree-family/replication.md b/docs/zh/engines/table-engines/mergetree-family/replication.md index 568484c6256..69d1cd9d942 100644 --- a/docs/zh/engines/table-engines/mergetree-family/replication.md +++ b/docs/zh/engines/table-engines/mergetree-family/replication.md @@ -277,4 +277,12 @@ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data 如果 ZooKeeper 中的数据丢失或损坏，如上所述，你可以通过将数据转移到非复制表来保存数据。 -[来源文章](https://clickhouse.com/docs/en/operations/table_engines/replication/) +**参考** + +- [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) +- [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) +- [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) +- [max_replicated_fetches_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_fetches_network_bandwidth) +- [max_replicated_sends_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_sends_network_bandwidth) + +[原始文章](https://clickhouse.com/docs/en/operations/table_engines/replication/) From 4bf1fc47188c7c158f6661352c608370d779909e Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Thu, 23 Jun 2022 10:28:31 +0300 Subject: [PATCH 052/123] Add error code and message displaying in exceptions of KerberosInit; Correct code style in KerberosInit --- src/Access/KerberosInit.cpp | 54 ++++++++++++++++++++++--------------- src/Access/KerberosInit.h | 2 +- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index d3be559cba7..9ca45b12531 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #if USE_KRB5 #include #include @@ -36,23 +37,34 @@ struct K5Data class KerberosInit : boost::noncopyable { public: - int init(const String & keytab_file, const String & principal, const String & cache_name = ""); + void init(const String & keytab_file, const String & principal, const String & cache_name = ""); ~KerberosInit(); private: - struct K5Data k5; + struct K5Data k5 {}; krb5_ccache defcache = nullptr; krb5_get_init_creds_opt * options = nullptr; // Credentials structure including ticket, session key, and lifetime info. krb5_creds my_creds; krb5_keytab keytab = nullptr; krb5_principal defcache_princ = nullptr; + String fmtError(krb5_error_code code); }; } -int KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) + +String KerberosInit::fmtError(krb5_error_code code) +{ + const char *msg; + msg = krb5_get_error_message(k5.ctx, code); + String fmt_error = fmt::format(" ({}, {})", code, msg); + krb5_free_error_message(k5.ctx, msg); + return fmt_error; +} + +void KerberosInit::init(const String & keytab_file, const String & principal, const String & cache_name) { auto * log = &Poco::Logger::get("KerberosInit"); - LOG_TRACE(log,"Trying to authenticate to Kerberos v5"); + LOG_TRACE(log,"Trying to authenticate with Kerberos v5"); krb5_error_code ret; @@ -61,16 +73,15 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con if (!std::filesystem::exists(keytab_file)) throw Exception("Keytab file does not exist", ErrorCodes::KERBEROS_ERROR); - memset(&k5, 0, sizeof(k5)); ret = krb5_init_context(&k5.ctx); if (ret) - throw Exception("Error while initializing Kerberos 5 library", ErrorCodes::KERBEROS_ERROR); + throw Exception(fmt::format("Error while initializing Kerberos 5 library ({})", ret), ErrorCodes::KERBEROS_ERROR); if (!cache_name.empty()) { ret = krb5_cc_resolve(k5.ctx, cache_name.c_str(), &k5.out_cc); if (ret) - throw Exception("Error in resolving cache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error in resolving cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Resolved cache"); } else @@ -78,7 +89,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Resolve the default cache and get its type and default principal (if it is initialized). ret = krb5_cc_default(k5.ctx, &defcache); if (ret) - throw Exception("Error while getting default cache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error while getting default cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Resolved default cache"); deftype = krb5_cc_get_type(k5.ctx, defcache); if (krb5_cc_get_principal(k5.ctx, defcache, &defcache_princ) != 0) @@ -88,7 +99,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Use the specified principal name. ret = krb5_parse_name_flags(k5.ctx, principal.c_str(), 0, &k5.me); if (ret) - throw Exception("Error when parsing principal name " + principal, ErrorCodes::KERBEROS_ERROR); + throw Exception("Error when parsing principal name " + principal + fmtError(ret), ErrorCodes::KERBEROS_ERROR); // Cache related commands if (k5.out_cc == nullptr && krb5_cc_support_switch(k5.ctx, deftype)) @@ -96,8 +107,8 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Use an existing cache for the client principal if we can. ret = krb5_cc_cache_match(k5.ctx, k5.me, &k5.out_cc); if (ret && ret != KRB5_CC_NOTFOUND) - throw Exception("Error while searching for cache for " + principal, ErrorCodes::KERBEROS_ERROR); - if (!ret) + throw Exception("Error while searching for cache for " + principal + fmtError(ret), ErrorCodes::KERBEROS_ERROR); + if (0 == ret) { LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); k5.switch_to_cache = 1; @@ -107,7 +118,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Create a new cache to avoid overwriting the initialized default cache. ret = krb5_cc_new_unique(k5.ctx, deftype, nullptr, &k5.out_cc); if (ret) - throw Exception("Error while generating new cache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error while generating new cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Using default cache: {}", krb5_cc_get_name(k5.ctx, k5.out_cc)); k5.switch_to_cache = 1; } @@ -123,24 +134,24 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con ret = krb5_unparse_name(k5.ctx, k5.me, &k5.name); if (ret) - throw Exception("Error when unparsing name", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error when unparsing name" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Using principal: {}", k5.name); // Allocate a new initial credential options structure. ret = krb5_get_init_creds_opt_alloc(k5.ctx, &options); if (ret) - throw Exception("Error in options allocation", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error in options allocation" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); // Resolve keytab ret = krb5_kt_resolve(k5.ctx, keytab_file.c_str(), &keytab); if (ret) - throw Exception("Error in resolving keytab "+keytab_file, ErrorCodes::KERBEROS_ERROR); + throw Exception("Error in resolving keytab "+keytab_file + fmtError(ret), ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Using keytab: {}", keytab_file); // Set an output credential cache in initial credential options. ret = krb5_get_init_creds_opt_set_out_ccache(k5.ctx, options, k5.out_cc); if (ret) - throw Exception("Error in setting output credential cache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error in setting output credential cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); // Action: init or renew LOG_TRACE(log,"Trying to renew credentials"); @@ -153,7 +164,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Request KDC for an initial credentials using keytab. ret = krb5_get_init_creds_keytab(k5.ctx, &my_creds, k5.me, keytab, 0, nullptr, options); if (ret) - throw Exception("Error in getting initial credentials", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error in getting initial credentials" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); else LOG_TRACE(log,"Got initial credentials"); } @@ -163,7 +174,7 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Initialize a credential cache. Destroy any existing contents of cache and initialize it for the default principal. ret = krb5_cc_initialize(k5.ctx, k5.out_cc, k5.me); if (ret) - throw Exception("Error when initializing cache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error when initializing cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); LOG_TRACE(log,"Initialized cache"); // Store credentials in a credential cache. ret = krb5_cc_store_cred(k5.ctx, k5.out_cc, &my_creds); @@ -177,11 +188,10 @@ int KerberosInit::init(const String & keytab_file, const String & principal, con // Make a credential cache the primary cache for its collection. ret = krb5_cc_switch(k5.ctx, k5.out_cc); if (ret) - throw Exception("Error while switching to new cache", ErrorCodes::KERBEROS_ERROR); + throw Exception("Error while switching to new cache" + fmtError(ret), ErrorCodes::KERBEROS_ERROR); } LOG_TRACE(log,"Authenticated to Kerberos v5"); - return 0; } KerberosInit::~KerberosInit() @@ -208,12 +218,12 @@ KerberosInit::~KerberosInit() } } -int kerberosInit(const String & keytab_file, const String & principal, const String & cache_name) +void kerberosInit(const String & keytab_file, const String & principal, const String & cache_name) { // Using mutex to prevent cache file corruptions static std::mutex kinit_mtx; std::unique_lock lck(kinit_mtx); KerberosInit k_init; - return k_init.init(keytab_file, principal, cache_name); + k_init.init(keytab_file, principal, cache_name); } #endif // USE_KRB5 diff --git a/src/Access/KerberosInit.h b/src/Access/KerberosInit.h index 4731baf706f..5a11a275529 100644 --- a/src/Access/KerberosInit.h +++ b/src/Access/KerberosInit.h @@ -6,6 +6,6 @@ #if USE_KRB5 -int kerberosInit(const String & keytab_file, const String & principal, const String & cache_name = ""); +void kerberosInit(const String & keytab_file, const String & principal, const String & cache_name = ""); #endif // USE_KRB5 From d2f3c5b83643674c0bd6f9ae51b8fb48cfca94af Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 22 Jun 2022 23:45:54 +0200 Subject: [PATCH 053/123] Add sleeping after received `second rate limit exceeded` --- utils/changelog/changelog.py | 41 +++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/utils/changelog/changelog.py b/utils/changelog/changelog.py index e1797fb5dd3..9d1dabbabe1 100755 --- a/utils/changelog/changelog.py +++ b/utils/changelog/changelog.py @@ -10,11 +10,12 @@ from datetime import date, datetime, timedelta from queue import Empty, Queue from subprocess import CalledProcessError, DEVNULL from threading import Thread +from time import sleep from typing import Dict, List, Optional, TextIO from fuzzywuzzy.fuzz import ratio # type: ignore from github import Github -from github.GithubException import UnknownObjectException +from github.GithubException import RateLimitExceededException, UnknownObjectException from github.NamedUser import NamedUser from github.Issue import Issue from github.PullRequest import PullRequest @@ -67,10 +68,17 @@ class Description: r"\1[#\2](https://github.com/ClickHouse/ClickHouse/issues/\2)", entry, ) - try: - user_name = self.user.name if self.user.name else self.user.login - except UnknownObjectException: - user_name = self.user.login + # It's possible that we face a secondary rate limit. + # In this case we should sleep until we get it + while True: + try: + user_name = self.user.name if self.user.name else self.user.login + break + except UnknownObjectException: + user_name = self.user.login + break + except RateLimitExceededException: + sleep_on_rate_limit() return ( f"* {entry} [#{self.number}]({self.html_url}) " f"([{user_name}]({self.user.html_url}))." @@ -118,6 +126,11 @@ class Worker(Thread): self.queue.task_done() +def sleep_on_rate_limit(time: int = 20): + logging.warning("Faced rate limit, sleeping %s", time) + sleep(time) + + def get_pull_cached( repo: Repository, number: int, updated_at: Optional[datetime] = None ) -> PullRequest: @@ -130,7 +143,12 @@ def get_pull_cached( if cache_updated > updated_at: with open(pr_cache_file, "rb") as prfd: return GitHub.load(prfd) # type: ignore - pr = repo.get_pull(number) + while True: + try: + pr = repo.get_pull(number) + break + except RateLimitExceededException: + sleep_on_rate_limit() with open(pr_cache_file, "wb") as prfd: GitHub.dump(pr, prfd) # type: ignore return pr @@ -414,9 +432,16 @@ def main(): api_prs = GitHub.search_issues(query=query, sort="created") logging.info("Found %s PRs for the query: '%s'", api_prs.totalCount, query) - pr_numbers = list(api_prs) + issues = [] # type: List[Issue] + while True: + try: + for issue in api_prs: + issues.append(issue) + break + except RateLimitExceededException: + sleep_on_rate_limit() - descriptions = get_descriptions(repo, pr_numbers, args.jobs) + descriptions = get_descriptions(repo, issues, args.jobs) write_changelog(args.output, descriptions) From 0b871a20500f2c6ccab132a5196583479fd51bd6 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 22 Jun 2022 23:47:26 +0200 Subject: [PATCH 054/123] Regenerate v22.* with `not for changelog` category --- docs/changelogs/v22.1.1.2542-prestable.md | 248 ++++++++++++++++++++++ docs/changelogs/v22.2.1.2139-prestable.md | 161 ++++++++++++++ docs/changelogs/v22.2.3.5-stable.md | 4 + docs/changelogs/v22.3.1.1262-prestable.md | 121 +++++++++++ docs/changelogs/v22.3.3.44-lts.md | 8 + docs/changelogs/v22.3.6.5-lts.md | 4 + docs/changelogs/v22.3.7.28-lts.md | 7 + docs/changelogs/v22.4.1.2305-prestable.md | 205 ++++++++++++++++++ docs/changelogs/v22.4.4.7-stable.md | 4 + docs/changelogs/v22.4.5.9-stable.md | 4 + docs/changelogs/v22.5.1.2079-stable.md | 201 ++++++++++++++++++ docs/changelogs/v22.6.1.1985-stable.md | 161 ++++++++++++++ 12 files changed, 1128 insertions(+) diff --git a/docs/changelogs/v22.1.1.2542-prestable.md b/docs/changelogs/v22.1.1.2542-prestable.md index cbe8e781d3c..a0c6f61d5a6 100644 --- a/docs/changelogs/v22.1.1.2542-prestable.md +++ b/docs/changelogs/v22.1.1.2542-prestable.md @@ -218,6 +218,254 @@ sidebar_label: 2022 * NO CL ENTRY: 'Added Superwall to adopters list'. [#33573](https://github.com/ClickHouse/ClickHouse/pull/33573) ([Justin Hilliard](https://github.com/jahilliard)). * NO CL ENTRY: 'Revert "Ignore parse failure of opentelemetry header"'. [#33594](https://github.com/ClickHouse/ClickHouse/pull/33594) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Merging [#26074](https://github.com/ClickHouse/ClickHouse/issues/26074) [#26559](https://github.com/ClickHouse/ClickHouse/pull/26559) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Add performance tests to new CI [#31467](https://github.com/ClickHouse/ClickHouse/pull/31467) ([alesapin](https://github.com/alesapin)). +* refactor CI tests [#31882](https://github.com/ClickHouse/ClickHouse/pull/31882) ([Constantine Peresypkin](https://github.com/pkit)). +* Add ability to drain connections synchronously [#31965](https://github.com/ClickHouse/ClickHouse/pull/31965) ([Azat Khuzhin](https://github.com/azat)). +* Add docker container for AWS lambda [#32205](https://github.com/ClickHouse/ClickHouse/pull/32205) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* test for [#28280](https://github.com/ClickHouse/ClickHouse/issues/28280) [#32235](https://github.com/ClickHouse/ClickHouse/pull/32235) ([Denny Crane](https://github.com/den-crane)). +* Rename window functions in WindowView [#32324](https://github.com/ClickHouse/ClickHouse/pull/32324) ([vxider](https://github.com/Vxider)). +* Follow-up to [#32140](https://github.com/ClickHouse/ClickHouse/issues/32140) [#32389](https://github.com/ClickHouse/ClickHouse/pull/32389) ([Alexander Tokmakov](https://github.com/tavplubix)). +* clickhouse-local: fix CREATE DATABASE with Atomic engine [#32417](https://github.com/ClickHouse/ClickHouse/pull/32417) ([Azat Khuzhin](https://github.com/azat)). +* Add a test [#16171](https://github.com/ClickHouse/ClickHouse/issues/16171) [#32421](https://github.com/ClickHouse/ClickHouse/pull/32421) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix images names in integration tests [#32438](https://github.com/ClickHouse/ClickHouse/pull/32438) ([alesapin](https://github.com/alesapin)). +* Fix build check empty report [#32440](https://github.com/ClickHouse/ClickHouse/pull/32440) ([alesapin](https://github.com/alesapin)). +* Fix ASTFuzzer [#32447](https://github.com/ClickHouse/ClickHouse/pull/32447) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Try fix attaching gdb in tests [#32448](https://github.com/ClickHouse/ClickHouse/pull/32448) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix unit tests (ubsan) on master [#32459](https://github.com/ClickHouse/ClickHouse/pull/32459) ([alesapin](https://github.com/alesapin)). +* Remove dependency between integration and functional tests [#32461](https://github.com/ClickHouse/ClickHouse/pull/32461) ([alesapin](https://github.com/alesapin)). +* Add automatic workflow rerun [#32462](https://github.com/ClickHouse/ClickHouse/pull/32462) ([alesapin](https://github.com/alesapin)). +* Fix crash in case of MATERIALIZE COLUMN with no default expression. [#32464](https://github.com/ClickHouse/ClickHouse/pull/32464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update odbc-bridge.md [#32475](https://github.com/ClickHouse/ClickHouse/pull/32475) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix test_prometheus_endpoint [#32476](https://github.com/ClickHouse/ClickHouse/pull/32476) ([Vladimir C](https://github.com/vdimir)). +* 02122_parallel_formatting: Address grep binary warnings [#32477](https://github.com/ClickHouse/ClickHouse/pull/32477) ([Raúl Marín](https://github.com/Algunenano)). +* 01950_kill_large_group_by_query: Increase timeout [#32479](https://github.com/ClickHouse/ClickHouse/pull/32479) ([Raúl Marín](https://github.com/Algunenano)). +* Fix backport workflow [#32485](https://github.com/ClickHouse/ClickHouse/pull/32485) ([alesapin](https://github.com/alesapin)). +* Make 01675_distributed_bytes_to_delay_insert less flaky (use http over client) [#32492](https://github.com/ClickHouse/ClickHouse/pull/32492) ([Azat Khuzhin](https://github.com/azat)). +* Fix 02050_client_profile_events flakiness [#32493](https://github.com/ClickHouse/ClickHouse/pull/32493) ([Azat Khuzhin](https://github.com/azat)). +* Fix integration tests docker images path [#32494](https://github.com/ClickHouse/ClickHouse/pull/32494) ([alesapin](https://github.com/alesapin)). +* Split long tests into multiple checks [#32496](https://github.com/ClickHouse/ClickHouse/pull/32496) ([alesapin](https://github.com/alesapin)). +* Add test for clickhouse-client history navigation [#32497](https://github.com/ClickHouse/ClickHouse/pull/32497) ([Amos Bird](https://github.com/amosbird)). +* Rename window-view function to time window function [#32498](https://github.com/ClickHouse/ClickHouse/pull/32498) ([vxider](https://github.com/Vxider)). +* Disable flaky tests with Window View [#32500](https://github.com/ClickHouse/ClickHouse/pull/32500) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix flaky window view tests [#32503](https://github.com/ClickHouse/ClickHouse/pull/32503) ([vxider](https://github.com/Vxider)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Try to fix attaching gdb 2 [#32511](https://github.com/ClickHouse/ClickHouse/pull/32511) ([Alexander Tokmakov](https://github.com/tavplubix)). +* clickhouse-test: use basename of the test for *.sh tests [#32533](https://github.com/ClickHouse/ClickHouse/pull/32533) ([Azat Khuzhin](https://github.com/azat)). +* Fix processing initial table (--table/stdin) in clickhouse-local [#32534](https://github.com/ClickHouse/ClickHouse/pull/32534) ([Azat Khuzhin](https://github.com/azat)). +* Suppress UBSan errors for avg() function [#32535](https://github.com/ClickHouse/ClickHouse/pull/32535) ([Azat Khuzhin](https://github.com/azat)). +* Fix LOGICAL_ERROR for MATERIALIZED VIEW over table functions (i.e. numbers()) [#32571](https://github.com/ClickHouse/ClickHouse/pull/32571) ([Azat Khuzhin](https://github.com/azat)). +* Improve quota's end-of-interval calculations. [#32575](https://github.com/ClickHouse/ClickHouse/pull/32575) ([Vitaly Baranov](https://github.com/vitlibar)). +* remove unused headers in test [#32578](https://github.com/ClickHouse/ClickHouse/pull/32578) ([Bharat Nallan](https://github.com/bharatnc)). +* Remove arcadia build support [#32580](https://github.com/ClickHouse/ClickHouse/pull/32580) ([Azat Khuzhin](https://github.com/azat)). +* Cleanup perf test runner [#32582](https://github.com/ClickHouse/ClickHouse/pull/32582) ([Azat Khuzhin](https://github.com/azat)). +* [RFC] perf: do not fail in case of slow queries (to avoid hiding possible issues) [#32583](https://github.com/ClickHouse/ClickHouse/pull/32583) ([Azat Khuzhin](https://github.com/azat)). +* Add test for issue 28316 [#32585](https://github.com/ClickHouse/ClickHouse/pull/32585) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix bad test [#32571](https://github.com/ClickHouse/ClickHouse/issues/32571) [#32588](https://github.com/ClickHouse/ClickHouse/pull/32588) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix perf test `reinterpret_as` [#32589](https://github.com/ClickHouse/ClickHouse/pull/32589) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix indecent error message [#32591](https://github.com/ClickHouse/ClickHouse/pull/32591) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix strange code in TCPHandler [#32592](https://github.com/ClickHouse/ClickHouse/pull/32592) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* CI: upload logs if task failed [#32618](https://github.com/ClickHouse/ClickHouse/pull/32618) ([Alexander Tokmakov](https://github.com/tavplubix)). +* clickhouse-test: avoid failing with UNKNOWN status in some cases [#32625](https://github.com/ClickHouse/ClickHouse/pull/32625) ([Azat Khuzhin](https://github.com/azat)). +* avoid inconsistent state in Block [#32641](https://github.com/ClickHouse/ClickHouse/pull/32641) ([gulige](https://github.com/gulige)). +* Rerun workflows more times [#32642](https://github.com/ClickHouse/ClickHouse/pull/32642) ([alesapin](https://github.com/alesapin)). +* Remove flaky check from master and split asan test [#32645](https://github.com/ClickHouse/ClickHouse/pull/32645) ([alesapin](https://github.com/alesapin)). +* Split database replicated [#32650](https://github.com/ClickHouse/ClickHouse/pull/32650) ([alesapin](https://github.com/alesapin)). +* Improve exceptions usage in access control [#32662](https://github.com/ClickHouse/ClickHouse/pull/32662) ([Vitaly Baranov](https://github.com/vitlibar)). +* Follow-up to [#32140](https://github.com/ClickHouse/ClickHouse/issues/32140) [#32667](https://github.com/ClickHouse/ClickHouse/pull/32667) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add test_s3_zero_copy_concurrent_merge [#32694](https://github.com/ClickHouse/ClickHouse/pull/32694) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Check trusted contributors in lowercase [#32696](https://github.com/ClickHouse/ClickHouse/pull/32696) ([Vladimir C](https://github.com/vdimir)). +* Mention ClickHouse CLA in CONTRIBUTING.md [#32697](https://github.com/ClickHouse/ClickHouse/pull/32697) ([Ivan Blinkov](https://github.com/blinkov)). +* Rerun release workflows [#32701](https://github.com/ClickHouse/ClickHouse/pull/32701) ([alesapin](https://github.com/alesapin)). +* plausible analytic blog post [#32719](https://github.com/ClickHouse/ClickHouse/pull/32719) ([Tom Risse](https://github.com/flickerbox-tom)). +* Update backport.py [#32720](https://github.com/ClickHouse/ClickHouse/pull/32720) ([Kruglov Pavel](https://github.com/Avogar)). +* Pass -no-pie to the linker call to disable it [#32731](https://github.com/ClickHouse/ClickHouse/pull/32731) ([Raúl Marín](https://github.com/Algunenano)). +* Add severity table to support case form [#32736](https://github.com/ClickHouse/ClickHouse/pull/32736) ([Cody Baker](https://github.com/codyrobert)). +* tests/integration: fix wait_start()/start_clickhouse() [#32739](https://github.com/ClickHouse/ClickHouse/pull/32739) ([Azat Khuzhin](https://github.com/azat)). +* tests/integration: fix waiting of mysql client container [#32740](https://github.com/ClickHouse/ClickHouse/pull/32740) ([Azat Khuzhin](https://github.com/azat)). +* Better control build artifacts [#32745](https://github.com/ClickHouse/ClickHouse/pull/32745) ([alesapin](https://github.com/alesapin)). +* Fix race in skipping index of type `hypothesis` [#32756](https://github.com/ClickHouse/ClickHouse/pull/32756) ([Anton Popov](https://github.com/CurtizJ)). +* Fix build performance [#32766](https://github.com/ClickHouse/ClickHouse/pull/32766) ([alesapin](https://github.com/alesapin)). +* Add test for issue 26912 [#32771](https://github.com/ClickHouse/ClickHouse/pull/32771) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add a test for CAST to named tuple just in case. [#32776](https://github.com/ClickHouse/ClickHouse/pull/32776) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix performance build [#32796](https://github.com/ClickHouse/ClickHouse/pull/32796) ([alesapin](https://github.com/alesapin)). +* Trying another stress servers [#32798](https://github.com/ClickHouse/ClickHouse/pull/32798) ([alesapin](https://github.com/alesapin)). +* More diagnostics in gdb script [#32801](https://github.com/ClickHouse/ClickHouse/pull/32801) ([Alexander Tokmakov](https://github.com/tavplubix)). +* CacheDictionary dictionary source access race fix [#32805](https://github.com/ClickHouse/ClickHouse/pull/32805) ([Maksim Kita](https://github.com/kitaisreal)). +* Update NuRaft [#32808](https://github.com/ClickHouse/ClickHouse/pull/32808) ([alesapin](https://github.com/alesapin)). +* Add retries to curl in performance tests. [#32809](https://github.com/ClickHouse/ClickHouse/pull/32809) ([alesapin](https://github.com/alesapin)). +* Made Azure blob storage to obey ENABLE_LIBRARIES flag [#32815](https://github.com/ClickHouse/ClickHouse/pull/32815) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove recursive submodules [#32820](https://github.com/ClickHouse/ClickHouse/pull/32820) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* test for summap nullable(0) [#32822](https://github.com/ClickHouse/ClickHouse/pull/32822) ([Denny Crane](https://github.com/den-crane)). +* Update CHANGELOG.md [#32823](https://github.com/ClickHouse/ClickHouse/pull/32823) ([Denny Crane](https://github.com/den-crane)). +* Make `test_storage_s3` less flaky [#32825](https://github.com/ClickHouse/ClickHouse/pull/32825) ([Azat Khuzhin](https://github.com/azat)). +* Fix error message for reading from url [#32826](https://github.com/ClickHouse/ClickHouse/pull/32826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Revert docs release [#32827](https://github.com/ClickHouse/ClickHouse/pull/32827) ([alesapin](https://github.com/alesapin)). +* Fix docs release one more time [#32828](https://github.com/ClickHouse/ClickHouse/pull/32828) ([alesapin](https://github.com/alesapin)). +* Trigger also for workflows changes [#32830](https://github.com/ClickHouse/ClickHouse/pull/32830) ([alesapin](https://github.com/alesapin)). +* Fix docs release one more time [#32831](https://github.com/ClickHouse/ClickHouse/pull/32831) ([alesapin](https://github.com/alesapin)). +* Add dispatch event to docs release [#32832](https://github.com/ClickHouse/ClickHouse/pull/32832) ([alesapin](https://github.com/alesapin)). +* Remove repository before checkout [#32834](https://github.com/ClickHouse/ClickHouse/pull/32834) ([alesapin](https://github.com/alesapin)). +* Fix release branches filter [#32837](https://github.com/ClickHouse/ClickHouse/pull/32837) ([alesapin](https://github.com/alesapin)). +* StorageLiveView fix function style [#32838](https://github.com/ClickHouse/ClickHouse/pull/32838) ([Maksim Kita](https://github.com/kitaisreal)). +* Try fix flaky test: order for test_s3_zero_copy_replication [#32846](https://github.com/ClickHouse/ClickHouse/pull/32846) ([Vladimir C](https://github.com/vdimir)). +* Update TRUSTED_CONTIBUTORS [#32850](https://github.com/ClickHouse/ClickHouse/pull/32850) ([Vladimir C](https://github.com/vdimir)). +* Followup docs check workflow [#32857](https://github.com/ClickHouse/ClickHouse/pull/32857) ([alesapin](https://github.com/alesapin)). +* Always apply const-condition-if optimization. [#32858](https://github.com/ClickHouse/ClickHouse/pull/32858) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Use a proper syntax for multiline env in GITHUB_ENV [#32864](https://github.com/ClickHouse/ClickHouse/pull/32864) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix multiline SSH_KEY in GITHUB_ENV file [#32865](https://github.com/ClickHouse/ClickHouse/pull/32865) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* warm up for test_kafka_json_as_string_no_kdc [#32885](https://github.com/ClickHouse/ClickHouse/pull/32885) ([Ilya Golshtein](https://github.com/ilejn)). +* Rename files and code from BlobStorage to AzureBlobStorage [#32886](https://github.com/ClickHouse/ClickHouse/pull/32886) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add dockerhub proxy to CI [#32905](https://github.com/ClickHouse/ClickHouse/pull/32905) ([alesapin](https://github.com/alesapin)). +* Update 01069_window_view_proc_tumble_watch.py [#32908](https://github.com/ClickHouse/ClickHouse/pull/32908) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update test azure_blob_storage::test_table_manipulations [#32909](https://github.com/ClickHouse/ClickHouse/pull/32909) ([Kseniia Sumarokova](https://github.com/kssenii)). +* More retries in perf tests. [#32910](https://github.com/ClickHouse/ClickHouse/pull/32910) ([alesapin](https://github.com/alesapin)). +* fix wrong testcase in gtest_Parser [#32912](https://github.com/ClickHouse/ClickHouse/pull/32912) ([SuperDJY](https://github.com/cmsxbc)). +* Disable percpu arena in jemalloc in case of non deterministic CPU count [#32916](https://github.com/ClickHouse/ClickHouse/pull/32916) ([Azat Khuzhin](https://github.com/azat)). +* [RFC] Add replxx/jemalloc into fasttest build of clickhouse [#32923](https://github.com/ClickHouse/ClickHouse/pull/32923) ([Azat Khuzhin](https://github.com/azat)). +* tests/integration/cleanup_environment: fix obtaining PYTEST_CLEANUP_CONTAINERS [#32931](https://github.com/ClickHouse/ClickHouse/pull/32931) ([Azat Khuzhin](https://github.com/azat)). +* fix typo [#32936](https://github.com/ClickHouse/ClickHouse/pull/32936) ([Denny Crane](https://github.com/den-crane)). +* Fix build issue related to azure blob storage [another try] [#32948](https://github.com/ClickHouse/ClickHouse/pull/32948) ([Amos Bird](https://github.com/amosbird)). +* test for base64encode_trailing_bytes [#31797](https://github.com/ClickHouse/ClickHouse/issues/31797) [#32954](https://github.com/ClickHouse/ClickHouse/pull/32954) ([Denny Crane](https://github.com/den-crane)). +* simplify looping in S2 functions [#32960](https://github.com/ClickHouse/ClickHouse/pull/32960) ([Bharat Nallan](https://github.com/bharatnc)). +* Sync release CI branches in master [#32967](https://github.com/ClickHouse/ClickHouse/pull/32967) ([alesapin](https://github.com/alesapin)). +* Loops remove postfix increment [#32970](https://github.com/ClickHouse/ClickHouse/pull/32970) ([Maksim Kita](https://github.com/kitaisreal)). +* Containers iteration fix erase [#32972](https://github.com/ClickHouse/ClickHouse/pull/32972) ([Maksim Kita](https://github.com/kitaisreal)). +* Merging [#32400](https://github.com/ClickHouse/ClickHouse/issues/32400) [#32974](https://github.com/ClickHouse/ClickHouse/pull/32974) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Decrease log level for some s3 messages. [#32979](https://github.com/ClickHouse/ClickHouse/pull/32979) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Trying to add woboq to new CI [#32983](https://github.com/ClickHouse/ClickHouse/pull/32983) ([alesapin](https://github.com/alesapin)). +* RabbitMQ tests fix [#32985](https://github.com/ClickHouse/ClickHouse/pull/32985) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix clang-tidy [#32987](https://github.com/ClickHouse/ClickHouse/pull/32987) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix typo [#32992](https://github.com/ClickHouse/ClickHouse/pull/32992) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix weird permission on log directories [#32994](https://github.com/ClickHouse/ClickHouse/pull/32994) ([Håvard Kvålen](https://github.com/havardk)). +* Add Keeper Jepsen check to new CI [#32998](https://github.com/ClickHouse/ClickHouse/pull/32998) ([alesapin](https://github.com/alesapin)). +* Add backport workflow rerun [#33005](https://github.com/ClickHouse/ClickHouse/pull/33005) ([alesapin](https://github.com/alesapin)). +* Remove the possibility of adding columns with table overrides [#33017](https://github.com/ClickHouse/ClickHouse/pull/33017) ([Stig Bakken](https://github.com/stigsb)). +* Merge [#33023](https://github.com/ClickHouse/ClickHouse/issues/33023) [#33027](https://github.com/ClickHouse/ClickHouse/pull/33027) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix cron expression [#33035](https://github.com/ClickHouse/ClickHouse/pull/33035) ([alesapin](https://github.com/alesapin)). +* Make `can be tested` great again [#33036](https://github.com/ClickHouse/ClickHouse/pull/33036) ([alesapin](https://github.com/alesapin)). +* Fix exception in azure write buffer desctructor [#33039](https://github.com/ClickHouse/ClickHouse/pull/33039) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add manual run for woboq [#33042](https://github.com/ClickHouse/ClickHouse/pull/33042) ([alesapin](https://github.com/alesapin)). +* Update Cassandra driver [#33043](https://github.com/ClickHouse/ClickHouse/pull/33043) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Decrease default timeout for integration test [#33044](https://github.com/ClickHouse/ClickHouse/pull/33044) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix merge [#33021](https://github.com/ClickHouse/ClickHouse/issues/33021) [#33049](https://github.com/ClickHouse/ClickHouse/pull/33049) ([Nikolay Degterinsky](https://github.com/evillique)). +* Update banner for january 20 release webinar [#33052](https://github.com/ClickHouse/ClickHouse/pull/33052) ([Cody Baker](https://github.com/codyrobert)). +* Update harmful library [#33058](https://github.com/ClickHouse/ClickHouse/pull/33058) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33025](https://github.com/ClickHouse/ClickHouse/issues/33025) [#33063](https://github.com/ClickHouse/ClickHouse/pull/33063) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33050](https://github.com/ClickHouse/ClickHouse/issues/33050) [#33065](https://github.com/ClickHouse/ClickHouse/pull/33065) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add alternative LRUCache version [#33098](https://github.com/ClickHouse/ClickHouse/pull/33098) ([lgbo](https://github.com/lgbo-ustc)). +* Fix test that was dependent on time zone [#33122](https://github.com/ClickHouse/ClickHouse/pull/33122) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix clang tidy [#33131](https://github.com/ClickHouse/ClickHouse/pull/33131) ([alesapin](https://github.com/alesapin)). +* Fix PVS check [#33135](https://github.com/ClickHouse/ClickHouse/pull/33135) ([alesapin](https://github.com/alesapin)). +* Upload build artifact in case of build failures [#33136](https://github.com/ClickHouse/ClickHouse/pull/33136) ([alesapin](https://github.com/alesapin)). +* Fix woboq codebrowser [#33137](https://github.com/ClickHouse/ClickHouse/pull/33137) ([alesapin](https://github.com/alesapin)). +* Longer timeout for server start in fuzzer [#33138](https://github.com/ClickHouse/ClickHouse/pull/33138) ([alesapin](https://github.com/alesapin)). +* Fix azure blob storage tests failures [#33140](https://github.com/ClickHouse/ClickHouse/pull/33140) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix functions with sparse arguments in case when result type doesn't support sparse serialization [#33146](https://github.com/ClickHouse/ClickHouse/pull/33146) ([Anton Popov](https://github.com/CurtizJ)). +* Add pretty print unknow event [#33148](https://github.com/ClickHouse/ClickHouse/pull/33148) ([alesapin](https://github.com/alesapin)). +* Better pr info for unknown events [#33149](https://github.com/ClickHouse/ClickHouse/pull/33149) ([alesapin](https://github.com/alesapin)). +* Better jepsen [#33150](https://github.com/ClickHouse/ClickHouse/pull/33150) ([alesapin](https://github.com/alesapin)). +* Debugging PR info [#33151](https://github.com/ClickHouse/ClickHouse/pull/33151) ([alesapin](https://github.com/alesapin)). +* Fix shared hermetic builds on Arch linux [#33153](https://github.com/ClickHouse/ClickHouse/pull/33153) ([Azat Khuzhin](https://github.com/azat)). +* Fix ccache with ENABLE_CHECK_HEAVY_BUILDS (ccache 4.0 and 4.1 only affected) [#33154](https://github.com/ClickHouse/ClickHouse/pull/33154) ([Azat Khuzhin](https://github.com/azat)). +* Cleanup trash from Kafka, HDFS and Azure [#33160](https://github.com/ClickHouse/ClickHouse/pull/33160) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix Date32 test [#33161](https://github.com/ClickHouse/ClickHouse/pull/33161) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make test_input_format_parallel_parsing_memory_tracking less flaky [#33163](https://github.com/ClickHouse/ClickHouse/pull/33163) ([Azat Khuzhin](https://github.com/azat)). +* Fix test_async_drain_connection flakiness [#33164](https://github.com/ClickHouse/ClickHouse/pull/33164) ([Azat Khuzhin](https://github.com/azat)). +* Fix clang-tidy [#33168](https://github.com/ClickHouse/ClickHouse/pull/33168) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test 01622_defaults_for_url_engine [#33169](https://github.com/ClickHouse/ClickHouse/pull/33169) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* materialized postgresql better startup [#33177](https://github.com/ClickHouse/ClickHouse/pull/33177) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible uncaught exception in clickhouse-local [#33189](https://github.com/ClickHouse/ClickHouse/pull/33189) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix tests with event_time/event_date = today(), and add a style check [#33198](https://github.com/ClickHouse/ClickHouse/pull/33198) ([Azat Khuzhin](https://github.com/azat)). +* Fix 01370_client_autocomplete_word_break_characters test logic [#33202](https://github.com/ClickHouse/ClickHouse/pull/33202) ([Azat Khuzhin](https://github.com/azat)). +* More debug in jepsen test [#33207](https://github.com/ClickHouse/ClickHouse/pull/33207) ([alesapin](https://github.com/alesapin)). +* Fix some flaky integration tests [#33215](https://github.com/ClickHouse/ClickHouse/pull/33215) ([alesapin](https://github.com/alesapin)). +* Remove old trash [#33217](https://github.com/ClickHouse/ClickHouse/pull/33217) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix cancel lambda [#33222](https://github.com/ClickHouse/ClickHouse/pull/33222) ([alesapin](https://github.com/alesapin)). +* Fix keeper log messages [#33224](https://github.com/ClickHouse/ClickHouse/pull/33224) ([alesapin](https://github.com/alesapin)). +* Fix clang tidy [#33235](https://github.com/ClickHouse/ClickHouse/pull/33235) ([alesapin](https://github.com/alesapin)). +* Fix flaky test 01650 [#33250](https://github.com/ClickHouse/ClickHouse/pull/33250) ([alesapin](https://github.com/alesapin)). +* Add retries for github api [#33252](https://github.com/ClickHouse/ClickHouse/pull/33252) ([alesapin](https://github.com/alesapin)). +* Add retries to AST fuzzer download and fix flaky test. [#33256](https://github.com/ClickHouse/ClickHouse/pull/33256) ([alesapin](https://github.com/alesapin)). +* Do not ignore eof in expect tests [#33263](https://github.com/ClickHouse/ClickHouse/pull/33263) ([Azat Khuzhin](https://github.com/azat)). +* Fix parsing symbols from resources (for shared builds) [#33264](https://github.com/ClickHouse/ClickHouse/pull/33264) ([Azat Khuzhin](https://github.com/azat)). +* test for [#24410](https://github.com/ClickHouse/ClickHouse/issues/24410) [#33265](https://github.com/ClickHouse/ClickHouse/pull/33265) ([Denny Crane](https://github.com/den-crane)). +* Wait for RabbitMQ container to actually start when it was restarted in test on purpose [#33266](https://github.com/ClickHouse/ClickHouse/pull/33266) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Mark max_alter_threads as obsolete [#33268](https://github.com/ClickHouse/ClickHouse/pull/33268) ([Denny Crane](https://github.com/den-crane)). +* Fix azure tests flackyness because of azure server closing connection [#33269](https://github.com/ClickHouse/ClickHouse/pull/33269) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Test for [#26920](https://github.com/ClickHouse/ClickHouse/issues/26920) [#33272](https://github.com/ClickHouse/ClickHouse/pull/33272) ([Denny Crane](https://github.com/den-crane)). +* Fix test_storage_kafka failures by adjusting retention.ms [#33278](https://github.com/ClickHouse/ClickHouse/pull/33278) ([Azat Khuzhin](https://github.com/azat)). +* Disable FunctionConvertFromString::canBeExecutedOnDefaultArguments [#33286](https://github.com/ClickHouse/ClickHouse/pull/33286) ([Vladimir C](https://github.com/vdimir)). +* Add IAST::QueryKind enum and use it in query limiter [#33294](https://github.com/ClickHouse/ClickHouse/pull/33294) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix clang tidy 3 [#33296](https://github.com/ClickHouse/ClickHouse/pull/33296) ([alesapin](https://github.com/alesapin)). +* Fix jepsen check [#33304](https://github.com/ClickHouse/ClickHouse/pull/33304) ([alesapin](https://github.com/alesapin)). +* Add test for broken connection [#33305](https://github.com/ClickHouse/ClickHouse/pull/33305) ([nvartolomei](https://github.com/nvartolomei)). +* Fix launcher not being defined for ccache < 4.0 and slightly cleanup [#33306](https://github.com/ClickHouse/ClickHouse/pull/33306) ([nvartolomei](https://github.com/nvartolomei)). +* Return early if azure blob storage is not used, otherwise it might with fail with irrelevant errors [#33307](https://github.com/ClickHouse/ClickHouse/pull/33307) ([nvartolomei](https://github.com/nvartolomei)). +* Minor refactor of read buffers with compression [#33318](https://github.com/ClickHouse/ClickHouse/pull/33318) ([Nikolay Degterinsky](https://github.com/evillique)). +* Rename Committed data part state to Active [#33327](https://github.com/ClickHouse/ClickHouse/pull/33327) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Tiny follow-up to pr 33098 [#33334](https://github.com/ClickHouse/ClickHouse/pull/33334) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix inner table parser in window view [#33340](https://github.com/ClickHouse/ClickHouse/pull/33340) ([vxider](https://github.com/Vxider)). +* Cover CustomSeparated/Template formats for Kafka [#33343](https://github.com/ClickHouse/ClickHouse/pull/33343) ([Azat Khuzhin](https://github.com/azat)). +* update copyright in docs [#33353](https://github.com/ClickHouse/ClickHouse/pull/33353) ([Bharat Nallan](https://github.com/bharatnc)). +* UserDefinedExecutableFunction fix exception [#33362](https://github.com/ClickHouse/ClickHouse/pull/33362) ([Maksim Kita](https://github.com/kitaisreal)). +* Trying to merge [#23230](https://github.com/ClickHouse/ClickHouse/issues/23230) [#33368](https://github.com/ClickHouse/ClickHouse/pull/33368) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Print alter command names in EXPLAIN AST [#33383](https://github.com/ClickHouse/ClickHouse/pull/33383) ([Stig Bakken](https://github.com/stigsb)). +* DictionaryStructure fixes [#33388](https://github.com/ClickHouse/ClickHouse/pull/33388) ([Maksim Kita](https://github.com/kitaisreal)). +* tests/stress: add core dumps into artifacts [#33389](https://github.com/ClickHouse/ClickHouse/pull/33389) ([Azat Khuzhin](https://github.com/azat)). +* Update success.html [#33397](https://github.com/ClickHouse/ClickHouse/pull/33397) ([Adri Fernandez](https://github.com/iladriano)). +* Remove obsolete and unsupported Docker Builder [#33345](https://github.com/ClickHouse/ClickHouse/issues/33345) [#33399](https://github.com/ClickHouse/ClickHouse/pull/33399) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Maybe improve robustness of the "query profiler" test [#33422](https://github.com/ClickHouse/ClickHouse/pull/33422) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix misleading log message in StorageWindowView [#33432](https://github.com/ClickHouse/ClickHouse/pull/33432) ([flynn](https://github.com/ucasfl)). +* Preparation for [#33167](https://github.com/ClickHouse/ClickHouse/issues/33167). [#33444](https://github.com/ClickHouse/ClickHouse/pull/33444) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Rerun docs release check as well [#33447](https://github.com/ClickHouse/ClickHouse/pull/33447) ([alesapin](https://github.com/alesapin)). +* Update AsynchronousReadIndirectBufferFromRemoteFS.h [#33452](https://github.com/ClickHouse/ClickHouse/pull/33452) ([xiedeyantu](https://github.com/xiedeyantu)). +* Tests visualizer better [#33453](https://github.com/ClickHouse/ClickHouse/pull/33453) ([Dmitry Khorkin](https://github.com/DimaAmega)). +* Function monthName small fix [#33464](https://github.com/ClickHouse/ClickHouse/pull/33464) ([Maksim Kita](https://github.com/kitaisreal)). +* Add a test for operator priorities in SQL [#33487](https://github.com/ClickHouse/ClickHouse/pull/33487) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add new team members to company page [#33506](https://github.com/ClickHouse/ClickHouse/pull/33506) ([Cody Baker](https://github.com/codyrobert)). +* fix variable name in h3 misc funcs [#33510](https://github.com/ClickHouse/ClickHouse/pull/33510) ([Bharat Nallan](https://github.com/bharatnc)). +* H3 functions bad arguments possible crash fix [#33512](https://github.com/ClickHouse/ClickHouse/pull/33512) ([Maksim Kita](https://github.com/kitaisreal)). +* Build with ENABLE_TESTS fix [#33513](https://github.com/ClickHouse/ClickHouse/pull/33513) ([Maksim Kita](https://github.com/kitaisreal)). +* IAST QueryKind style fix [#33514](https://github.com/ClickHouse/ClickHouse/pull/33514) ([Maksim Kita](https://github.com/kitaisreal)). +* Don't print exception twice in client in case of exception in parallel parsing [#33524](https://github.com/ClickHouse/ClickHouse/pull/33524) ([Kruglov Pavel](https://github.com/Avogar)). +* HashTable constant fix [#33525](https://github.com/ClickHouse/ClickHouse/pull/33525) ([Maksim Kita](https://github.com/kitaisreal)). +* Add admixer blog post [#33528](https://github.com/ClickHouse/ClickHouse/pull/33528) ([Cody Baker](https://github.com/codyrobert)). +* Fix clickhouse local interactive exception case [#33535](https://github.com/ClickHouse/ClickHouse/pull/33535) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix parsing queries with FROM INFILE statement again [#33556](https://github.com/ClickHouse/ClickHouse/pull/33556) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix logic for upload upload_master_static_binaries [#33560](https://github.com/ClickHouse/ClickHouse/pull/33560) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Don't upload splitted binaries to builds.clickhouse.com [#33562](https://github.com/ClickHouse/ClickHouse/pull/33562) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix minor bug in TTL and flaky test [#33564](https://github.com/ClickHouse/ClickHouse/pull/33564) ([Alexander Tokmakov](https://github.com/tavplubix)). +* skip MySQLCreateRewritten unit testsuite when enable_mysql is off [#33577](https://github.com/ClickHouse/ClickHouse/pull/33577) ([Suzy Wang](https://github.com/SuzyWangIBMer)). +* Fix test `02156_storage_merge_prewhere` [#33580](https://github.com/ClickHouse/ClickHouse/pull/33580) ([Anton Popov](https://github.com/CurtizJ)). +* Update mongodb.md [#33585](https://github.com/ClickHouse/ClickHouse/pull/33585) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Restore existing static builds links [#33597](https://github.com/ClickHouse/ClickHouse/pull/33597) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix pylint for run_check.py [#33600](https://github.com/ClickHouse/ClickHouse/pull/33600) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix flacky test_dictionaries_postgresql/ [#33601](https://github.com/ClickHouse/ClickHouse/pull/33601) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Make ZooKeeper client better interpret keeper server connection reject [#33602](https://github.com/ClickHouse/ClickHouse/pull/33602) ([alesapin](https://github.com/alesapin)). +* Fix broken workflow dependencies [#33608](https://github.com/ClickHouse/ClickHouse/pull/33608) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Force rebuild images in CI [#33609](https://github.com/ClickHouse/ClickHouse/pull/33609) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* YetAnother workflow fix [#33610](https://github.com/ClickHouse/ClickHouse/pull/33610) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Metrics lambda: fix deletion [#33616](https://github.com/ClickHouse/ClickHouse/pull/33616) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add an exception message example [#33623](https://github.com/ClickHouse/ClickHouse/pull/33623) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix two flaky integration tests [#33625](https://github.com/ClickHouse/ClickHouse/pull/33625) ([alesapin](https://github.com/alesapin)). +* Update version to 22.1 not 21.13 [#33626](https://github.com/ClickHouse/ClickHouse/pull/33626) ([alesapin](https://github.com/alesapin)). +* do not construct std::string if there is no error [#33628](https://github.com/ClickHouse/ClickHouse/pull/33628) ([Sergei Trifonov](https://github.com/serxa)). +* Fix test 00900_long_parquet_load [#33629](https://github.com/ClickHouse/ClickHouse/pull/33629) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix error codes order [#33633](https://github.com/ClickHouse/ClickHouse/pull/33633) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix s3 integration tests [#33641](https://github.com/ClickHouse/ClickHouse/pull/33641) ([Kruglov Pavel](https://github.com/Avogar)). +* test for [#33592](https://github.com/ClickHouse/ClickHouse/issues/33592) [#33658](https://github.com/ClickHouse/ClickHouse/pull/33658) ([Denny Crane](https://github.com/den-crane)). +* Update "gosu" version [#33660](https://github.com/ClickHouse/ClickHouse/pull/33660) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix 01814_distributed_push_down_limit flakiness [#33662](https://github.com/ClickHouse/ClickHouse/pull/33662) ([Azat Khuzhin](https://github.com/azat)). +* Add amosbird to trusted user. [#33663](https://github.com/ClickHouse/ClickHouse/pull/33663) ([Amos Bird](https://github.com/amosbird)). +* Fix pr ci restart [#33667](https://github.com/ClickHouse/ClickHouse/pull/33667) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + #### New Feature / New Tool * Tool for collecting diagnostics data. [#33175](https://github.com/ClickHouse/ClickHouse/pull/33175) ([Alexander Burmak](https://github.com/Alex-Burmak)). diff --git a/docs/changelogs/v22.2.1.2139-prestable.md b/docs/changelogs/v22.2.1.2139-prestable.md index f93a3a690c4..379fb8988c8 100644 --- a/docs/changelogs/v22.2.1.2139-prestable.md +++ b/docs/changelogs/v22.2.1.2139-prestable.md @@ -215,3 +215,164 @@ sidebar_label: 2022 * NO CL ENTRY: 'Add support agreement page and snippets.'. [#34512](https://github.com/ClickHouse/ClickHouse/pull/34512) ([Tom Risse](https://github.com/flickerbox-tom)). * NO CL ENTRY: 'Add Gigasheet to adopters'. [#34589](https://github.com/ClickHouse/ClickHouse/pull/34589) ([Brian Hunter](https://github.com/bjhunter)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix status of SKIPPED integration tests (was incorrectly marked as ERROR) [#33162](https://github.com/ClickHouse/ClickHouse/pull/33162) ([Azat Khuzhin](https://github.com/azat)). +* Client interactive suggest (extract info from CREATE queries) [#33201](https://github.com/ClickHouse/ClickHouse/pull/33201) ([Azat Khuzhin](https://github.com/azat)). +* Fix flushing of in-memory parts [#33234](https://github.com/ClickHouse/ClickHouse/pull/33234) ([Anton Popov](https://github.com/CurtizJ)). +* improvements to tests for h3kRing and h3ToChildren funcs [#33311](https://github.com/ClickHouse/ClickHouse/pull/33311) ([Bharat Nallan](https://github.com/bharatnc)). +* Simplify different block structure (i.e. after ALTER) support for Buffer [#33324](https://github.com/ClickHouse/ClickHouse/pull/33324) ([Azat Khuzhin](https://github.com/azat)). +* Dictionary rename fix [#33526](https://github.com/ClickHouse/ClickHouse/pull/33526) ([Maksim Kita](https://github.com/kitaisreal)). +* RFC: Split headers, move SystemLog into module, more forward declarations [#33534](https://github.com/ClickHouse/ClickHouse/pull/33534) ([Azat Khuzhin](https://github.com/azat)). +* add check for h3 empty column arguments [#33552](https://github.com/ClickHouse/ClickHouse/pull/33552) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix testflows tests [#33575](https://github.com/ClickHouse/ClickHouse/pull/33575) ([Vitaly Baranov](https://github.com/vitlibar)). +* Ignore parse failure of opentelemetry header. Another try. [#33595](https://github.com/ClickHouse/ClickHouse/pull/33595) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix mongo no_auth compose [#33634](https://github.com/ClickHouse/ClickHouse/pull/33634) ([Ilya Yatsishin](https://github.com/qoega)). +* AsynchronousMetrics: Ignore inaccessible sensors [#33639](https://github.com/ClickHouse/ClickHouse/pull/33639) ([Raúl Marín](https://github.com/Algunenano)). +* Slightly reduce memory usage for parsing collections (by using move ctor) [#33665](https://github.com/ClickHouse/ClickHouse/pull/33665) ([Azat Khuzhin](https://github.com/azat)). +* Merge [#33563](https://github.com/ClickHouse/ClickHouse/issues/33563) [#33669](https://github.com/ClickHouse/ClickHouse/pull/33669) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version after release 22.1 [#33673](https://github.com/ClickHouse/ClickHouse/pull/33673) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix sumIf rewrite [#33677](https://github.com/ClickHouse/ClickHouse/pull/33677) ([flynn](https://github.com/ucasfl)). +* DictionarySourceCoordinator update interface [#33682](https://github.com/ClickHouse/ClickHouse/pull/33682) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix LDAP and Kerberos config handling [#33689](https://github.com/ClickHouse/ClickHouse/pull/33689) ([Denis Glazachev](https://github.com/traceon)). +* Add some helping comments for API endpoints [#33700](https://github.com/ClickHouse/ClickHouse/pull/33700) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix hive tests [#33703](https://github.com/ClickHouse/ClickHouse/pull/33703) ([Kseniia Sumarokova](https://github.com/kssenii)). +* H3 remove functions degsToRads, radsToDegs [#33707](https://github.com/ClickHouse/ClickHouse/pull/33707) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove unused variable [#33731](https://github.com/ClickHouse/ClickHouse/pull/33731) ([vxider](https://github.com/Vxider)). +* Add test issue_31009 [#33739](https://github.com/ClickHouse/ClickHouse/pull/33739) ([Vladimir C](https://github.com/vdimir)). +* Cleanup build: .gitignore more debian directories, libpqxx-cmake without configure_file [#33742](https://github.com/ClickHouse/ClickHouse/pull/33742) ([Ilya Yatsishin](https://github.com/qoega)). +* Better exception text on suspicious broken parts [#33743](https://github.com/ClickHouse/ClickHouse/pull/33743) ([alesapin](https://github.com/alesapin)). +* Fix release_branches workflow for some cases [#33744](https://github.com/ClickHouse/ClickHouse/pull/33744) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add new team members to company page [#33755](https://github.com/ClickHouse/ClickHouse/pull/33755) ([Cody Baker](https://github.com/codyrobert)). +* Remove duplicated header [#33760](https://github.com/ClickHouse/ClickHouse/pull/33760) ([vxider](https://github.com/Vxider)). +* Hotfix of missing header [#33765](https://github.com/ClickHouse/ClickHouse/pull/33765) ([Amos Bird](https://github.com/amosbird)). +* rewrite bitHammingDistance with FunctionBinaryArithmetic [#33772](https://github.com/ClickHouse/ClickHouse/pull/33772) ([flynn](https://github.com/ucasfl)). +* Use tb64senc for base64Decode on aarch64 [#33779](https://github.com/ClickHouse/ClickHouse/pull/33779) ([Vladimir C](https://github.com/vdimir)). +* Support USE_* build flags in tests' tags [#33780](https://github.com/ClickHouse/ClickHouse/pull/33780) ([Vladimir C](https://github.com/vdimir)). +* Use workflow names in approve lambda [#33789](https://github.com/ClickHouse/ClickHouse/pull/33789) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Dictionaries remove unnecessary copy of keys during read [#33791](https://github.com/ClickHouse/ClickHouse/pull/33791) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove debug logging from TableFunctionFile [#33796](https://github.com/ClickHouse/ClickHouse/pull/33796) ([Kruglov Pavel](https://github.com/Avogar)). +* RangeHashedDictionary handle invalid intervals [#33827](https://github.com/ClickHouse/ClickHouse/pull/33827) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionaries added Date32 type support [#33831](https://github.com/ClickHouse/ClickHouse/pull/33831) ([Maksim Kita](https://github.com/kitaisreal)). +* TypeId better naming [#33832](https://github.com/ClickHouse/ClickHouse/pull/33832) ([Maksim Kita](https://github.com/kitaisreal)). +* FunctionsConversion fix typo [#33841](https://github.com/ClickHouse/ClickHouse/pull/33841) ([Maksim Kita](https://github.com/kitaisreal)). +* More cmake external modules cleanups [#33842](https://github.com/ClickHouse/ClickHouse/pull/33842) ([Azat Khuzhin](https://github.com/azat)). +* Remove harmful code and check what will happen [#33844](https://github.com/ClickHouse/ClickHouse/pull/33844) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix DESCRIBE TABLE query formatting [#33846](https://github.com/ClickHouse/ClickHouse/pull/33846) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix build with ENABLE_EXAMPLES [#33848](https://github.com/ClickHouse/ClickHouse/pull/33848) ([Maksim Kita](https://github.com/kitaisreal)). +* Add Trademark Policy Page [#33851](https://github.com/ClickHouse/ClickHouse/pull/33851) ([Cody Baker](https://github.com/codyrobert)). +* updates to min2 and max2 funcs [#33852](https://github.com/ClickHouse/ClickHouse/pull/33852) ([Bharat Nallan](https://github.com/bharatnc)). +* Use correct logging level [#33857](https://github.com/ClickHouse/ClickHouse/pull/33857) ([李扬](https://github.com/taiyang-li)). +* Fix zookeeper library dependency from interpreters (by marking library STATIC) [#33860](https://github.com/ClickHouse/ClickHouse/pull/33860) ([Azat Khuzhin](https://github.com/azat)). +* FunctionBase64Conversion warning fix [#33863](https://github.com/ClickHouse/ClickHouse/pull/33863) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionary ATTACH, DETACH added test [#33870](https://github.com/ClickHouse/ClickHouse/pull/33870) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix perf comparison rerun [#33872](https://github.com/ClickHouse/ClickHouse/pull/33872) ([alesapin](https://github.com/alesapin)). +* Improve postgresql integration test [#33880](https://github.com/ClickHouse/ClickHouse/pull/33880) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix cmake for mac. [#33882](https://github.com/ClickHouse/ClickHouse/pull/33882) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* SQLUserDefinedFunctions invalid lambda additional fixes [#33889](https://github.com/ClickHouse/ClickHouse/pull/33889) ([Maksim Kita](https://github.com/kitaisreal)). +* Update upcoming and past webinar links on homepage [#33890](https://github.com/ClickHouse/ClickHouse/pull/33890) ([Cody Baker](https://github.com/codyrobert)). +* disable animation on docs menu [#33903](https://github.com/ClickHouse/ClickHouse/pull/33903) ([SuperDJY](https://github.com/cmsxbc)). +* Add more retries in CI for AWS "spot" instances [#33907](https://github.com/ClickHouse/ClickHouse/pull/33907) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* FunctionMathUnary remove macro usage [#33916](https://github.com/ClickHouse/ClickHouse/pull/33916) ([Maksim Kita](https://github.com/kitaisreal)). +* explicitly checkAndGetColumn for remaining H3 funcs [#33921](https://github.com/ClickHouse/ClickHouse/pull/33921) ([Bharat Nallan](https://github.com/bharatnc)). +* SQLUserDefinedFunctions invalid lambda additional cases [#33924](https://github.com/ClickHouse/ClickHouse/pull/33924) ([Maksim Kita](https://github.com/kitaisreal)). +* when not DEBUG set USE_DEBUG_HELPERS OFF [#33925](https://github.com/ClickHouse/ClickHouse/pull/33925) ([Ben](https://github.com/benbiti)). +* Fix test_replica_is_active flaky test [#33926](https://github.com/ClickHouse/ClickHouse/pull/33926) ([alesapin](https://github.com/alesapin)). +* Fix bug in keeper which can lead to inconsistent snapshots [#33941](https://github.com/ClickHouse/ClickHouse/pull/33941) ([alesapin](https://github.com/alesapin)). +* Fix keeper data dumper build [#33942](https://github.com/ClickHouse/ClickHouse/pull/33942) ([alesapin](https://github.com/alesapin)). +* Remove MAKE_STATIC_LIBRARIES (in favor of USE_STATIC_LIBRARIES) [#33946](https://github.com/ClickHouse/ClickHouse/pull/33946) ([Azat Khuzhin](https://github.com/azat)). +* Revert glibc compatibility (via .symver) in favor of hermetic build (bundled libc) [#33950](https://github.com/ClickHouse/ClickHouse/pull/33950) ([Azat Khuzhin](https://github.com/azat)). +* add c++expr script for C++ one-liners [#33962](https://github.com/ClickHouse/ClickHouse/pull/33962) ([Sergei Trifonov](https://github.com/serxa)). +* docker: fix root squashed data dirs [#33963](https://github.com/ClickHouse/ClickHouse/pull/33963) ([Constantine Peresypkin](https://github.com/pkit)). +* Disable parallel run for 00985_merge_stack_overflow [#33976](https://github.com/ClickHouse/ClickHouse/pull/33976) ([Azat Khuzhin](https://github.com/azat)). +* Cancel lambda [#33990](https://github.com/ClickHouse/ClickHouse/pull/33990) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* reduce signal_pipe_buf_size [#33996](https://github.com/ClickHouse/ClickHouse/pull/33996) ([save-my-heart](https://github.com/save-my-heart)). +* make systemd to use EnvironmentFile [#34024](https://github.com/ClickHouse/ClickHouse/pull/34024) ([Denny Crane](https://github.com/den-crane)). +* CurrentlyExecuting: Require mutex usage explicitly [#34034](https://github.com/ClickHouse/ClickHouse/pull/34034) ([Raúl Marín](https://github.com/Algunenano)). +* Add symlinks to keeper [#34042](https://github.com/ClickHouse/ClickHouse/pull/34042) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove DecimalPaddedPODArray [#34052](https://github.com/ClickHouse/ClickHouse/pull/34052) ([Maksim Kita](https://github.com/kitaisreal)). +* Less flaky test_inconsistent_parts_if_drop_while_replica_not_active [#34057](https://github.com/ClickHouse/ClickHouse/pull/34057) ([Raúl Marín](https://github.com/Algunenano)). +* Small test for `FINAL` [#34058](https://github.com/ClickHouse/ClickHouse/pull/34058) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Slightly optimize ColumnArray::get()/operator[] (by using reserve over resize) [#34074](https://github.com/ClickHouse/ClickHouse/pull/34074) ([Azat Khuzhin](https://github.com/azat)). +* Tiny cleanup of AggregateFunctionSimpleState/AggregateFunctionState [#34075](https://github.com/ClickHouse/ClickHouse/pull/34075) ([Azat Khuzhin](https://github.com/azat)). +* Fix builds [#34090](https://github.com/ClickHouse/ClickHouse/pull/34090) ([Alexander Tokmakov](https://github.com/tavplubix)). +* add c++expr script examples [#34112](https://github.com/ClickHouse/ClickHouse/pull/34112) ([Sergei Trifonov](https://github.com/serxa)). +* Make clickhouse-diagnostics also work for altinity release [#34116](https://github.com/ClickHouse/ClickHouse/pull/34116) ([Ramazan Polat](https://github.com/ramazanpolat)). +* Small improvement in schema inference from stdin in local [#34117](https://github.com/ClickHouse/ClickHouse/pull/34117) ([Kruglov Pavel](https://github.com/Avogar)). +* Slightly optimize Array/Tuple/Map [#34126](https://github.com/ClickHouse/ClickHouse/pull/34126) ([Azat Khuzhin](https://github.com/azat)). +* Sort block refactoring [#34143](https://github.com/ClickHouse/ClickHouse/pull/34143) ([Maksim Kita](https://github.com/kitaisreal)). +* pdqsort performance check [#34145](https://github.com/ClickHouse/ClickHouse/pull/34145) ([Maksim Kita](https://github.com/kitaisreal)). +* Revert [#33957](https://github.com/ClickHouse/ClickHouse/issues/33957) [#34146](https://github.com/ClickHouse/ClickHouse/pull/34146) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add func tests run with s3 [#34153](https://github.com/ClickHouse/ClickHouse/pull/34153) ([alesapin](https://github.com/alesapin)). +* Close [#10197](https://github.com/ClickHouse/ClickHouse/issues/10197) [#34159](https://github.com/ClickHouse/ClickHouse/pull/34159) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Cmake leftovers cleanup [#34167](https://github.com/ClickHouse/ClickHouse/pull/34167) ([Azat Khuzhin](https://github.com/azat)). +* bitsetsort peformance check [#34175](https://github.com/ClickHouse/ClickHouse/pull/34175) ([Maksim Kita](https://github.com/kitaisreal)). +* Add authorisation for dockerhub proxy container [#34183](https://github.com/ClickHouse/ClickHouse/pull/34183) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Additionally check remote_fs_execute_merges_on_single_replica_time_threshold inside ReplicatedMergeTreeQueue [#34189](https://github.com/ClickHouse/ClickHouse/pull/34189) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix some perf tests [#34191](https://github.com/ClickHouse/ClickHouse/pull/34191) ([Kruglov Pavel](https://github.com/Avogar)). +* Clean up: insert_deduplication_token setting for INSERT statement [#34192](https://github.com/ClickHouse/ClickHouse/pull/34192) ([Igor Nikonov](https://github.com/devcrafter)). +* Add func tests run with s3 and fix several bugs [#34215](https://github.com/ClickHouse/ClickHouse/pull/34215) ([alesapin](https://github.com/alesapin)). +* Revert "Revert "Add pool to WriteBufferFromS3"" [#34219](https://github.com/ClickHouse/ClickHouse/pull/34219) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Revert "Revert "Additionally check remote_fs_execute_merges_on_single_replica_time_threshold inside ReplicatedMergeTreeQueue"" [#34221](https://github.com/ClickHouse/ClickHouse/pull/34221) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add const to make clang-tidy happy [#34222](https://github.com/ClickHouse/ClickHouse/pull/34222) ([Vitaly Baranov](https://github.com/vitlibar)). +* Bump fmtlib from 7.0.0 to 8.1.1 [#34223](https://github.com/ClickHouse/ClickHouse/pull/34223) ([Azat Khuzhin](https://github.com/azat)). +* Add submodule minizip [#34226](https://github.com/ClickHouse/ClickHouse/pull/34226) ([Vitaly Baranov](https://github.com/vitlibar)). +* Update list-versions.sh, update version_date.tsv [#34240](https://github.com/ClickHouse/ClickHouse/pull/34240) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add missing fmt::runtime() in MergeTreeBackgroundExecutor (fixes the build) [#34245](https://github.com/ClickHouse/ClickHouse/pull/34245) ([Azat Khuzhin](https://github.com/azat)). +* Update clickhouse-keeper.md [#34264](https://github.com/ClickHouse/ClickHouse/pull/34264) ([Andrew](https://github.com/andycol)). +* print query id when using `--interactive` with `--queries-file` in client/local [#34265](https://github.com/ClickHouse/ClickHouse/pull/34265) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update and add new team members [#34268](https://github.com/ClickHouse/ClickHouse/pull/34268) ([Cody Baker](https://github.com/codyrobert)). +* Add a test for shebang [#34274](https://github.com/ClickHouse/ClickHouse/pull/34274) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix fractional_progress_bar test [#34282](https://github.com/ClickHouse/ClickHouse/pull/34282) ([Maksim Kita](https://github.com/kitaisreal)). +* Minor fixes for [#34267](https://github.com/ClickHouse/ClickHouse/issues/34267) [#34284](https://github.com/ClickHouse/ClickHouse/pull/34284) ([Anton Popov](https://github.com/CurtizJ)). +* Add support policy page content. [#34309](https://github.com/ClickHouse/ClickHouse/pull/34309) ([Tom Risse](https://github.com/flickerbox-tom)). +* Probably fix data race in WriteBufferFromS3 destructor. [#34316](https://github.com/ClickHouse/ClickHouse/pull/34316) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Decrease severity for "Reading ... ranges ..." log message to Trace [#34319](https://github.com/ClickHouse/ClickHouse/pull/34319) ([Azat Khuzhin](https://github.com/azat)). +* use LowCardinality type for _file and _path in some Storages, continue of [#34317](https://github.com/ClickHouse/ClickHouse/issues/34317) [#34333](https://github.com/ClickHouse/ClickHouse/pull/34333) ([flynn](https://github.com/ucasfl)). +* Function mapPopulateSeries added additional performance test [#34339](https://github.com/ClickHouse/ClickHouse/pull/34339) ([Maksim Kita](https://github.com/kitaisreal)). +* Add test for propagating OpenTelemetry context via gRPC protocol [#34341](https://github.com/ClickHouse/ClickHouse/pull/34341) ([Vitaly Baranov](https://github.com/vitlibar)). +* explicitly check and get columns for s2 funcs [#34344](https://github.com/ClickHouse/ClickHouse/pull/34344) ([Bharat Nallan](https://github.com/bharatnc)). +* Small improvements [#34351](https://github.com/ClickHouse/ClickHouse/pull/34351) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix USE database for clickhouse-local [#34357](https://github.com/ClickHouse/ClickHouse/pull/34357) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix clang-tidy issue [#34365](https://github.com/ClickHouse/ClickHouse/pull/34365) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Delete empty file DateOrDateTimeFunctionsConvertion.cpp [#34371](https://github.com/ClickHouse/ClickHouse/pull/34371) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix consecutive backward seeks in seekable read buffers [#34376](https://github.com/ClickHouse/ClickHouse/pull/34376) ([Anton Popov](https://github.com/CurtizJ)). +* Enable one more check for clang-tidy [#34388](https://github.com/ClickHouse/ClickHouse/pull/34388) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Method called on already moved [#34398](https://github.com/ClickHouse/ClickHouse/pull/34398) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* Fix wrong destruction order in CreatingSetsTransform. [#34406](https://github.com/ClickHouse/ClickHouse/pull/34406) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Split and rename compression fields in gRPC [#34408](https://github.com/ClickHouse/ClickHouse/pull/34408) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixing test_storage_postgresql [#34410](https://github.com/ClickHouse/ClickHouse/pull/34410) ([Kseniia Sumarokova](https://github.com/kssenii)). +* asynchronous_inserts engine AsynchronousInserts -> SystemAsynchronousInserts [#34429](https://github.com/ClickHouse/ClickHouse/pull/34429) ([filimonov](https://github.com/filimonov)). +* clang-tidy move fix build [#34431](https://github.com/ClickHouse/ClickHouse/pull/34431) ([Maksim Kita](https://github.com/kitaisreal)). +* `static-files-disk-uploader`: add a mode to create symlinks [#34432](https://github.com/ClickHouse/ClickHouse/pull/34432) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bug in URL engine [#34448](https://github.com/ClickHouse/ClickHouse/pull/34448) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix style [#34450](https://github.com/ClickHouse/ClickHouse/pull/34450) ([alesapin](https://github.com/alesapin)). +* Added test 33734 [#34454](https://github.com/ClickHouse/ClickHouse/pull/34454) ([Maksim Kita](https://github.com/kitaisreal)). +* Update http_max_tries setting default [#34457](https://github.com/ClickHouse/ClickHouse/pull/34457) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Use `cpp_bin_float_double` in `set_multiplier` for `wide_integer_from_builtin` for aarch64 [#34463](https://github.com/ClickHouse/ClickHouse/pull/34463) ([Vladimir C](https://github.com/vdimir)). +* test for [#13907](https://github.com/ClickHouse/ClickHouse/issues/13907) toColumnTypeName_toLowCardinality_const [#34471](https://github.com/ClickHouse/ClickHouse/pull/34471) ([Denny Crane](https://github.com/den-crane)). +* Remove invalid IOS setting for RocksDB CMAKE to fix Apple M1 build [#34472](https://github.com/ClickHouse/ClickHouse/pull/34472) ([Geoff Genz](https://github.com/genzgd)). +* clang-tidy reported potential chance for divide by zero exception [#34473](https://github.com/ClickHouse/ClickHouse/pull/34473) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* accessing nested_column after already moved to data [#34475](https://github.com/ClickHouse/ClickHouse/pull/34475) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* Avoid unnecessary copying of `Settings` [#34476](https://github.com/ClickHouse/ClickHouse/pull/34476) ([Anton Popov](https://github.com/CurtizJ)). +* Update buildPushingToViewsChain.h [#34515](https://github.com/ClickHouse/ClickHouse/pull/34515) ([William.Walliams.Wong](https://github.com/wangzhen11aaa)). +* More gdb introspection on CI [#34517](https://github.com/ClickHouse/ClickHouse/pull/34517) ([Azat Khuzhin](https://github.com/azat)). +* add BeforeLambdaBody to .clang-format [#34533](https://github.com/ClickHouse/ClickHouse/pull/34533) ([Vladimir C](https://github.com/vdimir)). +* Remove very old cruft [#34547](https://github.com/ClickHouse/ClickHouse/pull/34547) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Tiny fixes for client/local suggestions [#34550](https://github.com/ClickHouse/ClickHouse/pull/34550) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix gtest_archive_reader_and_writer in case of !USE_MINIZIP [#34554](https://github.com/ClickHouse/ClickHouse/pull/34554) ([Azat Khuzhin](https://github.com/azat)). +* TableFunctionFile added performance test [#34555](https://github.com/ClickHouse/ClickHouse/pull/34555) ([Maksim Kita](https://github.com/kitaisreal)). +* check and get columns in geoToH3 func [#34557](https://github.com/ClickHouse/ClickHouse/pull/34557) ([Bharat Nallan](https://github.com/bharatnc)). +* try fix data race in StorageLog [#34558](https://github.com/ClickHouse/ClickHouse/pull/34558) ([flynn](https://github.com/ucasfl)). +* Postpone a bit nightly builds to the least loaded time [#34569](https://github.com/ClickHouse/ClickHouse/pull/34569) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Always remove unused actions from addMissingDefaults [#34577](https://github.com/ClickHouse/ClickHouse/pull/34577) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix indentation in ARRAY JOIN formatting [#34578](https://github.com/ClickHouse/ClickHouse/pull/34578) ([Azat Khuzhin](https://github.com/azat)). +* Fix deadlock in OvercommitTracker [#34591](https://github.com/ClickHouse/ClickHouse/pull/34591) ([Dmitry Novik](https://github.com/novikd)). +* ASTCreateQuery: Remove usused `tables` member [#34610](https://github.com/ClickHouse/ClickHouse/pull/34610) ([Raúl Marín](https://github.com/Algunenano)). +* Add test for [#19222](https://github.com/ClickHouse/ClickHouse/issues/19222) [#34615](https://github.com/ClickHouse/ClickHouse/pull/34615) ([Raúl Marín](https://github.com/Algunenano)). +* Attempt to fix freeBSD build [#34617](https://github.com/ClickHouse/ClickHouse/pull/34617) ([Raúl Marín](https://github.com/Algunenano)). +* Fix quadratic complexity while adding subcolumns [#34623](https://github.com/ClickHouse/ClickHouse/pull/34623) ([Anton Popov](https://github.com/CurtizJ)). +* Fix typo in welchttest.md [#34634](https://github.com/ClickHouse/ClickHouse/pull/34634) ([achimbab](https://github.com/achimbab)). +* Improve the release.py script [#34659](https://github.com/ClickHouse/ClickHouse/pull/34659) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Disable memory overcommit if timeout is not set [#34663](https://github.com/ClickHouse/ClickHouse/pull/34663) ([Dmitry Novik](https://github.com/novikd)). + diff --git a/docs/changelogs/v22.2.3.5-stable.md b/docs/changelogs/v22.2.3.5-stable.md index 24b535a7f91..2eb8a6b63cc 100644 --- a/docs/changelogs/v22.2.3.5-stable.md +++ b/docs/changelogs/v22.2.3.5-stable.md @@ -11,3 +11,7 @@ sidebar_label: 2022 * Backported in [#34848](https://github.com/ClickHouse/ClickHouse/issues/34848): Fix possible failures in S2 functions when queries contain const columns. [#34745](https://github.com/ClickHouse/ClickHouse/pull/34745) ([Bharat Nallan](https://github.com/bharatnc)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix using host/port from config for client [#34791](https://github.com/ClickHouse/ClickHouse/pull/34791) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v22.3.1.1262-prestable.md b/docs/changelogs/v22.3.1.1262-prestable.md index d2f976f4517..0058396d634 100644 --- a/docs/changelogs/v22.3.1.1262-prestable.md +++ b/docs/changelogs/v22.3.1.1262-prestable.md @@ -151,3 +151,124 @@ sidebar_label: 2022 * NO CL ENTRY: 'Revert "Change timezone in Docker"'. [#35243](https://github.com/ClickHouse/ClickHouse/pull/35243) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * NO CL ENTRY: 'Revert "Fix 00900_long_parquet_load"'. [#35301](https://github.com/ClickHouse/ClickHouse/pull/35301) ([Vladimir C](https://github.com/vdimir)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* [RFC] Update jemalloc to 5.3RC [#33057](https://github.com/ClickHouse/ClickHouse/pull/33057) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless setting experimental_query_deduplication_send_all_part_uuids [#34387](https://github.com/ClickHouse/ClickHouse/pull/34387) ([nvartolomei](https://github.com/nvartolomei)). +* Apply join_use_nulls on types before join [#34529](https://github.com/ClickHouse/ClickHouse/pull/34529) ([Vladimir C](https://github.com/vdimir)). +* Adjust 01681_arg_min_max_if_fix [#34612](https://github.com/ClickHouse/ClickHouse/pull/34612) ([Vladimir C](https://github.com/vdimir)). +* Check overflow in addSeconds/Minues/etc functions [#34619](https://github.com/ClickHouse/ClickHouse/pull/34619) ([Vladimir C](https://github.com/vdimir)). +* Update version after release [#34677](https://github.com/ClickHouse/ClickHouse/pull/34677) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix `positionUTF8` on aarch64 [#34683](https://github.com/ClickHouse/ClickHouse/pull/34683) ([Vladimir C](https://github.com/vdimir)). +* Update version_date.tsv after v22.2.2.1-stable [#34685](https://github.com/ClickHouse/ClickHouse/pull/34685) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Fix ZooKepper paths in zero_copy_schema_converter.py [#34686](https://github.com/ClickHouse/ClickHouse/pull/34686) ([ianton-ru](https://github.com/ianton-ru)). +* Changelog for version 22.2 [#34687](https://github.com/ClickHouse/ClickHouse/pull/34687) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Follow up to PR [#33698](https://github.com/ClickHouse/ClickHouse/issues/33698) [#34689](https://github.com/ClickHouse/ClickHouse/pull/34689) ([Vladimir C](https://github.com/vdimir)). +* Always update ProfileEvents (even on exceptions) [#34690](https://github.com/ClickHouse/ClickHouse/pull/34690) ([nvartolomei](https://github.com/nvartolomei)). +* Function encodeURLComponent minor fixes [#34695](https://github.com/ClickHouse/ClickHouse/pull/34695) ([Maksim Kita](https://github.com/kitaisreal)). +* CMake llvm add status message if not used [#34698](https://github.com/ClickHouse/ClickHouse/pull/34698) ([Maksim Kita](https://github.com/kitaisreal)). +* Adding noexcept for move constructor SharedContextHolder [#34699](https://github.com/ClickHouse/ClickHouse/pull/34699) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* Add new team members to company page [#34701](https://github.com/ClickHouse/ClickHouse/pull/34701) ([Cody Baker](https://github.com/codyrobert)). +* Fix flaky 00502_custom_partitioning_local [#34714](https://github.com/ClickHouse/ClickHouse/pull/34714) ([Azat Khuzhin](https://github.com/azat)). +* Workaround for a bug in NuRaft library [#34715](https://github.com/ClickHouse/ClickHouse/pull/34715) ([Azat Khuzhin](https://github.com/azat)). +* Fix possible memory_tracker use-after-free (for async s3 writes) for merges/mutations [#34717](https://github.com/ClickHouse/ClickHouse/pull/34717) ([Azat Khuzhin](https://github.com/azat)). +* Add taiyang-li to trusted contributors [#34729](https://github.com/ClickHouse/ClickHouse/pull/34729) ([Vladimir C](https://github.com/vdimir)). +* rename name in ci check [#34730](https://github.com/ClickHouse/ClickHouse/pull/34730) ([flynn](https://github.com/ucasfl)). +* Performance tests fix H3 [#34731](https://github.com/ClickHouse/ClickHouse/pull/34731) ([Maksim Kita](https://github.com/kitaisreal)). +* Enable async writes to S3. [#34734](https://github.com/ClickHouse/ClickHouse/pull/34734) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add a patch release for stable/lts packages to release.py [#34740](https://github.com/ClickHouse/ClickHouse/pull/34740) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Print only total profile events for --profile-events-delay-ms=-1 [#34749](https://github.com/ClickHouse/ClickHouse/pull/34749) ([Azat Khuzhin](https://github.com/azat)). +* Fix `parallel_reading_from_replicas` with `clickhouse-bechmark` [#34751](https://github.com/ClickHouse/ClickHouse/pull/34751) ([Azat Khuzhin](https://github.com/azat)). +* Submodules are mandatory [#34753](https://github.com/ClickHouse/ClickHouse/pull/34753) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Changelog for version 22.2 [#34758](https://github.com/ClickHouse/ClickHouse/pull/34758) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Sort additional_html_urls in test report in CI [#34770](https://github.com/ClickHouse/ClickHouse/pull/34770) ([Vladimir C](https://github.com/vdimir)). +* Fix undefined __pthread_mutex_lock/unlock for glibc 2.34+/DISABLE_HERMETIC_BUILD [#34771](https://github.com/ClickHouse/ClickHouse/pull/34771) ([Azat Khuzhin](https://github.com/azat)). +* jemalloc: fix includes order to avoid overlaps [#34783](https://github.com/ClickHouse/ClickHouse/pull/34783) ([Azat Khuzhin](https://github.com/azat)). +* Fix using host/port from config for client [#34791](https://github.com/ClickHouse/ClickHouse/pull/34791) ([Vladimir C](https://github.com/vdimir)). +* Update links for 22.2 and 22.3 release webinars [#34814](https://github.com/ClickHouse/ClickHouse/pull/34814) ([Cody Baker](https://github.com/codyrobert)). +* Add opensee blog post [#34817](https://github.com/ClickHouse/ClickHouse/pull/34817) ([Cody Baker](https://github.com/codyrobert)). +* Try remove trash from StorageFileLog [#34819](https://github.com/ClickHouse/ClickHouse/pull/34819) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix generating system.build_options [#34823](https://github.com/ClickHouse/ClickHouse/pull/34823) ([Azat Khuzhin](https://github.com/azat)). +* Comment output header for team keys [#34828](https://github.com/ClickHouse/ClickHouse/pull/34828) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix untuple condition in IN function [#34836](https://github.com/ClickHouse/ClickHouse/pull/34836) ([Vladimir C](https://github.com/vdimir)). +* Improve certificate reloader [#34887](https://github.com/ClickHouse/ClickHouse/pull/34887) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Update DiskS3.cpp [#34889](https://github.com/ClickHouse/ClickHouse/pull/34889) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Clang tidy improvements [#34893](https://github.com/ClickHouse/ClickHouse/pull/34893) ([Maksim Kita](https://github.com/kitaisreal)). +* Update version_date.tsv after v22.2.3.5-stable [#34898](https://github.com/ClickHouse/ClickHouse/pull/34898) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Add check_prerequisites to release.py [#34901](https://github.com/ClickHouse/ClickHouse/pull/34901) ([Vladimir C](https://github.com/vdimir)). +* Fix filelog storage data paths [#34912](https://github.com/ClickHouse/ClickHouse/pull/34912) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update version_date.tsv after v22.1.4.30-stable [#34913](https://github.com/ClickHouse/ClickHouse/pull/34913) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Fix alignment for aligned_alloc() to fix build for glibc2.35+ [#34919](https://github.com/ClickHouse/ClickHouse/pull/34919) ([Azat Khuzhin](https://github.com/azat)). +* Stop processing multiqueries in clickhouse-client/local on SIGINT [#34923](https://github.com/ClickHouse/ClickHouse/pull/34923) ([Azat Khuzhin](https://github.com/azat)). +* Fix some code comments style [#34935](https://github.com/ClickHouse/ClickHouse/pull/34935) ([hongbin](https://github.com/xlwh)). +* Adjust couple of tests for aarch64 [#34951](https://github.com/ClickHouse/ClickHouse/pull/34951) ([Vladimir C](https://github.com/vdimir)). +* Add benchmarks [#34960](https://github.com/ClickHouse/ClickHouse/pull/34960) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Catch exception in ~WriteBufferFromS3 [#34963](https://github.com/ClickHouse/ClickHouse/pull/34963) ([Vladimir C](https://github.com/vdimir)). +* Disable memory checking test with thread sanitizer [#34964](https://github.com/ClickHouse/ClickHouse/pull/34964) ([alesapin](https://github.com/alesapin)). +* Print number of available logical cores in configuration [#34972](https://github.com/ClickHouse/ClickHouse/pull/34972) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix possible "nmalloc >= ndalloc" jemalloc assertion [#34973](https://github.com/ClickHouse/ClickHouse/pull/34973) ([Azat Khuzhin](https://github.com/azat)). +* Add some benchmarks [#34974](https://github.com/ClickHouse/ClickHouse/pull/34974) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Fix typo [#34975](https://github.com/ClickHouse/ClickHouse/pull/34975) ([Nikita Evsyukov](https://github.com/NikitaEvs)). +* Fix signal-unsafe TSan report in client [#34988](https://github.com/ClickHouse/ClickHouse/pull/34988) ([Azat Khuzhin](https://github.com/azat)). +* Fix non-MergeTree engines for system.*_log [#34989](https://github.com/ClickHouse/ClickHouse/pull/34989) ([Azat Khuzhin](https://github.com/azat)). +* Fix test_storage_postgresql::test_concurrent_queries [#34991](https://github.com/ClickHouse/ClickHouse/pull/34991) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix race between INSERT async_insert=1 and system.asynchronous_inserts [#34992](https://github.com/ClickHouse/ClickHouse/pull/34992) ([Azat Khuzhin](https://github.com/azat)). +* Add option to clickhouse-test to skip aarch64 build [#34995](https://github.com/ClickHouse/ClickHouse/pull/34995) ([Vladimir C](https://github.com/vdimir)). +* clickhouse obfuscator aarch64 fix [#35005](https://github.com/ClickHouse/ClickHouse/pull/35005) ([Vladimir C](https://github.com/vdimir)). +* Fix tutorial link [#35014](https://github.com/ClickHouse/ClickHouse/pull/35014) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* ExecutableUDF function deterministic in scope of query fix [#35018](https://github.com/ClickHouse/ClickHouse/pull/35018) ([Maksim Kita](https://github.com/kitaisreal)). +* fasttest: add timings for "Test time" column in the results table [#35019](https://github.com/ClickHouse/ClickHouse/pull/35019) ([Azat Khuzhin](https://github.com/azat)). +* Always write preprocessed config in XML format [#35020](https://github.com/ClickHouse/ClickHouse/pull/35020) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix read old records from logs [#35021](https://github.com/ClickHouse/ClickHouse/pull/35021) ([alesapin](https://github.com/alesapin)). +* Fix trim function [#35046](https://github.com/ClickHouse/ClickHouse/pull/35046) ([Vladimir C](https://github.com/vdimir)). +* Skip 01086_odbc_roundtrip for aarch, disable force_tests [#35058](https://github.com/ClickHouse/ClickHouse/pull/35058) ([Vladimir C](https://github.com/vdimir)). +* Remove useless define [#35066](https://github.com/ClickHouse/ClickHouse/pull/35066) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Take flush_time into account for scheduling background flush of the Buffer [#35071](https://github.com/ClickHouse/ClickHouse/pull/35071) ([Azat Khuzhin](https://github.com/azat)). +* Try to fix failed tests [#35073](https://github.com/ClickHouse/ClickHouse/pull/35073) ([Kruglov Pavel](https://github.com/Avogar)). +* Do not hide exceptions during mutations [#35080](https://github.com/ClickHouse/ClickHouse/pull/35080) ([Azat Khuzhin](https://github.com/azat)). +* Put downloaded artifacts to a temorary path [#35088](https://github.com/ClickHouse/ClickHouse/pull/35088) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Get rid of duplicate query planing. [#35094](https://github.com/ClickHouse/ClickHouse/pull/35094) ([Amos Bird](https://github.com/amosbird)). +* Temporary supress move partition long for storage S3 [#35097](https://github.com/ClickHouse/ClickHouse/pull/35097) ([alesapin](https://github.com/alesapin)). +* Fix inconsistency in DiskLocal [#35099](https://github.com/ClickHouse/ClickHouse/pull/35099) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Try to build llvm for Aarch64 [#35103](https://github.com/ClickHouse/ClickHouse/pull/35103) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fix build fail with gcc [#35123](https://github.com/ClickHouse/ClickHouse/pull/35123) ([zhanghuajie](https://github.com/zhanghuajieHIT)). +* Fix hardcoded page size [#35129](https://github.com/ClickHouse/ClickHouse/pull/35129) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Delete nodes with attributes `remove` or `replace` if they didn't merge [#35141](https://github.com/ClickHouse/ClickHouse/pull/35141) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Upgrade icu to icu-release-70-1 [#35142](https://github.com/ClickHouse/ClickHouse/pull/35142) ([Yong Wang](https://github.com/kashwy)). +* Merging [#33398](https://github.com/ClickHouse/ClickHouse/issues/33398) [#35157](https://github.com/ClickHouse/ClickHouse/pull/35157) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fix typos [#35174](https://github.com/ClickHouse/ClickHouse/pull/35174) ([Zhang Yifan](https://github.com/zhangyifan27)). +* Adjust max_memory_usage in external_aggregation.sql [#35194](https://github.com/ClickHouse/ClickHouse/pull/35194) ([Vladimir C](https://github.com/vdimir)). +* MaterializedMySQL protocol example fix [#35198](https://github.com/ClickHouse/ClickHouse/pull/35198) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove useless files [#35199](https://github.com/ClickHouse/ClickHouse/pull/35199) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless file [#35200](https://github.com/ClickHouse/ClickHouse/pull/35200) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add Segment script to of docs pages [#35203](https://github.com/ClickHouse/ClickHouse/pull/35203) ([Rich Raposa](https://github.com/rfraposa)). +* Cache fix [#35209](https://github.com/ClickHouse/ClickHouse/pull/35209) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Use C++14 aliases for some type traits [#35212](https://github.com/ClickHouse/ClickHouse/pull/35212) ([Robert Schulze](https://github.com/rschu1ze)). +* Remove redundant configs for TestFlows [#35223](https://github.com/ClickHouse/ClickHouse/pull/35223) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Correct some integration tests [#35224](https://github.com/ClickHouse/ClickHouse/pull/35224) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change timezone in Docker [#35225](https://github.com/ClickHouse/ClickHouse/pull/35225) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change timezone example in server config [#35226](https://github.com/ClickHouse/ClickHouse/pull/35226) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Adjust timezone in performance tests [#35227](https://github.com/ClickHouse/ClickHouse/pull/35227) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove "bugs" that do not exist anymore [#35228](https://github.com/ClickHouse/ClickHouse/pull/35228) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change timezone in stateful tests. [#35229](https://github.com/ClickHouse/ClickHouse/pull/35229) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Moved submodules from ClickHouse-Extras to ClickHouse [#35232](https://github.com/ClickHouse/ClickHouse/pull/35232) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove "bugs" that do not exist anymore [#35242](https://github.com/ClickHouse/ClickHouse/pull/35242) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update CachedReadBufferFromRemoteFS.cpp [#35245](https://github.com/ClickHouse/ClickHouse/pull/35245) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add a test for [#34682](https://github.com/ClickHouse/ClickHouse/issues/34682) [#35247](https://github.com/ClickHouse/ClickHouse/pull/35247) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable clang tidy for header files [#35248](https://github.com/ClickHouse/ClickHouse/pull/35248) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix possible 01506_buffer_table_alter_block_structure_2 flakiness [#35249](https://github.com/ClickHouse/ClickHouse/pull/35249) ([Azat Khuzhin](https://github.com/azat)). +* Mark build action as failed if it was retried and there was no succeeded builds [#35260](https://github.com/ClickHouse/ClickHouse/pull/35260) ([Azat Khuzhin](https://github.com/azat)). +* Fix flaky test [#35261](https://github.com/ClickHouse/ClickHouse/pull/35261) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix logical error in remote fs cache [#35275](https://github.com/ClickHouse/ClickHouse/pull/35275) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix code style and other minor corrections after implementing allow_no_password. [#35276](https://github.com/ClickHouse/ClickHouse/pull/35276) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add remote fs cache optimization [#35278](https://github.com/ClickHouse/ClickHouse/pull/35278) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove utils/github-hook/hook.py [#35280](https://github.com/ClickHouse/ClickHouse/pull/35280) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix test [#35282](https://github.com/ClickHouse/ClickHouse/pull/35282) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix 00284_external_aggregation.sql [#35293](https://github.com/ClickHouse/ClickHouse/pull/35293) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 00900_long_parquet_load [#35299](https://github.com/ClickHouse/ClickHouse/pull/35299) ([Vladimir C](https://github.com/vdimir)). +* Maybe fix use-after-free inside S3 upload thread [#35305](https://github.com/ClickHouse/ClickHouse/pull/35305) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Modified python packages [#35306](https://github.com/ClickHouse/ClickHouse/pull/35306) ([Lalit Srikant](https://github.com/LAL2211)). +* Fix disappeared host and port from client help message [#35309](https://github.com/ClickHouse/ClickHouse/pull/35309) ([Kruglov Pavel](https://github.com/Avogar)). +* Changelog 22.3 [#35344](https://github.com/ClickHouse/ClickHouse/pull/35344) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Replace a few uses of enable_if for SFINAE by concepts [#35347](https://github.com/ClickHouse/ClickHouse/pull/35347) ([Robert Schulze](https://github.com/rschu1ze)). + diff --git a/docs/changelogs/v22.3.3.44-lts.md b/docs/changelogs/v22.3.3.44-lts.md index 4fa98940290..4cd48eefa5a 100644 --- a/docs/changelogs/v22.3.3.44-lts.md +++ b/docs/changelogs/v22.3.3.44-lts.md @@ -19,3 +19,11 @@ sidebar_label: 2022 * Backported in [#35856](https://github.com/ClickHouse/ClickHouse/issues/35856): Respect only quota & period from groups, ignore shares (which are not really limit the number of the cores which can be used). [#35815](https://github.com/ClickHouse/ClickHouse/pull/35815) ([filimonov](https://github.com/filimonov)). * Backported in [#35938](https://github.com/ClickHouse/ClickHouse/issues/35938): Avoid processing per-column TTL multiple times. [#35820](https://github.com/ClickHouse/ClickHouse/pull/35820) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Slightly better performance of inserts to `Object` type [#35388](https://github.com/ClickHouse/ClickHouse/pull/35388) ([Anton Popov](https://github.com/CurtizJ)). +* Fix race in data type `Object` [#35409](https://github.com/ClickHouse/ClickHouse/pull/35409) ([Anton Popov](https://github.com/CurtizJ)). +* Fix crash with enabled `optimize_functions_to_subcolumns` [#35512](https://github.com/ClickHouse/ClickHouse/pull/35512) ([Anton Popov](https://github.com/CurtizJ)). +* Fix enable LLVM for JIT compilation in CMake [#35770](https://github.com/ClickHouse/ClickHouse/pull/35770) ([Maksim Kita](https://github.com/kitaisreal)). +* Backport release to 22.3 [#35909](https://github.com/ClickHouse/ClickHouse/pull/35909) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/changelogs/v22.3.6.5-lts.md b/docs/changelogs/v22.3.6.5-lts.md index 9d86d9e786c..4b4772c611a 100644 --- a/docs/changelogs/v22.3.6.5-lts.md +++ b/docs/changelogs/v22.3.6.5-lts.md @@ -11,3 +11,7 @@ sidebar_label: 2022 * Backported in [#36795](https://github.com/ClickHouse/ClickHouse/issues/36795): Fix vertical merges in wide parts. Previously an exception `There is no column` can be thrown during merge. [#36707](https://github.com/ClickHouse/ClickHouse/pull/36707) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add passphrase for certificates [#36487](https://github.com/ClickHouse/ClickHouse/pull/36487) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + diff --git a/docs/changelogs/v22.3.7.28-lts.md b/docs/changelogs/v22.3.7.28-lts.md index ecfd0ce2cf6..14cb8628f09 100644 --- a/docs/changelogs/v22.3.7.28-lts.md +++ b/docs/changelogs/v22.3.7.28-lts.md @@ -22,3 +22,10 @@ sidebar_label: 2022 * Backported in [#36925](https://github.com/ClickHouse/ClickHouse/issues/36925): Fix bug in clickhouse-keeper which can lead to corrupted compressed log files in case of small load and restarts. [#36910](https://github.com/ClickHouse/ClickHouse/pull/36910) ([alesapin](https://github.com/alesapin)). * Backported in [#37364](https://github.com/ClickHouse/ClickHouse/issues/37364): Fixed problem with infs in `quantileTDigest`. Fixes [#32107](https://github.com/ClickHouse/ClickHouse/issues/32107). [#37021](https://github.com/ClickHouse/ClickHouse/pull/37021) ([Vladimir Chebotarev](https://github.com/excitoon)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix mongodb test with new cert [#36161](https://github.com/ClickHouse/ClickHouse/pull/36161) ([alesapin](https://github.com/alesapin)). +* Integration tests [#36866](https://github.com/ClickHouse/ClickHouse/pull/36866) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update protobuf files for kafka and rabbitmq [fix integration tests] [#37884](https://github.com/ClickHouse/ClickHouse/pull/37884) ([Nikita Taranov](https://github.com/nickitat)). +* Try fix `test_grpc_protocol/test.py::test_progress` [#37908](https://github.com/ClickHouse/ClickHouse/pull/37908) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v22.4.1.2305-prestable.md b/docs/changelogs/v22.4.1.2305-prestable.md index 5c6948251b3..c202b0b9331 100644 --- a/docs/changelogs/v22.4.1.2305-prestable.md +++ b/docs/changelogs/v22.4.1.2305-prestable.md @@ -243,3 +243,208 @@ sidebar_label: 2022 * NO CL ENTRY: 'Revert "clang-tidy report issues with Medium priority"'. [#35941](https://github.com/ClickHouse/ClickHouse/pull/35941) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * NO CL ENTRY: 'Revert "Fix crash in ParallelReadBuffer"'. [#36210](https://github.com/ClickHouse/ClickHouse/pull/36210) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Improve backup and restore (experimental) [#33985](https://github.com/ClickHouse/ClickHouse/pull/33985) ([Vitaly Baranov](https://github.com/vitlibar)). +* Do not leave any queries after test finish (and add a check into clickhouse-test) [#34924](https://github.com/ClickHouse/ClickHouse/pull/34924) ([Azat Khuzhin](https://github.com/azat)). +* libxml2 package under contrib directory is upgraded to 2.9.13 [#35034](https://github.com/ClickHouse/ClickHouse/pull/35034) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* Add bugfix validate check [#35124](https://github.com/ClickHouse/ClickHouse/pull/35124) ([Vladimir C](https://github.com/vdimir)). +* curl package upgraded to 7.81.0 [#35130](https://github.com/ClickHouse/ClickHouse/pull/35130) ([Deleted user](https://github.com/ghost)). +* Add test for [#26965](https://github.com/ClickHouse/ClickHouse/issues/26965) [#35159](https://github.com/ClickHouse/ClickHouse/pull/35159) ([palegre-tiny](https://github.com/palegre-tiny)). +* clang-tidy report issues with Medium priority [#35184](https://github.com/ClickHouse/ClickHouse/pull/35184) ([Rajkumar Varada](https://github.com/varadarajkumar)). +* Add build with GCC [#35204](https://github.com/ClickHouse/ClickHouse/pull/35204) ([Azat Khuzhin](https://github.com/azat)). +* Fix-Clang-Tidy-Errors [#35297](https://github.com/ClickHouse/ClickHouse/pull/35297) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Other fix for 00900_long_parquet_load [#35302](https://github.com/ClickHouse/ClickHouse/pull/35302) ([Vladimir C](https://github.com/vdimir)). +* Add more checks with remoteHostsFilter [#35355](https://github.com/ClickHouse/ClickHouse/pull/35355) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix clang-tidy [#35356](https://github.com/ClickHouse/ClickHouse/pull/35356) ([Anton Popov](https://github.com/CurtizJ)). +* Function h3GetRes0Indexes crash fix [#35358](https://github.com/ClickHouse/ClickHouse/pull/35358) ([Maksim Kita](https://github.com/kitaisreal)). +* Function proporationsZTest formatting fix [#35369](https://github.com/ClickHouse/ClickHouse/pull/35369) ([Maksim Kita](https://github.com/kitaisreal)). +* Update version after release [#35374](https://github.com/ClickHouse/ClickHouse/pull/35374) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update SECURITY.md [#35375](https://github.com/ClickHouse/ClickHouse/pull/35375) ([Ivan Blinkov](https://github.com/blinkov)). +* Update version_date.tsv after v22.3.2.2-lts [#35377](https://github.com/ClickHouse/ClickHouse/pull/35377) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Function proportionsZTest fix style check [#35380](https://github.com/ClickHouse/ClickHouse/pull/35380) ([Maksim Kita](https://github.com/kitaisreal)). +* Replace more uses of std::enable_if for SFINAE by concepts [#35383](https://github.com/ClickHouse/ClickHouse/pull/35383) ([Robert Schulze](https://github.com/rschu1ze)). +* Slightly better performance of inserts to `Object` type [#35388](https://github.com/ClickHouse/ClickHouse/pull/35388) ([Anton Popov](https://github.com/CurtizJ)). +* Validate some thoughts over making sets [#35395](https://github.com/ClickHouse/ClickHouse/pull/35395) ([Amos Bird](https://github.com/amosbird)). +* Fix race in data type `Object` [#35409](https://github.com/ClickHouse/ClickHouse/pull/35409) ([Anton Popov](https://github.com/CurtizJ)). +* Rename some variables in keeper [#35431](https://github.com/ClickHouse/ClickHouse/pull/35431) ([alesapin](https://github.com/alesapin)). +* fix 02177_issue_31009_pt2.sql [#35445](https://github.com/ClickHouse/ClickHouse/pull/35445) ([Vladimir C](https://github.com/vdimir)). +* mysqlxx PoolWithFailover style fix [#35462](https://github.com/ClickHouse/ClickHouse/pull/35462) ([Maksim Kita](https://github.com/kitaisreal)). +* Resubmit [#21474](https://github.com/ClickHouse/ClickHouse/issues/21474) [#35467](https://github.com/ClickHouse/ClickHouse/pull/35467) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Explicitly set allow_introspection_functions to 0 [#35470](https://github.com/ClickHouse/ClickHouse/pull/35470) ([Nikita Taranov](https://github.com/nickitat)). +* Merging [#30325](https://github.com/ClickHouse/ClickHouse/issues/30325) [#35478](https://github.com/ClickHouse/ClickHouse/pull/35478) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix show create for information_schema [#35480](https://github.com/ClickHouse/ClickHouse/pull/35480) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add retries in backward compatibility check [#35482](https://github.com/ClickHouse/ClickHouse/pull/35482) ([Kruglov Pavel](https://github.com/Avogar)). +* Improve backward compatibility check and stress tests [#35499](https://github.com/ClickHouse/ClickHouse/pull/35499) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix too strict assertion in DDLWorker [#35503](https://github.com/ClickHouse/ClickHouse/pull/35503) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add more validations in mask operations [#35507](https://github.com/ClickHouse/ClickHouse/pull/35507) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix crash with enabled `optimize_functions_to_subcolumns` [#35512](https://github.com/ClickHouse/ClickHouse/pull/35512) ([Anton Popov](https://github.com/CurtizJ)). +* Don't put red cross if jepsen check couldn't wait for build [#35522](https://github.com/ClickHouse/ClickHouse/pull/35522) ([alesapin](https://github.com/alesapin)). +* Fix ClickHouse name typo in caches.md [#35526](https://github.com/ClickHouse/ClickHouse/pull/35526) ([erikbaan](https://github.com/erikbaan)). +* Simplify strip for new packages [#35533](https://github.com/ClickHouse/ClickHouse/pull/35533) ([alesapin](https://github.com/alesapin)). +* Add workflow dispatch [#35535](https://github.com/ClickHouse/ClickHouse/pull/35535) ([alesapin](https://github.com/alesapin)). +* fix clang tidy warning, add nullptr check [#35540](https://github.com/ClickHouse/ClickHouse/pull/35540) ([Suzy Wang](https://github.com/SuzyWangIBMer)). +* Slightly better integration tests: test_backup_with_other_granularity test_azure_blob_storage_zero_copy_replication [#35543](https://github.com/ClickHouse/ClickHouse/pull/35543) ([Ilya Yatsishin](https://github.com/qoega)). +* ExternalModelsLoader refactoring [#35546](https://github.com/ClickHouse/ClickHouse/pull/35546) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed style check [#35550](https://github.com/ClickHouse/ClickHouse/pull/35550) ([Maksim Kita](https://github.com/kitaisreal)). +* Cleanup test data [#35555](https://github.com/ClickHouse/ClickHouse/pull/35555) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Revert "Decrease data_type_max_parse_depth a little to avoid stack overflow in coroutines" [#35556](https://github.com/ClickHouse/ClickHouse/pull/35556) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix test `test_s3_zero_copy_replication` [#35574](https://github.com/ClickHouse/ClickHouse/pull/35574) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix logging in `test_distributed_respect_user_timeouts` [#35575](https://github.com/ClickHouse/ClickHouse/pull/35575) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Increase fiber stack size a bit in attempt to fix stack overflow in tests with address sanitizer [#35578](https://github.com/ClickHouse/ClickHouse/pull/35578) ([Kruglov Pavel](https://github.com/Avogar)). +* Don't run 01318_long_unsuccessful_mutation_zookeeper test in backward compatibility check [#35580](https://github.com/ClickHouse/ClickHouse/pull/35580) ([Kruglov Pavel](https://github.com/Avogar)). +* Make some tests more stable [#35599](https://github.com/ClickHouse/ClickHouse/pull/35599) ([Kruglov Pavel](https://github.com/Avogar)). +* Check all logs for crashes, logical errors, etc in backward compatibility check [#35613](https://github.com/ClickHouse/ClickHouse/pull/35613) ([Kruglov Pavel](https://github.com/Avogar)). +* Update comment about mismatching checksums [#35621](https://github.com/ClickHouse/ClickHouse/pull/35621) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Push only to the new CI DB [#35622](https://github.com/ClickHouse/ClickHouse/pull/35622) ([alesapin](https://github.com/alesapin)). +* Fix optin.cplusplus.UninitializedObject issue [#35626](https://github.com/ClickHouse/ClickHouse/pull/35626) ([larryluogit](https://github.com/larryluogit)). +* test for partition_by using ignore() [#35636](https://github.com/ClickHouse/ClickHouse/pull/35636) ([Denny Crane](https://github.com/den-crane)). +* test for crash _join_with_nullable_lowcardinality [#35551](https://github.com/ClickHouse/ClickHouse/issues/35551) [#35638](https://github.com/ClickHouse/ClickHouse/pull/35638) ([Denny Crane](https://github.com/den-crane)). +* Use compile-time check for `Exception` messages [#35655](https://github.com/ClickHouse/ClickHouse/pull/35655) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix flaky test `01532_primary_key_without_order_by_zookeeper` [#35657](https://github.com/ClickHouse/ClickHouse/pull/35657) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix multiple flaky tests [#35659](https://github.com/ClickHouse/ClickHouse/pull/35659) ([alesapin](https://github.com/alesapin)). +* Fix flaky test `01091_num_threads` [#35663](https://github.com/ClickHouse/ClickHouse/pull/35663) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Improve black check: show diff in the output [#35665](https://github.com/ClickHouse/ClickHouse/pull/35665) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Use float devision for avg after optimize_fuse_sum_count_avg [#35666](https://github.com/ClickHouse/ClickHouse/pull/35666) ([Vladimir C](https://github.com/vdimir)). +* Disable random settings in Fast Test [#35669](https://github.com/ClickHouse/ClickHouse/pull/35669) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix randomization of max_read_buffer_size [#35675](https://github.com/ClickHouse/ClickHouse/pull/35675) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove outdated links from CI [#35676](https://github.com/ClickHouse/ClickHouse/pull/35676) ([alesapin](https://github.com/alesapin)). +* Fix flaky tests 02149_read_in_order_fixed_prefix and 02177_issue_31009 [#35679](https://github.com/ClickHouse/ClickHouse/pull/35679) ([Kruglov Pavel](https://github.com/Avogar)). +* Rerun failed jobs only for failed workflowa [#35685](https://github.com/ClickHouse/ClickHouse/pull/35685) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix flaky 01037_polygon_dicts_correctness_fast [#35688](https://github.com/ClickHouse/ClickHouse/pull/35688) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fail CI checks in case of errors in checks (not failed tests) [#35718](https://github.com/ClickHouse/ClickHouse/pull/35718) ([alesapin](https://github.com/alesapin)). +* Try to fix test_global_overcommit_tracker flakyness [#35719](https://github.com/ClickHouse/ClickHouse/pull/35719) ([Dmitry Novik](https://github.com/novikd)). +* Try to run stateful tests in parallel [#35720](https://github.com/ClickHouse/ClickHouse/pull/35720) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Format `test_allowed_url_from_config/test.py` with black [#35721](https://github.com/ClickHouse/ClickHouse/pull/35721) ([Antonio Andelic](https://github.com/antonio2368)). +* Resurrect automatic labelling [#35722](https://github.com/ClickHouse/ClickHouse/pull/35722) ([alesapin](https://github.com/alesapin)). +* Revert "Fix enable LLVM for JIT compilation in CMake" [#35725](https://github.com/ClickHouse/ClickHouse/pull/35725) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Do not require writable source directory for generating krb5 error tables [#35734](https://github.com/ClickHouse/ClickHouse/pull/35734) ([Azat Khuzhin](https://github.com/azat)). +* Update ci checks server. [#35737](https://github.com/ClickHouse/ClickHouse/pull/35737) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Tests fixes [#35745](https://github.com/ClickHouse/ClickHouse/pull/35745) ([Azat Khuzhin](https://github.com/azat)). +* Use common IOThreadPool for S3 and URL [#35746](https://github.com/ClickHouse/ClickHouse/pull/35746) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix stylecheck [#35754](https://github.com/ClickHouse/ClickHouse/pull/35754) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* extended a description of clickhouse-client compression parameter [#35758](https://github.com/ClickHouse/ClickHouse/pull/35758) ([Denny Crane](https://github.com/den-crane)). +* Resurrect build hash [#35766](https://github.com/ClickHouse/ClickHouse/pull/35766) ([alesapin](https://github.com/alesapin)). +* Fix 00484_preferred_max_column_in_block_size_bytes [#35768](https://github.com/ClickHouse/ClickHouse/pull/35768) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix enable LLVM for JIT compilation in CMake [#35770](https://github.com/ClickHouse/ClickHouse/pull/35770) ([Maksim Kita](https://github.com/kitaisreal)). +* clickhouse-keeper: correctly handle closed client connection [#35772](https://github.com/ClickHouse/ClickHouse/pull/35772) ([Azat Khuzhin](https://github.com/azat)). +* ci: replace directory system log tables artifacts with tsv [#35773](https://github.com/ClickHouse/ClickHouse/pull/35773) ([Azat Khuzhin](https://github.com/azat)). +* One more try to resurrect build hash [#35774](https://github.com/ClickHouse/ClickHouse/pull/35774) ([alesapin](https://github.com/alesapin)). +* Refactoring QueryPipeline [#35789](https://github.com/ClickHouse/ClickHouse/pull/35789) ([Amos Bird](https://github.com/amosbird)). +* Delete duplicate code [#35798](https://github.com/ClickHouse/ClickHouse/pull/35798) ([xiedeyantu](https://github.com/xiedeyantu)). +* remove unused variable [#35800](https://github.com/ClickHouse/ClickHouse/pull/35800) ([flynn](https://github.com/ucasfl)). +* Make `SortDescription::column_name` always non-empty [#35805](https://github.com/ClickHouse/ClickHouse/pull/35805) ([Nikita Taranov](https://github.com/nickitat)). +* Fix latest_error referenced before assignment [#35807](https://github.com/ClickHouse/ClickHouse/pull/35807) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Try to fix some integration tests [#35808](https://github.com/ClickHouse/ClickHouse/pull/35808) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Better error message for incorrect *MergeTree arguments [#35814](https://github.com/ClickHouse/ClickHouse/pull/35814) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix missing `noexcept(false)` flag on destructor which causes builds to fail [#35817](https://github.com/ClickHouse/ClickHouse/pull/35817) ([tcoyvwac](https://github.com/tcoyvwac)). +* Try remove unneed variable [#35833](https://github.com/ClickHouse/ClickHouse/pull/35833) ([flynn](https://github.com/ucasfl)). +* Refactoring of hints for column descriptor [#35852](https://github.com/ClickHouse/ClickHouse/pull/35852) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix automatic bucket prefix for master [#35854](https://github.com/ClickHouse/ClickHouse/pull/35854) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Format changes for new docs [#35858](https://github.com/ClickHouse/ClickHouse/pull/35858) ([Rich Raposa](https://github.com/rfraposa)). +* Properly cancel the query after client format error [#35867](https://github.com/ClickHouse/ClickHouse/pull/35867) ([Azat Khuzhin](https://github.com/azat)). +* Drop modernize-replace-auto-ptr from .clang-tidy [#35868](https://github.com/ClickHouse/ClickHouse/pull/35868) ([Robert Schulze](https://github.com/rschu1ze)). +* fix service start with systemd [#35869](https://github.com/ClickHouse/ClickHouse/pull/35869) ([Denny Crane](https://github.com/den-crane)). +* fix postgres test [#35885](https://github.com/ClickHouse/ClickHouse/pull/35885) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update poco: Fix polling of socket with negative timeout (when poll() interrupted by EINTR) [#35899](https://github.com/ClickHouse/ClickHouse/pull/35899) ([Azat Khuzhin](https://github.com/azat)). +* More logs on unsuccessful part removal [#35904](https://github.com/ClickHouse/ClickHouse/pull/35904) ([alesapin](https://github.com/alesapin)). +* Executable user defined functions prevent executing during analysis [#35917](https://github.com/ClickHouse/ClickHouse/pull/35917) ([Maksim Kita](https://github.com/kitaisreal)). +* JIT ProfileEvents added test [#35918](https://github.com/ClickHouse/ClickHouse/pull/35918) ([Maksim Kita](https://github.com/kitaisreal)). +* Clang tidy issues [#35919](https://github.com/ClickHouse/ClickHouse/pull/35919) ([Deleted user](https://github.com/ghost)). +* Fix race in cached buffer [#35922](https://github.com/ClickHouse/ClickHouse/pull/35922) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix building ubuntu image from deb-repo [#35931](https://github.com/ClickHouse/ClickHouse/pull/35931) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Move credentials for integration tests into a separate file [#35936](https://github.com/ClickHouse/ClickHouse/pull/35936) ([Lalit Srikant](https://github.com/LAL2211)). +* [Snyk] Security upgrade node from 8 to 16.14.2 [#35942](https://github.com/ClickHouse/ClickHouse/pull/35942) ([Snyk bot](https://github.com/snyk-bot)). +* Fix keeper image for `arm64` [#35945](https://github.com/ClickHouse/ClickHouse/pull/35945) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Make some replicated DDL queries faster [#35946](https://github.com/ClickHouse/ClickHouse/pull/35946) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update Contentsquare company case [#35964](https://github.com/ClickHouse/ClickHouse/pull/35964) ([François Violette](https://github.com/fviolette)). +* Small code changes in ZooKeeper client/Keeper code [#35967](https://github.com/ClickHouse/ClickHouse/pull/35967) ([Antonio Andelic](https://github.com/antonio2368)). +* parallel reading files for FileLog Engine [#35980](https://github.com/ClickHouse/ClickHouse/pull/35980) ([flynn](https://github.com/ucasfl)). +* Fix data race in StorageURL [#35984](https://github.com/ClickHouse/ClickHouse/pull/35984) ([Antonio Andelic](https://github.com/antonio2368)). +* Improve logs analysis in stress test checks [#35985](https://github.com/ClickHouse/ClickHouse/pull/35985) ([Azat Khuzhin](https://github.com/azat)). +* Print labels to log in run_check.py [#35991](https://github.com/ClickHouse/ClickHouse/pull/35991) ([Vladimir C](https://github.com/vdimir)). +* Set `ENABLE_BUILD_PATH_MAPPING` to `OFF` by default, if `CMAKE_BUILD_TYPE` is set to `Debug` [#35998](https://github.com/ClickHouse/ClickHouse/pull/35998) ([Denis Glazachev](https://github.com/traceon)). +* Make test 00159_parallel_formatting_tsv_and_friends.sh more stable [#36001](https://github.com/ClickHouse/ClickHouse/pull/36001) ([Kruglov Pavel](https://github.com/Avogar)). +* Update version_date.tsv after v22.3.3.44-lts [#36002](https://github.com/ClickHouse/ClickHouse/pull/36002) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Fix flaky test `test_system_merges/test.py::test_merge_simple` [#36004](https://github.com/ClickHouse/ClickHouse/pull/36004) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Another fix for settings randomization [#36005](https://github.com/ClickHouse/ClickHouse/pull/36005) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove some TestFlows that are irrelevant. [#36006](https://github.com/ClickHouse/ClickHouse/pull/36006) ([Lalit Srikant](https://github.com/LAL2211)). +* Fix 02248_nullable_custom_types_to_string [#36008](https://github.com/ClickHouse/ClickHouse/pull/36008) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix flaky test 00155_long_merges [#36009](https://github.com/ClickHouse/ClickHouse/pull/36009) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove unused code from TestFlows [#36010](https://github.com/ClickHouse/ClickHouse/pull/36010) ([Lalit Srikant](https://github.com/LAL2211)). +* Fix data race in StorgeFileLog [#36015](https://github.com/ClickHouse/ClickHouse/pull/36015) ([flynn](https://github.com/ucasfl)). +* Fix Backward comapatibility check [#36026](https://github.com/ClickHouse/ClickHouse/pull/36026) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Use storage_snapshot for projection analysis [#36034](https://github.com/ClickHouse/ClickHouse/pull/36034) ([Amos Bird](https://github.com/amosbird)). +* Nightly coverity [#36044](https://github.com/ClickHouse/ClickHouse/pull/36044) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix typo [#36047](https://github.com/ClickHouse/ClickHouse/pull/36047) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test [#36054](https://github.com/ClickHouse/ClickHouse/pull/36054) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove trailing whitespace in default client_name [#36056](https://github.com/ClickHouse/ClickHouse/pull/36056) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-client: fix query cancellation if any result was not received yet [#36057](https://github.com/ClickHouse/ClickHouse/pull/36057) ([Azat Khuzhin](https://github.com/azat)). +* Add debug and fix cancel_rerun lambda [#36064](https://github.com/ClickHouse/ClickHouse/pull/36064) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix style in run_check.py [#36065](https://github.com/ClickHouse/ClickHouse/pull/36065) ([Vladimir C](https://github.com/vdimir)). +* Try to turn off always green backward compatibility checks [#36067](https://github.com/ClickHouse/ClickHouse/pull/36067) ([Kruglov Pavel](https://github.com/Avogar)). +* Less dependencies from disks in buffers [#36070](https://github.com/ClickHouse/ClickHouse/pull/36070) ([alesapin](https://github.com/alesapin)). +* Backups: improve arguments handling and file removing [#36072](https://github.com/ClickHouse/ClickHouse/pull/36072) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add tonickkozlov to trusted users for auto ci run [#36080](https://github.com/ClickHouse/ClickHouse/pull/36080) ([nvartolomei](https://github.com/nvartolomei)). +* Try to avoid timeoutes in parallel parsing tests [#36083](https://github.com/ClickHouse/ClickHouse/pull/36083) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix BUILD_NAME issue in build jobs [#36085](https://github.com/ClickHouse/ClickHouse/pull/36085) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Remove incorrect assertion [#36086](https://github.com/ClickHouse/ClickHouse/pull/36086) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Get rid of fs paths keeper [#36106](https://github.com/ClickHouse/ClickHouse/pull/36106) ([alesapin](https://github.com/alesapin)). +* fix typo [#36110](https://github.com/ClickHouse/ClickHouse/pull/36110) ([Denny Crane](https://github.com/den-crane)). +* Fix scan report filename [#36112](https://github.com/ClickHouse/ClickHouse/pull/36112) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add new/delete overloads with align [#36125](https://github.com/ClickHouse/ClickHouse/pull/36125) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix code style in registerBackupEnginesFileAndDisk.cpp [#36127](https://github.com/ClickHouse/ClickHouse/pull/36127) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix flaky tests 00971 and 01003 [#36128](https://github.com/ClickHouse/ClickHouse/pull/36128) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update 01171_mv_select_insert_isolation_long.sh [#36131](https://github.com/ClickHouse/ClickHouse/pull/36131) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Build report [#36132](https://github.com/ClickHouse/ClickHouse/pull/36132) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix after [#35475](https://github.com/ClickHouse/ClickHouse/issues/35475) [#36135](https://github.com/ClickHouse/ClickHouse/pull/36135) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor fix for transactions [#36136](https://github.com/ClickHouse/ClickHouse/pull/36136) ([Alexander Tokmakov](https://github.com/tavplubix)). +* ProfileEvents fixes [#36137](https://github.com/ClickHouse/ClickHouse/pull/36137) ([Azat Khuzhin](https://github.com/azat)). +* Minor improvement for hung check [#36138](https://github.com/ClickHouse/ClickHouse/pull/36138) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Upload ccache for the first run in PR [#36139](https://github.com/ClickHouse/ClickHouse/pull/36139) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix test 02241_remote_filesystem_cache_on_insert for database Ordinary [#36150](https://github.com/ClickHouse/ClickHouse/pull/36150) ([Kseniia Sumarokova](https://github.com/kssenii)). +* remove unused array in h3Res0Indexes func [#36154](https://github.com/ClickHouse/ClickHouse/pull/36154) ([Bharat Nallan](https://github.com/bharatnc)). +* fix typo in cmake message [#36155](https://github.com/ClickHouse/ClickHouse/pull/36155) ([Bharat Nallan](https://github.com/bharatnc)). +* Fix UT error sometimes [#36157](https://github.com/ClickHouse/ClickHouse/pull/36157) ([zhanglistar](https://github.com/zhanglistar)). +* Fix a logical error left after debugging [#36159](https://github.com/ClickHouse/ClickHouse/pull/36159) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix mongodb test with new cert [#36161](https://github.com/ClickHouse/ClickHouse/pull/36161) ([alesapin](https://github.com/alesapin)). +* Some fixes for ReplicatedMergeTree [#36163](https://github.com/ClickHouse/ClickHouse/pull/36163) ([Alexander Tokmakov](https://github.com/tavplubix)). +* clickhouse-client: properly cancel query in case of error during formatting data [#36164](https://github.com/ClickHouse/ClickHouse/pull/36164) ([Azat Khuzhin](https://github.com/azat)). +* Fix flacky test 01161_all_system_tables under s3 storage [#36175](https://github.com/ClickHouse/ClickHouse/pull/36175) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Revert "Fix possible mutation stuck due to race with DROP_RANGE" [#36190](https://github.com/ClickHouse/ClickHouse/pull/36190) ([Azat Khuzhin](https://github.com/azat)). +* Use atomic instead of mutex + condvar in ParallelReadBuffer [#36192](https://github.com/ClickHouse/ClickHouse/pull/36192) ([Kruglov Pavel](https://github.com/Avogar)). +* Follow-up to [#36138](https://github.com/ClickHouse/ClickHouse/issues/36138) [#36194](https://github.com/ClickHouse/ClickHouse/pull/36194) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Revert reverting "Fix crash in ParallelReadBuffer" [#36212](https://github.com/ClickHouse/ClickHouse/pull/36212) ([Kruglov Pavel](https://github.com/Avogar)). +* Make stateless tests with s3 always green [#36214](https://github.com/ClickHouse/ClickHouse/pull/36214) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add Tyler Hannan to contributors [#36216](https://github.com/ClickHouse/ClickHouse/pull/36216) ([Tyler Hannan](https://github.com/tylerhannan)). +* Fix the repeated call of func to get the table when drop table [#36248](https://github.com/ClickHouse/ClickHouse/pull/36248) ([xiedeyantu](https://github.com/xiedeyantu)). +* Split test 01675_data_type_coroutine into 2 tests to prevent possible timeouts [#36250](https://github.com/ClickHouse/ClickHouse/pull/36250) ([Kruglov Pavel](https://github.com/Avogar)). +* Merge TRUSTED_CONTRIBUTORS in lambda and import in check [#36252](https://github.com/ClickHouse/ClickHouse/pull/36252) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix exception "File segment can be completed only by downloader" in tests [#36253](https://github.com/ClickHouse/ClickHouse/pull/36253) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix integration tests report parser [#36257](https://github.com/ClickHouse/ClickHouse/pull/36257) ([alesapin](https://github.com/alesapin)). +* fix crash when you use clickhouse-git-import with invalid parameter [#36262](https://github.com/ClickHouse/ClickHouse/pull/36262) ([zhanghuajie](https://github.com/zhanghuajieHIT)). +* Fix cancel-lambda for closed PRs [#36269](https://github.com/ClickHouse/ClickHouse/pull/36269) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Place new logic of schema inference in insert select from table function under setting [#36275](https://github.com/ClickHouse/ClickHouse/pull/36275) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove trash from CPUID [#36310](https://github.com/ClickHouse/ClickHouse/pull/36310) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove remaining parts of Arcadia [#36312](https://github.com/ClickHouse/ClickHouse/pull/36312) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix clang-tidy-14 (part 1) [#36320](https://github.com/ClickHouse/ClickHouse/pull/36320) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add dispatch trigger for debug CI [#36329](https://github.com/ClickHouse/ClickHouse/pull/36329) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Revert "support UNSIGNED modifier with unused parameters of INT" [#36337](https://github.com/ClickHouse/ClickHouse/pull/36337) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Simplify perf scripts by using schema inference [#36346](https://github.com/ClickHouse/ClickHouse/pull/36346) ([Azat Khuzhin](https://github.com/azat)). +* Do not randomize "priority" setting [#36358](https://github.com/ClickHouse/ClickHouse/pull/36358) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix integration tests [#36361](https://github.com/ClickHouse/ClickHouse/pull/36361) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove abbreviation [#36362](https://github.com/ClickHouse/ClickHouse/pull/36362) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix slow test [#36363](https://github.com/ClickHouse/ClickHouse/pull/36363) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix error in sanity checks [#36365](https://github.com/ClickHouse/ClickHouse/pull/36365) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add nodiscard attribute to immutable methods of `IColumn` to avoid errors [#36368](https://github.com/ClickHouse/ClickHouse/pull/36368) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix strange trash in Keeper [#36369](https://github.com/ClickHouse/ClickHouse/pull/36369) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix typo [#36374](https://github.com/ClickHouse/ClickHouse/pull/36374) ([flynn](https://github.com/ucasfl)). +* Fix test 01161_all_system_tables under s3 storage [#36388](https://github.com/ClickHouse/ClickHouse/pull/36388) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor refactor to prefer C++ Standard Algorithms [#36393](https://github.com/ClickHouse/ClickHouse/pull/36393) ([tcoyvwac](https://github.com/tcoyvwac)). +* jemalloc: enable logging in debug build [#36398](https://github.com/ClickHouse/ClickHouse/pull/36398) ([Azat Khuzhin](https://github.com/azat)). +* Respect library type for contrib libraries [#36399](https://github.com/ClickHouse/ClickHouse/pull/36399) ([Azat Khuzhin](https://github.com/azat)). +* Add more harmful variables for OSX [#36400](https://github.com/ClickHouse/ClickHouse/pull/36400) ([Azat Khuzhin](https://github.com/azat)). +* Activate clang-tidy warning "readability-container-contains" [#36402](https://github.com/ClickHouse/ClickHouse/pull/36402) ([Robert Schulze](https://github.com/rschu1ze)). +* Move down data_range construction in filterMarksUsingIndex and filterMarksUsingMergedIndex [#36410](https://github.com/ClickHouse/ClickHouse/pull/36410) ([Tian Xinhui](https://github.com/xinhuitian)). +* Fix stress test [#36450](https://github.com/ClickHouse/ClickHouse/pull/36450) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v22.4.4.7-stable.md b/docs/changelogs/v22.4.4.7-stable.md index 525c0de1341..71e077ac071 100644 --- a/docs/changelogs/v22.4.4.7-stable.md +++ b/docs/changelogs/v22.4.4.7-stable.md @@ -12,3 +12,7 @@ sidebar_label: 2022 * Backported in [#36524](https://github.com/ClickHouse/ClickHouse/issues/36524): Queries with aliases inside special operators returned parsing error (was broken in 22.1). Example: `SELECT substring('test' AS t, 1, 1)`. [#36167](https://github.com/ClickHouse/ClickHouse/pull/36167) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#36673](https://github.com/ClickHouse/ClickHouse/issues/36673): Fix merges of wide parts with type `Object`. [#36637](https://github.com/ClickHouse/ClickHouse/pull/36637) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add passphrase for certificates [#36487](https://github.com/ClickHouse/ClickHouse/pull/36487) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + diff --git a/docs/changelogs/v22.4.5.9-stable.md b/docs/changelogs/v22.4.5.9-stable.md index b5fffa56494..636ad2ed3ac 100644 --- a/docs/changelogs/v22.4.5.9-stable.md +++ b/docs/changelogs/v22.4.5.9-stable.md @@ -13,3 +13,7 @@ sidebar_label: 2022 * Backported in [#36794](https://github.com/ClickHouse/ClickHouse/issues/36794): Fix vertical merges in wide parts. Previously an exception `There is no column` can be thrown during merge. [#36707](https://github.com/ClickHouse/ClickHouse/pull/36707) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#36926](https://github.com/ClickHouse/ClickHouse/issues/36926): Fix bug in clickhouse-keeper which can lead to corrupted compressed log files in case of small load and restarts. [#36910](https://github.com/ClickHouse/ClickHouse/pull/36910) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Some fixes for replicated merge tree [#36909](https://github.com/ClickHouse/ClickHouse/pull/36909) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v22.5.1.2079-stable.md b/docs/changelogs/v22.5.1.2079-stable.md index 7614d678a2a..dfdcad64561 100644 --- a/docs/changelogs/v22.5.1.2079-stable.md +++ b/docs/changelogs/v22.5.1.2079-stable.md @@ -187,3 +187,204 @@ sidebar_label: 2022 * NO CL ENTRY: 'Revert "BLAKE3 hash function documentation"'. [#37092](https://github.com/ClickHouse/ClickHouse/pull/37092) ([Rich Raposa](https://github.com/rfraposa)). * NO CL ENTRY: 'Revert "Remove height restrictions from the query div in play web tool."'. [#37261](https://github.com/ClickHouse/ClickHouse/pull/37261) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Cleanup GROUPING SETS formating [#32999](https://github.com/ClickHouse/ClickHouse/pull/32999) ([Dmitry Novik](https://github.com/novikd)). +* Minor renames [#35272](https://github.com/ClickHouse/ClickHouse/pull/35272) ([Anton Popov](https://github.com/CurtizJ)). +* clickhouse-test: fix left-queries-check, to fix test log parser [#35865](https://github.com/ClickHouse/ClickHouse/pull/35865) ([Azat Khuzhin](https://github.com/azat)). +* Regression test for CHECKSUM_DOESNT_MATCH error because of per-column TTL bug [#35971](https://github.com/ClickHouse/ClickHouse/pull/35971) ([Azat Khuzhin](https://github.com/azat)). +* Fix performance tests [#35976](https://github.com/ClickHouse/ClickHouse/pull/35976) ([Maksim Kita](https://github.com/kitaisreal)). +* Backup for replicated tables and other improvements [#36198](https://github.com/ClickHouse/ClickHouse/pull/36198) ([Vitaly Baranov](https://github.com/vitlibar)). +* Trying to fix some trash in zero copy replication [#36299](https://github.com/ClickHouse/ClickHouse/pull/36299) ([alesapin](https://github.com/alesapin)). +* Speed up build a little [#36319](https://github.com/ClickHouse/ClickHouse/pull/36319) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Replace remove-erase idiom by C++20 erase()/erase_if() [#36348](https://github.com/ClickHouse/ClickHouse/pull/36348) ([Robert Schulze](https://github.com/rschu1ze)). +* Strict taskstats parser [#36351](https://github.com/ClickHouse/ClickHouse/pull/36351) ([Azat Khuzhin](https://github.com/azat)). +* Draft changelog for version 22.4 [#36397](https://github.com/ClickHouse/ClickHouse/pull/36397) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update int-uint.md [#36404](https://github.com/ClickHouse/ClickHouse/pull/36404) ([hardstep33](https://github.com/hardstep33)). +* Return back [#36126](https://github.com/ClickHouse/ClickHouse/issues/36126) [#36423](https://github.com/ClickHouse/ClickHouse/pull/36423) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed warnings of clang-tidy check "bugprone-branch-clone" [#36431](https://github.com/ClickHouse/ClickHouse/pull/36431) ([Robert Schulze](https://github.com/rschu1ze)). +* Clang tidy fixes [#36444](https://github.com/ClickHouse/ClickHouse/pull/36444) ([Robert Schulze](https://github.com/rschu1ze)). +* fixed /common/example cow_compositions.cpp clone the inner column when there is no need [#36453](https://github.com/ClickHouse/ClickHouse/pull/36453) ([zombee0](https://github.com/zombee0)). +* Refactoring dependency for ParserAttachAccessEntity [#36468](https://github.com/ClickHouse/ClickHouse/pull/36468) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* more specific warning text about low disk space [#36472](https://github.com/ClickHouse/ClickHouse/pull/36472) ([Sergei Trifonov](https://github.com/serxa)). +* Fixed missing enum values for ClientInfo::Interface [#36482](https://github.com/ClickHouse/ClickHouse/pull/36482) ([Vasily Nemkov](https://github.com/Enmk)). +* Add passphrase for certificates [#36487](https://github.com/ClickHouse/ClickHouse/pull/36487) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Update version after release [#36502](https://github.com/ClickHouse/ClickHouse/pull/36502) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Followup on [#36172](https://github.com/ClickHouse/ClickHouse/issues/36172) password hash salt feature [#36510](https://github.com/ClickHouse/ClickHouse/pull/36510) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Update version_date.tsv after v22.4.2.1-stable [#36533](https://github.com/ClickHouse/ClickHouse/pull/36533) ([github-actions[bot]](https://github.com/apps/github-actions)). +* fix log should print 'from' path [#36535](https://github.com/ClickHouse/ClickHouse/pull/36535) ([xiedeyantu](https://github.com/xiedeyantu)). +* Add function bin tests for Int/UInt128/UInt256 [#36537](https://github.com/ClickHouse/ClickHouse/pull/36537) ([Memo](https://github.com/Joeywzr)). +* Fix 01161_all_system_tables [#36539](https://github.com/ClickHouse/ClickHouse/pull/36539) ([Antonio Andelic](https://github.com/antonio2368)). +* Update PULL_REQUEST_TEMPLATE.md [#36543](https://github.com/ClickHouse/ClickHouse/pull/36543) ([Ivan Blinkov](https://github.com/blinkov)). +* Fixed integer overflow in toStartOfInterval [#36546](https://github.com/ClickHouse/ClickHouse/pull/36546) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix issue with broken git ownership [#36548](https://github.com/ClickHouse/ClickHouse/pull/36548) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix version autodetection for docker_server.py [#36552](https://github.com/ClickHouse/ClickHouse/pull/36552) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Snyk fixes [#36554](https://github.com/ClickHouse/ClickHouse/pull/36554) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Use just index to split performance tests by group [#36559](https://github.com/ClickHouse/ClickHouse/pull/36559) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update quantiledeterministic.md [#36560](https://github.com/ClickHouse/ClickHouse/pull/36560) ([ifinik](https://github.com/ifinik)). +* CodeQL Experiment - Exclude contrib Dependencies [#36561](https://github.com/ClickHouse/ClickHouse/pull/36561) ([Julio Jimenez](https://github.com/juliojimenez)). +* Small refactoring of Processors and QueryPipeline [#36579](https://github.com/ClickHouse/ClickHouse/pull/36579) ([Amos Bird](https://github.com/amosbird)). +* Simplify 01834_alias_columns_laziness_filimonov test [#36585](https://github.com/ClickHouse/ClickHouse/pull/36585) ([Azat Khuzhin](https://github.com/azat)). +* bash-completion: add completion for send_logs_level [#36586](https://github.com/ClickHouse/ClickHouse/pull/36586) ([Azat Khuzhin](https://github.com/azat)). +* client: add a message on reconnect (under warning log level) [#36587](https://github.com/ClickHouse/ClickHouse/pull/36587) ([Azat Khuzhin](https://github.com/azat)). +* Fix strange whitespace (or I do not know YAML) [#36590](https://github.com/ClickHouse/ClickHouse/pull/36590) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Strict taskstats parser (resubmit) [#36602](https://github.com/ClickHouse/ClickHouse/pull/36602) ([Azat Khuzhin](https://github.com/azat)). +* Exclude test `02271_fix_column_matcher_and_column_transformer` from bc check [#36607](https://github.com/ClickHouse/ClickHouse/pull/36607) ([alesapin](https://github.com/alesapin)). +* Ancient cmake version cleanup [#36612](https://github.com/ClickHouse/ClickHouse/pull/36612) ([Robert Schulze](https://github.com/rschu1ze)). +* Cleanup clang-tidy integration. [#36613](https://github.com/ClickHouse/ClickHouse/pull/36613) ([Robert Schulze](https://github.com/rschu1ze)). +* ParallelReadBuffer small improvements [#36619](https://github.com/ClickHouse/ClickHouse/pull/36619) ([Antonio Andelic](https://github.com/antonio2368)). +* Lambda cancel sync [#36622](https://github.com/ClickHouse/ClickHouse/pull/36622) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Version history benchmarks [#36628](https://github.com/ClickHouse/ClickHouse/pull/36628) ([Ilya Yatsishin](https://github.com/qoega)). +* fix typo [#36629](https://github.com/ClickHouse/ClickHouse/pull/36629) ([Sergei Trifonov](https://github.com/serxa)). +* Tiny Mutator code cleanup [#36630](https://github.com/ClickHouse/ClickHouse/pull/36630) ([Azat Khuzhin](https://github.com/azat)). +* Disble test postgresql replica with asan [#36631](https://github.com/ClickHouse/ClickHouse/pull/36631) ([alesapin](https://github.com/alesapin)). +* Minor Coverity defects fixes [#36632](https://github.com/ClickHouse/ClickHouse/pull/36632) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Properly wait for queries in 01502_long_log_tinylog_deadlock_race test [#36634](https://github.com/ClickHouse/ClickHouse/pull/36634) ([Azat Khuzhin](https://github.com/azat)). +* remove unneeded if statement [#36636](https://github.com/ClickHouse/ClickHouse/pull/36636) ([flynn](https://github.com/ucasfl)). +* Fix SortingStep::updateOutputStream() [#36638](https://github.com/ClickHouse/ClickHouse/pull/36638) ([Nikita Taranov](https://github.com/nickitat)). +* CodeQL - Run Daily Analysis [#36640](https://github.com/ClickHouse/ClickHouse/pull/36640) ([Julio Jimenez](https://github.com/juliojimenez)). +* Simplify check_branch, prefetch target branches [#36641](https://github.com/ClickHouse/ClickHouse/pull/36641) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Make backward compatible integration tests runner [#36643](https://github.com/ClickHouse/ClickHouse/pull/36643) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix some flaky tests [#36644](https://github.com/ClickHouse/ClickHouse/pull/36644) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update version_date.tsv after v22.4.3.3-stable [#36651](https://github.com/ClickHouse/ClickHouse/pull/36651) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Fix flaky test [#36652](https://github.com/ClickHouse/ClickHouse/pull/36652) ([Amos Bird](https://github.com/amosbird)). +* add missing pandas package [#36653](https://github.com/ClickHouse/ClickHouse/pull/36653) ([Ramazan Polat](https://github.com/ramazanpolat)). +* Fix style issue reported by black formatter [#36655](https://github.com/ClickHouse/ClickHouse/pull/36655) ([Alexander Gololobov](https://github.com/davenger)). +* Fix formatting in drop cache system query [#36658](https://github.com/ClickHouse/ClickHouse/pull/36658) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix stress test after 36639 [#36660](https://github.com/ClickHouse/ClickHouse/pull/36660) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update version_date.tsv after v22.3.4.20-lts [#36668](https://github.com/ClickHouse/ClickHouse/pull/36668) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Update 00170_s3_cache.sql [#36669](https://github.com/ClickHouse/ClickHouse/pull/36669) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better version of SeekableReadBufferWithSize [#36676](https://github.com/ClickHouse/ClickHouse/pull/36676) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better logs for virtual parts [#36680](https://github.com/ClickHouse/ClickHouse/pull/36680) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Check socket is connected in HTTPSession [#36683](https://github.com/ClickHouse/ClickHouse/pull/36683) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fix typo [#36684](https://github.com/ClickHouse/ClickHouse/pull/36684) ([flynn](https://github.com/ucasfl)). +* Remove excessive logging from S3 [#36689](https://github.com/ClickHouse/ClickHouse/pull/36689) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Tests report [#36701](https://github.com/ClickHouse/ClickHouse/pull/36701) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Use correct nodes in test_keeper_znode_time [#36711](https://github.com/ClickHouse/ClickHouse/pull/36711) ([Antonio Andelic](https://github.com/antonio2368)). +* Reorganize source files so that base won't depend on Common [#36715](https://github.com/ClickHouse/ClickHouse/pull/36715) ([Amos Bird](https://github.com/amosbird)). +* tests: fix 02015_async_inserts_stress_long flakiness [#36731](https://github.com/ClickHouse/ClickHouse/pull/36731) ([Azat Khuzhin](https://github.com/azat)). +* Add an extra check for RAFT config change [#36736](https://github.com/ClickHouse/ClickHouse/pull/36736) ([Antonio Andelic](https://github.com/antonio2368)). +* Update 00170_s3_cache.sql [#36743](https://github.com/ClickHouse/ClickHouse/pull/36743) ([Kseniia Sumarokova](https://github.com/kssenii)). +* ClickHouseDictionarySource context copy [#36744](https://github.com/ClickHouse/ClickHouse/pull/36744) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix build SeekableReadBufferWithSize -> SeekableReadBuffer [#36745](https://github.com/ClickHouse/ClickHouse/pull/36745) ([Vladimir C](https://github.com/vdimir)). +* Better error message from NuRaft in case of CRC mismatch [#36746](https://github.com/ClickHouse/ClickHouse/pull/36746) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix certs (finishing pr [#36457](https://github.com/ClickHouse/ClickHouse/issues/36457)) [#36747](https://github.com/ClickHouse/ClickHouse/pull/36747) ([Nikita Taranov](https://github.com/nickitat)). +* Another fix for Hung Check [#36752](https://github.com/ClickHouse/ClickHouse/pull/36752) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Deduce `UInt8` type for bools from json instead of `UInt64` [#36756](https://github.com/ClickHouse/ClickHouse/pull/36756) ([Anton Popov](https://github.com/CurtizJ)). +* Add small script for keeper check [#36758](https://github.com/ClickHouse/ClickHouse/pull/36758) ([alesapin](https://github.com/alesapin)). +* Fix ungrammatical error message [#36760](https://github.com/ClickHouse/ClickHouse/pull/36760) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix flaky test `test_ddl_worker_non_leader` [#36765](https://github.com/ClickHouse/ClickHouse/pull/36765) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix evaluateConstantExpression for subqueries [#36766](https://github.com/ClickHouse/ClickHouse/pull/36766) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix clickhouse-test for server without `is_all_data_sent` in system.processes [#36767](https://github.com/ClickHouse/ClickHouse/pull/36767) ([Azat Khuzhin](https://github.com/azat)). +* Init thread pools for clickhouse-local [#36778](https://github.com/ClickHouse/ClickHouse/pull/36778) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix performance test [#36779](https://github.com/ClickHouse/ClickHouse/pull/36779) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better exception messages while socket timeouts [#36781](https://github.com/ClickHouse/ClickHouse/pull/36781) ([Kruglov Pavel](https://github.com/Avogar)). +* Improve docs_check to be used in clickhouse-docs [#36796](https://github.com/ClickHouse/ClickHouse/pull/36796) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* pull poco #58 [#36798](https://github.com/ClickHouse/ClickHouse/pull/36798) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Tiny cleanup [#36799](https://github.com/ClickHouse/ClickHouse/pull/36799) ([Azat Khuzhin](https://github.com/azat)). +* Use mutex per worker in ParallelReadBuffer [#36801](https://github.com/ClickHouse/ClickHouse/pull/36801) ([Antonio Andelic](https://github.com/antonio2368)). +* Update version_date.tsv after v22.3.5.5-lts [#36805](https://github.com/ClickHouse/ClickHouse/pull/36805) ([github-actions[bot]](https://github.com/apps/github-actions)). +* play.html: add button to transpose table [#36811](https://github.com/ClickHouse/ClickHouse/pull/36811) ([Vladimir C](https://github.com/vdimir)). +* Fix style error in DocsCheck [#36813](https://github.com/ClickHouse/ClickHouse/pull/36813) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Set is_all_data_sent on exceptions too [#36816](https://github.com/ClickHouse/ClickHouse/pull/36816) ([Azat Khuzhin](https://github.com/azat)). +* Clang -Tidy Fixes [#36817](https://github.com/ClickHouse/ClickHouse/pull/36817) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Protection from incorrect build [#36819](https://github.com/ClickHouse/ClickHouse/pull/36819) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid recreation of system.asynchronous_metric_log (due to difference in codec) [#36820](https://github.com/ClickHouse/ClickHouse/pull/36820) ([Azat Khuzhin](https://github.com/azat)). +* Removed forceful drop cache command, fix detached status state [#36825](https://github.com/ClickHouse/ClickHouse/pull/36825) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Log hash table's cache messages with TRACE level [#36830](https://github.com/ClickHouse/ClickHouse/pull/36830) ([Nikita Taranov](https://github.com/nickitat)). +* Cleanup: Remove switches for obsolete GCC version [#36831](https://github.com/ClickHouse/ClickHouse/pull/36831) ([Robert Schulze](https://github.com/rschu1ze)). +* Add functions from CREATE FUNCTION to completion [#36834](https://github.com/ClickHouse/ClickHouse/pull/36834) ([Azat Khuzhin](https://github.com/azat)). +* Validate that function had been passed in CREATE FUNCTION [#36835](https://github.com/ClickHouse/ClickHouse/pull/36835) ([Azat Khuzhin](https://github.com/azat)). +* [RFC] Remove unimplemented RAID1 support [#36836](https://github.com/ClickHouse/ClickHouse/pull/36836) ([Azat Khuzhin](https://github.com/azat)). +* Disable `merge_tree_metadata_cache` by default [#36838](https://github.com/ClickHouse/ClickHouse/pull/36838) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Replace uses of SFINAE by C++20 concepts [#36839](https://github.com/ClickHouse/ClickHouse/pull/36839) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix performance test (4) [#36840](https://github.com/ClickHouse/ClickHouse/pull/36840) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove "preconditions" from performance tests (overengineering, unneeded feature) [#36841](https://github.com/ClickHouse/ClickHouse/pull/36841) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove inherited create() method + disallow copying of storages [#36844](https://github.com/ClickHouse/ClickHouse/pull/36844) ([Robert Schulze](https://github.com/rschu1ze)). +* tests: disable 02260_alter_compact_part_drop_nested_column for <=22.4 [#36845](https://github.com/ClickHouse/ClickHouse/pull/36845) ([Azat Khuzhin](https://github.com/azat)). +* Fix check black [#36850](https://github.com/ClickHouse/ClickHouse/pull/36850) ([Kruglov Pavel](https://github.com/Avogar)). +* Aggregator JIT compilation lock fix [#36852](https://github.com/ClickHouse/ClickHouse/pull/36852) ([Maksim Kita](https://github.com/kitaisreal)). +* Add Other Query Time Microseconds Profile Event [#36853](https://github.com/ClickHouse/ClickHouse/pull/36853) ([Ilya Yatsishin](https://github.com/qoega)). +* Reproduce and a little bit better fix for LC dict right offset. [#36856](https://github.com/ClickHouse/ClickHouse/pull/36856) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Terminate if S3 buffer is not finalized [#36857](https://github.com/ClickHouse/ClickHouse/pull/36857) ([alesapin](https://github.com/alesapin)). +* Activated a bunch of LLVM 12/13/14 clang-tidy warnings [#36862](https://github.com/ClickHouse/ClickHouse/pull/36862) ([Robert Schulze](https://github.com/rschu1ze)). +* Replace `Timeout` with `Tests not finished` [#36863](https://github.com/ClickHouse/ClickHouse/pull/36863) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backup improvements [#36864](https://github.com/ClickHouse/ClickHouse/pull/36864) ([Vitaly Baranov](https://github.com/vitlibar)). +* Integration tests [#36866](https://github.com/ClickHouse/ClickHouse/pull/36866) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Relax cmake check for CFLAGS/CXXFLAGS/LDFLAGS [#36869](https://github.com/ClickHouse/ClickHouse/pull/36869) ([Azat Khuzhin](https://github.com/azat)). +* Fix insertion of complex JSONs with nested arrays [2] [#36873](https://github.com/ClickHouse/ClickHouse/pull/36873) ([Anton Popov](https://github.com/CurtizJ)). +* Add column to system.filesystem_cache_log [#36874](https://github.com/ClickHouse/ClickHouse/pull/36874) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Log into filesystem_cache_log when cache not even attempted [#36876](https://github.com/ClickHouse/ClickHouse/pull/36876) ([Kseniia Sumarokova](https://github.com/kssenii)). +* fix typo in comment [#36880](https://github.com/ClickHouse/ClickHouse/pull/36880) ([Sergei Trifonov](https://github.com/serxa)). +* Add some CurrentMetrics for fs cache [#36882](https://github.com/ClickHouse/ClickHouse/pull/36882) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add some asynchronous metrics for fs cache [#36883](https://github.com/ClickHouse/ClickHouse/pull/36883) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Save system logs in functionsl tests if server crashed [#36885](https://github.com/ClickHouse/ClickHouse/pull/36885) ([Alexander Tokmakov](https://github.com/tavplubix)). +* quick tmp fix for stress test [#36900](https://github.com/ClickHouse/ClickHouse/pull/36900) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Some fixes for replicated merge tree [#36909](https://github.com/ClickHouse/ClickHouse/pull/36909) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Invert .clang-tidy [#36911](https://github.com/ClickHouse/ClickHouse/pull/36911) ([Robert Schulze](https://github.com/rschu1ze)). +* Replace make_pair()/make_tuple() by pair()/tuple() [#36913](https://github.com/ClickHouse/ClickHouse/pull/36913) ([Robert Schulze](https://github.com/rschu1ze)). +* Remove log message on client reconnects (reverts [#36587](https://github.com/ClickHouse/ClickHouse/issues/36587)) [#36915](https://github.com/ClickHouse/ClickHouse/pull/36915) ([Azat Khuzhin](https://github.com/azat)). +* Fix profile events in fs cached buffer [#36916](https://github.com/ClickHouse/ClickHouse/pull/36916) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Benchmark was not loaded properly [#36918](https://github.com/ClickHouse/ClickHouse/pull/36918) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Force keeper snapshot equality on different replicas [#36947](https://github.com/ClickHouse/ClickHouse/pull/36947) ([alesapin](https://github.com/alesapin)). +* Remove strange code [#36951](https://github.com/ClickHouse/ClickHouse/pull/36951) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove nested exception [#36952](https://github.com/ClickHouse/ClickHouse/pull/36952) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable clangtidies [#36960](https://github.com/ClickHouse/ClickHouse/pull/36960) ([Robert Schulze](https://github.com/rschu1ze)). +* Improve stress tests report a little bit [#36961](https://github.com/ClickHouse/ClickHouse/pull/36961) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Follow up for ConcurrentHashJoin [#36970](https://github.com/ClickHouse/ClickHouse/pull/36970) ([Vladimir C](https://github.com/vdimir)). +* Fix formats docs [#36972](https://github.com/ClickHouse/ClickHouse/pull/36972) ([Kruglov Pavel](https://github.com/Avogar)). +* Support secure connection in clickhouse-test [#36977](https://github.com/ClickHouse/ClickHouse/pull/36977) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix installation_id filter, minimize docker images diff [#36978](https://github.com/ClickHouse/ClickHouse/pull/36978) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update version_date.tsv after v22.3.6.5-lts [#36985](https://github.com/ClickHouse/ClickHouse/pull/36985) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Update version_date.tsv after v22.4.5.9-stable [#36986](https://github.com/ClickHouse/ClickHouse/pull/36986) ([github-actions[bot]](https://github.com/apps/github-actions)). +* Removing ReplacingWindowColumnTransform in WindowView [#36998](https://github.com/ClickHouse/ClickHouse/pull/36998) ([vxider](https://github.com/Vxider)). +* Merging [#34765](https://github.com/ClickHouse/ClickHouse/issues/34765) [#37002](https://github.com/ClickHouse/ClickHouse/pull/37002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove CodeQL [#37006](https://github.com/ClickHouse/ClickHouse/pull/37006) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove obsolete code [#37009](https://github.com/ClickHouse/ClickHouse/pull/37009) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Try fix flaky test [#37010](https://github.com/ClickHouse/ClickHouse/pull/37010) ([Nikita Taranov](https://github.com/nickitat)). +* fix output error in LabelsCheck [#37016](https://github.com/ClickHouse/ClickHouse/pull/37016) ([wuxiaobai24](https://github.com/wuxiaobai24)). +* remove useless code [#37020](https://github.com/ClickHouse/ClickHouse/pull/37020) ([flynn](https://github.com/ucasfl)). +* Merging [#34932](https://github.com/ClickHouse/ClickHouse/issues/34932). [#37023](https://github.com/ClickHouse/ClickHouse/pull/37023) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix benchmark build code [#37025](https://github.com/ClickHouse/ClickHouse/pull/37025) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add benchmark script for testing clouds [#37027](https://github.com/ClickHouse/ClickHouse/pull/37027) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky integration mongodb test [#37035](https://github.com/ClickHouse/ClickHouse/pull/37035) ([Kruglov Pavel](https://github.com/Avogar)). +* Print stacks if we cannot terminate server in stress tests [#37052](https://github.com/ClickHouse/ClickHouse/pull/37052) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove last mentions of data streams [#37053](https://github.com/ClickHouse/ClickHouse/pull/37053) ([Anton Popov](https://github.com/CurtizJ)). +* Changelog script [#37057](https://github.com/ClickHouse/ClickHouse/pull/37057) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* This evening I started using Grammarly. [#37058](https://github.com/ClickHouse/ClickHouse/pull/37058) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* update sanity checks warning message text [#37063](https://github.com/ClickHouse/ClickHouse/pull/37063) ([Sergei Trifonov](https://github.com/serxa)). +* Continue fixing [#36199](https://github.com/ClickHouse/ClickHouse/issues/36199). [#37071](https://github.com/ClickHouse/ClickHouse/pull/37071) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* just fix a number [#37072](https://github.com/ClickHouse/ClickHouse/pull/37072) ([jiahui-97](https://github.com/jiahui-97)). +* Disable thread fuzzer after server restart [#37079](https://github.com/ClickHouse/ClickHouse/pull/37079) ([alesapin](https://github.com/alesapin)). +* Option to force cross_to_inner_join_rewrite [#37085](https://github.com/ClickHouse/ClickHouse/pull/37085) ([Vladimir C](https://github.com/vdimir)). +* fix wrong argument in proxy resolver of DiskS3 [#37100](https://github.com/ClickHouse/ClickHouse/pull/37100) ([flynn](https://github.com/ucasfl)). +* tests: fix 01119_optimize_trivial_insert_select (due to max_threads randomization) [#37101](https://github.com/ClickHouse/ClickHouse/pull/37101) ([Azat Khuzhin](https://github.com/azat)). +* Support SELECT query in WindowView [#37105](https://github.com/ClickHouse/ClickHouse/pull/37105) ([vxider](https://github.com/Vxider)). +* Fix workflow style check [#37113](https://github.com/ClickHouse/ClickHouse/pull/37113) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add check for columns size in Block::cloneWithColumns [#37124](https://github.com/ClickHouse/ClickHouse/pull/37124) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update documentation and defaults for memory overcommit [#37129](https://github.com/ClickHouse/ClickHouse/pull/37129) ([Dmitry Novik](https://github.com/novikd)). +* Update default remote fs read method in ReadSettings [#37130](https://github.com/ClickHouse/ClickHouse/pull/37130) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update Exception Message for allowed auth types [#37132](https://github.com/ClickHouse/ClickHouse/pull/37132) ([Marcelo Rodriguez](https://github.com/e-mars)). +* tests/integration: fix possible race for iptables user rules inside containers [#37138](https://github.com/ClickHouse/ClickHouse/pull/37138) ([Azat Khuzhin](https://github.com/azat)). +* Fix fasttest ccache permissions [#37143](https://github.com/ClickHouse/ClickHouse/pull/37143) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add const qualifier for few methods from Context [#37154](https://github.com/ClickHouse/ClickHouse/pull/37154) ([Azat Khuzhin](https://github.com/azat)). +* Reload listen_reuse_port/listen_backlog from config [#37156](https://github.com/ClickHouse/ClickHouse/pull/37156) ([Azat Khuzhin](https://github.com/azat)). +* Enable DNS cache for HTTPSClientSession. [#37157](https://github.com/ClickHouse/ClickHouse/pull/37157) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* fix async reads from remote fs internal setting not being always turned on [#37164](https://github.com/ClickHouse/ClickHouse/pull/37164) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Backups Improvements 5 [#37168](https://github.com/ClickHouse/ClickHouse/pull/37168) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add S3Requests metric [#37200](https://github.com/ClickHouse/ClickHouse/pull/37200) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix coverity build problem with LD_PRELOAD [#37203](https://github.com/ClickHouse/ClickHouse/pull/37203) ([Boris Kuschel](https://github.com/bkuschel)). +* Relax log level for some checks in check thread [#37208](https://github.com/ClickHouse/ClickHouse/pull/37208) ([alesapin](https://github.com/alesapin)). +* update poco [#37209](https://github.com/ClickHouse/ClickHouse/pull/37209) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Ignore harmful env variables in clickhouse binaries (reexec w/o them) [#37211](https://github.com/ClickHouse/ClickHouse/pull/37211) ([Azat Khuzhin](https://github.com/azat)). +* Fix wrong comment in IVolume.h [#37218](https://github.com/ClickHouse/ClickHouse/pull/37218) ([Sergei Trifonov](https://github.com/serxa)). +* Cmake cleanup pt2 [#37222](https://github.com/ClickHouse/ClickHouse/pull/37222) ([Robert Schulze](https://github.com/rschu1ze)). +* Temporarily fix flaky test `01825_type_json_insert_select.sql` [#37245](https://github.com/ClickHouse/ClickHouse/pull/37245) ([Anton Popov](https://github.com/CurtizJ)). +* Pass need_filter, has_null_map to joinRightColumns [#37256](https://github.com/ClickHouse/ClickHouse/pull/37256) ([Vladimir C](https://github.com/vdimir)). +* Activate more clangtidies [#37259](https://github.com/ClickHouse/ClickHouse/pull/37259) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix docker cleaner in workflows [#37271](https://github.com/ClickHouse/ClickHouse/pull/37271) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* tidy build fix [#37291](https://github.com/ClickHouse/ClickHouse/pull/37291) ([Alexander Gololobov](https://github.com/davenger)). +* Update run-check.py to match PR template, add comments [#37301](https://github.com/ClickHouse/ClickHouse/pull/37301) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Release without prestable [#37306](https://github.com/ClickHouse/ClickHouse/pull/37306) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fixed typos [#37322](https://github.com/ClickHouse/ClickHouse/pull/37322) ([Alexander Gololobov](https://github.com/davenger)). + diff --git a/docs/changelogs/v22.6.1.1985-stable.md b/docs/changelogs/v22.6.1.1985-stable.md index 9c7ecc1dae3..eeb4078eb04 100644 --- a/docs/changelogs/v22.6.1.1985-stable.md +++ b/docs/changelogs/v22.6.1.1985-stable.md @@ -197,3 +197,164 @@ sidebar_label: 2022 * NO CL ENTRY: 'Revert "More parallel execution for queries with `FINAL`"'. [#38094](https://github.com/ClickHouse/ClickHouse/pull/38094) ([Alexander Tokmakov](https://github.com/tavplubix)). * NO CL ENTRY: 'Revert "Revert "add d3js based trace visualizer as gantt chart""'. [#38129](https://github.com/ClickHouse/ClickHouse/pull/38129) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix collision of S3 operation log revision [#34915](https://github.com/ClickHouse/ClickHouse/pull/34915) ([ianton-ru](https://github.com/ianton-ru)). +* Change timezone in stateless tests [#35231](https://github.com/ClickHouse/ClickHouse/pull/35231) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code in ReplicatedMergeTreeRestartingThread [#36113](https://github.com/ClickHouse/ClickHouse/pull/36113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge release and performance builds [#36538](https://github.com/ClickHouse/ClickHouse/pull/36538) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add support for preprocessing ZooKeeper operations in `clickhouse-keeper` [#37036](https://github.com/ClickHouse/ClickHouse/pull/37036) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix jemalloc compatibility with LLVM libunwind [#37078](https://github.com/ClickHouse/ClickHouse/pull/37078) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Separate object storage operations from disks [#37139](https://github.com/ClickHouse/ClickHouse/pull/37139) ([alesapin](https://github.com/alesapin)). +* Add test for WATCH LIMIT query in WindowView [#37219](https://github.com/ClickHouse/ClickHouse/pull/37219) ([vxider](https://github.com/Vxider)). +* Improve changelog script [#37249](https://github.com/ClickHouse/ClickHouse/pull/37249) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Rework AccessControl's notifications. [#37269](https://github.com/ClickHouse/ClickHouse/pull/37269) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add ability to pass QueryKind via clickhouse-client/local (useful for debugging) [#37290](https://github.com/ClickHouse/ClickHouse/pull/37290) ([Azat Khuzhin](https://github.com/azat)). +* Various cmake cleanups [#37300](https://github.com/ClickHouse/ClickHouse/pull/37300) ([Robert Schulze](https://github.com/rschu1ze)). +* Try to fix some trash [#37303](https://github.com/ClickHouse/ClickHouse/pull/37303) ([Alexander Tokmakov](https://github.com/tavplubix)). +* [bug-fix] root_dir is not set in copyThroughBuffers [#37319](https://github.com/ClickHouse/ClickHouse/pull/37319) ([lingo-xp](https://github.com/lingo-xp)). +* Speed up test 00157_cache_dictionary [#37320](https://github.com/ClickHouse/ClickHouse/pull/37320) ([Kruglov Pavel](https://github.com/Avogar)). +* Add changelog for 22.5 [#37339](https://github.com/ClickHouse/ClickHouse/pull/37339) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version after release [#37346](https://github.com/ClickHouse/ClickHouse/pull/37346) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Disable clang-tidy readability-identifier-length [#37347](https://github.com/ClickHouse/ClickHouse/pull/37347) ([Robert Schulze](https://github.com/rschu1ze)). +* Tags and release [#37348](https://github.com/ClickHouse/ClickHouse/pull/37348) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* New stable tag [#37349](https://github.com/ClickHouse/ClickHouse/pull/37349) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fixes for eigen library build [#37369](https://github.com/ClickHouse/ClickHouse/pull/37369) ([Alexander Gololobov](https://github.com/davenger)). +* Do not fail CI if events clickhouse is down [#37371](https://github.com/ClickHouse/ClickHouse/pull/37371) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Ugly hacks for performance.tgz artifacts [#37373](https://github.com/ClickHouse/ClickHouse/pull/37373) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix docker server images building [#37374](https://github.com/ClickHouse/ClickHouse/pull/37374) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix debug symbols in packages [#37379](https://github.com/ClickHouse/ClickHouse/pull/37379) ([Azat Khuzhin](https://github.com/azat)). +* Enable -Wc++98-compat-extra-semi [#37382](https://github.com/ClickHouse/ClickHouse/pull/37382) ([Robert Schulze](https://github.com/rschu1ze)). +* Mark all `operator bool()` as explicit [#37388](https://github.com/ClickHouse/ClickHouse/pull/37388) ([Anton Popov](https://github.com/CurtizJ)). +* Fixes for transactions [#37398](https://github.com/ClickHouse/ClickHouse/pull/37398) ([Alexander Tokmakov](https://github.com/tavplubix)). +* WindowTransform::moveRowNumber fix [#37400](https://github.com/ClickHouse/ClickHouse/pull/37400) ([Nikolay](https://github.com/ndchikin)). +* Fix possible memory leaks in system.certificates implementation [#37407](https://github.com/ClickHouse/ClickHouse/pull/37407) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Remove unused code in WindowView [#37426](https://github.com/ClickHouse/ClickHouse/pull/37426) ([vxider](https://github.com/Vxider)). +* Add join_algorithm='parallel_hash' to stress test [#37436](https://github.com/ClickHouse/ClickHouse/pull/37436) ([Vladimir C](https://github.com/vdimir)). +* Perf test for Norm and Distance functions for arrays and tuples [#37437](https://github.com/ClickHouse/ClickHouse/pull/37437) ([Alexander Gololobov](https://github.com/davenger)). +* FunctionBinaryRepresentation style fixes [#37438](https://github.com/ClickHouse/ClickHouse/pull/37438) ([Maksim Kita](https://github.com/kitaisreal)). +* CompressedWriteBuffer added comment [#37442](https://github.com/ClickHouse/ClickHouse/pull/37442) ([Maksim Kita](https://github.com/kitaisreal)). +* BinaryFunctionVectorized remove macro [#37447](https://github.com/ClickHouse/ClickHouse/pull/37447) ([Maksim Kita](https://github.com/kitaisreal)). +* RangeHashedDictionary added test [#37449](https://github.com/ClickHouse/ClickHouse/pull/37449) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix failed thread stateless tests [#37455](https://github.com/ClickHouse/ClickHouse/pull/37455) ([Kruglov Pavel](https://github.com/Avogar)). +* tests: fix table in 01710_projection_aggregation_in_order [#37468](https://github.com/ClickHouse/ClickHouse/pull/37468) ([Azat Khuzhin](https://github.com/azat)). +* Sync workflows paths for PR and DocsCheck [#37477](https://github.com/ClickHouse/ClickHouse/pull/37477) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Set compiler flags for ASM same as for C/CXX [#37479](https://github.com/ClickHouse/ClickHouse/pull/37479) ([Azat Khuzhin](https://github.com/azat)). +* Dynamic dispatch infrastructure style fixes [#37480](https://github.com/ClickHouse/ClickHouse/pull/37480) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactorings of LIKE/MATCH code [#37491](https://github.com/ClickHouse/ClickHouse/pull/37491) ([Robert Schulze](https://github.com/rschu1ze)). +* Improve changelog.py script, get changelogs for 2021 [#37496](https://github.com/ClickHouse/ClickHouse/pull/37496) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Try to run aarch64 performance tests [#37497](https://github.com/ClickHouse/ClickHouse/pull/37497) ([Ilya Yatsishin](https://github.com/qoega)). +* Renamed arrayXXNorm/arrayXXDistance functions to XXNorm/XXDistance and fixed some overflow cases [#37502](https://github.com/ClickHouse/ClickHouse/pull/37502) ([Alexander Gololobov](https://github.com/davenger)). +* Bump cctz to 2022-05-15 [#37518](https://github.com/ClickHouse/ClickHouse/pull/37518) ([Robert Schulze](https://github.com/rschu1ze)). +* Add failed builds to the build report [#37527](https://github.com/ClickHouse/ClickHouse/pull/37527) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Functions normalizeUTF8 unstable performance tests fix [#37528](https://github.com/ClickHouse/ClickHouse/pull/37528) ([Maksim Kita](https://github.com/kitaisreal)). +* Use a separate mutex for query_factories_info in Context. [#37532](https://github.com/ClickHouse/ClickHouse/pull/37532) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update SECURITY.md [#37541](https://github.com/ClickHouse/ClickHouse/pull/37541) ([Ivan Blinkov](https://github.com/blinkov)). +* Refactor read metrics and callbacks [#37543](https://github.com/ClickHouse/ClickHouse/pull/37543) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Follow-up to [#37398](https://github.com/ClickHouse/ClickHouse/issues/37398) [#37547](https://github.com/ClickHouse/ClickHouse/pull/37547) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix root CMakeLists.txt search [#37552](https://github.com/ClickHouse/ClickHouse/pull/37552) ([Sergei Trifonov](https://github.com/serxa)). +* Cleanup StorageHDFS (unused variables prevent build with clang 12) [#37554](https://github.com/ClickHouse/ClickHouse/pull/37554) ([Michail Safronov](https://github.com/msaf1980)). +* Fix failed assertion in cache [#37566](https://github.com/ClickHouse/ClickHouse/pull/37566) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Slightly better jepsen tests [#37568](https://github.com/ClickHouse/ClickHouse/pull/37568) ([alesapin](https://github.com/alesapin)). +* Remove unused MergeTreeDataMergerMutator::chooseMergeAlgorithm() [#37574](https://github.com/ClickHouse/ClickHouse/pull/37574) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless files [#37579](https://github.com/ClickHouse/ClickHouse/pull/37579) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Flush and shutdown temporary table before drop [#37585](https://github.com/ClickHouse/ClickHouse/pull/37585) ([vxider](https://github.com/Vxider)). +* tests: fix 01317_no_password_in_command_line flakiness (and make it race free) [#37597](https://github.com/ClickHouse/ClickHouse/pull/37597) ([Azat Khuzhin](https://github.com/azat)). +* Use Jepsen worklow directly in PR workflow [#37599](https://github.com/ClickHouse/ClickHouse/pull/37599) ([Antonio Andelic](https://github.com/antonio2368)). +* Added LpNorm and LpDistance functions for arrays [#37601](https://github.com/ClickHouse/ClickHouse/pull/37601) ([Alexander Gololobov](https://github.com/davenger)). +* Turn on s3 tests to red mode [#37604](https://github.com/ClickHouse/ClickHouse/pull/37604) ([alesapin](https://github.com/alesapin)). +* Update FileCache.cpp [#37606](https://github.com/ClickHouse/ClickHouse/pull/37606) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix Clang-Tidy: remove std::move() from trivially-copyable object [#37609](https://github.com/ClickHouse/ClickHouse/pull/37609) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Fix clang-tidy-14, part 1 [#37612](https://github.com/ClickHouse/ClickHouse/pull/37612) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove margin in test reports and change the font [#37614](https://github.com/ClickHouse/ClickHouse/pull/37614) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bugs in WindowView when using table identifier [#37619](https://github.com/ClickHouse/ClickHouse/pull/37619) ([vxider](https://github.com/Vxider)). +* Try fix tests [#37622](https://github.com/ClickHouse/ClickHouse/pull/37622) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix refactoring issue [#37629](https://github.com/ClickHouse/ClickHouse/pull/37629) ([alesapin](https://github.com/alesapin)). +* fix typo [#37637](https://github.com/ClickHouse/ClickHouse/pull/37637) ([flynn](https://github.com/ucasfl)). +* Fix excessive LIST requests to coordinator for transactions [#37640](https://github.com/ClickHouse/ClickHouse/pull/37640) ([Azat Khuzhin](https://github.com/azat)). +* Less flaky jbod rebalancer test [#37642](https://github.com/ClickHouse/ClickHouse/pull/37642) ([Amos Bird](https://github.com/amosbird)). +* Avoid useless context copy when building query interpreters [#37643](https://github.com/ClickHouse/ClickHouse/pull/37643) ([Amos Bird](https://github.com/amosbird)). +* Disable amqp-cpp and cassandra build if libuv is disabled [#37644](https://github.com/ClickHouse/ClickHouse/pull/37644) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix errors of CheckTriviallyCopyableMove type [#37647](https://github.com/ClickHouse/ClickHouse/pull/37647) ([Heena Bansal](https://github.com/HeenaBansal2009)). +* Fix hung check [#37655](https://github.com/ClickHouse/ClickHouse/pull/37655) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix flaky test test_row_policy [#37658](https://github.com/ClickHouse/ClickHouse/pull/37658) ([Vitaly Baranov](https://github.com/vitlibar)). +* Trying to remove prewhere_info.alias_actions as they seem to always be included in prewhere_actions [#37661](https://github.com/ClickHouse/ClickHouse/pull/37661) ([Alexander Gololobov](https://github.com/davenger)). +* test for [#36995](https://github.com/ClickHouse/ClickHouse/issues/36995) [#37668](https://github.com/ClickHouse/ClickHouse/pull/37668) ([Denny Crane](https://github.com/den-crane)). +* Cleanup unused file [#37674](https://github.com/ClickHouse/ClickHouse/pull/37674) ([hongbin](https://github.com/xlwh)). +* more verbose sanity checks [#37676](https://github.com/ClickHouse/ClickHouse/pull/37676) ([Sergei Trifonov](https://github.com/serxa)). +* Fix `test_keeper_force_recovery*` tests [#37679](https://github.com/ClickHouse/ClickHouse/pull/37679) ([Antonio Andelic](https://github.com/antonio2368)). +* Display entires for failed tests at the top of report by default [#37704](https://github.com/ClickHouse/ClickHouse/pull/37704) ([Vladimir C](https://github.com/vdimir)). +* Make GROUPING function skip constant folding [#37710](https://github.com/ClickHouse/ClickHouse/pull/37710) ([Dmitry Novik](https://github.com/novikd)). +* Get rid of duplicate download_previous_release [#37712](https://github.com/ClickHouse/ClickHouse/pull/37712) ([Vladimir C](https://github.com/vdimir)). +* Fix possible flaky 00814_replicated_minimalistic_part_header_zookeeper [#37714](https://github.com/ClickHouse/ClickHouse/pull/37714) ([Vladimir C](https://github.com/vdimir)). +* Fix build with -DENABLE_LIBRARIES=0 [#37719](https://github.com/ClickHouse/ClickHouse/pull/37719) ([Robert Schulze](https://github.com/rschu1ze)). +* Update grpc submodule to PR 9 [#37721](https://github.com/ClickHouse/ClickHouse/pull/37721) ([Harry Lee](https://github.com/HarryLeeIBM)). +* Initialize `ParallelReadBuffer` after construction [#37726](https://github.com/ClickHouse/ClickHouse/pull/37726) ([Antonio Andelic](https://github.com/antonio2368)). +* Some fixes for tests [#37740](https://github.com/ClickHouse/ClickHouse/pull/37740) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add test 02315_pmj_union_ubsan_35857 [#37744](https://github.com/ClickHouse/ClickHouse/pull/37744) ([Vladimir C](https://github.com/vdimir)). +* Upgrade libxml2 to 2.9.14 [#37745](https://github.com/ClickHouse/ClickHouse/pull/37745) ([Suzy Wang](https://github.com/SuzyWangIBMer)). +* Implement new submodule sync to fix nightly scan [#37750](https://github.com/ClickHouse/ClickHouse/pull/37750) ([Boris Kuschel](https://github.com/bkuschel)). +* Return back [#37266](https://github.com/ClickHouse/ClickHouse/issues/37266) [#37755](https://github.com/ClickHouse/ClickHouse/pull/37755) ([Anton Popov](https://github.com/CurtizJ)). +* Fix flaky 01154_move_partition_long [#37763](https://github.com/ClickHouse/ClickHouse/pull/37763) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Moved `ClientConfigurationPerRequest` to ClickHouse [#37767](https://github.com/ClickHouse/ClickHouse/pull/37767) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Got rid of `S3AuthSigner` [#37769](https://github.com/ClickHouse/ClickHouse/pull/37769) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix empty target table id in WindowView [#37770](https://github.com/ClickHouse/ClickHouse/pull/37770) ([vxider](https://github.com/Vxider)). +* Typos [#37773](https://github.com/ClickHouse/ClickHouse/pull/37773) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Use multitarget framework for numZerosInTail implementation [#37777](https://github.com/ClickHouse/ClickHouse/pull/37777) ([Alexander Gololobov](https://github.com/davenger)). +* Dictionaries improve exception messages [#37778](https://github.com/ClickHouse/ClickHouse/pull/37778) ([Maksim Kita](https://github.com/kitaisreal)). +* Ignore PART_IS_TEMPORARILY_LOCKED error in BC check [#37788](https://github.com/ClickHouse/ClickHouse/pull/37788) ([Kruglov Pavel](https://github.com/Avogar)). +* Use clang compiler by default and cleanup cmake preload [#37796](https://github.com/ClickHouse/ClickHouse/pull/37796) ([Azat Khuzhin](https://github.com/azat)). +* Fix reading of empty S3 files [#37801](https://github.com/ClickHouse/ClickHouse/pull/37801) ([Azat Khuzhin](https://github.com/azat)). +* Metadata storage abstraction for Disks [#37804](https://github.com/ClickHouse/ClickHouse/pull/37804) ([alesapin](https://github.com/alesapin)). +* Minor follow-up to cache table: std::{vector-->array} [#37806](https://github.com/ClickHouse/ClickHouse/pull/37806) ([Robert Schulze](https://github.com/rschu1ze)). +* Delay schedule of the cleaning task in WindowView [#37807](https://github.com/ClickHouse/ClickHouse/pull/37807) ([vxider](https://github.com/Vxider)). +* Fix submodule changed label condition [#37825](https://github.com/ClickHouse/ClickHouse/pull/37825) ([Vladimir C](https://github.com/vdimir)). +* Fix build (packager) [#37829](https://github.com/ClickHouse/ClickHouse/pull/37829) ([Nikita Taranov](https://github.com/nickitat)). +* Executable task forward job callback [#37830](https://github.com/ClickHouse/ClickHouse/pull/37830) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix docs check (remove git clean for yarn.lock) [#37836](https://github.com/ClickHouse/ClickHouse/pull/37836) ([Vladimir C](https://github.com/vdimir)). +* Opentracing minimal changes for processors [#37837](https://github.com/ClickHouse/ClickHouse/pull/37837) ([Ilya Yatsishin](https://github.com/qoega)). +* Do not count covered unexpected parts on sanity check [#37839](https://github.com/ClickHouse/ClickHouse/pull/37839) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Print query in one line on fatal errors [#37840](https://github.com/ClickHouse/ClickHouse/pull/37840) ([Azat Khuzhin](https://github.com/azat)). +* Hotfix for minio in functional tests [#37845](https://github.com/ClickHouse/ClickHouse/pull/37845) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Hotfix for minio in functional tests 2 [#37846](https://github.com/ClickHouse/ClickHouse/pull/37846) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fire and clean windows in WindowView only when data is inserted [#37853](https://github.com/ClickHouse/ClickHouse/pull/37853) ([vxider](https://github.com/Vxider)). +* Don't try to kill empty list of containers in `integration/runner` [#37854](https://github.com/ClickHouse/ClickHouse/pull/37854) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Minor cleanup [#37866](https://github.com/ClickHouse/ClickHouse/pull/37866) ([Igor Nikonov](https://github.com/devcrafter)). +* Update protobuf files for kafka and rabbitmq [fix integration tests] [#37884](https://github.com/ClickHouse/ClickHouse/pull/37884) ([Nikita Taranov](https://github.com/nickitat)). +* Fix keeper converter test [#37890](https://github.com/ClickHouse/ClickHouse/pull/37890) ([alesapin](https://github.com/alesapin)). +* Fix stress hung because of attached MV with sleep() [#37892](https://github.com/ClickHouse/ClickHouse/pull/37892) ([Azat Khuzhin](https://github.com/azat)). +* Fix build [#37903](https://github.com/ClickHouse/ClickHouse/pull/37903) ([Nikita Taranov](https://github.com/nickitat)). +* Try fix `test_grpc_protocol/test.py::test_progress` [#37908](https://github.com/ClickHouse/ClickHouse/pull/37908) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable flaky tests with MaterializedPostgreSQL [#37924](https://github.com/ClickHouse/ClickHouse/pull/37924) ([Alexander Tokmakov](https://github.com/tavplubix)). +* ProcfsMetricsProvider fix typo [#37927](https://github.com/ClickHouse/ClickHouse/pull/37927) ([Maksim Kita](https://github.com/kitaisreal)). +* Function dictGet check arguments size [#37930](https://github.com/ClickHouse/ClickHouse/pull/37930) ([Maksim Kita](https://github.com/kitaisreal)). +* Hierarchical dictionaries performance test fix [#37953](https://github.com/ClickHouse/ClickHouse/pull/37953) ([Maksim Kita](https://github.com/kitaisreal)). +* Normalize UTF8 performance test fix [#37954](https://github.com/ClickHouse/ClickHouse/pull/37954) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix flaky `test_parts_delete_zookeeper` [#37957](https://github.com/ClickHouse/ClickHouse/pull/37957) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Better parsing of `versionId` in `S3::URI::URI` [#37964](https://github.com/ClickHouse/ClickHouse/pull/37964) ([Vladimir Chebotarev](https://github.com/excitoon)). +* More consistent use of platform macros [#37969](https://github.com/ClickHouse/ClickHouse/pull/37969) ([Robert Schulze](https://github.com/rschu1ze)). +* Always disable --color-diagnostics for LLVM [#37970](https://github.com/ClickHouse/ClickHouse/pull/37970) ([Robert Schulze](https://github.com/rschu1ze)). +* Try fix `test_consistent_parts_after_clone_replica` [#37976](https://github.com/ClickHouse/ClickHouse/pull/37976) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Generalize setup_minio.sh [#37979](https://github.com/ClickHouse/ClickHouse/pull/37979) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add checks for numChildren in Keeper [#37980](https://github.com/ClickHouse/ClickHouse/pull/37980) ([alesapin](https://github.com/alesapin)). +* PartialSortingTransform refactoring [#37992](https://github.com/ClickHouse/ClickHouse/pull/37992) ([Maksim Kita](https://github.com/kitaisreal)). +* Follow up on self-extracting-executable [#38011](https://github.com/ClickHouse/ClickHouse/pull/38011) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Update docker run client command in docs [#38015](https://github.com/ClickHouse/ClickHouse/pull/38015) ([Vladimir C](https://github.com/vdimir)). +* Add another useful `docker exec` option [#38016](https://github.com/ClickHouse/ClickHouse/pull/38016) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add release commit hash to image labels [#38020](https://github.com/ClickHouse/ClickHouse/pull/38020) ([Pavel Novitskiy](https://github.com/pnovitskiy)). +* Use correct version in `DiskObjectStorageMetadata` [#38021](https://github.com/ClickHouse/ClickHouse/pull/38021) ([Antonio Andelic](https://github.com/antonio2368)). +* Examples coroutines build fix [#38023](https://github.com/ClickHouse/ClickHouse/pull/38023) ([Maksim Kita](https://github.com/kitaisreal)). +* Use pdqsort instead of standard sort [#38025](https://github.com/ClickHouse/ClickHouse/pull/38025) ([Maksim Kita](https://github.com/kitaisreal)). +* Decimal: noexcept move constructor/assignment operator [#38026](https://github.com/ClickHouse/ClickHouse/pull/38026) ([Igor Nikonov](https://github.com/devcrafter)). +* tests: avoid "_csv.Error: field larger than field limit (131072)" error [#38030](https://github.com/ClickHouse/ClickHouse/pull/38030) ([Azat Khuzhin](https://github.com/azat)). +* More consts for disks [#38033](https://github.com/ClickHouse/ClickHouse/pull/38033) ([alesapin](https://github.com/alesapin)). +* Remove link to past event [#38045](https://github.com/ClickHouse/ClickHouse/pull/38045) ([Ivan Blinkov](https://github.com/blinkov)). +* Use workflow URL as "link" ref [#38057](https://github.com/ClickHouse/ClickHouse/pull/38057) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix a compile errors in self-extracting-executable (de)compressor [#38087](https://github.com/ClickHouse/ClickHouse/pull/38087) ([Robert Schulze](https://github.com/rschu1ze)). +* Small follow-up for FPC codec [#38089](https://github.com/ClickHouse/ClickHouse/pull/38089) ([Robert Schulze](https://github.com/rschu1ze)). +* Add dispatch to CherryPick action [#38095](https://github.com/ClickHouse/ClickHouse/pull/38095) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix: build error [#38098](https://github.com/ClickHouse/ClickHouse/pull/38098) ([Igor Nikonov](https://github.com/devcrafter)). +* Try random container name in integration tests runner [#38107](https://github.com/ClickHouse/ClickHouse/pull/38107) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Try to fix fpc codec [#38117](https://github.com/ClickHouse/ClickHouse/pull/38117) ([Anton Popov](https://github.com/CurtizJ)). +* Remove unused class member [#38120](https://github.com/ClickHouse/ClickHouse/pull/38120) ([Daohui Wang](https://github.com/wangdh15)). + From 45e3b9c2f3dab75c2a11bbe8d1f3882d967da253 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 23 Jun 2022 12:12:54 +0200 Subject: [PATCH 055/123] Regenerate v21.* changelogs with headers and `not for changelog` --- docs/changelogs/v21.1.1.5646-prestable.md | 204 +++++++++++++++ docs/changelogs/v21.1.2.15-stable.md | 7 + docs/changelogs/v21.1.3.32-stable.md | 11 + docs/changelogs/v21.1.4.46-stable.md | 12 + docs/changelogs/v21.1.5.4-stable.md | 7 + docs/changelogs/v21.1.6.13-stable.md | 7 + docs/changelogs/v21.1.7.1-stable.md | 7 + docs/changelogs/v21.1.8.30-stable.md | 12 + docs/changelogs/v21.1.9.41-stable.md | 12 + docs/changelogs/v21.10.1.8013-prestable.md | 130 ++++++++++ docs/changelogs/v21.10.2.15-stable.md | 21 ++ docs/changelogs/v21.10.3.9-stable.md | 17 ++ docs/changelogs/v21.10.4.26-stable.md | 11 + docs/changelogs/v21.10.5.3-stable.md | 7 + docs/changelogs/v21.10.6.2-stable.md | 17 ++ docs/changelogs/v21.11.1.8636-prestable.md | 282 ++++++++++++++++++++- docs/changelogs/v21.11.10.1-stable.md | 7 + docs/changelogs/v21.11.11.1-stable.md | 7 + docs/changelogs/v21.11.2.2-stable.md | 11 + docs/changelogs/v21.11.3.6-stable.md | 7 + docs/changelogs/v21.11.4.14-stable.md | 13 + docs/changelogs/v21.11.5.33-stable.md | 11 + docs/changelogs/v21.11.6.7-stable.md | 12 + docs/changelogs/v21.11.7.9-stable.md | 13 + docs/changelogs/v21.11.8.4-stable.md | 12 + docs/changelogs/v21.11.9.1-stable.md | 13 + docs/changelogs/v21.12.1.9017-prestable.md | 224 +++++++++++++++- docs/changelogs/v21.12.2.17-stable.md | 15 ++ docs/changelogs/v21.12.3.32-stable.md | 16 ++ docs/changelogs/v21.12.4.1-stable.md | 7 + docs/changelogs/v21.2.1.5869-prestable.md | 120 +++++++++ docs/changelogs/v21.2.10.48-stable.md | 11 + docs/changelogs/v21.2.2.8-stable.md | 123 ++++++++- docs/changelogs/v21.2.3.15-stable.md | 11 + docs/changelogs/v21.2.4.6-stable.md | 7 + docs/changelogs/v21.2.5.5-stable.md | 7 + docs/changelogs/v21.2.6.1-stable.md | 7 + docs/changelogs/v21.2.7.11-stable.md | 11 + docs/changelogs/v21.2.8.31-stable.md | 11 + docs/changelogs/v21.2.9.41-stable.md | 12 + docs/changelogs/v21.3.1.6185-prestable.md | 166 ++++++++++++ docs/changelogs/v21.3.10.1-lts.md | 11 + docs/changelogs/v21.3.11.5-lts.md | 7 + docs/changelogs/v21.3.12.2-lts.md | 13 + docs/changelogs/v21.3.13.9-lts.md | 12 + docs/changelogs/v21.3.14.1-lts.md | 11 + docs/changelogs/v21.3.15.4-stable.md | 11 + docs/changelogs/v21.3.16.5-lts.md | 19 ++ docs/changelogs/v21.3.17.2-lts.md | 7 + docs/changelogs/v21.3.18.4-lts.md | 14 + docs/changelogs/v21.3.19.1-lts.md | 12 + docs/changelogs/v21.3.2.5-lts.md | 168 ++++++++++++ docs/changelogs/v21.3.20.1-lts.md | 17 ++ docs/changelogs/v21.3.3.14-lts.md | 11 + docs/changelogs/v21.3.4.25-lts.md | 12 + docs/changelogs/v21.3.5.42-lts.md | 11 + docs/changelogs/v21.3.6.55-lts.md | 11 + docs/changelogs/v21.3.7.62-stable.md | 11 + docs/changelogs/v21.3.8.76-lts.md | 11 + docs/changelogs/v21.3.9.83-lts.md | 7 + docs/changelogs/v21.4.1.6422-prestable.md | 118 +++++++++ docs/changelogs/v21.4.2.10-prestable.md | 115 +++++++++ docs/changelogs/v21.4.3.21-stable.md | 11 + docs/changelogs/v21.4.4.30-stable.md | 7 + docs/changelogs/v21.4.5.46-stable.md | 13 + docs/changelogs/v21.4.6.55-stable.md | 7 + docs/changelogs/v21.4.7.3-stable.md | 11 + docs/changelogs/v21.5.1.6601-prestable.md | 102 +++++++- docs/changelogs/v21.5.2.25-prestable.md | 14 + docs/changelogs/v21.5.3.1-prestable.md | 11 + docs/changelogs/v21.5.4.6-prestable.md | 7 + docs/changelogs/v21.5.5.12-stable.md | 7 + docs/changelogs/v21.5.6.6-stable.md | 13 + docs/changelogs/v21.5.7.9-stable.md | 13 + docs/changelogs/v21.5.8.21-stable.md | 12 + docs/changelogs/v21.5.9.4-stable.md | 12 + docs/changelogs/v21.6.1.6891-prestable.md | 170 +++++++++++++ docs/changelogs/v21.6.2.7-prestable.md | 13 + docs/changelogs/v21.6.3.14-stable.md | 7 + docs/changelogs/v21.6.4.26-stable.md | 12 + docs/changelogs/v21.6.5.37-stable.md | 12 + docs/changelogs/v21.6.6.51-stable.md | 11 + docs/changelogs/v21.6.7.57-stable.md | 11 + docs/changelogs/v21.6.8.62-stable.md | 11 + docs/changelogs/v21.6.9.7-stable.md | 23 ++ docs/changelogs/v21.7.1.7283-prestable.md | 237 +++++++++++++++++ docs/changelogs/v21.7.10.4-stable.md | 15 ++ docs/changelogs/v21.7.11.3-stable.md | 13 + docs/changelogs/v21.7.2.7-stable.md | 12 + docs/changelogs/v21.7.3.14-stable.md | 11 + docs/changelogs/v21.7.4.18-stable.md | 7 + docs/changelogs/v21.7.5.29-stable.md | 14 + docs/changelogs/v21.7.6.39-stable.md | 11 + docs/changelogs/v21.7.7.47-stable.md | 16 ++ docs/changelogs/v21.7.8.58-stable.md | 12 + docs/changelogs/v21.7.9.7-stable.md | 19 ++ docs/changelogs/v21.8.1.7409-prestable.md | 66 +++++ docs/changelogs/v21.8.10.19-lts.md | 11 + docs/changelogs/v21.8.11.4-lts.md | 14 + docs/changelogs/v21.8.12.29-lts.md | 11 + docs/changelogs/v21.8.13.6-lts.md | 19 ++ docs/changelogs/v21.8.14.5-lts.md | 7 + docs/changelogs/v21.8.15.7-lts.md | 7 + docs/changelogs/v21.8.2.19-prestable.md | 14 + docs/changelogs/v21.8.3.44-lts.md | 21 ++ docs/changelogs/v21.8.4.51-lts.md | 7 + docs/changelogs/v21.8.5.7-lts.md | 20 ++ docs/changelogs/v21.8.6.15-lts.md | 16 ++ docs/changelogs/v21.8.7.22-lts.md | 13 + docs/changelogs/v21.8.8.29-lts.md | 7 + docs/changelogs/v21.8.9.13-lts.md | 13 + docs/changelogs/v21.9.1.8000-prestable.md | 213 ++++++++++++++++ docs/changelogs/v21.9.2.17-stable.md | 19 ++ docs/changelogs/v21.9.3.30-stable.md | 14 + docs/changelogs/v21.9.4.35-stable.md | 13 + docs/changelogs/v21.9.5.16-stable.md | 13 + docs/changelogs/v21.9.6.24-stable.md | 14 + 117 files changed, 3631 insertions(+), 5 deletions(-) diff --git a/docs/changelogs/v21.1.1.5646-prestable.md b/docs/changelogs/v21.1.1.5646-prestable.md index ec8abc8a05b..97b645c45f9 100644 --- a/docs/changelogs/v21.1.1.5646-prestable.md +++ b/docs/changelogs/v21.1.1.5646-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.1.5646-prestable FIXME as compared to v20.12.1.5236-prestable #### Backward Incompatible Change @@ -257,3 +264,200 @@ * NO CL ENTRY: 'Revert "Add metrics for part number in MergeTree in ClickHouse"'. [#18834](https://github.com/ClickHouse/ClickHouse/pull/18834) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * NO CL ENTRY: 'Fixed typo in metrics.md'. [#18920](https://github.com/ClickHouse/ClickHouse/pull/18920) ([Mark Frost](https://github.com/frostmark)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Minor change in query profiler [#16899](https://github.com/ClickHouse/ClickHouse/pull/16899) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove escaping from toString(std::string) [#17206](https://github.com/ClickHouse/ClickHouse/pull/17206) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix libunwind build for cmake 3.19+ [#17271](https://github.com/ClickHouse/ClickHouse/pull/17271) ([Azat Khuzhin](https://github.com/azat)). +* Fix AST formatting in log messages [#17274](https://github.com/ClickHouse/ClickHouse/pull/17274) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Tiny cleanup [#17275](https://github.com/ClickHouse/ClickHouse/pull/17275) ([Azat Khuzhin](https://github.com/azat)). +* Add a test for [#13990](https://github.com/ClickHouse/ClickHouse/issues/13990) [#17298](https://github.com/ClickHouse/ClickHouse/pull/17298) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed flaky test_storage_s3::test_custom_auth_headers [#17299](https://github.com/ClickHouse/ClickHouse/pull/17299) ([Pavel Kovalenko](https://github.com/Jokser)). +* Minor changes for ODBC storage [#17301](https://github.com/ClickHouse/ClickHouse/pull/17301) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merging [#16309](https://github.com/ClickHouse/ClickHouse/issues/16309) [#17309](https://github.com/ClickHouse/ClickHouse/pull/17309) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix some flaky tests [#17311](https://github.com/ClickHouse/ClickHouse/pull/17311) ([alesapin](https://github.com/alesapin)). +* Remove outdated test [#17361](https://github.com/ClickHouse/ClickHouse/pull/17361) ([Anton Popov](https://github.com/CurtizJ)). +* Added a test for what was always working [#17375](https://github.com/ClickHouse/ClickHouse/pull/17375) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for mutation with empty partition [#17410](https://github.com/ClickHouse/ClickHouse/pull/17410) ([alesapin](https://github.com/alesapin)). +* Implement GRPC protocol (corrections) [#17435](https://github.com/ClickHouse/ClickHouse/pull/17435) ([Vitaly Baranov](https://github.com/vitlibar)). +* Drop include of the removed libbtrie in cmake rules [#17454](https://github.com/ClickHouse/ClickHouse/pull/17454) ([Azat Khuzhin](https://github.com/azat)). +* Merge expressions [#17458](https://github.com/ClickHouse/ClickHouse/pull/17458) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fail queries on unknown settings in perf tests [#17460](https://github.com/ClickHouse/ClickHouse/pull/17460) ([Azat Khuzhin](https://github.com/azat)). +* Adjust perf test thresholds [#17485](https://github.com/ClickHouse/ClickHouse/pull/17485) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix more flaky integration tests [#17486](https://github.com/ClickHouse/ClickHouse/pull/17486) ([alesapin](https://github.com/alesapin)). +* Fix data race on global BlockStreamProfileInfo in PullingAsyncPipelineExecutor [#17498](https://github.com/ClickHouse/ClickHouse/pull/17498) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix CMake generation and build for native Xcode and AppleClang [#17501](https://github.com/ClickHouse/ClickHouse/pull/17501) ([Denis Glazachev](https://github.com/traceon)). +* Fix bad test 01317_no_password_in_command_line.sh [#17506](https://github.com/ClickHouse/ClickHouse/pull/17506) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix spelling errors [#17527](https://github.com/ClickHouse/ClickHouse/pull/17527) ([flynn](https://github.com/ucasfl)). +* Fix toUnixTimestamp(Date()) error (use type name not column type name) [#17536](https://github.com/ClickHouse/ClickHouse/pull/17536) ([Azat Khuzhin](https://github.com/azat)). +* Add a test for StorageJoin and UUID [#17541](https://github.com/ClickHouse/ClickHouse/pull/17541) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Cassandra for BoringSSL [#17544](https://github.com/ClickHouse/ClickHouse/pull/17544) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Kafka for BoringSSL [#17545](https://github.com/ClickHouse/ClickHouse/pull/17545) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update MariaDB for BoringSSL [#17546](https://github.com/ClickHouse/ClickHouse/pull/17546) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update OpenLDAP for BoringSSL [#17547](https://github.com/ClickHouse/ClickHouse/pull/17547) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update AMQP for BoringSSL [#17548](https://github.com/ClickHouse/ClickHouse/pull/17548) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* getMemoryAmount: make code worse [#17556](https://github.com/ClickHouse/ClickHouse/pull/17556) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#12297](https://github.com/ClickHouse/ClickHouse/issues/12297) [#17557](https://github.com/ClickHouse/ClickHouse/pull/17557) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#9490](https://github.com/ClickHouse/ClickHouse/issues/9490) [#17561](https://github.com/ClickHouse/ClickHouse/pull/17561) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#11803](https://github.com/ClickHouse/ClickHouse/issues/11803) [#17562](https://github.com/ClickHouse/ClickHouse/pull/17562) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix mixed statements in 01018_ip_dictionary.sql [#17570](https://github.com/ClickHouse/ClickHouse/pull/17570) ([Vladimir C](https://github.com/vdimir)). +* Fix GRPC tests [#17597](https://github.com/ClickHouse/ClickHouse/pull/17597) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Port ClickHouse code to BoringSSL [#17606](https://github.com/ClickHouse/ClickHouse/pull/17606) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix the OpenTelemetry test [#17621](https://github.com/ClickHouse/ClickHouse/pull/17621) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Try to fix Yandex Synchronization check [#17634](https://github.com/ClickHouse/ClickHouse/pull/17634) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Use feature testing macro to test if char8_t is supported [#17645](https://github.com/ClickHouse/ClickHouse/pull/17645) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Make LockExceptionInThread/BlockerInThread nested [#17658](https://github.com/ClickHouse/ClickHouse/pull/17658) ([Azat Khuzhin](https://github.com/azat)). +* Mark grpc protocol's tests as flaky. [#17662](https://github.com/ClickHouse/ClickHouse/pull/17662) ([Vitaly Baranov](https://github.com/vitlibar)). +* Use feature testing macro once more [#17685](https://github.com/ClickHouse/ClickHouse/pull/17685) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Fix sequential number in TestKeeper [#17700](https://github.com/ClickHouse/ClickHouse/pull/17700) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Try fix arcadia build [#17720](https://github.com/ClickHouse/ClickHouse/pull/17720) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix arcadian build [#17781](https://github.com/ClickHouse/ClickHouse/pull/17781) ([Ivan](https://github.com/abyss7)). +* Fix a typo [#17791](https://github.com/ClickHouse/ClickHouse/pull/17791) ([achimbab](https://github.com/achimbab)). +* Attempt to use IOStream in AWS SDK [#17794](https://github.com/ClickHouse/ClickHouse/pull/17794) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Update StorageReplicatedMergeTree::waitForAllReplicasToProcessLogEntry to support waiting on foreign shards / tables [#17800](https://github.com/ClickHouse/ClickHouse/pull/17800) ([nvartolomei](https://github.com/nvartolomei)). +* Fix flaky test_ttl_move [#17805](https://github.com/ClickHouse/ClickHouse/pull/17805) ([Azat Khuzhin](https://github.com/azat)). +* Merging data type Map [#15806](https://github.com/ClickHouse/ClickHouse/issues/15806) [#17829](https://github.com/ClickHouse/ClickHouse/pull/17829) ([Anton Popov](https://github.com/CurtizJ)). +* Kill network container with retries in integration tests [#17856](https://github.com/ClickHouse/ClickHouse/pull/17856) ([alesapin](https://github.com/alesapin)). +* Merging [#17750](https://github.com/ClickHouse/ClickHouse/issues/17750) [#17874](https://github.com/ClickHouse/ClickHouse/pull/17874) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix bad code [#17878](https://github.com/ClickHouse/ClickHouse/pull/17878) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Retrieve logs from grpc [#17888](https://github.com/ClickHouse/ClickHouse/pull/17888) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixed typo in log message format [#17900](https://github.com/ClickHouse/ClickHouse/pull/17900) ([Alexander Kazakov](https://github.com/Akazz)). +* Perf tests fixes [#17907](https://github.com/ClickHouse/ClickHouse/pull/17907) ([Azat Khuzhin](https://github.com/azat)). +* Better exception message for MaterializeMySQL [#17915](https://github.com/ClickHouse/ClickHouse/pull/17915) ([Winter Zhang](https://github.com/zhang2014)). +* Add additional columns size check for MergeTree in debug mode [#17919](https://github.com/ClickHouse/ClickHouse/pull/17919) ([alesapin](https://github.com/alesapin)). +* Small simplification of MergeTreeDataWriter [#17943](https://github.com/ClickHouse/ClickHouse/pull/17943) ([alesapin](https://github.com/alesapin)). +* Use ryu instead of dragonbox in Arcadia [#17963](https://github.com/ClickHouse/ClickHouse/pull/17963) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Tiny build changes [#17982](https://github.com/ClickHouse/ClickHouse/pull/17982) ([Azat Khuzhin](https://github.com/azat)). +* Fix arcadia [#17984](https://github.com/ClickHouse/ClickHouse/pull/17984) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* More forward declaration for generic headers [#17986](https://github.com/ClickHouse/ClickHouse/pull/17986) ([Azat Khuzhin](https://github.com/azat)). +* Remove some redundant includes to speed up build [#17988](https://github.com/ClickHouse/ClickHouse/pull/17988) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix flappy test_log_family_s3 [#18027](https://github.com/ClickHouse/ClickHouse/pull/18027) ([Pavel Kovalenko](https://github.com/Jokser)). +* encodeXMLComponent: rename files after [#17659](https://github.com/ClickHouse/ClickHouse/issues/17659) [#18033](https://github.com/ClickHouse/ClickHouse/pull/18033) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* HashTable erase added tests [#18047](https://github.com/ClickHouse/ClickHouse/pull/18047) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove obsolete settings [#18054](https://github.com/ClickHouse/ClickHouse/pull/18054) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix 01600_remerge_sort_lowered_memory_bytes_ratio flap [#18057](https://github.com/ClickHouse/ClickHouse/pull/18057) ([Azat Khuzhin](https://github.com/azat)). +* Remove test_keeper_server in perf tests [#18058](https://github.com/ClickHouse/ClickHouse/pull/18058) ([Azat Khuzhin](https://github.com/azat)). +* Add changelog for 20.12 [#18062](https://github.com/ClickHouse/ClickHouse/pull/18062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed unsafe ast rewrite in InterpreterSelectWithUnionQuery [#18064](https://github.com/ClickHouse/ClickHouse/pull/18064) ([Alexander Kazakov](https://github.com/Akazz)). +* Build utils in CI, at least in split build [#18066](https://github.com/ClickHouse/ClickHouse/pull/18066) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Do not pass -fsanitize-blacklist for gcc (it does not support it) for UBSAN [#18081](https://github.com/ClickHouse/ClickHouse/pull/18081) ([Azat Khuzhin](https://github.com/azat)). +* Try to fix Arcadia [#18084](https://github.com/ClickHouse/ClickHouse/pull/18084) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix dependencies for docker stateful_with_coverage [#18105](https://github.com/ClickHouse/ClickHouse/pull/18105) ([alesapin](https://github.com/alesapin)). +* Avoid using symlinks for top_level_domains [#18113](https://github.com/ClickHouse/ClickHouse/pull/18113) ([Azat Khuzhin](https://github.com/azat)). +* gcc10 sanitizers support [#18114](https://github.com/ClickHouse/ClickHouse/pull/18114) ([Azat Khuzhin](https://github.com/azat)). +* Port Kerberos to BoringSSL [#18128](https://github.com/ClickHouse/ClickHouse/pull/18128) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Try fix integration tests. [#18132](https://github.com/ClickHouse/ClickHouse/pull/18132) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Trying to fix 00620_optimize_on_nonleader_test [#18140](https://github.com/ClickHouse/ClickHouse/pull/18140) ([alesapin](https://github.com/alesapin)). +* Suppress error in 00993_system_parts_race_condition_drop_zookeeper [#18148](https://github.com/ClickHouse/ClickHouse/pull/18148) ([alesapin](https://github.com/alesapin)). +* Better exception message for unknown function [#18168](https://github.com/ClickHouse/ClickHouse/pull/18168) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Test watches for in-memory zookeeper [#18191](https://github.com/ClickHouse/ClickHouse/pull/18191) ([alesapin](https://github.com/alesapin)). +* Add support for LTS branches in backport automation [#18195](https://github.com/ClickHouse/ClickHouse/pull/18195) ([Ivan](https://github.com/abyss7)). +* Enable optimize_on_insert for MaterializeMySQL [#18198](https://github.com/ClickHouse/ClickHouse/pull/18198) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add one more case in zk test util [#18199](https://github.com/ClickHouse/ClickHouse/pull/18199) ([alesapin](https://github.com/alesapin)). +* More logs during quorum insert [#18200](https://github.com/ClickHouse/ClickHouse/pull/18200) ([alesapin](https://github.com/alesapin)). +* Update libc headers [#18202](https://github.com/ClickHouse/ClickHouse/pull/18202) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix watches processing in TestKeeper [#18217](https://github.com/ClickHouse/ClickHouse/pull/18217) ([alesapin](https://github.com/alesapin)). +* Add test for fixed bug with skip indices [#18219](https://github.com/ClickHouse/ClickHouse/pull/18219) ([Anton Popov](https://github.com/CurtizJ)). +* [wip] a prototype for window functions [#18222](https://github.com/ClickHouse/ClickHouse/pull/18222) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Return clickhouse-git-import [#18234](https://github.com/ClickHouse/ClickHouse/pull/18234) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* MADV_DONTNEED check in runtime for qemu (via patching jemalloc) [#18238](https://github.com/ClickHouse/ClickHouse/pull/18238) ([Azat Khuzhin](https://github.com/azat)). +* New Year preparations [#18274](https://github.com/ClickHouse/ClickHouse/pull/18274) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add powf and powl to glibc-compatibility [#18279](https://github.com/ClickHouse/ClickHouse/pull/18279) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#18286](https://github.com/ClickHouse/ClickHouse/pull/18286) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky 01076_parallel_alter test [#18293](https://github.com/ClickHouse/ClickHouse/pull/18293) ([alesapin](https://github.com/alesapin)). +* Make single image for building coverage report [#18312](https://github.com/ClickHouse/ClickHouse/pull/18312) ([alesapin](https://github.com/alesapin)). +* Fixed flaky test [#18313](https://github.com/ClickHouse/ClickHouse/pull/18313) ([Vasily Nemkov](https://github.com/Enmk)). +* Perf test for ColumnMap [#18317](https://github.com/ClickHouse/ClickHouse/pull/18317) ([Vasily Nemkov](https://github.com/Enmk)). +* Add more tests to skip-list [#18318](https://github.com/ClickHouse/ClickHouse/pull/18318) ([Ivan](https://github.com/abyss7)). +* Add support for ANTLR inside clickhouse-test [#18319](https://github.com/ClickHouse/ClickHouse/pull/18319) ([Ivan](https://github.com/abyss7)). +* Fix clickhouse-test [#18330](https://github.com/ClickHouse/ClickHouse/pull/18330) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Provide extra constructor for Async metrics [#18331](https://github.com/ClickHouse/ClickHouse/pull/18331) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* tests: remove unused configs/scripts [#18334](https://github.com/ClickHouse/ClickHouse/pull/18334) ([Azat Khuzhin](https://github.com/azat)). +* Fix log message for memory tracking drift [#18335](https://github.com/ClickHouse/ClickHouse/pull/18335) ([Azat Khuzhin](https://github.com/azat)). +* try to pass ninja flags in deb package [#18348](https://github.com/ClickHouse/ClickHouse/pull/18348) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove useless code [#18350](https://github.com/ClickHouse/ClickHouse/pull/18350) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove unused code [#18366](https://github.com/ClickHouse/ClickHouse/pull/18366) ([Anton Popov](https://github.com/CurtizJ)). +* Return back some configs, that are used in Arcadia [#18370](https://github.com/ClickHouse/ClickHouse/pull/18370) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Return clickhouse-test-server for Arcadia needs [#18372](https://github.com/ClickHouse/ClickHouse/pull/18372) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Drop libnsl dependency (because of gRPC -> c-ares) [#18375](https://github.com/ClickHouse/ClickHouse/pull/18375) ([Azat Khuzhin](https://github.com/azat)). +* bump compatibility level to 10 for debian manifests [#18376](https://github.com/ClickHouse/ClickHouse/pull/18376) ([Azat Khuzhin](https://github.com/azat)). +* Do not override RULE_LAUNCH_COMPILE/RULE_LAUNCH_LINK in rocksdb [#18378](https://github.com/ClickHouse/ClickHouse/pull/18378) ([Azat Khuzhin](https://github.com/azat)). +* Make some perf tests faster on slower machines [#18386](https://github.com/ClickHouse/ClickHouse/pull/18386) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix exception text from Pipe.cpp [#18396](https://github.com/ClickHouse/ClickHouse/pull/18396) ([filimonov](https://github.com/filimonov)). +* Add test for already working code [#18405](https://github.com/ClickHouse/ClickHouse/pull/18405) ([alesapin](https://github.com/alesapin)). +* Try fix ya.make [#18409](https://github.com/ClickHouse/ClickHouse/pull/18409) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Use Port::Data instead of Chunk in LazyOutputFormat. [#18411](https://github.com/ClickHouse/ClickHouse/pull/18411) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Do not use watchdog when server is run from tty [#18433](https://github.com/ClickHouse/ClickHouse/pull/18433) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Raise an error if more than one key is provided to ip_dictionary [#18435](https://github.com/ClickHouse/ClickHouse/pull/18435) ([Vladimir C](https://github.com/vdimir)). +* Remove unused code [#18436](https://github.com/ClickHouse/ClickHouse/pull/18436) ([Anton Popov](https://github.com/CurtizJ)). +* Ignore SOURCE_DATE_EPOCH for newer ccache (4+) [#18441](https://github.com/ClickHouse/ClickHouse/pull/18441) ([Azat Khuzhin](https://github.com/azat)). +* Poco build fixes [#18443](https://github.com/ClickHouse/ClickHouse/pull/18443) ([Azat Khuzhin](https://github.com/azat)). +* [wip] some window function fixes [#18455](https://github.com/ClickHouse/ClickHouse/pull/18455) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Merging [#17858](https://github.com/ClickHouse/ClickHouse/issues/17858) [#18475](https://github.com/ClickHouse/ClickHouse/pull/18475) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Allow AppleClang builds [#18488](https://github.com/ClickHouse/ClickHouse/pull/18488) ([Denis Glazachev](https://github.com/traceon)). +* Build job pool tiny fixes [#18489](https://github.com/ClickHouse/ClickHouse/pull/18489) ([Azat Khuzhin](https://github.com/azat)). +* Add NuRaft to contrib [#18491](https://github.com/ClickHouse/ClickHouse/pull/18491) ([alesapin](https://github.com/alesapin)). +* Fix flaky test 01584_distributed_buffer_cannot_find_column [#18493](https://github.com/ClickHouse/ClickHouse/pull/18493) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* More correct error code on incorrect interserver protocol [#18515](https://github.com/ClickHouse/ClickHouse/pull/18515) ([alesapin](https://github.com/alesapin)). +* Merging [#18188](https://github.com/ClickHouse/ClickHouse/issues/18188) [#18516](https://github.com/ClickHouse/ClickHouse/pull/18516) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix usage of concurrent bounded queue in test keeper storage [#18522](https://github.com/ClickHouse/ClickHouse/pull/18522) ([alesapin](https://github.com/alesapin)). +* Fix cast to map from tuple of arrays with unequal sizes [#18523](https://github.com/ClickHouse/ClickHouse/pull/18523) ([Anton Popov](https://github.com/CurtizJ)). +* Sim/Min Hash fixes [#18524](https://github.com/ClickHouse/ClickHouse/pull/18524) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Use RLIMIT_DATA/RLIMIT_AS over RLIMIT_RSS for ENABLE_CHECK_HEAVY_BUILDS [#18537](https://github.com/ClickHouse/ClickHouse/pull/18537) ([Azat Khuzhin](https://github.com/azat)). +* Do not throw logical error from IPAddressDictionary ctor [#18548](https://github.com/ClickHouse/ClickHouse/pull/18548) ([Vladimir C](https://github.com/vdimir)). +* Check for CLICKHOUSE_CLIENT_OPT env before setting it [#18574](https://github.com/ClickHouse/ClickHouse/pull/18574) ([Ivan](https://github.com/abyss7)). +* Minor fixes for min/sim hash [#18595](https://github.com/ClickHouse/ClickHouse/pull/18595) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Change union_default_mode to throw exception [#18615](https://github.com/ClickHouse/ClickHouse/pull/18615) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fixed exit code of watchdog [#18616](https://github.com/ClickHouse/ClickHouse/pull/18616) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Small utils improvements (check-marks and compressor) [#18619](https://github.com/ClickHouse/ClickHouse/pull/18619) ([Azat Khuzhin](https://github.com/azat)). +* Fix too long perf test [#18634](https://github.com/ClickHouse/ClickHouse/pull/18634) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* More correct words about parser [#18646](https://github.com/ClickHouse/ClickHouse/pull/18646) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless support for symbolic port names [#18647](https://github.com/ClickHouse/ClickHouse/pull/18647) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Send fatal logs in all tests [#18648](https://github.com/ClickHouse/ClickHouse/pull/18648) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix AppleClang compilation - Remove auto in function parameters [#18674](https://github.com/ClickHouse/ClickHouse/pull/18674) ([Denis Glazachev](https://github.com/traceon)). +* compressor: remove extra check for seeking of input [#18675](https://github.com/ClickHouse/ClickHouse/pull/18675) ([Azat Khuzhin](https://github.com/azat)). +* Better linker name matcher [#18678](https://github.com/ClickHouse/ClickHouse/pull/18678) ([Amos Bird](https://github.com/amosbird)). +* Remove "harmful" function from mariadbclient [#18682](https://github.com/ClickHouse/ClickHouse/pull/18682) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix performance comparison [#18686](https://github.com/ClickHouse/ClickHouse/pull/18686) ([Azat Khuzhin](https://github.com/azat)). +* Simplify code of function "bar" [#18687](https://github.com/ClickHouse/ClickHouse/pull/18687) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge byteSize function [#18688](https://github.com/ClickHouse/ClickHouse/pull/18688) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow Replicated tables in Arcadia [#18697](https://github.com/ClickHouse/ClickHouse/pull/18697) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Bump replxx to fix loading of multi-line entries from the history [#18700](https://github.com/ClickHouse/ClickHouse/pull/18700) ([Azat Khuzhin](https://github.com/azat)). +* Add a test for already fixed issue [#18702](https://github.com/ClickHouse/ClickHouse/pull/18702) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report: applying non-zero offset to nullptr [#18703](https://github.com/ClickHouse/ClickHouse/pull/18703) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable more tests and see what will happen [#18704](https://github.com/ClickHouse/ClickHouse/pull/18704) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added a test for [#13477](https://github.com/ClickHouse/ClickHouse/issues/13477) [#18708](https://github.com/ClickHouse/ClickHouse/pull/18708) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix fuzz errors in sumMap [#18710](https://github.com/ClickHouse/ClickHouse/pull/18710) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use relaxed for flag in RemoteQueryExecutorReadContext. [#18715](https://github.com/ClickHouse/ClickHouse/pull/18715) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Do not throw from Parser [#18745](https://github.com/ClickHouse/ClickHouse/pull/18745) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove pink screen with confusing questions about Kerberos [#18748](https://github.com/ClickHouse/ClickHouse/pull/18748) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Don't allow conversion between UUID and numeric types [#18749](https://github.com/ClickHouse/ClickHouse/pull/18749) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove bad code in HashJoin [#18750](https://github.com/ClickHouse/ClickHouse/pull/18750) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* More robust stateful test [#18751](https://github.com/ClickHouse/ClickHouse/pull/18751) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test from [#15641](https://github.com/ClickHouse/ClickHouse/issues/15641) [#18753](https://github.com/ClickHouse/ClickHouse/pull/18753) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Mark some TestFlows as flaky [#18757](https://github.com/ClickHouse/ClickHouse/pull/18757) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove some headers [#18758](https://github.com/ClickHouse/ClickHouse/pull/18758) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a regression test for marks corruption [#18763](https://github.com/ClickHouse/ClickHouse/pull/18763) ([Azat Khuzhin](https://github.com/azat)). +* Use sigdescr_np() over sys_siglist (fixes glibc 2.32+ unbundled build) [#18764](https://github.com/ClickHouse/ClickHouse/pull/18764) ([Azat Khuzhin](https://github.com/azat)). +* Do not materialize block for FetchColumns header. [#18768](https://github.com/ClickHouse/ClickHouse/pull/18768) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Query Fuzzer: fix some cultural issues [#18770](https://github.com/ClickHouse/ClickHouse/pull/18770) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* more tests for simple aggregate functions in summingMT aggregatingMT try2 [#18771](https://github.com/ClickHouse/ClickHouse/pull/18771) ([Denny Crane](https://github.com/den-crane)). +* Check if XCODE_IDE is true and avoid enforcing ninja in that case [#18773](https://github.com/ClickHouse/ClickHouse/pull/18773) ([Denis Glazachev](https://github.com/traceon)). +* Respect memory tracker blocker level during deallocations [#18774](https://github.com/ClickHouse/ClickHouse/pull/18774) ([Azat Khuzhin](https://github.com/azat)). +* Do not allow Fuzzer to enable LLVM [#18777](https://github.com/ClickHouse/ClickHouse/pull/18777) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add sanity checks for Sim/Min hash arguments [#18803](https://github.com/ClickHouse/ClickHouse/pull/18803) ([Azat Khuzhin](https://github.com/azat)). +* Respect MINSIGSTKSZ for alternative stack to fix under aarch64 [#18832](https://github.com/ClickHouse/ClickHouse/pull/18832) ([Azat Khuzhin](https://github.com/azat)). +* Do not check bit flips for big buffers (since the size can be corrupted) [#18852](https://github.com/ClickHouse/ClickHouse/pull/18852) ([Azat Khuzhin](https://github.com/azat)). +* Correctly override default settings remotely [#18857](https://github.com/ClickHouse/ClickHouse/pull/18857) ([Amos Bird](https://github.com/amosbird)). +* Import strsignal from Musl [#18858](https://github.com/ClickHouse/ClickHouse/pull/18858) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Storage features improvements [#18870](https://github.com/ClickHouse/ClickHouse/pull/18870) ([Azat Khuzhin](https://github.com/azat)). +* Fix integrity check [#18871](https://github.com/ClickHouse/ClickHouse/pull/18871) ([Azat Khuzhin](https://github.com/azat)). +* Minor fix in backport script [#18873](https://github.com/ClickHouse/ClickHouse/pull/18873) ([Ivan](https://github.com/abyss7)). +* Query Fuzzer: return fail fast semantics [#18880](https://github.com/ClickHouse/ClickHouse/pull/18880) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless headers [#18881](https://github.com/ClickHouse/ClickHouse/pull/18881) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use time_macros over unset SOURCE_DATE_EPOCH in ccache 4.2 (unreleased) [#18885](https://github.com/ClickHouse/ClickHouse/pull/18885) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless code [#18886](https://github.com/ClickHouse/ClickHouse/pull/18886) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove TestFlows due to timeouts [#18888](https://github.com/ClickHouse/ClickHouse/pull/18888) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless headers [#18892](https://github.com/ClickHouse/ClickHouse/pull/18892) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Drop RESTART REPLICAS from stateless tests to avoid locking lots of mutexes [#18897](https://github.com/ClickHouse/ClickHouse/pull/18897) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v21.1.2.15-stable.md b/docs/changelogs/v21.1.2.15-stable.md index 205794b94c2..ac76c79ea4b 100644 --- a/docs/changelogs/v21.1.2.15-stable.md +++ b/docs/changelogs/v21.1.2.15-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.2.15-stable FIXME as compared to v21.1.1.5646-prestable #### Improvement diff --git a/docs/changelogs/v21.1.3.32-stable.md b/docs/changelogs/v21.1.3.32-stable.md index ea4c9fd0fe6..85e7162a509 100644 --- a/docs/changelogs/v21.1.3.32-stable.md +++ b/docs/changelogs/v21.1.3.32-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.3.32-stable FIXME as compared to v21.1.2.15-stable #### Bug Fix @@ -29,3 +36,7 @@ * Backported in [#19938](https://github.com/ClickHouse/ClickHouse/issues/19938): Deadlock was possible if system.text_log is enabled. This fixes [#19874](https://github.com/ClickHouse/ClickHouse/issues/19874). [#19875](https://github.com/ClickHouse/ClickHouse/pull/19875) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#19934](https://github.com/ClickHouse/ClickHouse/issues/19934): BloomFilter index crash fix. Fixes [#19757](https://github.com/ClickHouse/ClickHouse/issues/19757). [#19884](https://github.com/ClickHouse/ClickHouse/pull/19884) ([Maksim Kita](https://github.com/kitaisreal)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Update Dragonbox [#19218](https://github.com/ClickHouse/ClickHouse/pull/19218) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.1.4.46-stable.md b/docs/changelogs/v21.1.4.46-stable.md index 3033c5edd21..cc9010880e4 100644 --- a/docs/changelogs/v21.1.4.46-stable.md +++ b/docs/changelogs/v21.1.4.46-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.4.46-stable FIXME as compared to v21.1.3.32-stable #### Bug Fix @@ -20,3 +27,8 @@ * NO CL ENTRY: 'Revert "Backport [#20224](https://github.com/ClickHouse/ClickHouse/issues/20224) to 21.1: Fix access control manager destruction order"'. [#20395](https://github.com/ClickHouse/ClickHouse/pull/20395) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Use fixed version of confluent-kafka library in integration tests [#20124](https://github.com/ClickHouse/ClickHouse/pull/20124) ([alesapin](https://github.com/alesapin)). +* Handle syntax error for ARRAY JOIN with no args [#20223](https://github.com/ClickHouse/ClickHouse/pull/20223) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.1.5.4-stable.md b/docs/changelogs/v21.1.5.4-stable.md index c67724a4512..277814b74be 100644 --- a/docs/changelogs/v21.1.5.4-stable.md +++ b/docs/changelogs/v21.1.5.4-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.5.4-stable FIXME as compared to v21.1.4.46-stable #### Bug Fix diff --git a/docs/changelogs/v21.1.6.13-stable.md b/docs/changelogs/v21.1.6.13-stable.md index 547cd38a06f..5308dc43dea 100644 --- a/docs/changelogs/v21.1.6.13-stable.md +++ b/docs/changelogs/v21.1.6.13-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.6.13-stable FIXME as compared to v21.1.5.4-stable #### Bug Fix diff --git a/docs/changelogs/v21.1.7.1-stable.md b/docs/changelogs/v21.1.7.1-stable.md index 371efb8d5df..0e66e1bc623 100644 --- a/docs/changelogs/v21.1.7.1-stable.md +++ b/docs/changelogs/v21.1.7.1-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.7.1-stable FIXME as compared to v21.1.6.13-stable #### Bug Fix diff --git a/docs/changelogs/v21.1.8.30-stable.md b/docs/changelogs/v21.1.8.30-stable.md index 0859cc8ccbd..f9776ce4810 100644 --- a/docs/changelogs/v21.1.8.30-stable.md +++ b/docs/changelogs/v21.1.8.30-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.8.30-stable FIXME as compared to v21.1.7.1-stable #### Bug Fix @@ -27,3 +34,8 @@ * Backported in [#22700](https://github.com/ClickHouse/ClickHouse/issues/22700): Fix wait for mutations on several replicas for ReplicatedMergeTree table engines. Previously, mutation/alter query may finish before mutation actually executed on other replicas. [#22669](https://github.com/ClickHouse/ClickHouse/pull/22669) ([alesapin](https://github.com/alesapin)). * Backported in [#22739](https://github.com/ClickHouse/ClickHouse/issues/22739): Fix possible hangs in zk requests in case of OOM exception. Fixes [#22438](https://github.com/ClickHouse/ClickHouse/issues/22438). [#22684](https://github.com/ClickHouse/ClickHouse/pull/22684) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* LRUCache fix exception unsafe element insertion [#21891](https://github.com/ClickHouse/ClickHouse/pull/21891) ([Maksim Kita](https://github.com/kitaisreal)). +* Mannual backport of [#21429](https://github.com/ClickHouse/ClickHouse/issues/21429) in 21.1 [#22506](https://github.com/ClickHouse/ClickHouse/pull/22506) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.1.9.41-stable.md b/docs/changelogs/v21.1.9.41-stable.md index 2a94073b810..bef4385e2cb 100644 --- a/docs/changelogs/v21.1.9.41-stable.md +++ b/docs/changelogs/v21.1.9.41-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.1.9.41-stable FIXME as compared to v21.1.8.30-stable #### Improvement @@ -15,3 +22,8 @@ #### Build/Testing/Packaging Improvement * Backported in [#22813](https://github.com/ClickHouse/ClickHouse/issues/22813): Allow to start up with modified binary under gdb. In previous version if you set up breakpoint in gdb before start, server will refuse to start up due to failed integrity check. [#21258](https://github.com/ClickHouse/ClickHouse/pull/21258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Check for EINTR in epoll_wait [#20958](https://github.com/ClickHouse/ClickHouse/pull/20958) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* FileDictionarySource fix absolute file path [#22822](https://github.com/ClickHouse/ClickHouse/pull/22822) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.10.1.8013-prestable.md b/docs/changelogs/v21.10.1.8013-prestable.md index d3e06c056cf..506f3642287 100644 --- a/docs/changelogs/v21.10.1.8013-prestable.md +++ b/docs/changelogs/v21.10.1.8013-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.1.8013-prestable FIXME as compared to v21.9.1.7770-prestable #### Backward Incompatible Change @@ -136,3 +143,126 @@ * NO CL ENTRY: 'Revert "Add test for [#13398](https://github.com/ClickHouse/ClickHouse/issues/13398)"'. [#28274](https://github.com/ClickHouse/ClickHouse/pull/28274) ([Alexander Tokmakov](https://github.com/tavplubix)). * NO CL ENTRY: 'fix minor typo'. [#28629](https://github.com/ClickHouse/ClickHouse/pull/28629) ([flynn](https://github.com/ucasfl)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Experiment with asynchronous readers [#26791](https://github.com/ClickHouse/ClickHouse/pull/26791) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Introduce sessions [#26864](https://github.com/ClickHouse/ClickHouse/pull/26864) ([Vitaly Baranov](https://github.com/vitlibar)). +* FILTER clause for aggregate functions [#27036](https://github.com/ClickHouse/ClickHouse/pull/27036) ([Nikita Taranov](https://github.com/nickitat)). +* Add test for parsing maps with integer keys [#27146](https://github.com/ClickHouse/ClickHouse/pull/27146) ([Anton Popov](https://github.com/CurtizJ)). +* S3 disk unstable reads test [#27176](https://github.com/ClickHouse/ClickHouse/pull/27176) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Refactor NotJoined [#27299](https://github.com/ClickHouse/ClickHouse/pull/27299) ([Vladimir C](https://github.com/vdimir)). +* Accept error code by error name in client test hints [#27430](https://github.com/ClickHouse/ClickHouse/pull/27430) ([Azat Khuzhin](https://github.com/azat)). +* Break some tests [#27529](https://github.com/ClickHouse/ClickHouse/pull/27529) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove the remains of ANTLR in the tests [#27637](https://github.com/ClickHouse/ClickHouse/pull/27637) ([Raúl Marín](https://github.com/Algunenano)). +* Disable memory tracking for roaring bitmaps on Mac OS [#27681](https://github.com/ClickHouse/ClickHouse/pull/27681) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use only SSE2 in "unbundled" build [#27683](https://github.com/ClickHouse/ClickHouse/pull/27683) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove trash [#27685](https://github.com/ClickHouse/ClickHouse/pull/27685) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix stress test in `~CompressedWriteBuffer` [#27686](https://github.com/ClickHouse/ClickHouse/pull/27686) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Mark tests for `DatabaseReplicated` as green [#27688](https://github.com/ClickHouse/ClickHouse/pull/27688) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Removed DenseHashMap, DenseHashSet [#27690](https://github.com/ClickHouse/ClickHouse/pull/27690) ([Maksim Kita](https://github.com/kitaisreal)). +* Map data type parsing tests [#27692](https://github.com/ClickHouse/ClickHouse/pull/27692) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactor arrayJoin check on partition expressions [#27733](https://github.com/ClickHouse/ClickHouse/pull/27733) ([Raúl Marín](https://github.com/Algunenano)). +* Fix test 01014_lazy_database_concurrent_recreate_reattach_and_show_tables [#27734](https://github.com/ClickHouse/ClickHouse/pull/27734) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Better code around decompression [2] [#27743](https://github.com/ClickHouse/ClickHouse/pull/27743) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Disable jemalloc under OSX [#27751](https://github.com/ClickHouse/ClickHouse/pull/27751) ([Raúl Marín](https://github.com/Algunenano)). +* try to collect some core dumps in perf tests [#27752](https://github.com/ClickHouse/ClickHouse/pull/27752) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix jemalloc under osx (zone_register() had been optimized out again) [#27753](https://github.com/ClickHouse/ClickHouse/pull/27753) ([Azat Khuzhin](https://github.com/azat)). +* Merging [#20089](https://github.com/ClickHouse/ClickHouse/issues/20089) [#27755](https://github.com/ClickHouse/ClickHouse/pull/27755) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix intersect/except with limit [#27757](https://github.com/ClickHouse/ClickHouse/pull/27757) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add HTTP string parsing test [#27762](https://github.com/ClickHouse/ClickHouse/pull/27762) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix some tests [#27785](https://github.com/ClickHouse/ClickHouse/pull/27785) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set function divide as suitable for short-circuit in case of Nullable(Decimal) [#27788](https://github.com/ClickHouse/ClickHouse/pull/27788) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove unnecessary files [#27789](https://github.com/ClickHouse/ClickHouse/pull/27789) ([Kruglov Pavel](https://github.com/Avogar)). +* Revert "Mark tests for `DatabaseReplicated` as green" [#27791](https://github.com/ClickHouse/ClickHouse/pull/27791) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove hardening for watches in DDLWorker [#27792](https://github.com/ClickHouse/ClickHouse/pull/27792) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Stateless test: Cleanup leftovers [#27793](https://github.com/ClickHouse/ClickHouse/pull/27793) ([Raúl Marín](https://github.com/Algunenano)). +* Dictionaries key types refactoring [#27795](https://github.com/ClickHouse/ClickHouse/pull/27795) ([Maksim Kita](https://github.com/kitaisreal)). +* Update 01822_short_circuit.reference (after merging [#27680](https://github.com/ClickHouse/ClickHouse/issues/27680)) [#27802](https://github.com/ClickHouse/ClickHouse/pull/27802) ([Azat Khuzhin](https://github.com/azat)). +* Proper shutdown global context [#27804](https://github.com/ClickHouse/ClickHouse/pull/27804) ([Amos Bird](https://github.com/amosbird)). +* 01766_todatetime64_no_timezone_arg: Use a date without timezone changes [#27810](https://github.com/ClickHouse/ClickHouse/pull/27810) ([Raúl Marín](https://github.com/Algunenano)). +* Use sessions more [#27817](https://github.com/ClickHouse/ClickHouse/pull/27817) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add test for clickhouse-keeper start after conversion [#27818](https://github.com/ClickHouse/ClickHouse/pull/27818) ([alesapin](https://github.com/alesapin)). +* Fix setting name "allow_experimental_database_materialized_postgresql" in the error message [#27824](https://github.com/ClickHouse/ClickHouse/pull/27824) ([Denny Crane](https://github.com/den-crane)). +* Fix bug in short-circuit found by fuzzer [#27826](https://github.com/ClickHouse/ClickHouse/pull/27826) ([Kruglov Pavel](https://github.com/Avogar)). +* Add more checks for LC in native protocol. [#27827](https://github.com/ClickHouse/ClickHouse/pull/27827) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix test 00443_preferred_block_size_bytes.sh [#27846](https://github.com/ClickHouse/ClickHouse/pull/27846) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Mute some integration tests until failures are fixed [#27862](https://github.com/ClickHouse/ClickHouse/pull/27862) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix bad cast in insertPostgreSQLValue [#27869](https://github.com/ClickHouse/ClickHouse/pull/27869) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add fuzzers for codecs [#27872](https://github.com/ClickHouse/ClickHouse/pull/27872) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove tmp folders from tests [#27878](https://github.com/ClickHouse/ClickHouse/pull/27878) ([Kruglov Pavel](https://github.com/Avogar)). +* blog article about perf tests [#27879](https://github.com/ClickHouse/ClickHouse/pull/27879) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* make the sql-standard window functions case insensitive [#27880](https://github.com/ClickHouse/ClickHouse/pull/27880) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Bump librdkafka (fixes use of an invalid/destroyed mutex) [#27883](https://github.com/ClickHouse/ClickHouse/pull/27883) ([Azat Khuzhin](https://github.com/azat)). +* fix decimal formatting settings in perf test [#27884](https://github.com/ClickHouse/ClickHouse/pull/27884) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add separate constants for interfaces LOCAL and TCP_INTERSERVER. [#27886](https://github.com/ClickHouse/ClickHouse/pull/27886) ([Vitaly Baranov](https://github.com/vitlibar)). +* Build fuzzers with clang-tidy [#27895](https://github.com/ClickHouse/ClickHouse/pull/27895) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Allow implicit cast bool to Field [#27921](https://github.com/ClickHouse/ClickHouse/pull/27921) ([Vitaly Baranov](https://github.com/vitlibar)). +* Improve server logs checking in integration tests [#27934](https://github.com/ClickHouse/ClickHouse/pull/27934) ([Azat Khuzhin](https://github.com/azat)). +* Get rid of mutable value in FunctionGetSetting. [#27982](https://github.com/ClickHouse/ClickHouse/pull/27982) ([Vitaly Baranov](https://github.com/vitlibar)). +* Disable fuzzers build with clang-tidy [#27985](https://github.com/ClickHouse/ClickHouse/pull/27985) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Build fuzzers in CI [#27990](https://github.com/ClickHouse/ClickHouse/pull/27990) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix file progress for local [#27991](https://github.com/ClickHouse/ClickHouse/pull/27991) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update libunwind [#27993](https://github.com/ClickHouse/ClickHouse/pull/27993) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix style typos [#28020](https://github.com/ClickHouse/ClickHouse/pull/28020) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Bump replxx [#28039](https://github.com/ClickHouse/ClickHouse/pull/28039) ([Azat Khuzhin](https://github.com/azat)). +* Add test for [#13398](https://github.com/ClickHouse/ClickHouse/issues/13398) [#28054](https://github.com/ClickHouse/ClickHouse/pull/28054) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix zookeeper secure client test [#28066](https://github.com/ClickHouse/ClickHouse/pull/28066) ([alesapin](https://github.com/alesapin)). +* Fix typo in docs [#28077](https://github.com/ClickHouse/ClickHouse/pull/28077) ([Kruglov Pavel](https://github.com/Avogar)). +* Fixed a typo in comments to `SinkToStorage` [#28078](https://github.com/ClickHouse/ClickHouse/pull/28078) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Use jinja template tests in fuzzer [#28079](https://github.com/ClickHouse/ClickHouse/pull/28079) ([Vladimir C](https://github.com/vdimir)). +* Clickhouse-keeper: renames and comments [#28080](https://github.com/ClickHouse/ClickHouse/pull/28080) ([alesapin](https://github.com/alesapin)). +* Update nanodbc [#28084](https://github.com/ClickHouse/ClickHouse/pull/28084) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Improve 01730_distributed_group_by_no_merge_order_by_long [#28123](https://github.com/ClickHouse/ClickHouse/pull/28123) ([Azat Khuzhin](https://github.com/azat)). +* Get rid of useless projection columns during merge [#28135](https://github.com/ClickHouse/ClickHouse/pull/28135) ([Amos Bird](https://github.com/amosbird)). +* Fix style [#28140](https://github.com/ClickHouse/ClickHouse/pull/28140) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Fix clickhouse keeper jepsen tests [#28143](https://github.com/ClickHouse/ClickHouse/pull/28143) ([alesapin](https://github.com/alesapin)). +* Updated ya.make files [#28157](https://github.com/ClickHouse/ClickHouse/pull/28157) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Better detection of the default interface in replicated fetches tests [#28184](https://github.com/ClickHouse/ClickHouse/pull/28184) ([alesapin](https://github.com/alesapin)). +* Reserve protocol number for ALTER PRIMARY KEY. [#28193](https://github.com/ClickHouse/ClickHouse/pull/28193) ([Amos Bird](https://github.com/amosbird)). +* Maybe fix livelock in ZooKeeper client [#28195](https://github.com/ClickHouse/ClickHouse/pull/28195) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Guard UDF container with a lock [#28211](https://github.com/ClickHouse/ClickHouse/pull/28211) ([Azat Khuzhin](https://github.com/azat)). +* Fix error codes [#28234](https://github.com/ClickHouse/ClickHouse/pull/28234) ([Kseniia Sumarokova](https://github.com/kssenii)). +* CHJIT custom memory manager [#28236](https://github.com/ClickHouse/ClickHouse/pull/28236) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionaries small fixes [#28249](https://github.com/ClickHouse/ClickHouse/pull/28249) ([Maksim Kita](https://github.com/kitaisreal)). +* Better nullable primary key implementation [#28269](https://github.com/ClickHouse/ClickHouse/pull/28269) ([Amos Bird](https://github.com/amosbird)). +* ODBC connection holder fix dangling reference [#28298](https://github.com/ClickHouse/ClickHouse/pull/28298) ([Maksim Kita](https://github.com/kitaisreal)). +* test/stress: fix patterns for filtering out Raft messages [#28303](https://github.com/ClickHouse/ClickHouse/pull/28303) ([Azat Khuzhin](https://github.com/azat)). +* Rename system.views columns [#28319](https://github.com/ClickHouse/ClickHouse/pull/28319) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Try to fix arcadia build (generate ya.make) [#28326](https://github.com/ClickHouse/ClickHouse/pull/28326) ([DimasKovas](https://github.com/DimasKovas)). +* Try to fix arcadia build [#28333](https://github.com/ClickHouse/ClickHouse/pull/28333) ([DimasKovas](https://github.com/DimasKovas)). +* Fix test_storage_s3/test_put_get_with_globs (cleanup after test) [#28336](https://github.com/ClickHouse/ClickHouse/pull/28336) ([ianton-ru](https://github.com/ianton-ru)). +* Fix sed argument in test/fuzzer/run-fuzzer.sh [#28350](https://github.com/ClickHouse/ClickHouse/pull/28350) ([Vladimir C](https://github.com/vdimir)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Rework kafka topic creation. [#28354](https://github.com/ClickHouse/ClickHouse/pull/28354) ([Ilya Yatsishin](https://github.com/qoega)). +* Add compat between SinkToStorage and BlockOutputStream [#28361](https://github.com/ClickHouse/ClickHouse/pull/28361) ([DimasKovas](https://github.com/DimasKovas)). +* Try to fix arcadia build (generate ya.make) [#28382](https://github.com/ClickHouse/ClickHouse/pull/28382) ([DimasKovas](https://github.com/DimasKovas)). +* Add a test for a friend [#28396](https://github.com/ClickHouse/ClickHouse/pull/28396) ([alesapin](https://github.com/alesapin)). +* Update ya.make [#28403](https://github.com/ClickHouse/ClickHouse/pull/28403) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix disk with static files a little [#28411](https://github.com/ClickHouse/ClickHouse/pull/28411) ([Kseniia Sumarokova](https://github.com/kssenii)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27980](https://github.com/ClickHouse/ClickHouse/issues/27980) [#28413](https://github.com/ClickHouse/ClickHouse/pull/28413) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 01457_create_as_table_function_structure [#28428](https://github.com/ClickHouse/ClickHouse/pull/28428) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove some rename tests [#28437](https://github.com/ClickHouse/ClickHouse/pull/28437) ([alesapin](https://github.com/alesapin)). +* Fix race in UDF (follow up) [#28438](https://github.com/ClickHouse/ClickHouse/pull/28438) ([Azat Khuzhin](https://github.com/azat)). +* Executable multiple pipes added test [#28503](https://github.com/ClickHouse/ClickHouse/pull/28503) ([Maksim Kita](https://github.com/kitaisreal)). +* UserDefinedFunctionFactory added comments [#28516](https://github.com/ClickHouse/ClickHouse/pull/28516) ([Maksim Kita](https://github.com/kitaisreal)). +* BorrowedObjectPool fix style [#28523](https://github.com/ClickHouse/ClickHouse/pull/28523) ([Maksim Kita](https://github.com/kitaisreal)). +* Add test for keeper 2 node configuration [#28526](https://github.com/ClickHouse/ClickHouse/pull/28526) ([alesapin](https://github.com/alesapin)). +* Function dictGet default implementation for nulls [#28530](https://github.com/ClickHouse/ClickHouse/pull/28530) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix race in zlib [#28534](https://github.com/ClickHouse/ClickHouse/pull/28534) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Bump poco to remove getpid() calls [#28537](https://github.com/ClickHouse/ClickHouse/pull/28537) ([Azat Khuzhin](https://github.com/azat)). +* Fix broken kafka test [#28542](https://github.com/ClickHouse/ClickHouse/pull/28542) ([alesapin](https://github.com/alesapin)). +* Fix format names in docs [#28557](https://github.com/ClickHouse/ClickHouse/pull/28557) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix wrong header of minmax_count projection [#28560](https://github.com/ClickHouse/ClickHouse/pull/28560) ([Amos Bird](https://github.com/amosbird)). +* remove recursion in ZstdInflatingReadBuffer [#28561](https://github.com/ClickHouse/ClickHouse/pull/28561) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Improve [C|T]SV errors [#28579](https://github.com/ClickHouse/ClickHouse/pull/28579) ([Raúl Marín](https://github.com/Algunenano)). +* Function dictGet small fix [#28615](https://github.com/ClickHouse/ClickHouse/pull/28615) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix arcadia build [#28640](https://github.com/ClickHouse/ClickHouse/pull/28640) ([DimasKovas](https://github.com/DimasKovas)). +* Add missed log level into TextLog [#28648](https://github.com/ClickHouse/ClickHouse/pull/28648) ([alesapin](https://github.com/alesapin)). +* Revert [#28082](https://github.com/ClickHouse/ClickHouse/issues/28082) [#28665](https://github.com/ClickHouse/ClickHouse/pull/28665) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Revert [#28397](https://github.com/ClickHouse/ClickHouse/issues/28397) [#28667](https://github.com/ClickHouse/ClickHouse/pull/28667) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.10.2.15-stable.md b/docs/changelogs/v21.10.2.15-stable.md index 05e278f03ae..d23ff95307c 100644 --- a/docs/changelogs/v21.10.2.15-stable.md +++ b/docs/changelogs/v21.10.2.15-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.2.15-stable FIXME as compared to v21.10.1.8013-prestable #### Improvement @@ -63,3 +70,17 @@ * Avoid deadlocks when reading and writting on JOIN Engine tables at the same time. [#30182](https://github.com/ClickHouse/ClickHouse/pull/30182) ([Raúl Marín](https://github.com/Algunenano)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix usage of nested columns with non-array columns with the same prefix [2] [#28762](https://github.com/ClickHouse/ClickHouse/pull/28762) ([Anton Popov](https://github.com/CurtizJ)). +* Lower compiled_expression_cache_size to 128MB [#28816](https://github.com/ClickHouse/ClickHouse/pull/28816) ([Maksim Kita](https://github.com/kitaisreal)). +* Column default dictGet identifier fix [#28863](https://github.com/ClickHouse/ClickHouse/pull/28863) ([Maksim Kita](https://github.com/kitaisreal)). +* Don not add const group by key for query with only having. [#28975](https://github.com/ClickHouse/ClickHouse/pull/28975) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27963](https://github.com/ClickHouse/ClickHouse/issues/27963) [#29063](https://github.com/ClickHouse/ClickHouse/pull/29063) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix terminate on uncaught exception [#29216](https://github.com/ClickHouse/ClickHouse/pull/29216) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix arcadia (pre)stable 21.10 build [#29250](https://github.com/ClickHouse/ClickHouse/pull/29250) ([DimasKovas](https://github.com/DimasKovas)). +* May be fix s3 tests [#29762](https://github.com/ClickHouse/ClickHouse/pull/29762) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove unnecessary PEERDIR to libcxx-filesystem [#29798](https://github.com/ClickHouse/ClickHouse/pull/29798) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Update BoringSSL [#29998](https://github.com/ClickHouse/ClickHouse/pull/29998) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* SQL user defined functions fix alias [#30075](https://github.com/ClickHouse/ClickHouse/pull/30075) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.10.3.9-stable.md b/docs/changelogs/v21.10.3.9-stable.md index 78240367d55..8cccd01e77a 100644 --- a/docs/changelogs/v21.10.3.9-stable.md +++ b/docs/changelogs/v21.10.3.9-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.3.9-stable FIXME as compared to v21.10.2.15-stable #### New Feature @@ -43,3 +50,13 @@ * Backported in [#31255](https://github.com/ClickHouse/ClickHouse/issues/31255): Fix bug in Keeper which can lead to inability to start when some coordination logs was lost and we have more fresh snapshot than our latest log. [#31150](https://github.com/ClickHouse/ClickHouse/pull/31150) ([alesapin](https://github.com/alesapin)). * Backported in [#31436](https://github.com/ClickHouse/ClickHouse/issues/31436): Fix bug with group by and positional arguments. Closes [#31280](https://github.com/ClickHouse/ClickHouse/issues/31280)#issuecomment-968696186. [#31420](https://github.com/ClickHouse/ClickHouse/pull/31420) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* AddDefaultDatabaseVisitor support dictGet [#29650](https://github.com/ClickHouse/ClickHouse/pull/29650) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageExecutable fix small issues [#30352](https://github.com/ClickHouse/ClickHouse/pull/30352) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix ca-bundle.crt in kerberized_hadoop/Dockerfile [#30358](https://github.com/ClickHouse/ClickHouse/pull/30358) ([Vladimir C](https://github.com/vdimir)). +* SQLUserDefinedFunctions composition fix [#30483](https://github.com/ClickHouse/ClickHouse/pull/30483) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageDictionary fix potential configuration race [#30502](https://github.com/ClickHouse/ClickHouse/pull/30502) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix typo in USE_MYSQL check [#31226](https://github.com/ClickHouse/ClickHouse/pull/31226) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.10.4.26-stable.md b/docs/changelogs/v21.10.4.26-stable.md index b2089fad0c4..06eab149e6c 100644 --- a/docs/changelogs/v21.10.4.26-stable.md +++ b/docs/changelogs/v21.10.4.26-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.4.26-stable FIXME as compared to v21.10.3.9-stable #### Performance Improvement @@ -23,3 +30,7 @@ * Backported in [#31939](https://github.com/ClickHouse/ClickHouse/issues/31939): - Change configuration path from `keeper_server.session_timeout_ms` to `keeper_server.coordination_settings.session_timeout_ms` when constructing a `KeeperTCPHandler` - Same with `operation_timeout`. [#31859](https://github.com/ClickHouse/ClickHouse/pull/31859) ([JackyWoo](https://github.com/JackyWoo)). * Backported in [#31909](https://github.com/ClickHouse/ClickHouse/issues/31909): Fix functions `empty` and `notEmpty` with arguments of `UUID` type. Fixes [#31819](https://github.com/ClickHouse/ClickHouse/issues/31819). [#31883](https://github.com/ClickHouse/ClickHouse/pull/31883) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.10.5.3-stable.md b/docs/changelogs/v21.10.5.3-stable.md index a591c4a07a8..7a87b349324 100644 --- a/docs/changelogs/v21.10.5.3-stable.md +++ b/docs/changelogs/v21.10.5.3-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.5.3-stable FIXME as compared to v21.10.4.26-stable #### Bug Fix diff --git a/docs/changelogs/v21.10.6.2-stable.md b/docs/changelogs/v21.10.6.2-stable.md index da146ee364d..c9afcfae75e 100644 --- a/docs/changelogs/v21.10.6.2-stable.md +++ b/docs/changelogs/v21.10.6.2-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.10.6.2-stable FIXME as compared to v21.10.5.3-stable #### Bug Fix @@ -20,3 +27,13 @@ * Backported in [#32657](https://github.com/ClickHouse/ClickHouse/issues/32657): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in case of MATERIALIZE COLUMN with no default expression. [#32464](https://github.com/ClickHouse/ClickHouse/pull/32464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.11.1.8636-prestable.md b/docs/changelogs/v21.11.1.8636-prestable.md index 2aab0293223..95c982a2a2e 100644 --- a/docs/changelogs/v21.11.1.8636-prestable.md +++ b/docs/changelogs/v21.11.1.8636-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.1.8636-prestable FIXME as compared to v21.10.1.8013-prestable #### Backward Incompatible Change @@ -13,7 +20,7 @@ * Users now can set comments to database in `CREATE DATABASE` statement ... [#29429](https://github.com/ClickHouse/ClickHouse/pull/29429) ([Vasily Nemkov](https://github.com/Enmk)). * New function` mapContainsKeyLike` to get the map that key matches a simple regular expression. [#29471](https://github.com/ClickHouse/ClickHouse/pull/29471) ([凌涛](https://github.com/lingtaolf)). * Huawei OBS Storage support. Closes [#24294](https://github.com/ClickHouse/ClickHouse/issues/24294). [#29511](https://github.com/ClickHouse/ClickHouse/pull/29511) ([kevin wan](https://github.com/MaxWk)). -* ClickHouse HTTP Server can enable HSTS by set `hsts_max_age` in config.xml with a positive number. [#29516](https://github.com/ClickHouse/ClickHouse/pull/29516) ([凌涛](https://github.com/lingtaolf)). +* Clickhouse HTTP Server can enable HSTS by set `hsts_max_age` in config.xml with a positive number. [#29516](https://github.com/ClickHouse/ClickHouse/pull/29516) ([凌涛](https://github.com/lingtaolf)). * - Added MD4 and SHA384 functions. [#29602](https://github.com/ClickHouse/ClickHouse/pull/29602) ([Nikita Tikhomirov](https://github.com/NSTikhomirov)). * Support EXISTS(subquery). Closes [#6852](https://github.com/ClickHouse/ClickHouse/issues/6852). [#29731](https://github.com/ClickHouse/ClickHouse/pull/29731) ([Kseniia Sumarokova](https://github.com/kssenii)). * Added function `ngram`. Closes [#29699](https://github.com/ClickHouse/ClickHouse/issues/29699). [#29738](https://github.com/ClickHouse/ClickHouse/pull/29738) ([Maksim Kita](https://github.com/kitaisreal)). @@ -190,6 +197,279 @@ * NO CL ENTRY: 'Revert "Revert "Improve usability of `remote_url_allow_hosts`""'. [#30708](https://github.com/ClickHouse/ClickHouse/pull/30708) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * NO CL ENTRY: 'remove some unneeded header files'. [#30722](https://github.com/ClickHouse/ClickHouse/pull/30722) ([flynn](https://github.com/ucasfl)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix race in StorageSystemRocksDB [#29289](https://github.com/ClickHouse/ClickHouse/pull/29289) ([Vladimir C](https://github.com/vdimir)). +* Fixed logging level for message in `S3Common.cpp` [#29308](https://github.com/ClickHouse/ClickHouse/pull/29308) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Revert parse number binary literal [#29309](https://github.com/ClickHouse/ClickHouse/pull/29309) ([Maksim Kita](https://github.com/kitaisreal)). +* Parser number binary literal update [#29310](https://github.com/ClickHouse/ClickHouse/pull/29310) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix assertion in DDLDependencyVisitor [#29323](https://github.com/ClickHouse/ClickHouse/pull/29323) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Function h3GetFaces optimization [#29335](https://github.com/ClickHouse/ClickHouse/pull/29335) ([Maksim Kita](https://github.com/kitaisreal)). +* Less sleeps in integration tests. [#29338](https://github.com/ClickHouse/ClickHouse/pull/29338) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix trivial mistake in DiskWebServer [#29340](https://github.com/ClickHouse/ClickHouse/pull/29340) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix integration test for RocksDB [#29341](https://github.com/ClickHouse/ClickHouse/pull/29341) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix typo in comment [#29342](https://github.com/ClickHouse/ClickHouse/pull/29342) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add lld into Build-Depends for debian package [#29343](https://github.com/ClickHouse/ClickHouse/pull/29343) ([Azat Khuzhin](https://github.com/azat)). +* Apply a patch from Azat [#29344](https://github.com/ClickHouse/ClickHouse/pull/29344) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update libhdfs3 [#29345](https://github.com/ClickHouse/ClickHouse/pull/29345) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove ya.make [#29346](https://github.com/ClickHouse/ClickHouse/pull/29346) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix assert in table function `merge` with database regexp [#29355](https://github.com/ClickHouse/ClickHouse/pull/29355) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless files related to pytest [#29361](https://github.com/ClickHouse/ClickHouse/pull/29361) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove obsolete non-automated tests [#29362](https://github.com/ClickHouse/ClickHouse/pull/29362) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix integration test [#29365](https://github.com/ClickHouse/ClickHouse/pull/29365) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix H3 function name (h3ResIsClassIII to h3IsResClassIII) [#29368](https://github.com/ClickHouse/ClickHouse/pull/29368) ([Bharat Nallan](https://github.com/bharatnc)). +* RFC: tests: purge unused configs [#29375](https://github.com/ClickHouse/ClickHouse/pull/29375) ([Azat Khuzhin](https://github.com/azat)). +* Fix ugly typo [#29379](https://github.com/ClickHouse/ClickHouse/pull/29379) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix comments in AggregateFunctionFactory [#29380](https://github.com/ClickHouse/ClickHouse/pull/29380) ([Bharat Nallan](https://github.com/bharatnc)). +* Add GraphCDN in adopter list [#29387](https://github.com/ClickHouse/ClickHouse/pull/29387) ([Mohamad Fadhil](https://github.com/sdil)). +* Fix 02015_async_inserts_2 flakiness [#29390](https://github.com/ClickHouse/ClickHouse/pull/29390) ([Azat Khuzhin](https://github.com/azat)). +* test for [#23634](https://github.com/ClickHouse/ClickHouse/issues/23634) ( nullable PK and negate cond ) [#29392](https://github.com/ClickHouse/ClickHouse/pull/29392) ([Denny Crane](https://github.com/den-crane)). +* Merge EmbeddedRocksDBBlockInputStream and EmbeddedRocksDBSource [#29428](https://github.com/ClickHouse/ClickHouse/pull/29428) ([Vladimir C](https://github.com/vdimir)). +* Map bloom filter index mapValues equals function support [#29431](https://github.com/ClickHouse/ClickHouse/pull/29431) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionary Array nested Map added test [#29432](https://github.com/ClickHouse/ClickHouse/pull/29432) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionary view different database added test [#29443](https://github.com/ClickHouse/ClickHouse/pull/29443) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix deadlock in concurrent async inserts and truncates [#29444](https://github.com/ClickHouse/ClickHouse/pull/29444) ([Anton Popov](https://github.com/CurtizJ)). +* Fix flaky test 01158_zookeeper_log_long [#29445](https://github.com/ClickHouse/ClickHouse/pull/29445) ([Alexander Tokmakov](https://github.com/tavplubix)). +* clickhouse-test: fix long tag check for flaky check (--test-runs > 1) [#29449](https://github.com/ClickHouse/ClickHouse/pull/29449) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test: fix shared list object (by fixing manager lifetime) [#29452](https://github.com/ClickHouse/ClickHouse/pull/29452) ([Azat Khuzhin](https://github.com/azat)). +* Enable memory profiler on CI [#29453](https://github.com/ClickHouse/ClickHouse/pull/29453) ([Azat Khuzhin](https://github.com/azat)). +* test for [#23634](https://github.com/ClickHouse/ClickHouse/issues/23634) added tests for tuples [#29460](https://github.com/ClickHouse/ClickHouse/pull/29460) ([Denny Crane](https://github.com/den-crane)). +* Fix 2024_merge_regexp_assert [#29461](https://github.com/ClickHouse/ClickHouse/pull/29461) ([Azat Khuzhin](https://github.com/azat)). +* Add std::cerr/std::cout style check [#29464](https://github.com/ClickHouse/ClickHouse/pull/29464) ([Azat Khuzhin](https://github.com/azat)). +* [github] we're switching to CLA based on Apache CLA [#29466](https://github.com/ClickHouse/ClickHouse/pull/29466) ([ClickHouse Admin](https://github.com/clickhouse-admin)). +* Bloom filter indexes updated tests [#29474](https://github.com/ClickHouse/ClickHouse/pull/29474) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove more data streams [#29491](https://github.com/ClickHouse/ClickHouse/pull/29491) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible hang in PushingAsyncPipelineExecutor. [#29494](https://github.com/ClickHouse/ClickHouse/pull/29494) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Use `max_parse_depth` in fuzzers [#29497](https://github.com/ClickHouse/ClickHouse/pull/29497) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Introducing Fn concept for function signature checking, simplifying SimpleCache [#29500](https://github.com/ClickHouse/ClickHouse/pull/29500) ([Mike Kot](https://github.com/myrrc)). +* Fix LOGICAL_ERROR for INSERT with concurrent ALTER [#29502](https://github.com/ClickHouse/ClickHouse/pull/29502) ([Azat Khuzhin](https://github.com/azat)). +* Log queries to external databases (since they may be rewritten) [#29503](https://github.com/ClickHouse/ClickHouse/pull/29503) ([Azat Khuzhin](https://github.com/azat)). +* Fix server pid (hence exit code and attaching with gdb) in fuzzer tests [#29513](https://github.com/ClickHouse/ClickHouse/pull/29513) ([Azat Khuzhin](https://github.com/azat)). +* Bump cmake minimum required version to 3.14 [#29515](https://github.com/ClickHouse/ClickHouse/pull/29515) ([Azat Khuzhin](https://github.com/azat)). +* Changelog heredoc added backward incompatible change [#29530](https://github.com/ClickHouse/ClickHouse/pull/29530) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix test names [#29539](https://github.com/ClickHouse/ClickHouse/pull/29539) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Making Monotonicity an aggregate to use with designated initializers [#29540](https://github.com/ClickHouse/ClickHouse/pull/29540) ([Mike Kot](https://github.com/myrrc)). +* Generalize code in IColumn::permute [#29545](https://github.com/ClickHouse/ClickHouse/pull/29545) ([Anton Popov](https://github.com/CurtizJ)). +* Add error for multiple keys without current_key_id [#29546](https://github.com/ClickHouse/ClickHouse/pull/29546) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* support conversion between date32 and datetime64 [#29552](https://github.com/ClickHouse/ClickHouse/pull/29552) ([kevin wan](https://github.com/MaxWk)). +* Fixed test with dictGet without database name as column default value [#29568](https://github.com/ClickHouse/ClickHouse/pull/29568) ([Maksim Kita](https://github.com/kitaisreal)). +* Add FAIL message to test_results.tsv [#29583](https://github.com/ClickHouse/ClickHouse/pull/29583) ([Dmitry Novik](https://github.com/novikd)). +* Generalize code in `IColumn::updatePermutation` [#29595](https://github.com/ClickHouse/ClickHouse/pull/29595) ([Anton Popov](https://github.com/CurtizJ)). +* Add fuzzer for `executeQuery` function [#29596](https://github.com/ClickHouse/ClickHouse/pull/29596) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix docker/test/util declaration [#29597](https://github.com/ClickHouse/ClickHouse/pull/29597) ([Dmitry Novik](https://github.com/novikd)). +* Skip test for executable table function under MSan [#29600](https://github.com/ClickHouse/ClickHouse/pull/29600) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve some more integration tests [#29603](https://github.com/ClickHouse/ClickHouse/pull/29603) ([Ilya Yatsishin](https://github.com/qoega)). +* Improve CompressedReadBuffer [#29605](https://github.com/ClickHouse/ClickHouse/pull/29605) ([Ilya Yatsishin](https://github.com/qoega)). +* Do not send many signals at server restart (integration tests). [#29608](https://github.com/ClickHouse/ClickHouse/pull/29608) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between MergeTask and storage destruction [#29614](https://github.com/ClickHouse/ClickHouse/pull/29614) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* BloomFilter indexes map data type support fix unsafe identifier cast [#29623](https://github.com/ClickHouse/ClickHouse/pull/29623) ([Maksim Kita](https://github.com/kitaisreal)). +* Follow-up for [#26231](https://github.com/ClickHouse/ClickHouse/issues/26231) [#29626](https://github.com/ClickHouse/ClickHouse/pull/29626) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Map get non const key added test [#29636](https://github.com/ClickHouse/ClickHouse/pull/29636) ([Maksim Kita](https://github.com/kitaisreal)). +* Add columns in columns.sql for INFORMATION_SCHEMA [#29637](https://github.com/ClickHouse/ClickHouse/pull/29637) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Merging [#29376](https://github.com/ClickHouse/ClickHouse/issues/29376) [#29649](https://github.com/ClickHouse/ClickHouse/pull/29649) ([Anton Popov](https://github.com/CurtizJ)). +* AddDefaultDatabaseVisitor support dictGet [#29650](https://github.com/ClickHouse/ClickHouse/pull/29650) ([Maksim Kita](https://github.com/kitaisreal)). +* Do not try to fuzz `USE`/`SET` queries and print stacktrace [#29660](https://github.com/ClickHouse/ClickHouse/pull/29660) ([Azat Khuzhin](https://github.com/azat)). +* Rename `common` to `base` [#29661](https://github.com/ClickHouse/ClickHouse/pull/29661) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Move modelEvaluate function to its own file [#29662](https://github.com/ClickHouse/ClickHouse/pull/29662) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Perf fixes [#29663](https://github.com/ClickHouse/ClickHouse/pull/29663) ([Azat Khuzhin](https://github.com/azat)). +* UserDefinedExecutableFunctions added implicit cast [#29666](https://github.com/ClickHouse/ClickHouse/pull/29666) ([Maksim Kita](https://github.com/kitaisreal)). +* Cleanup common defines [#29680](https://github.com/ClickHouse/ClickHouse/pull/29680) ([Azat Khuzhin](https://github.com/azat)). +* Fix Xcode 13 build [#29682](https://github.com/ClickHouse/ClickHouse/pull/29682) ([Denis Glazachev](https://github.com/traceon)). +* Non-recursive implementation for type list and its functions [#29683](https://github.com/ClickHouse/ClickHouse/pull/29683) ([Mike Kot](https://github.com/myrrc)). +* Reorganize contrib/ IDE folders [#29684](https://github.com/ClickHouse/ClickHouse/pull/29684) ([Denis Glazachev](https://github.com/traceon)). +* Cleanup unbundled image [#29689](https://github.com/ClickHouse/ClickHouse/pull/29689) ([Azat Khuzhin](https://github.com/azat)). +* Fix memory tracking for merges and mutations [#29691](https://github.com/ClickHouse/ClickHouse/pull/29691) ([Azat Khuzhin](https://github.com/azat)). +* Fix data-race in WriteIndirectBuffer (used in DiskMemory) [#29692](https://github.com/ClickHouse/ClickHouse/pull/29692) ([Azat Khuzhin](https://github.com/azat)). +* Fix flacky test [#29706](https://github.com/ClickHouse/ClickHouse/pull/29706) ([Kseniia Sumarokova](https://github.com/kssenii)). +* BorrowedObjectPool condition variable notify fix [#29722](https://github.com/ClickHouse/ClickHouse/pull/29722) ([Maksim Kita](https://github.com/kitaisreal)). +* Better exception message for local interactive [#29737](https://github.com/ClickHouse/ClickHouse/pull/29737) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix --stage for clickhouse-local [#29745](https://github.com/ClickHouse/ClickHouse/pull/29745) ([Azat Khuzhin](https://github.com/azat)). +* Forbid Nullable for JSONExtract* (JSONExtract() still supports Nullable), leads to SIGSEGV before [#29746](https://github.com/ClickHouse/ClickHouse/pull/29746) ([Azat Khuzhin](https://github.com/azat)). +* TableFunctionDictionary fix comment [#29747](https://github.com/ClickHouse/ClickHouse/pull/29747) ([Maksim Kita](https://github.com/kitaisreal)). +* May be fix s3 tests [#29762](https://github.com/ClickHouse/ClickHouse/pull/29762) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove some merging streams [#29768](https://github.com/ClickHouse/ClickHouse/pull/29768) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Parse query from metadata exception throw fix [#29779](https://github.com/ClickHouse/ClickHouse/pull/29779) ([Maksim Kita](https://github.com/kitaisreal)). +* Simplify obtaining of server pid in fuzzer tests [#29781](https://github.com/ClickHouse/ClickHouse/pull/29781) ([Azat Khuzhin](https://github.com/azat)). +* Fix data-race between fatal error handler and progress packets [#29783](https://github.com/ClickHouse/ClickHouse/pull/29783) ([Azat Khuzhin](https://github.com/azat)). +* Fix parallel_view_processing [#29786](https://github.com/ClickHouse/ClickHouse/pull/29786) ([Azat Khuzhin](https://github.com/azat)). +* Function reinterpretAs improve readability [#29796](https://github.com/ClickHouse/ClickHouse/pull/29796) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactor ConcurrentBoundedQueue [#29801](https://github.com/ClickHouse/ClickHouse/pull/29801) ([Maksim Kita](https://github.com/kitaisreal)). +* Improve usability of error messages when error is caused by sophisticated interventions [#29803](https://github.com/ClickHouse/ClickHouse/pull/29803) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improvement for [#28373](https://github.com/ClickHouse/ClickHouse/issues/28373) [#29804](https://github.com/ClickHouse/ClickHouse/pull/29804) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Better diagnostic for OPTIMIZE [#29812](https://github.com/ClickHouse/ClickHouse/pull/29812) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add listen_backlog to documentation [#29813](https://github.com/ClickHouse/ClickHouse/pull/29813) ([Azat Khuzhin](https://github.com/azat)). +* Dictionary attributes updated documentation [#29816](https://github.com/ClickHouse/ClickHouse/pull/29816) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix compilation with glibc 2.34 (MINSIGSTKSZ defined as sysconf(_SC_SIGSTKSZ)) [#29820](https://github.com/ClickHouse/ClickHouse/pull/29820) ([Azat Khuzhin](https://github.com/azat)). +* Make memory_profiler_step API cleaner [#29825](https://github.com/ClickHouse/ClickHouse/pull/29825) ([Azat Khuzhin](https://github.com/azat)). +* Add coroutines example. [#29827](https://github.com/ClickHouse/ClickHouse/pull/29827) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add coroutines example. [#29841](https://github.com/ClickHouse/ClickHouse/pull/29841) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update changelog to include caveats about Nullable types in data skipping indexes [#29855](https://github.com/ClickHouse/ClickHouse/pull/29855) ([Azat Khuzhin](https://github.com/azat)). +* Rewrite clickhouse-test to use python clickhouse_driver [#29856](https://github.com/ClickHouse/ClickHouse/pull/29856) ([Azat Khuzhin](https://github.com/azat)). +* Fix client [#29864](https://github.com/ClickHouse/ClickHouse/pull/29864) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove some more streams. [#29898](https://github.com/ClickHouse/ClickHouse/pull/29898) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add logging in ZooKeeper client [#29901](https://github.com/ClickHouse/ClickHouse/pull/29901) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix some flacky tests [#29902](https://github.com/ClickHouse/ClickHouse/pull/29902) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Grep server log even if it contains binary data [#29903](https://github.com/ClickHouse/ClickHouse/pull/29903) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Cosmetic refactoring of server constants. [#29913](https://github.com/ClickHouse/ClickHouse/pull/29913) ([Amos Bird](https://github.com/amosbird)). +* Format improvement of AlterQuery [#29916](https://github.com/ClickHouse/ClickHouse/pull/29916) ([flynn](https://github.com/ucasfl)). +* Fix flaky integration tests (test_backup_restore/test_input_format_parallel_parsing_memory_tracking) [#29919](https://github.com/ClickHouse/ClickHouse/pull/29919) ([Azat Khuzhin](https://github.com/azat)). +* Fix some flaky tests [#29923](https://github.com/ClickHouse/ClickHouse/pull/29923) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix build due to conflicts in serverConstants [#29931](https://github.com/ClickHouse/ClickHouse/pull/29931) ([Azat Khuzhin](https://github.com/azat)). +* Minor changes [#29932](https://github.com/ClickHouse/ClickHouse/pull/29932) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove trash from SentryWriter [#29933](https://github.com/ClickHouse/ClickHouse/pull/29933) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove `printf` function usage. [#29935](https://github.com/ClickHouse/ClickHouse/pull/29935) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Stateless flaky tests fixes [#29947](https://github.com/ClickHouse/ClickHouse/pull/29947) ([Azat Khuzhin](https://github.com/azat)). +* Fix filtering by tuple (some conditions was lost during analyzing) [#29956](https://github.com/ClickHouse/ClickHouse/pull/29956) ([Azat Khuzhin](https://github.com/azat)). +* Allow memory profiler under sanitizers (ASan/UBsan/MSan only) [#29979](https://github.com/ClickHouse/ClickHouse/pull/29979) ([Azat Khuzhin](https://github.com/azat)). +* Tests naming fix [#29982](https://github.com/ClickHouse/ClickHouse/pull/29982) ([Maksim Kita](https://github.com/kitaisreal)). +* More timeouts in test scripts [#29992](https://github.com/ClickHouse/ClickHouse/pull/29992) ([alesapin](https://github.com/alesapin)). +* Update BoringSSL [#29998](https://github.com/ClickHouse/ClickHouse/pull/29998) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Remove streams from formats. [#30001](https://github.com/ClickHouse/ClickHouse/pull/30001) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Follow-up to [#29901](https://github.com/ClickHouse/ClickHouse/issues/29901) [#30003](https://github.com/ClickHouse/ClickHouse/pull/30003) ([Alexander Tokmakov](https://github.com/tavplubix)). +* More strict check for intersecting parts [#30005](https://github.com/ClickHouse/ClickHouse/pull/30005) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Ignore parallel removing warning in 00992_system_parts_race_condition_zookeeper_long [#30007](https://github.com/ClickHouse/ClickHouse/pull/30007) ([Azat Khuzhin](https://github.com/azat)). +* Try remove excessive logging [#30008](https://github.com/ClickHouse/ClickHouse/pull/30008) ([Alexander Tokmakov](https://github.com/tavplubix)). +* remove redundant dot in exception message [#30017](https://github.com/ClickHouse/ClickHouse/pull/30017) ([flynn](https://github.com/ucasfl)). +* Fix build [#30028](https://github.com/ClickHouse/ClickHouse/pull/30028) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Attach trace_log to stateless tests report as is [#30030](https://github.com/ClickHouse/ClickHouse/pull/30030) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove unused headers and handle exception 'unrecognised option' in clickhouse-local [#30053](https://github.com/ClickHouse/ClickHouse/pull/30053) ([Kruglov Pavel](https://github.com/Avogar)). +* clickhouse-test: replace clickhouse-driver with http interface (via http.client) [#30065](https://github.com/ClickHouse/ClickHouse/pull/30065) ([Azat Khuzhin](https://github.com/azat)). +* Fix hardware utilization info printing in client [#30072](https://github.com/ClickHouse/ClickHouse/pull/30072) ([Dmitry Novik](https://github.com/novikd)). +* cmake generator: unlink before creating a link [#30073](https://github.com/ClickHouse/ClickHouse/pull/30073) ([lehasm](https://github.com/lehasm)). +* SQL user defined functions fix alias [#30075](https://github.com/ClickHouse/ClickHouse/pull/30075) ([Maksim Kita](https://github.com/kitaisreal)). +* Add Greenhouse careers page to website [#30077](https://github.com/ClickHouse/ClickHouse/pull/30077) ([Cody Baker](https://github.com/codyrobert)). +* Update team photos on website [#30078](https://github.com/ClickHouse/ClickHouse/pull/30078) ([Cody Baker](https://github.com/codyrobert)). +* Add webinar signup promo to website homepage [#30079](https://github.com/ClickHouse/ClickHouse/pull/30079) ([Cody Baker](https://github.com/codyrobert)). +* test for rename atomic hanging [#30080](https://github.com/ClickHouse/ClickHouse/pull/30080) ([Denny Crane](https://github.com/den-crane)). +* Modify ConnectionPoolWithFailover get_priority ROUND_ROBIN comments [#30092](https://github.com/ClickHouse/ClickHouse/pull/30092) ([小路](https://github.com/nicelulu)). +* Fix flaky test 01939_network_send_bytes_metrics [#30134](https://github.com/ClickHouse/ClickHouse/pull/30134) ([Dmitry Novik](https://github.com/novikd)). +* System data skipping indices size fix test names [#30141](https://github.com/ClickHouse/ClickHouse/pull/30141) ([Maksim Kita](https://github.com/kitaisreal)). +* FunctionsJSON avoid copying object element during iteration [#30145](https://github.com/ClickHouse/ClickHouse/pull/30145) ([Maksim Kita](https://github.com/kitaisreal)). +* Disable fsync_metadata on CI [#30149](https://github.com/ClickHouse/ClickHouse/pull/30149) ([Azat Khuzhin](https://github.com/azat)). +* Make test_MemoryTracking::test_http not flaky [#30150](https://github.com/ClickHouse/ClickHouse/pull/30150) ([Azat Khuzhin](https://github.com/azat)). +* Add careers menu item to navigation [#30151](https://github.com/ClickHouse/ClickHouse/pull/30151) ([Cody Baker](https://github.com/codyrobert)). +* Update yandex logo on homepage [#30152](https://github.com/ClickHouse/ClickHouse/pull/30152) ([Cody Baker](https://github.com/codyrobert)). +* Remove stream interfaces [#30171](https://github.com/ClickHouse/ClickHouse/pull/30171) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better interfaces for `IDataType` and `ISerialization` [#30174](https://github.com/ClickHouse/ClickHouse/pull/30174) ([Anton Popov](https://github.com/CurtizJ)). +* Add blog post for v21.10 release [#30186](https://github.com/ClickHouse/ClickHouse/pull/30186) ([Cody Baker](https://github.com/codyrobert)). +* Smaller smoothing window in throttler. [#30193](https://github.com/ClickHouse/ClickHouse/pull/30193) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix clang-tidy warnings in FunctionsJSON code [#30217](https://github.com/ClickHouse/ClickHouse/pull/30217) ([Azat Khuzhin](https://github.com/azat)). +* Fix --hung-check in clickhouse-test [#30218](https://github.com/ClickHouse/ClickHouse/pull/30218) ([Azat Khuzhin](https://github.com/azat)). +* FunctionsJSON updated [#30228](https://github.com/ClickHouse/ClickHouse/pull/30228) ([Maksim Kita](https://github.com/kitaisreal)). +* PolygonDictionary fix bytes_allocated [#30239](https://github.com/ClickHouse/ClickHouse/pull/30239) ([Maksim Kita](https://github.com/kitaisreal)). +* ComplexKeyHashedDictionary fix keys copy [#30241](https://github.com/ClickHouse/ClickHouse/pull/30241) ([Maksim Kita](https://github.com/kitaisreal)). +* Removing data streams folder [#30247](https://github.com/ClickHouse/ClickHouse/pull/30247) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* --stage for clickhouse-local [#30275](https://github.com/ClickHouse/ClickHouse/pull/30275) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Use forward declaration for Buffer<> in generic headers [#30276](https://github.com/ClickHouse/ClickHouse/pull/30276) ([Azat Khuzhin](https://github.com/azat)). +* Less threads in clickhouse-local, fix Ok. printing [#30282](https://github.com/ClickHouse/ClickHouse/pull/30282) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Adjust resource heavy tests [#30284](https://github.com/ClickHouse/ClickHouse/pull/30284) ([Raúl Marín](https://github.com/Algunenano)). +* Fix printing stacktraces for clickhouse-local [#30285](https://github.com/ClickHouse/ClickHouse/pull/30285) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix throwing syntax exception in clickhouse-local [#30288](https://github.com/ClickHouse/ClickHouse/pull/30288) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Various fixes to install procedure [#30295](https://github.com/ClickHouse/ClickHouse/pull/30295) ([Denis Glazachev](https://github.com/traceon)). +* Fix clickhouse-local break on timeout [#30297](https://github.com/ClickHouse/ClickHouse/pull/30297) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add RISC-V build [#30298](https://github.com/ClickHouse/ClickHouse/pull/30298) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Move SquashingTransform to Interpreters (to fix split build) [#30318](https://github.com/ClickHouse/ClickHouse/pull/30318) ([Azat Khuzhin](https://github.com/azat)). +* Increase default wait of the server start in clickhouse-test [#30320](https://github.com/ClickHouse/ClickHouse/pull/30320) ([Azat Khuzhin](https://github.com/azat)). +* Update memory optimisation for MergingSorted. [#30322](https://github.com/ClickHouse/ClickHouse/pull/30322) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Single sorting step [#30335](https://github.com/ClickHouse/ClickHouse/pull/30335) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Send table columns in clickhouse-local [#30336](https://github.com/ClickHouse/ClickHouse/pull/30336) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix consumed memory indication in clickhouse-client [#30337](https://github.com/ClickHouse/ClickHouse/pull/30337) ([Dmitry Novik](https://github.com/novikd)). +* Fix fuzzer build [#30344](https://github.com/ClickHouse/ClickHouse/pull/30344) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* ExecutablePool dictionary source fix borrow timeout milliseconds [#30345](https://github.com/ClickHouse/ClickHouse/pull/30345) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageExecutable fix small issues [#30352](https://github.com/ClickHouse/ClickHouse/pull/30352) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix ca-bundle.crt in kerberized_hadoop/Dockerfile [#30358](https://github.com/ClickHouse/ClickHouse/pull/30358) ([Vladimir C](https://github.com/vdimir)). +* Update obsolete comments. [#30359](https://github.com/ClickHouse/ClickHouse/pull/30359) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Rename readWkt* functions to readWKT*, added an alias SVG for svg function [#30361](https://github.com/ClickHouse/ClickHouse/pull/30361) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix crash when minmax_count projection is used with primary key in partition expr [#30384](https://github.com/ClickHouse/ClickHouse/pull/30384) ([Amos Bird](https://github.com/amosbird)). +* Follow-up for [#30282](https://github.com/ClickHouse/ClickHouse/issues/30282) [#30412](https://github.com/ClickHouse/ClickHouse/pull/30412) ([Kseniia Sumarokova](https://github.com/kssenii)). +* ExecutableUDF example [#30436](https://github.com/ClickHouse/ClickHouse/pull/30436) ([Maksim Kita](https://github.com/kitaisreal)). +* Use robot token in actions for statuses [#30439](https://github.com/ClickHouse/ClickHouse/pull/30439) ([alesapin](https://github.com/alesapin)). +* Remove statuses from actions [#30444](https://github.com/ClickHouse/ClickHouse/pull/30444) ([alesapin](https://github.com/alesapin)). +* Fix s3 for github actions [#30447](https://github.com/ClickHouse/ClickHouse/pull/30447) ([alesapin](https://github.com/alesapin)). +* ExecutableUDF example fix style check [#30451](https://github.com/ClickHouse/ClickHouse/pull/30451) ([Maksim Kita](https://github.com/kitaisreal)). +* Support VALUES format in async inserts [#30456](https://github.com/ClickHouse/ClickHouse/pull/30456) ([Anton Popov](https://github.com/CurtizJ)). +* Update release date and add training link [#30475](https://github.com/ClickHouse/ClickHouse/pull/30475) ([Cody Baker](https://github.com/codyrobert)). +* Fix horizontal scroll bar [#30476](https://github.com/ClickHouse/ClickHouse/pull/30476) ([Cody Baker](https://github.com/codyrobert)). +* SQLUserDefinedFunctions composition fix [#30483](https://github.com/ClickHouse/ClickHouse/pull/30483) ([Maksim Kita](https://github.com/kitaisreal)). +* Trying builds on github actions [#30493](https://github.com/ClickHouse/ClickHouse/pull/30493) ([alesapin](https://github.com/alesapin)). +* HashedArrayDictionary optimize read multiple attributes [#30501](https://github.com/ClickHouse/ClickHouse/pull/30501) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageDictionary fix potential configuration race [#30502](https://github.com/ClickHouse/ClickHouse/pull/30502) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix style regressions on benchmark page [#30506](https://github.com/ClickHouse/ClickHouse/pull/30506) ([Cody Baker](https://github.com/codyrobert)). +* Change link in nav from community -> learn [#30508](https://github.com/ClickHouse/ClickHouse/pull/30508) ([Cody Baker](https://github.com/codyrobert)). +* Add placeholder to play.html inputs [#30509](https://github.com/ClickHouse/ClickHouse/pull/30509) ([Vitaly Orlov](https://github.com/orloffv)). +* Add bytes to stats and human readable rows to play.html [#30511](https://github.com/ClickHouse/ClickHouse/pull/30511) ([Vitaly Orlov](https://github.com/orloffv)). +* clickhouse-local: fix block lost in interactive mode [#30521](https://github.com/ClickHouse/ClickHouse/pull/30521) ([Azat Khuzhin](https://github.com/azat)). +* Remove check_columns argument from MergeTree code (false was never passed) [#30522](https://github.com/ClickHouse/ClickHouse/pull/30522) ([Azat Khuzhin](https://github.com/azat)). +* Fix typo and update NuRaft [#30550](https://github.com/ClickHouse/ClickHouse/pull/30550) ([alesapin](https://github.com/alesapin)). +* Refactoring in codec encrypted [#30564](https://github.com/ClickHouse/ClickHouse/pull/30564) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Try to fix [#30397](https://github.com/ClickHouse/ClickHouse/issues/30397) [#30565](https://github.com/ClickHouse/ClickHouse/pull/30565) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Support Nullable(String) type in arrayStringConcat [#30566](https://github.com/ClickHouse/ClickHouse/pull/30566) ([Nikita Taranov](https://github.com/nickitat)). +* Function ngrams fixed tests [#30567](https://github.com/ClickHouse/ClickHouse/pull/30567) ([Maksim Kita](https://github.com/kitaisreal)). +* Test clickhouse local columns description fix number [#30568](https://github.com/ClickHouse/ClickHouse/pull/30568) ([Maksim Kita](https://github.com/kitaisreal)). +* Update documentation for distributed_push_down_limit [#30577](https://github.com/ClickHouse/ClickHouse/pull/30577) ([Azat Khuzhin](https://github.com/azat)). +* Fix tests that relies on checking stack size under TSan [#30579](https://github.com/ClickHouse/ClickHouse/pull/30579) ([Azat Khuzhin](https://github.com/azat)). +* Add metadata for *_log into tests artifacts [#30589](https://github.com/ClickHouse/ClickHouse/pull/30589) ([Azat Khuzhin](https://github.com/azat)). +* Fix LOGICAL_ERROR on connection draining in case of ECONNRESET [#30594](https://github.com/ClickHouse/ClickHouse/pull/30594) ([Azat Khuzhin](https://github.com/azat)). +* Adjust perf test for simdjson [#30596](https://github.com/ClickHouse/ClickHouse/pull/30596) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update minimum allowed compiler versions [#30597](https://github.com/ClickHouse/ClickHouse/pull/30597) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Replace estimated_size with optional for readFile/createReadBufferFromFileBase [#30611](https://github.com/ClickHouse/ClickHouse/pull/30611) ([Azat Khuzhin](https://github.com/azat)). +* RFC: Relax minimal clang version (set it to 12, was 13) [#30613](https://github.com/ClickHouse/ClickHouse/pull/30613) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test: increase delay for initial SELECT 1 check [#30619](https://github.com/ClickHouse/ClickHouse/pull/30619) ([Azat Khuzhin](https://github.com/azat)). +* Update Client.cpp [#30636](https://github.com/ClickHouse/ClickHouse/pull/30636) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update LocalServer.cpp [#30637](https://github.com/ClickHouse/ClickHouse/pull/30637) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Typo fix in play.html [#30638](https://github.com/ClickHouse/ClickHouse/pull/30638) ([Vitaly Orlov](https://github.com/orloffv)). +* Fix argument types for now and now64 [#30639](https://github.com/ClickHouse/ClickHouse/pull/30639) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix intersecting parts in `parts_to_do` [#30651](https://github.com/ClickHouse/ClickHouse/pull/30651) ([Alexander Tokmakov](https://github.com/tavplubix)). +* HashedDictionaries added read performance tests [#30653](https://github.com/ClickHouse/ClickHouse/pull/30653) ([Maksim Kita](https://github.com/kitaisreal)). +* Change `alter_lock` from `RWLock` to `std::timed_mutex` [#30658](https://github.com/ClickHouse/ClickHouse/pull/30658) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Initialize custom TLDs in clickhouse-local [#30660](https://github.com/ClickHouse/ClickHouse/pull/30660) ([Azat Khuzhin](https://github.com/azat)). +* Add QueryProfilerRuns profile event [#30661](https://github.com/ClickHouse/ClickHouse/pull/30661) ([Azat Khuzhin](https://github.com/azat)). +* Switch everything left from `` to `` [#30662](https://github.com/ClickHouse/ClickHouse/pull/30662) ([Azat Khuzhin](https://github.com/azat)). +* Improve usability of `remote_url_allow_hosts` [#30673](https://github.com/ClickHouse/ClickHouse/pull/30673) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix indecency [#30337](https://github.com/ClickHouse/ClickHouse/issues/30337) [#30674](https://github.com/ClickHouse/ClickHouse/pull/30674) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better wait for server start in integration tests [#30677](https://github.com/ClickHouse/ClickHouse/pull/30677) ([alesapin](https://github.com/alesapin)). +* Fix keeper zookeeper converter test [#30678](https://github.com/ClickHouse/ClickHouse/pull/30678) ([alesapin](https://github.com/alesapin)). +* Don't run build checks for documentation [#30681](https://github.com/ClickHouse/ClickHouse/pull/30681) ([alesapin](https://github.com/alesapin)). +* Refactoring of Log family [#30689](https://github.com/ClickHouse/ClickHouse/pull/30689) ([Vitaly Baranov](https://github.com/vitlibar)). +* Update hardware page colors [#30719](https://github.com/ClickHouse/ClickHouse/pull/30719) ([Cody Baker](https://github.com/codyrobert)). +* Fix test_part_uuid::test_part_uuid_wal [#30723](https://github.com/ClickHouse/ClickHouse/pull/30723) ([Azat Khuzhin](https://github.com/azat)). +* Fix gtest_disk_encrypted (for new readFile/createReadBufferFromFileBase() interfaces) [#30725](https://github.com/ClickHouse/ClickHouse/pull/30725) ([Azat Khuzhin](https://github.com/azat)). +* tests/ci/docker_images_check: add missing time import [#30727](https://github.com/ClickHouse/ClickHouse/pull/30727) ([Azat Khuzhin](https://github.com/azat)). +* Add functional tests to github actions [#30729](https://github.com/ClickHouse/ClickHouse/pull/30729) ([alesapin](https://github.com/alesapin)). +* Run pylint over python scripts for github actions [#30733](https://github.com/ClickHouse/ClickHouse/pull/30733) ([Azat Khuzhin](https://github.com/azat)). +* DictionarySource unknown column name in dictionary fix [#30736](https://github.com/ClickHouse/ClickHouse/pull/30736) ([Maksim Kita](https://github.com/kitaisreal)). +* Update 01083_expressions_in_engine_arguments.sql [#30741](https://github.com/ClickHouse/ClickHouse/pull/30741) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Performance tests tuning [#30773](https://github.com/ClickHouse/ClickHouse/pull/30773) ([Azat Khuzhin](https://github.com/azat)). +* Remove cruft [#30780](https://github.com/ClickHouse/ClickHouse/pull/30780) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix test 02022 [#30786](https://github.com/ClickHouse/ClickHouse/pull/30786) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Trying to make fuzzer check less hardcoded [#30795](https://github.com/ClickHouse/ClickHouse/pull/30795) ([alesapin](https://github.com/alesapin)). +* DictionarySource unknown column name fix [#30800](https://github.com/ClickHouse/ClickHouse/pull/30800) ([Maksim Kita](https://github.com/kitaisreal)). +* Add funding press release [#30806](https://github.com/ClickHouse/ClickHouse/pull/30806) ([Cody Baker](https://github.com/codyrobert)). +* Update installation success message [#30672](https://github.com/ClickHouse/ClickHouse/issues/30672) [#30830](https://github.com/ClickHouse/ClickHouse/pull/30830) ([Teja](https://github.com/tejasrivastav)). +* filelog engine tests improve [#30832](https://github.com/ClickHouse/ClickHouse/pull/30832) ([flynn](https://github.com/ucasfl)). +* Remove redundant from http buffer [#30837](https://github.com/ClickHouse/ClickHouse/pull/30837) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Docs check on github actions [#30839](https://github.com/ClickHouse/ClickHouse/pull/30839) ([alesapin](https://github.com/alesapin)). +* Fix fuzzer on master [#30841](https://github.com/ClickHouse/ClickHouse/pull/30841) ([alesapin](https://github.com/alesapin)). +* Disable check for ZooKeeper session uptime by default [#30847](https://github.com/ClickHouse/ClickHouse/pull/30847) ([Alexander Tokmakov](https://github.com/tavplubix)). +* enable modify table comment of rest table engine [#30852](https://github.com/ClickHouse/ClickHouse/pull/30852) ([flynn](https://github.com/ucasfl)). +* Fix typo [#30853](https://github.com/ClickHouse/ClickHouse/pull/30853) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixed `--disable-net-host` in `runner` [#30863](https://github.com/ClickHouse/ClickHouse/pull/30863) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix protocol revision. [#30864](https://github.com/ClickHouse/ClickHouse/pull/30864) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add docker cleanup after actions [#30868](https://github.com/ClickHouse/ClickHouse/pull/30868) ([alesapin](https://github.com/alesapin)). +* Fix error message in Keeper handler [#30880](https://github.com/ClickHouse/ClickHouse/pull/30880) ([alesapin](https://github.com/alesapin)). +* Better handling of `xtables.lock` in `runner` [#30892](https://github.com/ClickHouse/ClickHouse/pull/30892) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Reimplement [#28639](https://github.com/ClickHouse/ClickHouse/issues/28639) [#30903](https://github.com/ClickHouse/ClickHouse/pull/30903) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Miscellaneous [#30917](https://github.com/ClickHouse/ClickHouse/pull/30917) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix yml in docs release [#30928](https://github.com/ClickHouse/ClickHouse/pull/30928) ([alesapin](https://github.com/alesapin)). +* Debug github event [#30929](https://github.com/ClickHouse/ClickHouse/pull/30929) ([alesapin](https://github.com/alesapin)). +* Fix docs release [#30933](https://github.com/ClickHouse/ClickHouse/pull/30933) ([alesapin](https://github.com/alesapin)). +* Fix style check [#30937](https://github.com/ClickHouse/ClickHouse/pull/30937) ([alesapin](https://github.com/alesapin)). +* Fix file progress for clickhouse-local [#30938](https://github.com/ClickHouse/ClickHouse/pull/30938) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix flacky test [#30940](https://github.com/ClickHouse/ClickHouse/pull/30940) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix reading from TinyLog [#30941](https://github.com/ClickHouse/ClickHouse/pull/30941) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add github to known hosts in docs release [#30947](https://github.com/ClickHouse/ClickHouse/pull/30947) ([alesapin](https://github.com/alesapin)). +* Parse json from response in ci checks [#30948](https://github.com/ClickHouse/ClickHouse/pull/30948) ([alesapin](https://github.com/alesapin)). + #### Testing Improvement * Implemented structure-aware fuzzing approach in ClickHouse for select statement parser. [#30012](https://github.com/ClickHouse/ClickHouse/pull/30012) ([Paul](https://github.com/PaulCher)). diff --git a/docs/changelogs/v21.11.10.1-stable.md b/docs/changelogs/v21.11.10.1-stable.md index f07eccd30c5..c180918bb07 100644 --- a/docs/changelogs/v21.11.10.1-stable.md +++ b/docs/changelogs/v21.11.10.1-stable.md @@ -1,2 +1,9 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.10.1-stable FIXME as compared to v21.11.9.1-stable diff --git a/docs/changelogs/v21.11.11.1-stable.md b/docs/changelogs/v21.11.11.1-stable.md index e46f43c53e0..922245d031e 100644 --- a/docs/changelogs/v21.11.11.1-stable.md +++ b/docs/changelogs/v21.11.11.1-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.11.1-stable FIXME as compared to v21.11.10.1-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v21.11.2.2-stable.md b/docs/changelogs/v21.11.2.2-stable.md index f48f91a9b13..9a11dfd03c5 100644 --- a/docs/changelogs/v21.11.2.2-stable.md +++ b/docs/changelogs/v21.11.2.2-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.2.2-stable FIXME as compared to v21.11.1.8636-prestable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) @@ -6,3 +13,7 @@ * Backported in [#31027](https://github.com/ClickHouse/ClickHouse/issues/31027): Using `formatRow` function with not row formats led to segfault. Don't allow to use this function with such formats (because it doesn't make sense). [#31001](https://github.com/ClickHouse/ClickHouse/pull/31001) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#31026](https://github.com/ClickHouse/ClickHouse/issues/31026): Fix JSONValue/Query with quoted identifiers. This allows to have spaces in json path. Closes [#30971](https://github.com/ClickHouse/ClickHouse/issues/30971). [#31003](https://github.com/ClickHouse/ClickHouse/pull/31003) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Return back accidentally removed code [#30996](https://github.com/ClickHouse/ClickHouse/pull/30996) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + diff --git a/docs/changelogs/v21.11.3.6-stable.md b/docs/changelogs/v21.11.3.6-stable.md index bf46ecec590..5db65d42c6d 100644 --- a/docs/changelogs/v21.11.3.6-stable.md +++ b/docs/changelogs/v21.11.3.6-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.3.6-stable FIXME as compared to v21.11.2.2-stable #### Bug Fix diff --git a/docs/changelogs/v21.11.4.14-stable.md b/docs/changelogs/v21.11.4.14-stable.md index b3d44b8c193..be661ad244d 100644 --- a/docs/changelogs/v21.11.4.14-stable.md +++ b/docs/changelogs/v21.11.4.14-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.4.14-stable FIXME as compared to v21.11.3.6-stable #### Bug Fix @@ -15,3 +22,9 @@ * Backported in [#31283](https://github.com/ClickHouse/ClickHouse/issues/31283): Rename setting value `read_threadpool` to `threadpool` for setting `remote_filesystem_read_method`. [#31224](https://github.com/ClickHouse/ClickHouse/pull/31224) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix threadpool read for remote disks [#31112](https://github.com/ClickHouse/ClickHouse/pull/31112) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix check in async buffer [#31325](https://github.com/ClickHouse/ClickHouse/pull/31325) ([Kseniia Sumarokova](https://github.com/kssenii)). +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.11.5.33-stable.md b/docs/changelogs/v21.11.5.33-stable.md index 973c443d9f8..2895505715f 100644 --- a/docs/changelogs/v21.11.5.33-stable.md +++ b/docs/changelogs/v21.11.5.33-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.5.33-stable FIXME as compared to v21.11.4.14-stable #### Performance Improvement @@ -31,3 +38,7 @@ * Backported in [#32091](https://github.com/ClickHouse/ClickHouse/issues/32091): Some `GET_PART` entry might hang in replication queue if part is lost on all replicas and there are no other parts in the same partition. It's fixed in cases when partition key contains only columns of integer types or `Date[Time]`. Fixes [#31485](https://github.com/ClickHouse/ClickHouse/issues/31485). [#31887](https://github.com/ClickHouse/ClickHouse/pull/31887) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#32020](https://github.com/ClickHouse/ClickHouse/issues/32020): Fix FileLog engine unnesessary create meta data directory when create table failed. Fix [#31962](https://github.com/ClickHouse/ClickHouse/issues/31962). [#31967](https://github.com/ClickHouse/ClickHouse/pull/31967) ([flynn](https://github.com/ucasfl)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.11.6.7-stable.md b/docs/changelogs/v21.11.6.7-stable.md index 1f3df589466..c3127723a68 100644 --- a/docs/changelogs/v21.11.6.7-stable.md +++ b/docs/changelogs/v21.11.6.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.6.7-stable FIXME as compared to v21.11.5.33-stable #### Bug Fix @@ -22,3 +29,8 @@ * NO CL ENTRY: 'Manual backport of [#31766](https://github.com/ClickHouse/ClickHouse/issues/31766) into 21.11'. [#32202](https://github.com/ClickHouse/ClickHouse/pull/32202) ([Raúl Marín](https://github.com/Algunenano)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in `removePartAndEnqueueFetch(...)` [#32119](https://github.com/ClickHouse/ClickHouse/pull/32119) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.11.7.9-stable.md b/docs/changelogs/v21.11.7.9-stable.md index 1d907ad0ce1..5595e7bc0a9 100644 --- a/docs/changelogs/v21.11.7.9-stable.md +++ b/docs/changelogs/v21.11.7.9-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.7.9-stable FIXME as compared to v21.11.6.7-stable #### Bug Fix @@ -13,3 +20,9 @@ * Backported in [#32617](https://github.com/ClickHouse/ClickHouse/issues/32617): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in case of MATERIALIZE COLUMN with no default expression. [#32464](https://github.com/ClickHouse/ClickHouse/pull/32464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.11.8.4-stable.md b/docs/changelogs/v21.11.8.4-stable.md index 0826b473758..c670180a52e 100644 --- a/docs/changelogs/v21.11.8.4-stable.md +++ b/docs/changelogs/v21.11.8.4-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.8.4-stable FIXME as compared to v21.11.7.9-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) @@ -10,3 +17,8 @@ * Backported in [#32891](https://github.com/ClickHouse/ClickHouse/issues/32891): Fix LOGICAL_ERROR when the target of a materialized view is a JOIN or a SET table. [#32669](https://github.com/ClickHouse/ClickHouse/pull/32669) ([Raúl Marín](https://github.com/Algunenano)). * Backported in [#32792](https://github.com/ClickHouse/ClickHouse/issues/32792): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Always apply const-condition-if optimization. [#32858](https://github.com/ClickHouse/ClickHouse/pull/32858) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.11.9.1-stable.md b/docs/changelogs/v21.11.9.1-stable.md index c1754614c3c..bca99350b47 100644 --- a/docs/changelogs/v21.11.9.1-stable.md +++ b/docs/changelogs/v21.11.9.1-stable.md @@ -1,6 +1,19 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.11.9.1-stable FIXME as compared to v21.11.8.4-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) * Backported in [#33181](https://github.com/ClickHouse/ClickHouse/issues/33181): Server might fail to start if database with `MySQL` engine cannot connect to MySQL server, it's fixed. Fixes [#14441](https://github.com/ClickHouse/ClickHouse/issues/14441). [#32802](https://github.com/ClickHouse/ClickHouse/pull/32802) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33050](https://github.com/ClickHouse/ClickHouse/issues/33050) [#33065](https://github.com/ClickHouse/ClickHouse/pull/33065) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.12.1.9017-prestable.md b/docs/changelogs/v21.12.1.9017-prestable.md index f5e036c9c52..7ca0cbc3605 100644 --- a/docs/changelogs/v21.12.1.9017-prestable.md +++ b/docs/changelogs/v21.12.1.9017-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.12.1.9017-prestable FIXME as compared to v21.11.1.8636-prestable #### Backward Incompatible Change @@ -21,7 +28,7 @@ * Exposes all GlobalThreadPool configurations to the configuration files. [#31285](https://github.com/ClickHouse/ClickHouse/pull/31285) ([Tomáš Hromada](https://github.com/gyfis)). * Aliyun OSS Storage support. [#31286](https://github.com/ClickHouse/ClickHouse/pull/31286) ([cfcz48](https://github.com/cfcz48)). * Allow to print/parse names and types of colums in CustomSeparated input/output format. Add formats CustomSeparatedWithNames/WithNamesAndTypes similar to TSVWithNames/WithNamesAndTypes. [#31434](https://github.com/ClickHouse/ClickHouse/pull/31434) ([Kruglov Pavel](https://github.com/Avogar)). -* - Basic access authentication for http/url functions. [#31648](https://github.com/ClickHouse/ClickHouse/pull/31648) ([michael1589](https://github.com/michael1589)). +* - Basic access authentication for http/url functions. [#31648](https://github.com/ClickHouse/ClickHouse/pull/31648) ([Peng Liu](https://github.com/michael1589)). #### Performance Improvement * ... Allow to split GraphiteMergeTree rollup rules for plain/tagged metrics (optional rule_type field). [#25122](https://github.com/ClickHouse/ClickHouse/pull/25122) ([Michail Safronov](https://github.com/msaf1980)). @@ -54,7 +61,7 @@ * Add settings `merge_tree_min_rows_for_concurrent_read_for_remote_filesystem` and `merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem`. [#30970](https://github.com/ClickHouse/ClickHouse/pull/30970) ([Kseniia Sumarokova](https://github.com/kssenii)). * Do not allow to drop a table or dictionary if some tables or dictionaries depend on it. [#30977](https://github.com/ClickHouse/ClickHouse/pull/30977) ([Alexander Tokmakov](https://github.com/tavplubix)). * Only grab AlterLock when we do alter command. Let's see if the assumption is correct. [#31010](https://github.com/ClickHouse/ClickHouse/pull/31010) ([Amos Bird](https://github.com/amosbird)). -* The local session inside a ClickHouse dictionary source won't send its events to the session log anymore. This fixes a possible deadlock (tsan alert) on shutdown. Also this PR fixes flaky `test_dictionaries_dependency_xml/`. [#31013](https://github.com/ClickHouse/ClickHouse/pull/31013) ([Vitaly Baranov](https://github.com/vitlibar)). +* The local session inside a Clickhouse dictionary source won't send its events to the session log anymore. This fixes a possible deadlock (tsan alert) on shutdown. Also this PR fixes flaky `test_dictionaries_dependency_xml/`. [#31013](https://github.com/ClickHouse/ClickHouse/pull/31013) ([Vitaly Baranov](https://github.com/vitlibar)). * Cancel vertical merges when partition is dropped. This is a follow-up of https://github.com/ClickHouse/ClickHouse/pull/25684 and https://github.com/ClickHouse/ClickHouse/pull/30996. [#31057](https://github.com/ClickHouse/ClickHouse/pull/31057) ([Amos Bird](https://github.com/amosbird)). * Support `IF EXISTS` modifier for `RENAME DATABASE`/`TABLE`/`DICTIONARY` query, If this directive is used, one will not get an error if the DATABASE/TABLE/DICTIONARY to be renamed doesn't exist. [#31081](https://github.com/ClickHouse/ClickHouse/pull/31081) ([victorgao](https://github.com/kafka1991)). * Function name normalization for ALTER queries. This helps avoid metadata mismatch between creating table with indices/projections and adding indices/projections via alter commands. This is a follow-up PR of https://github.com/ClickHouse/ClickHouse/pull/20174. Mark as improvements as there are no bug reports and the senario is somehow rare. [#31095](https://github.com/ClickHouse/ClickHouse/pull/31095) ([Amos Bird](https://github.com/amosbird)). @@ -204,3 +211,216 @@ * NO CL ENTRY: 'Revert "Add a test with 20000 mutations in one query"'. [#32326](https://github.com/ClickHouse/ClickHouse/pull/32326) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * NO CL ENTRY: 'Revert "Revert "Add a test with 20000 mutations in one query""'. [#32327](https://github.com/ClickHouse/ClickHouse/pull/32327) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Refactor pipeline executor [#19587](https://github.com/ClickHouse/ClickHouse/pull/19587) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Support NULLS FIRST in KeyCondition [#29528](https://github.com/ClickHouse/ClickHouse/pull/29528) ([DimasKovas](https://github.com/DimasKovas)). +* Integration test improvements [#29806](https://github.com/ClickHouse/ClickHouse/pull/29806) ([Ilya Yatsishin](https://github.com/qoega)). +* Do not allow zero-length reads [#30190](https://github.com/ClickHouse/ClickHouse/pull/30190) ([Azat Khuzhin](https://github.com/azat)). +* Fix test_backward_compatibility [#30950](https://github.com/ClickHouse/ClickHouse/pull/30950) ([Ilya Yatsishin](https://github.com/qoega)). +* Add stress test to github actions [#30952](https://github.com/ClickHouse/ClickHouse/pull/30952) ([alesapin](https://github.com/alesapin)). +* Try smaller blacklist of non parallel integration tests [#30963](https://github.com/ClickHouse/ClickHouse/pull/30963) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix flacky test [#30967](https://github.com/ClickHouse/ClickHouse/pull/30967) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Move access-rights source code [#30973](https://github.com/ClickHouse/ClickHouse/pull/30973) ([Vitaly Baranov](https://github.com/vitlibar)). +* Set output_format_avro_rows_in_file default to 1 [#30990](https://github.com/ClickHouse/ClickHouse/pull/30990) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove remaining usages of Y_IGNORE [#30993](https://github.com/ClickHouse/ClickHouse/pull/30993) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Return back accidentally removed code [#30996](https://github.com/ClickHouse/ClickHouse/pull/30996) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Rename AccessControlManager [#30998](https://github.com/ClickHouse/ClickHouse/pull/30998) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add fuzzer check to actions [#31002](https://github.com/ClickHouse/ClickHouse/pull/31002) ([alesapin](https://github.com/alesapin)). +* Small refactoring in formats [#31004](https://github.com/ClickHouse/ClickHouse/pull/31004) ([Kruglov Pavel](https://github.com/Avogar)). +* Changelog for version 21.11 [#31014](https://github.com/ClickHouse/ClickHouse/pull/31014) ([Rich Raposa](https://github.com/rfraposa)). +* perf: add missing DROP TABLE queries [#31028](https://github.com/ClickHouse/ClickHouse/pull/31028) ([Azat Khuzhin](https://github.com/azat)). +* Tune perf tests configs/scripts [#31029](https://github.com/ClickHouse/ClickHouse/pull/31029) ([Azat Khuzhin](https://github.com/azat)). +* Remove metric_log/part_log overrides in tests (enabled by default) [#31030](https://github.com/ClickHouse/ClickHouse/pull/31030) ([Azat Khuzhin](https://github.com/azat)). +* improve error message while using OFFSET FETCH clause without ORDER BY [#31031](https://github.com/ClickHouse/ClickHouse/pull/31031) ([SuperDJY](https://github.com/cmsxbc)). +* Log size of remapped memory (remap_executable) [#31033](https://github.com/ClickHouse/ClickHouse/pull/31033) ([Azat Khuzhin](https://github.com/azat)). +* Separate option for enabling fuse syntax for sum, avg, count [#31035](https://github.com/ClickHouse/ClickHouse/pull/31035) ([Vladimir C](https://github.com/vdimir)). +* Add integration test on top of github actions [#31045](https://github.com/ClickHouse/ClickHouse/pull/31045) ([alesapin](https://github.com/alesapin)). +* Fix intersecting parts in `parts_to_do` 2 [#31060](https://github.com/ClickHouse/ClickHouse/pull/31060) ([Alexander Tokmakov](https://github.com/tavplubix)). +* perf: switch *_log tables to Memory engine (attempt to reduce cache misses) [#31063](https://github.com/ClickHouse/ClickHouse/pull/31063) ([Azat Khuzhin](https://github.com/azat)). +* Add new employee photos to team [#31084](https://github.com/ClickHouse/ClickHouse/pull/31084) ([Cody Baker](https://github.com/codyrobert)). +* Update button widths for responsive sizing [#31085](https://github.com/ClickHouse/ClickHouse/pull/31085) ([Cody Baker](https://github.com/codyrobert)). +* Add overflow to benchmark tables [#31086](https://github.com/ClickHouse/ClickHouse/pull/31086) ([Cody Baker](https://github.com/codyrobert)). +* Remove padding below greenhouse iframe [#31087](https://github.com/ClickHouse/ClickHouse/pull/31087) ([Cody Baker](https://github.com/codyrobert)). +* Crb update case study cards [#31088](https://github.com/ClickHouse/ClickHouse/pull/31088) ([Cody Baker](https://github.com/codyrobert)). +* Fix threadpool read for remote disks [#31112](https://github.com/ClickHouse/ClickHouse/pull/31112) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Doc build: remove single.md [#31118](https://github.com/ClickHouse/ClickHouse/pull/31118) ([lehasm](https://github.com/lehasm)). +* remove unnecessary assert in StorageFileLog [#31119](https://github.com/ClickHouse/ClickHouse/pull/31119) ([nauta](https://github.com/nautaa)). +* Add lambda for approve [#31139](https://github.com/ClickHouse/ClickHouse/pull/31139) ([alesapin](https://github.com/alesapin)). +* Do not include unnecessary experimental/type_traits [#31147](https://github.com/ClickHouse/ClickHouse/pull/31147) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Disable optimize_syntax_fuse_functions by default [#31149](https://github.com/ClickHouse/ClickHouse/pull/31149) ([Vladimir C](https://github.com/vdimir)). +* Add pvs studio to actions [#31156](https://github.com/ClickHouse/ClickHouse/pull/31156) ([alesapin](https://github.com/alesapin)). +* Add cherry-pick on github actions [#31158](https://github.com/ClickHouse/ClickHouse/pull/31158) ([alesapin](https://github.com/alesapin)). +* correct disk space calculations [#31159](https://github.com/ClickHouse/ClickHouse/pull/31159) ([Alexandre Snarskii](https://github.com/snar)). +* Fix typo [#31164](https://github.com/ClickHouse/ClickHouse/pull/31164) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update text on thank you page [#31166](https://github.com/ClickHouse/ClickHouse/pull/31166) ([Cody Baker](https://github.com/codyrobert)). +* Add unit tests to CI [#31175](https://github.com/ClickHouse/ClickHouse/pull/31175) ([alesapin](https://github.com/alesapin)). +* Debug cherry-pick CI [#31177](https://github.com/ClickHouse/ClickHouse/pull/31177) ([alesapin](https://github.com/alesapin)). +* Update docker_compose_postgres.yml [#31179](https://github.com/ClickHouse/ClickHouse/pull/31179) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix integration tests [#31223](https://github.com/ClickHouse/ClickHouse/pull/31223) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Relax test 02026_storage_filelog_largefile.sh [#31225](https://github.com/ClickHouse/ClickHouse/pull/31225) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix typo in USE_MYSQL check [#31226](https://github.com/ClickHouse/ClickHouse/pull/31226) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Download ccache if not exists in builds [#31227](https://github.com/ClickHouse/ClickHouse/pull/31227) ([alesapin](https://github.com/alesapin)). +* Disable fuzzer builds in CI [#31244](https://github.com/ClickHouse/ClickHouse/pull/31244) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add flaky check to new CI [#31248](https://github.com/ClickHouse/ClickHouse/pull/31248) ([alesapin](https://github.com/alesapin)). +* Fix test [#31250](https://github.com/ClickHouse/ClickHouse/pull/31250) ([Kseniia Sumarokova](https://github.com/kssenii)). +* move InputCreatorFunc to InputCreator [#31258](https://github.com/ClickHouse/ClickHouse/pull/31258) ([Alex Cao](https://github.com/cccgp)). +* Print warning during old directories cleanup in MergeTree only if it is old [#31259](https://github.com/ClickHouse/ClickHouse/pull/31259) ([Azat Khuzhin](https://github.com/azat)). +* Cleanup extern ProfileEvents/CurrentMetrics and add a style check [#31260](https://github.com/ClickHouse/ClickHouse/pull/31260) ([Azat Khuzhin](https://github.com/azat)). +* Fix and refactor WriteBiffer-s a little [#31265](https://github.com/ClickHouse/ClickHouse/pull/31265) ([Kruglov Pavel](https://github.com/Avogar)). +* Followup to ccache build [#31287](https://github.com/ClickHouse/ClickHouse/pull/31287) ([alesapin](https://github.com/alesapin)). +* Add compatibility check [#31294](https://github.com/ClickHouse/ClickHouse/pull/31294) ([alesapin](https://github.com/alesapin)). +* Add assertions to ZooKeeperLock [#31295](https://github.com/ClickHouse/ClickHouse/pull/31295) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add split build check [#31299](https://github.com/ClickHouse/ClickHouse/pull/31299) ([alesapin](https://github.com/alesapin)). +* Remove strange code from mutations [#31300](https://github.com/ClickHouse/ClickHouse/pull/31300) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable integration tests in new CI [#31301](https://github.com/ClickHouse/ClickHouse/pull/31301) ([alesapin](https://github.com/alesapin)). +* Merging [#31081](https://github.com/ClickHouse/ClickHouse/issues/31081) [#31305](https://github.com/ClickHouse/ClickHouse/pull/31305) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Debug rabbitmq tests [#31316](https://github.com/ClickHouse/ClickHouse/pull/31316) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible data-race in case of query cancellation with async_socket_for_remote [#31317](https://github.com/ClickHouse/ClickHouse/pull/31317) ([Azat Khuzhin](https://github.com/azat)). +* Improve fuzzer report in case of fuzzer killed [#31324](https://github.com/ClickHouse/ClickHouse/pull/31324) ([Azat Khuzhin](https://github.com/azat)). +* Fix check in async buffer [#31325](https://github.com/ClickHouse/ClickHouse/pull/31325) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add Blog Post for 21.11 Release Update [#31326](https://github.com/ClickHouse/ClickHouse/pull/31326) ([Cody Baker](https://github.com/codyrobert)). +* Add blog post for Moscow meetup [#31327](https://github.com/ClickHouse/ClickHouse/pull/31327) ([Cody Baker](https://github.com/codyrobert)). +* Do not try to resolve temporary tables from global context [#31333](https://github.com/ClickHouse/ClickHouse/pull/31333) ([Azat Khuzhin](https://github.com/azat)). +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove some copypaste from CI [#31340](https://github.com/ClickHouse/ClickHouse/pull/31340) ([alesapin](https://github.com/alesapin)). +* Fix test_kafka_insert_avro by pinning avro version [#31387](https://github.com/ClickHouse/ClickHouse/pull/31387) ([Azat Khuzhin](https://github.com/azat)). +* Fix QueryScope in MaterializedMySQLSyncThread [#31392](https://github.com/ClickHouse/ClickHouse/pull/31392) ([Azat Khuzhin](https://github.com/azat)). +* Check stderr is writable before reopining it (to avoid losing errors) [#31393](https://github.com/ClickHouse/ClickHouse/pull/31393) ([Azat Khuzhin](https://github.com/azat)). +* Remove thread_local std::string [#31400](https://github.com/ClickHouse/ClickHouse/pull/31400) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix client [#31403](https://github.com/ClickHouse/ClickHouse/pull/31403) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove excessive debug info from the log message in DDLWorker [#31406](https://github.com/ClickHouse/ClickHouse/pull/31406) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Turn on more CI checks [#31413](https://github.com/ClickHouse/ClickHouse/pull/31413) ([alesapin](https://github.com/alesapin)). +* Update description for webinar calendar links [#31433](https://github.com/ClickHouse/ClickHouse/pull/31433) ([Cody Baker](https://github.com/codyrobert)). +* Trying to debug integration tests [#31443](https://github.com/ClickHouse/ClickHouse/pull/31443) ([alesapin](https://github.com/alesapin)). +* Try increase `snapshot_distance` for functional tests [#31448](https://github.com/ClickHouse/ClickHouse/pull/31448) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Minor improvement for test_replicated_fetches_bandwidth [#31451](https://github.com/ClickHouse/ClickHouse/pull/31451) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging [#18787](https://github.com/ClickHouse/ClickHouse/issues/18787) (Constraints) [#31476](https://github.com/ClickHouse/ClickHouse/pull/31476) ([Anton Popov](https://github.com/CurtizJ)). +* Crb support page [#31490](https://github.com/ClickHouse/ClickHouse/pull/31490) ([Cody Baker](https://github.com/codyrobert)). +* Add new team members to company page [#31491](https://github.com/ClickHouse/ClickHouse/pull/31491) ([Cody Baker](https://github.com/codyrobert)). +* [ci] whitelist codyrobert [#31492](https://github.com/ClickHouse/ClickHouse/pull/31492) ([Ivan Blinkov](https://github.com/blinkov)). +* Reapply style changes to hardware page [#31506](https://github.com/ClickHouse/ClickHouse/pull/31506) ([Cody Baker](https://github.com/codyrobert)). +* Split row policy and quota headers [#31509](https://github.com/ClickHouse/ClickHouse/pull/31509) ([Vitaly Baranov](https://github.com/vitlibar)). +* Do not clean iptables rules in session-scope fixture [#31527](https://github.com/ClickHouse/ClickHouse/pull/31527) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Push tests results to CI database [#31540](https://github.com/ClickHouse/ClickHouse/pull/31540) ([alesapin](https://github.com/alesapin)). +* Remove strange multimap for mutations in StorageMergeTree [#31542](https://github.com/ClickHouse/ClickHouse/pull/31542) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove duplicated not compressed logs from CI (actions) [#31544](https://github.com/ClickHouse/ClickHouse/pull/31544) ([Azat Khuzhin](https://github.com/azat)). +* Fix 02010_lc_native flakiness (Query with id = 123456 is already running) [#31556](https://github.com/ClickHouse/ClickHouse/pull/31556) ([Azat Khuzhin](https://github.com/azat)). +* Make 01114_database_atomic more stable in debug builds [#31564](https://github.com/ClickHouse/ClickHouse/pull/31564) ([Azat Khuzhin](https://github.com/azat)). +* Fix MySQLWire format (this will also fix performance tests) [#31565](https://github.com/ClickHouse/ClickHouse/pull/31565) ([Azat Khuzhin](https://github.com/azat)). +* get Build ID via Section headers first [#31566](https://github.com/ClickHouse/ClickHouse/pull/31566) ([Ilya Golshtein](https://github.com/ilejn)). +* Try to debug expired sessions [#31584](https://github.com/ClickHouse/ClickHouse/pull/31584) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix cleanup in integration tests [#31605](https://github.com/ClickHouse/ClickHouse/pull/31605) ([Vitaly Baranov](https://github.com/vitlibar)). +* Stop all periodic reloading of all the configuration files on shutdown earlier [#31607](https://github.com/ClickHouse/ClickHouse/pull/31607) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix build requirements for unit tests [#31617](https://github.com/ClickHouse/ClickHouse/pull/31617) ([alesapin](https://github.com/alesapin)). +* Add workflow run for release branches [#31618](https://github.com/ClickHouse/ClickHouse/pull/31618) ([alesapin](https://github.com/alesapin)). +* Other event for release PR [#31619](https://github.com/ClickHouse/ClickHouse/pull/31619) ([alesapin](https://github.com/alesapin)). +* Trying push event again [#31623](https://github.com/ClickHouse/ClickHouse/pull/31623) ([alesapin](https://github.com/alesapin)). +* Add github actions for master [#31629](https://github.com/ClickHouse/ClickHouse/pull/31629) ([alesapin](https://github.com/alesapin)). +* Fix master yml [#31630](https://github.com/ClickHouse/ClickHouse/pull/31630) ([alesapin](https://github.com/alesapin)). +* fix kerberized_hadoop image [#31637](https://github.com/ClickHouse/ClickHouse/pull/31637) ([Constantine Peresypkin](https://github.com/pkit)). +* [ci] add flickerbox-tom to whitelist [#31651](https://github.com/ClickHouse/ClickHouse/pull/31651) ([Ivan Blinkov](https://github.com/blinkov)). +* Try to fix possible data race in RemoteQueryExecutorReadContext [#31652](https://github.com/ClickHouse/ClickHouse/pull/31652) ([Kruglov Pavel](https://github.com/Avogar)). +* Integration tests flaky check and small fixes [#31654](https://github.com/ClickHouse/ClickHouse/pull/31654) ([alesapin](https://github.com/alesapin)). +* Update base64 library [#31677](https://github.com/ClickHouse/ClickHouse/pull/31677) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix typo [#31678](https://github.com/ClickHouse/ClickHouse/pull/31678) ([flynn](https://github.com/ucasfl)). +* Try fix OOMs with TSAN [#31685](https://github.com/ClickHouse/ClickHouse/pull/31685) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix workflow in master [#31688](https://github.com/ClickHouse/ClickHouse/pull/31688) ([alesapin](https://github.com/alesapin)). +* Add perf test for writing valid UTF8 [#31695](https://github.com/ClickHouse/ClickHouse/pull/31695) ([Kruglov Pavel](https://github.com/Avogar)). +* hdfs disable stderr logging [#31703](https://github.com/ClickHouse/ClickHouse/pull/31703) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix possible Logical error: Cannot write to finalized buffer [#31712](https://github.com/ClickHouse/ClickHouse/pull/31712) ([Kruglov Pavel](https://github.com/Avogar)). +* 02050: Use CLICKHOUSE_TMP and delete files when finished [#31713](https://github.com/ClickHouse/ClickHouse/pull/31713) ([Raúl Marín](https://github.com/Algunenano)). +* Make 02112_with_fill_interval independent of the server timezone [#31714](https://github.com/ClickHouse/ClickHouse/pull/31714) ([Raúl Marín](https://github.com/Algunenano)). +* 02010_lc_native: Generate a new id for each query [#31720](https://github.com/ClickHouse/ClickHouse/pull/31720) ([Raúl Marín](https://github.com/Algunenano)). +* 00623_replicated_truncate_table_zookeeper_long: Wait for truncate in replicas [#31721](https://github.com/ClickHouse/ClickHouse/pull/31721) ([Raúl Marín](https://github.com/Algunenano)). +* Try to push data into another ci database. [#31724](https://github.com/ClickHouse/ClickHouse/pull/31724) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove OpenCL completely [#31744](https://github.com/ClickHouse/ClickHouse/pull/31744) ([Timur Magomedov](https://github.com/tmagomedov)). +* Fix diff for backports. [#31765](https://github.com/ClickHouse/ClickHouse/pull/31765) ([alesapin](https://github.com/alesapin)). +* Fail fasttest, builds and functional checks if some tests was not successful [#31767](https://github.com/ClickHouse/ClickHouse/pull/31767) ([alesapin](https://github.com/alesapin)). +* Improve how queries are output in the performance dashboard [#31780](https://github.com/ClickHouse/ClickHouse/pull/31780) ([Raúl Marín](https://github.com/Algunenano)). +* Use version from git describe in builds [#31782](https://github.com/ClickHouse/ClickHouse/pull/31782) ([alesapin](https://github.com/alesapin)). +* Fix stylecheck for tests/ci/push_to_artifactory.py [#31798](https://github.com/ClickHouse/ClickHouse/pull/31798) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Shorter stress and fuzzer tests [#31803](https://github.com/ClickHouse/ClickHouse/pull/31803) ([alesapin](https://github.com/alesapin)). +* Fix oss-fuzz build [#31818](https://github.com/ClickHouse/ClickHouse/pull/31818) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix check in approve lambda [#31821](https://github.com/ClickHouse/ClickHouse/pull/31821) ([alesapin](https://github.com/alesapin)). +* Cover query_views_log [#31825](https://github.com/ClickHouse/ClickHouse/pull/31825) ([Azat Khuzhin](https://github.com/azat)). +* Forbid files that differ only by character case [#31834](https://github.com/ClickHouse/ClickHouse/pull/31834) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added ubsan suppression for libprotobuf-mutator [#31835](https://github.com/ClickHouse/ClickHouse/pull/31835) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Update 01155_rename_move_materialized_view.sql [#31849](https://github.com/ClickHouse/ClickHouse/pull/31849) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix style. [#31850](https://github.com/ClickHouse/ClickHouse/pull/31850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Minor improvements to DUMP macro [#31858](https://github.com/ClickHouse/ClickHouse/pull/31858) ([Vasily Nemkov](https://github.com/Enmk)). +* Get rid of build numbers and simplify builds paths in S3 [#31861](https://github.com/ClickHouse/ClickHouse/pull/31861) ([alesapin](https://github.com/alesapin)). +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Added -no-sanitize=unsigned-integer-overflow build flag [#31881](https://github.com/ClickHouse/ClickHouse/pull/31881) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix typos [#31886](https://github.com/ClickHouse/ClickHouse/pull/31886) ([Anton Popov](https://github.com/CurtizJ)). +* Try to fix flacky test. [#31889](https://github.com/ClickHouse/ClickHouse/pull/31889) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Reduce the files that depend on parser headers [#31896](https://github.com/ClickHouse/ClickHouse/pull/31896) ([Raúl Marín](https://github.com/Algunenano)). +* Fix magic_enum for debug helpers (fixes build w/ USE_DEBUG_HELPERS) [#31922](https://github.com/ClickHouse/ClickHouse/pull/31922) ([Azat Khuzhin](https://github.com/azat)). +* Remove some trash from build [#31923](https://github.com/ClickHouse/ClickHouse/pull/31923) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add json type to changes in documentation [#31926](https://github.com/ClickHouse/ClickHouse/pull/31926) ([alesapin](https://github.com/alesapin)). +* fix some broken links [#31948](https://github.com/ClickHouse/ClickHouse/pull/31948) ([Ramazan Polat](https://github.com/ramazanpolat)). +* Kill container in integration tests if it's already running [#31950](https://github.com/ClickHouse/ClickHouse/pull/31950) ([alesapin](https://github.com/alesapin)). +* Drop libc-headers [#31951](https://github.com/ClickHouse/ClickHouse/pull/31951) ([Raúl Marín](https://github.com/Algunenano)). +* Give some love to macOS platform [#31957](https://github.com/ClickHouse/ClickHouse/pull/31957) ([Denis Glazachev](https://github.com/traceon)). +* Fix segfault in MaterializedMySQL [#31960](https://github.com/ClickHouse/ClickHouse/pull/31960) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix docs check [#31980](https://github.com/ClickHouse/ClickHouse/pull/31980) ([alesapin](https://github.com/alesapin)). +* Make stress tests slightly more stable [#31985](https://github.com/ClickHouse/ClickHouse/pull/31985) ([alesapin](https://github.com/alesapin)). +* Add rest functional tests to CI [#31987](https://github.com/ClickHouse/ClickHouse/pull/31987) ([alesapin](https://github.com/alesapin)). +* Add special builds to CI [#31991](https://github.com/ClickHouse/ClickHouse/pull/31991) ([alesapin](https://github.com/alesapin)). +* Run less tests for backport branches [#31992](https://github.com/ClickHouse/ClickHouse/pull/31992) ([alesapin](https://github.com/alesapin)). +* Fix race in ParallelFormattingOutputFormat constructor [#32004](https://github.com/ClickHouse/ClickHouse/pull/32004) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix build on master [#32009](https://github.com/ClickHouse/ClickHouse/pull/32009) ([alesapin](https://github.com/alesapin)). +* Fix style-check for ProfileEvents checking [#32033](https://github.com/ClickHouse/ClickHouse/pull/32033) ([Azat Khuzhin](https://github.com/azat)). +* Provide clickhouse binary w/o debug symbols (stripped) in fasttest [#32036](https://github.com/ClickHouse/ClickHouse/pull/32036) ([Azat Khuzhin](https://github.com/azat)). +* Minor fixes for `StorageMergeTree` [#32037](https://github.com/ClickHouse/ClickHouse/pull/32037) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Remove most of old checks [#32041](https://github.com/ClickHouse/ClickHouse/pull/32041) ([alesapin](https://github.com/alesapin)). +* Remove fast test from master [#32042](https://github.com/ClickHouse/ClickHouse/pull/32042) ([alesapin](https://github.com/alesapin)). +* Better scripts for runners [#32043](https://github.com/ClickHouse/ClickHouse/pull/32043) ([alesapin](https://github.com/alesapin)). +* Fix force tests label [#32044](https://github.com/ClickHouse/ClickHouse/pull/32044) ([alesapin](https://github.com/alesapin)). +* Don't run checks for label event [#32046](https://github.com/ClickHouse/ClickHouse/pull/32046) ([alesapin](https://github.com/alesapin)). +* Fix flaky test 00925 [#32050](https://github.com/ClickHouse/ClickHouse/pull/32050) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Cancel redundant checks with lambda [#32051](https://github.com/ClickHouse/ClickHouse/pull/32051) ([alesapin](https://github.com/alesapin)). +* Fix flaky integration test for MaterializedMySQL CREATE TABLE LIKE [#32052](https://github.com/ClickHouse/ClickHouse/pull/32052) ([Stig Bakken](https://github.com/stigsb)). +* Use functional test group for tests with thread sanitizer [#32062](https://github.com/ClickHouse/ClickHouse/pull/32062) ([alesapin](https://github.com/alesapin)). +* Add ability for lightweight checks rerun [#32064](https://github.com/ClickHouse/ClickHouse/pull/32064) ([alesapin](https://github.com/alesapin)). +* Increase length of random database in clickhouse-test [#32094](https://github.com/ClickHouse/ClickHouse/pull/32094) ([Azat Khuzhin](https://github.com/azat)). +* make looping in H3 funcs uniform [#32110](https://github.com/ClickHouse/ClickHouse/pull/32110) ([Bharat Nallan](https://github.com/bharatnc)). +* Remove PVS check from master [#32114](https://github.com/ClickHouse/ClickHouse/pull/32114) ([alesapin](https://github.com/alesapin)). +* Fix flaky keeper whitelist test [#32115](https://github.com/ClickHouse/ClickHouse/pull/32115) ([alesapin](https://github.com/alesapin)). +* Fix flacky test test_executable_storage_input [#32118](https://github.com/ClickHouse/ClickHouse/pull/32118) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix data race in `removePartAndEnqueueFetch(...)` [#32119](https://github.com/ClickHouse/ClickHouse/pull/32119) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Move fuzzers and unit tests to another group [#32120](https://github.com/ClickHouse/ClickHouse/pull/32120) ([alesapin](https://github.com/alesapin)). +* Add a test with 20000 mutations in one query [#32122](https://github.com/ClickHouse/ClickHouse/pull/32122) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Change test 02117_custom_separated_with_names_and_types [#32123](https://github.com/ClickHouse/ClickHouse/pull/32123) ([Kruglov Pavel](https://github.com/Avogar)). +* Use seq_cst semantic for MergeTreeBackgroundExecutor mertic. [#32125](https://github.com/ClickHouse/ClickHouse/pull/32125) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove 02116_global_in_time_limit. [#32126](https://github.com/ClickHouse/ClickHouse/pull/32126) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* fixing postgres tests [#32129](https://github.com/ClickHouse/ClickHouse/pull/32129) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Small improvements in lambda code [#32155](https://github.com/ClickHouse/ClickHouse/pull/32155) ([alesapin](https://github.com/alesapin)). +* Update featured image for 21.11 release blog post [#32156](https://github.com/ClickHouse/ClickHouse/pull/32156) ([Cody Baker](https://github.com/codyrobert)). +* Fix CI [#32159](https://github.com/ClickHouse/ClickHouse/pull/32159) ([alesapin](https://github.com/alesapin)). +* tests/ci: do not compress logs twice [#32162](https://github.com/ClickHouse/ClickHouse/pull/32162) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test: do not use random generator with shared state [#32163](https://github.com/ClickHouse/ClickHouse/pull/32163) ([Azat Khuzhin](https://github.com/azat)). +* Fix stress tests [#32164](https://github.com/ClickHouse/ClickHouse/pull/32164) ([Azat Khuzhin](https://github.com/azat)). +* Fix QueryProfiler (query_profiler_{cpu,real}_time_period_ns) reset [#32165](https://github.com/ClickHouse/ClickHouse/pull/32165) ([Azat Khuzhin](https://github.com/azat)). +* MaterializedMySQL support VARBINARY type [#32173](https://github.com/ClickHouse/ClickHouse/pull/32173) ([zzsmdfj](https://github.com/zzsmdfj)). +* perf: fix waiting of the server after running tests [#32174](https://github.com/ClickHouse/ClickHouse/pull/32174) ([Azat Khuzhin](https://github.com/azat)). +* Better output for some actions [#32175](https://github.com/ClickHouse/ClickHouse/pull/32175) ([alesapin](https://github.com/alesapin)). +* Use ccache in fast test [#32177](https://github.com/ClickHouse/ClickHouse/pull/32177) ([alesapin](https://github.com/alesapin)). +* Fix window view tests [#32178](https://github.com/ClickHouse/ClickHouse/pull/32178) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Function accurateCastOrDefault remove separate branch [#32184](https://github.com/ClickHouse/ClickHouse/pull/32184) ([Maksim Kita](https://github.com/kitaisreal)). +* Add test for [#32186](https://github.com/ClickHouse/ClickHouse/issues/32186) [#32203](https://github.com/ClickHouse/ClickHouse/pull/32203) ([Raúl Marín](https://github.com/Algunenano)). +* Fix uncaught exception in DatabaseLazy [#32206](https://github.com/ClickHouse/ClickHouse/pull/32206) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update ASTCreateQuery.cpp [#32208](https://github.com/ClickHouse/ClickHouse/pull/32208) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix flacky fileLog test (probably) [#32209](https://github.com/ClickHouse/ClickHouse/pull/32209) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix jemalloc under osx [#32219](https://github.com/ClickHouse/ClickHouse/pull/32219) ([Azat Khuzhin](https://github.com/azat)). +* Add missing timezones to some tests [#32222](https://github.com/ClickHouse/ClickHouse/pull/32222) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix versioning of aggregate functions (fixes performance tests) [#32236](https://github.com/ClickHouse/ClickHouse/pull/32236) ([Azat Khuzhin](https://github.com/azat)). +* Disable window view tests temporarily because still flacky [#32257](https://github.com/ClickHouse/ClickHouse/pull/32257) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix typo in tupleToNameValuePairs doc [#32262](https://github.com/ClickHouse/ClickHouse/pull/32262) ([Vladimir C](https://github.com/vdimir)). +* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix possible crash in DataTypeAggregateFunction [#32287](https://github.com/ClickHouse/ClickHouse/pull/32287) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Update backport.py [#32323](https://github.com/ClickHouse/ClickHouse/pull/32323) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix graphite-bench build [#32351](https://github.com/ClickHouse/ClickHouse/pull/32351) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Revert "graphite: split tagged/plain rollup rules (for merges perfoma… [#32376](https://github.com/ClickHouse/ClickHouse/pull/32376) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Another attempt to fix unit test Executor::RemoveTasksStress [#32390](https://github.com/ClickHouse/ClickHouse/pull/32390) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + diff --git a/docs/changelogs/v21.12.2.17-stable.md b/docs/changelogs/v21.12.2.17-stable.md index 909bc7917c7..a35006d9c57 100644 --- a/docs/changelogs/v21.12.2.17-stable.md +++ b/docs/changelogs/v21.12.2.17-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.12.2.17-stable FIXME as compared to v21.12.1.9017-prestable #### Bug Fix @@ -20,3 +27,11 @@ * Backported in [#32616](https://github.com/ClickHouse/ClickHouse/issues/32616): Fix possible crash (or incorrect result) in case of `LowCardinality` arguments of window function. Fixes [#31114](https://github.com/ClickHouse/ClickHouse/issues/31114). [#31888](https://github.com/ClickHouse/ClickHouse/pull/31888) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Follow-up to [#32140](https://github.com/ClickHouse/ClickHouse/issues/32140) [#32389](https://github.com/ClickHouse/ClickHouse/pull/32389) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash in case of MATERIALIZE COLUMN with no default expression. [#32464](https://github.com/ClickHouse/ClickHouse/pull/32464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.12.3.32-stable.md b/docs/changelogs/v21.12.3.32-stable.md index b650f62dd34..f14057981a2 100644 --- a/docs/changelogs/v21.12.3.32-stable.md +++ b/docs/changelogs/v21.12.3.32-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.12.3.32-stable FIXME as compared to v21.12.2.17-stable #### Bug Fix @@ -15,3 +22,12 @@ * Backported in [#33100](https://github.com/ClickHouse/ClickHouse/issues/33100): Fix Context leak in case of cancel_http_readonly_queries_on_client_close (i.e. leaking of external tables that had been uploaded the the server and other resources). [#32982](https://github.com/ClickHouse/ClickHouse/pull/32982) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#33123](https://github.com/ClickHouse/ClickHouse/issues/33123): Fix error `Invalid version for SerializationLowCardinality key column` in case of reading from `LowCardinality` column with `local_filesystem_read_prefetch` or `remote_filesystem_read_prefetch` enabled. [#33046](https://github.com/ClickHouse/ClickHouse/pull/33046) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Improve quota's end-of-interval calculations. [#32575](https://github.com/ClickHouse/ClickHouse/pull/32575) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix race in skipping index of type `hypothesis` [#32756](https://github.com/ClickHouse/ClickHouse/pull/32756) ([Anton Popov](https://github.com/CurtizJ)). +* Always apply const-condition-if optimization. [#32858](https://github.com/ClickHouse/ClickHouse/pull/32858) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33050](https://github.com/ClickHouse/ClickHouse/issues/33050) [#33065](https://github.com/ClickHouse/ClickHouse/pull/33065) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.12.4.1-stable.md b/docs/changelogs/v21.12.4.1-stable.md index 7c028592876..d08997378af 100644 --- a/docs/changelogs/v21.12.4.1-stable.md +++ b/docs/changelogs/v21.12.4.1-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.12.4.1-stable FIXME as compared to v21.12.3.32-stable #### Improvement diff --git a/docs/changelogs/v21.2.1.5869-prestable.md b/docs/changelogs/v21.2.1.5869-prestable.md index 43703bc13b9..ae5efdcf14f 100644 --- a/docs/changelogs/v21.2.1.5869-prestable.md +++ b/docs/changelogs/v21.2.1.5869-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.1.5869-prestable FIXME as compared to v21.1.1.5646-prestable #### Backward Incompatible Change @@ -154,3 +161,116 @@ * NO CL ENTRY: 'Remove useless codes'. [#19293](https://github.com/ClickHouse/ClickHouse/pull/19293) ([sundyli](https://github.com/sundy-li)). * NO CL ENTRY: 'Merging [#19387](https://github.com/ClickHouse/ClickHouse/issues/19387)'. [#19683](https://github.com/ClickHouse/ClickHouse/pull/19683) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add ANTLR tests check [#18624](https://github.com/ClickHouse/ClickHouse/pull/18624) ([Ivan](https://github.com/abyss7)). +* Add more debug info to PollingQueue exception. [#18922](https://github.com/ClickHouse/ClickHouse/pull/18922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More checks in merge tree writer wide [#18928](https://github.com/ClickHouse/ClickHouse/pull/18928) ([alesapin](https://github.com/alesapin)). +* Try to remove ActionsDAG::removeColumn [#18953](https://github.com/ClickHouse/ClickHouse/pull/18953) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix rarely flaky test 01459_manual_write_to_replicas_quorum [#18970](https://github.com/ClickHouse/ClickHouse/pull/18970) ([alesapin](https://github.com/alesapin)). +* fix some wrong words in comment [#18998](https://github.com/ClickHouse/ClickHouse/pull/18998) ([flynn](https://github.com/ucasfl)). +* Add tests from [#15889](https://github.com/ClickHouse/ClickHouse/issues/15889) [#19007](https://github.com/ClickHouse/ClickHouse/pull/19007) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix manipulators for common string types [#19011](https://github.com/ClickHouse/ClickHouse/pull/19011) ([alesapin](https://github.com/alesapin)). +* Fix misleading error message while inserting in a table function [#19013](https://github.com/ClickHouse/ClickHouse/pull/19013) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix duplicate UUIDs of LiveView on server startup [#19020](https://github.com/ClickHouse/ClickHouse/pull/19020) ([Alexander Tokmakov](https://github.com/tavplubix)). +* [wip] WINDOW clause [#19022](https://github.com/ClickHouse/ClickHouse/pull/19022) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Avoid redundant exception while dropping part [#19040](https://github.com/ClickHouse/ClickHouse/pull/19040) ([alesapin](https://github.com/alesapin)). +* More debug for stateless tests writer [#19055](https://github.com/ClickHouse/ClickHouse/pull/19055) ([alesapin](https://github.com/alesapin)). +* Update test containers [#19058](https://github.com/ClickHouse/ClickHouse/pull/19058) ([filimonov](https://github.com/filimonov)). +* Support operations with views in ANTLR parser [#19063](https://github.com/ClickHouse/ClickHouse/pull/19063) ([Anton Popov](https://github.com/CurtizJ)). +* Run more tests in SQLancer [#19077](https://github.com/ClickHouse/ClickHouse/pull/19077) ([Ilya Yatsishin](https://github.com/qoega)). +* Update Cassandra driver library [#19091](https://github.com/ClickHouse/ClickHouse/pull/19091) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add metrics for part number in MergeTree in ClickHouse [#19122](https://github.com/ClickHouse/ClickHouse/pull/19122) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#19136](https://github.com/ClickHouse/ClickHouse/pull/19136) ([Amos Bird](https://github.com/amosbird)). +* [ANTLR] Print errors through LOG_ERROR [#19137](https://github.com/ClickHouse/ClickHouse/pull/19137) ([Ivan](https://github.com/abyss7)). +* MemoryTracker: Do not ignore server memory limits during blocking by default [#19146](https://github.com/ClickHouse/ClickHouse/pull/19146) ([Azat Khuzhin](https://github.com/azat)). +* speed up some perf tests (for other machines) [#19154](https://github.com/ClickHouse/ClickHouse/pull/19154) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Cover distributed send with different headers [#19155](https://github.com/ClickHouse/ClickHouse/pull/19155) ([Azat Khuzhin](https://github.com/azat)). +* Add missing type check in dictHas for COMPLEX_KEY_SSD_CACHE layout [#19164](https://github.com/ClickHouse/ClickHouse/pull/19164) ([Azat Khuzhin](https://github.com/azat)). +* test for [#18839](https://github.com/ClickHouse/ClickHouse/issues/18839) Expand_macros_for_fetchPartition [#19200](https://github.com/ClickHouse/ClickHouse/pull/19200) ([Denny Crane](https://github.com/den-crane)). +* add MySQL Var check [#19205](https://github.com/ClickHouse/ClickHouse/pull/19205) ([TCeason](https://github.com/TCeason)). +* Tiny changes in DistributedBlockOutputStream [#19206](https://github.com/ClickHouse/ClickHouse/pull/19206) ([Azat Khuzhin](https://github.com/azat)). +* curl dependency tiny fixes [#19210](https://github.com/ClickHouse/ClickHouse/pull/19210) ([Azat Khuzhin](https://github.com/azat)). +* Fix MSan error in rocksdb [#19213](https://github.com/ClickHouse/ClickHouse/issues/19213) [#19214](https://github.com/ClickHouse/ClickHouse/pull/19214) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix MSan report in Kerberos library [#19215](https://github.com/ClickHouse/ClickHouse/pull/19215) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Dragonbox [#19218](https://github.com/ClickHouse/ClickHouse/pull/19218) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make symbolizers available in fuzzer Docker image [#19220](https://github.com/ClickHouse/ClickHouse/pull/19220) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add redundant test for Yandex banner system [#19235](https://github.com/ClickHouse/ClickHouse/pull/19235) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make `-- { echo }` hint preserve leading comments [#19236](https://github.com/ClickHouse/ClickHouse/pull/19236) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Disable msan for epoll array in libuv [#19244](https://github.com/ClickHouse/ClickHouse/pull/19244) ([alesapin](https://github.com/alesapin)). +* Remove tsan supression [#19250](https://github.com/ClickHouse/ClickHouse/pull/19250) ([alesapin](https://github.com/alesapin)). +* Consolidate the test hint handling [#19254](https://github.com/ClickHouse/ClickHouse/pull/19254) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Support 'keys' and 'values' subcolumns in type Map [#19273](https://github.com/ClickHouse/ClickHouse/pull/19273) ([Anton Popov](https://github.com/CurtizJ)). +* Split TestKeeperStorage and processing thread [#19284](https://github.com/ClickHouse/ClickHouse/pull/19284) ([alesapin](https://github.com/alesapin)). +* move ctr from private to delete [#19285](https://github.com/ClickHouse/ClickHouse/pull/19285) ([flynn](https://github.com/ucasfl)). +* Avoid mixing output from parallel test runs [#19298](https://github.com/ClickHouse/ClickHouse/pull/19298) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fixed race between copy-constructor and addQueryAccessInfo [#19313](https://github.com/ClickHouse/ClickHouse/pull/19313) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add Sanitizer report issue template [#19318](https://github.com/ClickHouse/ClickHouse/pull/19318) ([Ilya Yatsishin](https://github.com/qoega)). +* fix sleep with infinite input [#19343](https://github.com/ClickHouse/ClickHouse/pull/19343) ([flynn](https://github.com/ucasfl)). +* ISSUES-18684 fix MaterializeMySQL integration test failure [#19344](https://github.com/ClickHouse/ClickHouse/pull/19344) ([Winter Zhang](https://github.com/zhang2014)). +* Fix race condition in TestKeeperHandler on session finish [#19355](https://github.com/ClickHouse/ClickHouse/pull/19355) ([alesapin](https://github.com/alesapin)). +* Fix several cases, while reading subcolumns [#19358](https://github.com/ClickHouse/ClickHouse/pull/19358) ([Anton Popov](https://github.com/CurtizJ)). +* Reconnect after client errors [#19361](https://github.com/ClickHouse/ClickHouse/pull/19361) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* SQLancer binary changed its name [#19363](https://github.com/ClickHouse/ClickHouse/pull/19363) ([Ilya Yatsishin](https://github.com/qoega)). +* Better logging in MySQLHandler [#19365](https://github.com/ClickHouse/ClickHouse/pull/19365) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix IDisk::open parameters to match posix open [#19372](https://github.com/ClickHouse/ClickHouse/pull/19372) ([Alexander Gololobov](https://github.com/davenger)). +* MacOS fixed build issues [#19377](https://github.com/ClickHouse/ClickHouse/pull/19377) ([Maksim Kita](https://github.com/kitaisreal)). +* Add log message with elapsed time while pushing to view [#19378](https://github.com/ClickHouse/ClickHouse/pull/19378) ([Azat Khuzhin](https://github.com/azat)). +* Avoid UBSan report in aggregate function sum [#19385](https://github.com/ClickHouse/ClickHouse/pull/19385) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fail fast in incorrect usage of extractAllGroups [#19393](https://github.com/ClickHouse/ClickHouse/pull/19393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in quantileExactWeighted [#19394](https://github.com/ClickHouse/ClickHouse/pull/19394) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Limit max memory usage in fuzz testing [#19396](https://github.com/ClickHouse/ClickHouse/pull/19396) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Check literal types in RewriteSumIfFunctionMatcher [#19406](https://github.com/ClickHouse/ClickHouse/pull/19406) ([Vladimir C](https://github.com/vdimir)). +* mirror changes in code and comment [#19410](https://github.com/ClickHouse/ClickHouse/pull/19410) ([flynn](https://github.com/ucasfl)). +* Fix one more race in TestKeeper [#19412](https://github.com/ClickHouse/ClickHouse/pull/19412) ([alesapin](https://github.com/alesapin)). +* Continue fix for Block structure mismatch in PipelineExecuting stream [#19414](https://github.com/ClickHouse/ClickHouse/pull/19414) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Integration tests: print stderr of failed subprocess [#19431](https://github.com/ClickHouse/ClickHouse/pull/19431) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Don't dwell on the past in query fuzzer [#19442](https://github.com/ClickHouse/ClickHouse/pull/19442) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove outdated suppressions [#19444](https://github.com/ClickHouse/ClickHouse/pull/19444) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix build with unbundled poco [#19450](https://github.com/ClickHouse/ClickHouse/pull/19450) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in arraySlice and substring [#19459](https://github.com/ClickHouse/ClickHouse/pull/19459) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in runningDifference [#19460](https://github.com/ClickHouse/ClickHouse/pull/19460) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid UBSan report in greatCircleDistance [#19461](https://github.com/ClickHouse/ClickHouse/pull/19461) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* UBsan report fixes (arraySlice, addMinutes/addHours/addWeeks/addDays, sumWithOverflow(Decimal)) [#19466](https://github.com/ClickHouse/ClickHouse/pull/19466) ([Azat Khuzhin](https://github.com/azat)). +* Remove complications from FunctionsAES [#19467](https://github.com/ClickHouse/ClickHouse/pull/19467) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix insufficient args check in AES functions [#19474](https://github.com/ClickHouse/ClickHouse/pull/19474) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in avgWeighted [#19475](https://github.com/ClickHouse/ClickHouse/pull/19475) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* remove part of misleading exception message [#19487](https://github.com/ClickHouse/ClickHouse/pull/19487) ([flynn](https://github.com/ucasfl)). +* Bug fix : support const column processing in mapContains, mapKeys, mapValues functions [#19515](https://github.com/ClickHouse/ClickHouse/pull/19515) ([hexiaoting](https://github.com/hexiaoting)). +* More diagnostics in fuzzer [#19524](https://github.com/ClickHouse/ClickHouse/pull/19524) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix 01459_manual_write_to_replicas flaky test [#19551](https://github.com/ClickHouse/ClickHouse/pull/19551) ([alesapin](https://github.com/alesapin)). +* Check for hung queries or server hung in fast test [#19558](https://github.com/ClickHouse/ClickHouse/pull/19558) ([alesapin](https://github.com/alesapin)). +* DateLUTImpl::addYears(...): suppress UBSan [#19566](https://github.com/ClickHouse/ClickHouse/pull/19566) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging [#18549](https://github.com/ClickHouse/ClickHouse/issues/18549) [#19583](https://github.com/ClickHouse/ClickHouse/pull/19583) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Cassandra submodule with patch by @leshikus [#19590](https://github.com/ClickHouse/ClickHouse/pull/19590) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove unused include header [#19598](https://github.com/ClickHouse/ClickHouse/pull/19598) ([BohuTANG](https://github.com/BohuTANG)). +* Fix merge join constants [#19648](https://github.com/ClickHouse/ClickHouse/pull/19648) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better abstractions in disk interface [#19660](https://github.com/ClickHouse/ClickHouse/pull/19660) ([Anton Popov](https://github.com/CurtizJ)). +* Fix deadlock in testkeeper [#19661](https://github.com/ClickHouse/ClickHouse/pull/19661) ([alesapin](https://github.com/alesapin)). +* fix special build on clang11 [#19663](https://github.com/ClickHouse/ClickHouse/pull/19663) ([flynn](https://github.com/ucasfl)). +* Require current_database filter for tests with query_log/query_thread_log [#19675](https://github.com/ClickHouse/ClickHouse/pull/19675) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test tiny cleanup [#19676](https://github.com/ClickHouse/ClickHouse/pull/19676) ([Azat Khuzhin](https://github.com/azat)). +* Fix crash after merging ActionsDAG. [#19704](https://github.com/ClickHouse/ClickHouse/pull/19704) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix flaky test concat_nary_const_with_nonconst_segfault [#19711](https://github.com/ClickHouse/ClickHouse/pull/19711) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Run tests in parallel in flaky check [#19715](https://github.com/ClickHouse/ClickHouse/pull/19715) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix 01115_join_with_dictionary flakiness [#19723](https://github.com/ClickHouse/ClickHouse/pull/19723) ([Azat Khuzhin](https://github.com/azat)). +* add empty line after error messages in client [#19724](https://github.com/ClickHouse/ClickHouse/pull/19724) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Update QueryPlan tree optimization traverse. [#19725](https://github.com/ClickHouse/ClickHouse/pull/19725) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add missing lsof for fasttest docker image [#19751](https://github.com/ClickHouse/ClickHouse/pull/19751) ([Azat Khuzhin](https://github.com/azat)). +* Make Fuzzer more reliable [#19752](https://github.com/ClickHouse/ClickHouse/pull/19752) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor code improvement in JOIN [#19758](https://github.com/ClickHouse/ClickHouse/pull/19758) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make integration odbc tests idempotent [#19759](https://github.com/ClickHouse/ClickHouse/pull/19759) ([alesapin](https://github.com/alesapin)). +* Small fixes for fasttest [#19760](https://github.com/ClickHouse/ClickHouse/pull/19760) ([alesapin](https://github.com/alesapin)). +* LowCardinality UUID fix [#19767](https://github.com/ClickHouse/ClickHouse/pull/19767) ([Maksim Kita](https://github.com/kitaisreal)). +* Add log comment when running .sh tests [#19774](https://github.com/ClickHouse/ClickHouse/pull/19774) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in "round" [#19786](https://github.com/ClickHouse/ClickHouse/pull/19786) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test script for [#19794](https://github.com/ClickHouse/ClickHouse/issues/19794) [#19798](https://github.com/ClickHouse/ClickHouse/pull/19798) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix dependency on ODBC for Yandex internal build [#19804](https://github.com/ClickHouse/ClickHouse/pull/19804) ([Alexander Gololobov](https://github.com/davenger)). +* Don't run all stateless tests in parallel [#19806](https://github.com/ClickHouse/ClickHouse/pull/19806) ([alesapin](https://github.com/alesapin)). +* Avoid losing exception messages in logs under high memory pressure [#19824](https://github.com/ClickHouse/ClickHouse/pull/19824) ([Azat Khuzhin](https://github.com/azat)). +* Try to make test_dir.tar smaller [#19833](https://github.com/ClickHouse/ClickHouse/pull/19833) ([filimonov](https://github.com/filimonov)). +* style-check tiny fixes [#19834](https://github.com/ClickHouse/ClickHouse/pull/19834) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in DateTimeAddInterval [#19859](https://github.com/ClickHouse/ClickHouse/pull/19859) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix debug assertion in Hyperscan [#19860](https://github.com/ClickHouse/ClickHouse/pull/19860) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in GatherUtils [#19862](https://github.com/ClickHouse/ClickHouse/pull/19862) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.2.10.48-stable.md b/docs/changelogs/v21.2.10.48-stable.md index 11eea960931..f9d29374e91 100644 --- a/docs/changelogs/v21.2.10.48-stable.md +++ b/docs/changelogs/v21.2.10.48-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.10.48-stable FIXME as compared to v21.2.9.41-stable #### Improvement @@ -9,3 +16,7 @@ * Backported in [#23032](https://github.com/ClickHouse/ClickHouse/issues/23032): Fix error `Cannot find column in ActionsDAG result` which may happen if subquery uses `untuple`. Fixes [#22290](https://github.com/ClickHouse/ClickHouse/issues/22290). [#22991](https://github.com/ClickHouse/ClickHouse/pull/22991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#23171](https://github.com/ClickHouse/ClickHouse/issues/23171): Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* LibraryDictionarySource fix possible leak [#21686](https://github.com/ClickHouse/ClickHouse/pull/21686) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.2.2.8-stable.md b/docs/changelogs/v21.2.2.8-stable.md index 73baea91547..d3b1f43bcc3 100644 --- a/docs/changelogs/v21.2.2.8-stable.md +++ b/docs/changelogs/v21.2.2.8-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.2.8-stable FIXME as compared to v21.1.1.5646-prestable #### Backward Incompatible Change @@ -68,7 +75,7 @@ * Add separate pool for message brokers (RabbitMQ and Kafka). [#19722](https://github.com/ClickHouse/ClickHouse/pull/19722) ([Azat Khuzhin](https://github.com/azat)). * In distributed queries if the setting `async_socket_for_remote` is enabled, it was possible to get stack overflow at least in debug build configuration if very deeply nested data type is used in table (e.g. `Array(Array(Array(...more...)))`). This fixes [#19108](https://github.com/ClickHouse/ClickHouse/issues/19108). This change introduces minor backward incompatibility: excessive parenthesis in type definitions no longer supported, example: `Array((UInt8))`. [#19736](https://github.com/ClickHouse/ClickHouse/pull/19736) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Table function `S3` will use global region if the region can't be determined exactly. This closes [#10998](https://github.com/ClickHouse/ClickHouse/issues/10998). [#19750](https://github.com/ClickHouse/ClickHouse/pull/19750) ([Vladimir Chebotarev](https://github.com/excitoon)). -* ClickHouse client query param CTE added test. [#19762](https://github.com/ClickHouse/ClickHouse/pull/19762) ([Maksim Kita](https://github.com/kitaisreal)). +* Clickhouse client query param CTE added test. [#19762](https://github.com/ClickHouse/ClickHouse/pull/19762) ([Maksim Kita](https://github.com/kitaisreal)). * Correctly output infinite arguments for `formatReadableTimeDelta` function. In previous versions, there was implicit conversion to implementation specific integer value. [#19791](https://github.com/ClickHouse/ClickHouse/pull/19791) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * `S3` table function now supports `auto` compression mode (autodetect). This closes [#18754](https://github.com/ClickHouse/ClickHouse/issues/18754). [#19793](https://github.com/ClickHouse/ClickHouse/pull/19793) ([Vladimir Chebotarev](https://github.com/excitoon)). * Set charset to utf8mb4 when interacting with remote MySQL servers. Fixes [#19795](https://github.com/ClickHouse/ClickHouse/issues/19795). [#19800](https://github.com/ClickHouse/ClickHouse/pull/19800) ([Alexey Milovidov](https://github.com/alexey-milovidov)). @@ -164,3 +171,117 @@ * NO CL ENTRY: 'Remove useless codes'. [#19293](https://github.com/ClickHouse/ClickHouse/pull/19293) ([sundyli](https://github.com/sundy-li)). * NO CL ENTRY: 'Merging [#19387](https://github.com/ClickHouse/ClickHouse/issues/19387)'. [#19683](https://github.com/ClickHouse/ClickHouse/pull/19683) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add ANTLR tests check [#18624](https://github.com/ClickHouse/ClickHouse/pull/18624) ([Ivan](https://github.com/abyss7)). +* Add more debug info to PollingQueue exception. [#18922](https://github.com/ClickHouse/ClickHouse/pull/18922) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More checks in merge tree writer wide [#18928](https://github.com/ClickHouse/ClickHouse/pull/18928) ([alesapin](https://github.com/alesapin)). +* Try to remove ActionsDAG::removeColumn [#18953](https://github.com/ClickHouse/ClickHouse/pull/18953) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix rarely flaky test 01459_manual_write_to_replicas_quorum [#18970](https://github.com/ClickHouse/ClickHouse/pull/18970) ([alesapin](https://github.com/alesapin)). +* fix some wrong words in comment [#18998](https://github.com/ClickHouse/ClickHouse/pull/18998) ([flynn](https://github.com/ucasfl)). +* Add tests from [#15889](https://github.com/ClickHouse/ClickHouse/issues/15889) [#19007](https://github.com/ClickHouse/ClickHouse/pull/19007) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix manipulators for common string types [#19011](https://github.com/ClickHouse/ClickHouse/pull/19011) ([alesapin](https://github.com/alesapin)). +* Fix misleading error message while inserting in a table function [#19013](https://github.com/ClickHouse/ClickHouse/pull/19013) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix duplicate UUIDs of LiveView on server startup [#19020](https://github.com/ClickHouse/ClickHouse/pull/19020) ([Alexander Tokmakov](https://github.com/tavplubix)). +* [wip] WINDOW clause [#19022](https://github.com/ClickHouse/ClickHouse/pull/19022) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Avoid redundant exception while dropping part [#19040](https://github.com/ClickHouse/ClickHouse/pull/19040) ([alesapin](https://github.com/alesapin)). +* More debug for stateless tests writer [#19055](https://github.com/ClickHouse/ClickHouse/pull/19055) ([alesapin](https://github.com/alesapin)). +* Update test containers [#19058](https://github.com/ClickHouse/ClickHouse/pull/19058) ([filimonov](https://github.com/filimonov)). +* Support operations with views in ANTLR parser [#19063](https://github.com/ClickHouse/ClickHouse/pull/19063) ([Anton Popov](https://github.com/CurtizJ)). +* Run more tests in SQLancer [#19077](https://github.com/ClickHouse/ClickHouse/pull/19077) ([Ilya Yatsishin](https://github.com/qoega)). +* Update Cassandra driver library [#19091](https://github.com/ClickHouse/ClickHouse/pull/19091) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add metrics for part number in MergeTree in ClickHouse [#19122](https://github.com/ClickHouse/ClickHouse/pull/19122) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#19136](https://github.com/ClickHouse/ClickHouse/pull/19136) ([Amos Bird](https://github.com/amosbird)). +* [ANTLR] Print errors through LOG_ERROR [#19137](https://github.com/ClickHouse/ClickHouse/pull/19137) ([Ivan](https://github.com/abyss7)). +* MemoryTracker: Do not ignore server memory limits during blocking by default [#19146](https://github.com/ClickHouse/ClickHouse/pull/19146) ([Azat Khuzhin](https://github.com/azat)). +* speed up some perf tests (for other machines) [#19154](https://github.com/ClickHouse/ClickHouse/pull/19154) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Cover distributed send with different headers [#19155](https://github.com/ClickHouse/ClickHouse/pull/19155) ([Azat Khuzhin](https://github.com/azat)). +* Add missing type check in dictHas for COMPLEX_KEY_SSD_CACHE layout [#19164](https://github.com/ClickHouse/ClickHouse/pull/19164) ([Azat Khuzhin](https://github.com/azat)). +* test for [#18839](https://github.com/ClickHouse/ClickHouse/issues/18839) Expand_macros_for_fetchPartition [#19200](https://github.com/ClickHouse/ClickHouse/pull/19200) ([Denny Crane](https://github.com/den-crane)). +* add MySQL Var check [#19205](https://github.com/ClickHouse/ClickHouse/pull/19205) ([TCeason](https://github.com/TCeason)). +* Tiny changes in DistributedBlockOutputStream [#19206](https://github.com/ClickHouse/ClickHouse/pull/19206) ([Azat Khuzhin](https://github.com/azat)). +* curl dependency tiny fixes [#19210](https://github.com/ClickHouse/ClickHouse/pull/19210) ([Azat Khuzhin](https://github.com/azat)). +* Fix MSan error in rocksdb [#19213](https://github.com/ClickHouse/ClickHouse/issues/19213) [#19214](https://github.com/ClickHouse/ClickHouse/pull/19214) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix MSan report in Kerberos library [#19215](https://github.com/ClickHouse/ClickHouse/pull/19215) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Dragonbox [#19218](https://github.com/ClickHouse/ClickHouse/pull/19218) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make symbolizers available in fuzzer Docker image [#19220](https://github.com/ClickHouse/ClickHouse/pull/19220) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add redundant test for Yandex banner system [#19235](https://github.com/ClickHouse/ClickHouse/pull/19235) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make `-- { echo }` hint preserve leading comments [#19236](https://github.com/ClickHouse/ClickHouse/pull/19236) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Disable msan for epoll array in libuv [#19244](https://github.com/ClickHouse/ClickHouse/pull/19244) ([alesapin](https://github.com/alesapin)). +* Remove tsan supression [#19250](https://github.com/ClickHouse/ClickHouse/pull/19250) ([alesapin](https://github.com/alesapin)). +* Consolidate the test hint handling [#19254](https://github.com/ClickHouse/ClickHouse/pull/19254) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Support 'keys' and 'values' subcolumns in type Map [#19273](https://github.com/ClickHouse/ClickHouse/pull/19273) ([Anton Popov](https://github.com/CurtizJ)). +* Split TestKeeperStorage and processing thread [#19284](https://github.com/ClickHouse/ClickHouse/pull/19284) ([alesapin](https://github.com/alesapin)). +* move ctr from private to delete [#19285](https://github.com/ClickHouse/ClickHouse/pull/19285) ([flynn](https://github.com/ucasfl)). +* Avoid mixing output from parallel test runs [#19298](https://github.com/ClickHouse/ClickHouse/pull/19298) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fixed race between copy-constructor and addQueryAccessInfo [#19313](https://github.com/ClickHouse/ClickHouse/pull/19313) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add Sanitizer report issue template [#19318](https://github.com/ClickHouse/ClickHouse/pull/19318) ([Ilya Yatsishin](https://github.com/qoega)). +* fix sleep with infinite input [#19343](https://github.com/ClickHouse/ClickHouse/pull/19343) ([flynn](https://github.com/ucasfl)). +* ISSUES-18684 fix MaterializeMySQL integration test failure [#19344](https://github.com/ClickHouse/ClickHouse/pull/19344) ([Winter Zhang](https://github.com/zhang2014)). +* Fix race condition in TestKeeperHandler on session finish [#19355](https://github.com/ClickHouse/ClickHouse/pull/19355) ([alesapin](https://github.com/alesapin)). +* Fix several cases, while reading subcolumns [#19358](https://github.com/ClickHouse/ClickHouse/pull/19358) ([Anton Popov](https://github.com/CurtizJ)). +* Reconnect after client errors [#19361](https://github.com/ClickHouse/ClickHouse/pull/19361) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* SQLancer binary changed its name [#19363](https://github.com/ClickHouse/ClickHouse/pull/19363) ([Ilya Yatsishin](https://github.com/qoega)). +* Better logging in MySQLHandler [#19365](https://github.com/ClickHouse/ClickHouse/pull/19365) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix IDisk::open parameters to match posix open [#19372](https://github.com/ClickHouse/ClickHouse/pull/19372) ([Alexander Gololobov](https://github.com/davenger)). +* MacOS fixed build issues [#19377](https://github.com/ClickHouse/ClickHouse/pull/19377) ([Maksim Kita](https://github.com/kitaisreal)). +* Add log message with elapsed time while pushing to view [#19378](https://github.com/ClickHouse/ClickHouse/pull/19378) ([Azat Khuzhin](https://github.com/azat)). +* Avoid UBSan report in aggregate function sum [#19385](https://github.com/ClickHouse/ClickHouse/pull/19385) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fail fast in incorrect usage of extractAllGroups [#19393](https://github.com/ClickHouse/ClickHouse/pull/19393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in quantileExactWeighted [#19394](https://github.com/ClickHouse/ClickHouse/pull/19394) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Limit max memory usage in fuzz testing [#19396](https://github.com/ClickHouse/ClickHouse/pull/19396) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Check literal types in RewriteSumIfFunctionMatcher [#19406](https://github.com/ClickHouse/ClickHouse/pull/19406) ([Vladimir C](https://github.com/vdimir)). +* mirror changes in code and comment [#19410](https://github.com/ClickHouse/ClickHouse/pull/19410) ([flynn](https://github.com/ucasfl)). +* Fix one more race in TestKeeper [#19412](https://github.com/ClickHouse/ClickHouse/pull/19412) ([alesapin](https://github.com/alesapin)). +* Continue fix for Block structure mismatch in PipelineExecuting stream [#19414](https://github.com/ClickHouse/ClickHouse/pull/19414) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Integration tests: print stderr of failed subprocess [#19431](https://github.com/ClickHouse/ClickHouse/pull/19431) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Don't dwell on the past in query fuzzer [#19442](https://github.com/ClickHouse/ClickHouse/pull/19442) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove outdated suppressions [#19444](https://github.com/ClickHouse/ClickHouse/pull/19444) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix build with unbundled poco [#19450](https://github.com/ClickHouse/ClickHouse/pull/19450) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in arraySlice and substring [#19459](https://github.com/ClickHouse/ClickHouse/pull/19459) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in runningDifference [#19460](https://github.com/ClickHouse/ClickHouse/pull/19460) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Avoid UBSan report in greatCircleDistance [#19461](https://github.com/ClickHouse/ClickHouse/pull/19461) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* UBsan report fixes (arraySlice, addMinutes/addHours/addWeeks/addDays, sumWithOverflow(Decimal)) [#19466](https://github.com/ClickHouse/ClickHouse/pull/19466) ([Azat Khuzhin](https://github.com/azat)). +* Remove complications from FunctionsAES [#19467](https://github.com/ClickHouse/ClickHouse/pull/19467) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix insufficient args check in AES functions [#19474](https://github.com/ClickHouse/ClickHouse/pull/19474) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in avgWeighted [#19475](https://github.com/ClickHouse/ClickHouse/pull/19475) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* remove part of misleading exception message [#19487](https://github.com/ClickHouse/ClickHouse/pull/19487) ([flynn](https://github.com/ucasfl)). +* Bug fix : support const column processing in mapContains, mapKeys, mapValues functions [#19515](https://github.com/ClickHouse/ClickHouse/pull/19515) ([hexiaoting](https://github.com/hexiaoting)). +* More diagnostics in fuzzer [#19524](https://github.com/ClickHouse/ClickHouse/pull/19524) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix 01459_manual_write_to_replicas flaky test [#19551](https://github.com/ClickHouse/ClickHouse/pull/19551) ([alesapin](https://github.com/alesapin)). +* Check for hung queries or server hung in fast test [#19558](https://github.com/ClickHouse/ClickHouse/pull/19558) ([alesapin](https://github.com/alesapin)). +* DateLUTImpl::addYears(...): suppress UBSan [#19566](https://github.com/ClickHouse/ClickHouse/pull/19566) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging [#18549](https://github.com/ClickHouse/ClickHouse/issues/18549) [#19583](https://github.com/ClickHouse/ClickHouse/pull/19583) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Cassandra submodule with patch by @leshikus [#19590](https://github.com/ClickHouse/ClickHouse/pull/19590) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove unused include header [#19598](https://github.com/ClickHouse/ClickHouse/pull/19598) ([BohuTANG](https://github.com/BohuTANG)). +* Fix merge join constants [#19648](https://github.com/ClickHouse/ClickHouse/pull/19648) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better abstractions in disk interface [#19660](https://github.com/ClickHouse/ClickHouse/pull/19660) ([Anton Popov](https://github.com/CurtizJ)). +* Fix deadlock in testkeeper [#19661](https://github.com/ClickHouse/ClickHouse/pull/19661) ([alesapin](https://github.com/alesapin)). +* fix special build on clang11 [#19663](https://github.com/ClickHouse/ClickHouse/pull/19663) ([flynn](https://github.com/ucasfl)). +* Require current_database filter for tests with query_log/query_thread_log [#19675](https://github.com/ClickHouse/ClickHouse/pull/19675) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test tiny cleanup [#19676](https://github.com/ClickHouse/ClickHouse/pull/19676) ([Azat Khuzhin](https://github.com/azat)). +* Fix crash after merging ActionsDAG. [#19704](https://github.com/ClickHouse/ClickHouse/pull/19704) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix flaky test concat_nary_const_with_nonconst_segfault [#19711](https://github.com/ClickHouse/ClickHouse/pull/19711) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Run tests in parallel in flaky check [#19715](https://github.com/ClickHouse/ClickHouse/pull/19715) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix 01115_join_with_dictionary flakiness [#19723](https://github.com/ClickHouse/ClickHouse/pull/19723) ([Azat Khuzhin](https://github.com/azat)). +* add empty line after error messages in client [#19724](https://github.com/ClickHouse/ClickHouse/pull/19724) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Update QueryPlan tree optimization traverse. [#19725](https://github.com/ClickHouse/ClickHouse/pull/19725) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add missing lsof for fasttest docker image [#19751](https://github.com/ClickHouse/ClickHouse/pull/19751) ([Azat Khuzhin](https://github.com/azat)). +* Make Fuzzer more reliable [#19752](https://github.com/ClickHouse/ClickHouse/pull/19752) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor code improvement in JOIN [#19758](https://github.com/ClickHouse/ClickHouse/pull/19758) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Make integration odbc tests idempotent [#19759](https://github.com/ClickHouse/ClickHouse/pull/19759) ([alesapin](https://github.com/alesapin)). +* Small fixes for fasttest [#19760](https://github.com/ClickHouse/ClickHouse/pull/19760) ([alesapin](https://github.com/alesapin)). +* LowCardinality UUID fix [#19767](https://github.com/ClickHouse/ClickHouse/pull/19767) ([Maksim Kita](https://github.com/kitaisreal)). +* Add log comment when running .sh tests [#19774](https://github.com/ClickHouse/ClickHouse/pull/19774) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in "round" [#19786](https://github.com/ClickHouse/ClickHouse/pull/19786) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test script for [#19794](https://github.com/ClickHouse/ClickHouse/issues/19794) [#19798](https://github.com/ClickHouse/ClickHouse/pull/19798) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix dependency on ODBC for Yandex internal build [#19804](https://github.com/ClickHouse/ClickHouse/pull/19804) ([Alexander Gololobov](https://github.com/davenger)). +* Don't run all stateless tests in parallel [#19806](https://github.com/ClickHouse/ClickHouse/pull/19806) ([alesapin](https://github.com/alesapin)). +* Avoid losing exception messages in logs under high memory pressure [#19824](https://github.com/ClickHouse/ClickHouse/pull/19824) ([Azat Khuzhin](https://github.com/azat)). +* Try to make test_dir.tar smaller [#19833](https://github.com/ClickHouse/ClickHouse/pull/19833) ([filimonov](https://github.com/filimonov)). +* style-check tiny fixes [#19834](https://github.com/ClickHouse/ClickHouse/pull/19834) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in DateTimeAddInterval [#19859](https://github.com/ClickHouse/ClickHouse/pull/19859) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix debug assertion in Hyperscan [#19860](https://github.com/ClickHouse/ClickHouse/pull/19860) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in GatherUtils [#19862](https://github.com/ClickHouse/ClickHouse/pull/19862) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use fixed version of confluent-kafka library in integration tests [#20124](https://github.com/ClickHouse/ClickHouse/pull/20124) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v21.2.3.15-stable.md b/docs/changelogs/v21.2.3.15-stable.md index 26653e780bb..0471c499748 100644 --- a/docs/changelogs/v21.2.3.15-stable.md +++ b/docs/changelogs/v21.2.3.15-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.3.15-stable FIXME as compared to v21.2.2.8-stable #### Bug Fix @@ -17,3 +24,7 @@ * NO CL ENTRY: 'Revert "Backport [#20224](https://github.com/ClickHouse/ClickHouse/issues/20224) to 21.2: Fix access control manager destruction order"'. [#20397](https://github.com/ClickHouse/ClickHouse/pull/20397) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Handle syntax error for ARRAY JOIN with no args [#20223](https://github.com/ClickHouse/ClickHouse/pull/20223) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.2.4.6-stable.md b/docs/changelogs/v21.2.4.6-stable.md index 1605186701d..709abbcf730 100644 --- a/docs/changelogs/v21.2.4.6-stable.md +++ b/docs/changelogs/v21.2.4.6-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.4.6-stable FIXME as compared to v21.2.3.15-stable #### Bug Fix diff --git a/docs/changelogs/v21.2.5.5-stable.md b/docs/changelogs/v21.2.5.5-stable.md index b5275e89519..c4b16d8123e 100644 --- a/docs/changelogs/v21.2.5.5-stable.md +++ b/docs/changelogs/v21.2.5.5-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.5.5-stable FIXME as compared to v21.2.4.6-stable #### Bug Fix diff --git a/docs/changelogs/v21.2.6.1-stable.md b/docs/changelogs/v21.2.6.1-stable.md index 1f28c14c485..ef7b13e260e 100644 --- a/docs/changelogs/v21.2.6.1-stable.md +++ b/docs/changelogs/v21.2.6.1-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.6.1-stable FIXME as compared to v21.2.5.5-stable #### Bug Fix diff --git a/docs/changelogs/v21.2.7.11-stable.md b/docs/changelogs/v21.2.7.11-stable.md index 1f0f94ee0bf..277ddb9fcb3 100644 --- a/docs/changelogs/v21.2.7.11-stable.md +++ b/docs/changelogs/v21.2.7.11-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.7.11-stable FIXME as compared to v21.2.6.1-stable #### Bug Fix @@ -10,3 +17,7 @@ * Backported in [#21979](https://github.com/ClickHouse/ClickHouse/issues/21979): Reverted [#15454](https://github.com/ClickHouse/ClickHouse/issues/15454) that may cause significant increase in memory usage while loading external dictionaries of hashed type. This closes [#21935](https://github.com/ClickHouse/ClickHouse/issues/21935). [#21948](https://github.com/ClickHouse/ClickHouse/pull/21948) ([Maksim Kita](https://github.com/kitaisreal)). * Backported in [#22140](https://github.com/ClickHouse/ClickHouse/issues/22140): The function `decrypt` was lacking a check for the minimal size of data encrypted in AEAD mode. This closes [#21897](https://github.com/ClickHouse/ClickHouse/issues/21897). [#22064](https://github.com/ClickHouse/ClickHouse/pull/22064) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* LRUCache fix exception unsafe element insertion [#21891](https://github.com/ClickHouse/ClickHouse/pull/21891) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.2.8.31-stable.md b/docs/changelogs/v21.2.8.31-stable.md index 884dcb5a649..d7f7e5a7099 100644 --- a/docs/changelogs/v21.2.8.31-stable.md +++ b/docs/changelogs/v21.2.8.31-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.8.31-stable FIXME as compared to v21.2.7.11-stable #### Bug Fix @@ -28,3 +35,7 @@ * Backported in [#22699](https://github.com/ClickHouse/ClickHouse/issues/22699): Fix wait for mutations on several replicas for ReplicatedMergeTree table engines. Previously, mutation/alter query may finish before mutation actually executed on other replicas. [#22669](https://github.com/ClickHouse/ClickHouse/pull/22669) ([alesapin](https://github.com/alesapin)). * Backported in [#22740](https://github.com/ClickHouse/ClickHouse/issues/22740): Fix possible hangs in zk requests in case of OOM exception. Fixes [#22438](https://github.com/ClickHouse/ClickHouse/issues/22438). [#22684](https://github.com/ClickHouse/ClickHouse/pull/22684) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Mannual backport of [#21429](https://github.com/ClickHouse/ClickHouse/issues/21429) in 21.2 [#22505](https://github.com/ClickHouse/ClickHouse/pull/22505) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.2.9.41-stable.md b/docs/changelogs/v21.2.9.41-stable.md index ab4303aaa2a..d3a8644a879 100644 --- a/docs/changelogs/v21.2.9.41-stable.md +++ b/docs/changelogs/v21.2.9.41-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.2.9.41-stable FIXME as compared to v21.2.8.31-stable #### Improvement @@ -15,3 +22,8 @@ #### Build/Testing/Packaging Improvement * Backported in [#22814](https://github.com/ClickHouse/ClickHouse/issues/22814): Allow to start up with modified binary under gdb. In previous version if you set up breakpoint in gdb before start, server will refuse to start up due to failed integrity check. [#21258](https://github.com/ClickHouse/ClickHouse/pull/21258) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Check for EINTR in epoll_wait [#20958](https://github.com/ClickHouse/ClickHouse/pull/20958) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* FileDictionarySource fix absolute file path [#22822](https://github.com/ClickHouse/ClickHouse/pull/22822) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.1.6185-prestable.md b/docs/changelogs/v21.3.1.6185-prestable.md index dabfe3cfeb3..8d2a222f8aa 100644 --- a/docs/changelogs/v21.3.1.6185-prestable.md +++ b/docs/changelogs/v21.3.1.6185-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.1.6185-prestable FIXME as compared to v21.2.1.5869-prestable #### Backward Incompatible Change @@ -157,3 +164,162 @@ * NO CL ENTRY: 'Revert "Fix access control manager destruction order"'. [#20394](https://github.com/ClickHouse/ClickHouse/pull/20394) ([alesapin](https://github.com/alesapin)). * NO CL ENTRY: 'Update argmax.md '. [#20625](https://github.com/ClickHouse/ClickHouse/pull/20625) ([Marvin Taschenberger](https://github.com/Taschenbergerm)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Update Pytest check [#18972](https://github.com/ClickHouse/ClickHouse/pull/18972) ([Ivan](https://github.com/abyss7)). +* [wip] support RANGE frame for window functions [#19299](https://github.com/ClickHouse/ClickHouse/pull/19299) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* In memory coordination inside ClickHouse [#19580](https://github.com/ClickHouse/ClickHouse/pull/19580) ([alesapin](https://github.com/alesapin)). +* client: more suggestions [#19584](https://github.com/ClickHouse/ClickHouse/pull/19584) ([Azat Khuzhin](https://github.com/azat)). +* Allow to run all style checks in one file [#19726](https://github.com/ClickHouse/ClickHouse/pull/19726) ([Anton Popov](https://github.com/CurtizJ)). +* Fix [#19371](https://github.com/ClickHouse/ClickHouse/issues/19371) [#19765](https://github.com/ClickHouse/ClickHouse/pull/19765) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor code improvements around ThreadStatus [#19771](https://github.com/ClickHouse/ClickHouse/pull/19771) ([Alexander Tokmakov](https://github.com/tavplubix)). +* continue of [#19487](https://github.com/ClickHouse/ClickHouse/issues/19487) [#19801](https://github.com/ClickHouse/ClickHouse/pull/19801) ([flynn](https://github.com/ucasfl)). +* Fix UBSan report in intDiv [#19876](https://github.com/ClickHouse/ClickHouse/pull/19876) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow writing partial buffer [#19886](https://github.com/ClickHouse/ClickHouse/pull/19886) ([Azat Khuzhin](https://github.com/azat)). +* Remove an always-false condition from query parser [#19919](https://github.com/ClickHouse/ClickHouse/pull/19919) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* UNBOUNDED FOLLOWING frame end [#19921](https://github.com/ClickHouse/ClickHouse/pull/19921) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix logical error in INSERT VALUES [#19925](https://github.com/ClickHouse/ClickHouse/pull/19925) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix test keeper integration tests [#19942](https://github.com/ClickHouse/ClickHouse/pull/19942) ([alesapin](https://github.com/alesapin)). +* Fix build [#19948](https://github.com/ClickHouse/ClickHouse/pull/19948) ([Kruglov Pavel](https://github.com/Avogar)). +* CURRENT ROW and offset for start of ROWS frame [#19951](https://github.com/ClickHouse/ClickHouse/pull/19951) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix UBSan report in geoHashesInBox [#19956](https://github.com/ClickHouse/ClickHouse/pull/19956) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add MSan annotation for system.stack_trace [#19957](https://github.com/ClickHouse/ClickHouse/pull/19957) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky integration tests [#19971](https://github.com/ClickHouse/ClickHouse/pull/19971) ([Azat Khuzhin](https://github.com/azat)). +* Do not use inputs which values are known constants in ActionsDAG. [#19991](https://github.com/ClickHouse/ClickHouse/pull/19991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* update perf tests [#20000](https://github.com/ClickHouse/ClickHouse/pull/20000) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* UBsan fixes [#20011](https://github.com/ClickHouse/ClickHouse/pull/20011) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless code [#20029](https://github.com/ClickHouse/ClickHouse/pull/20029) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* update changelog [#20033](https://github.com/ClickHouse/ClickHouse/pull/20033) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Close input ports in DelayedPortsProcessor as soon as all outputs are finished [#20035](https://github.com/ClickHouse/ClickHouse/pull/20035) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* ReadBuffer: check for unread data on next() [#20037](https://github.com/ClickHouse/ClickHouse/pull/20037) ([Ivan](https://github.com/abyss7)). +* ReadBuffer: check that buffer position is not set beyond end [#20039](https://github.com/ClickHouse/ClickHouse/pull/20039) ([Ivan](https://github.com/abyss7)). +* CURRENT ROW frame start for RANGE frame [#20041](https://github.com/ClickHouse/ClickHouse/pull/20041) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix log message with elapsed time while pushing to views [#20047](https://github.com/ClickHouse/ClickHouse/pull/20047) ([Azat Khuzhin](https://github.com/azat)). +* Avoid UBSan report in pointInPolygon [#20049](https://github.com/ClickHouse/ClickHouse/pull/20049) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix tests *.reference files [#20050](https://github.com/ClickHouse/ClickHouse/pull/20050) ([Azat Khuzhin](https://github.com/azat)). +* ROWS OFFSET frame end [#20060](https://github.com/ClickHouse/ClickHouse/pull/20060) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add more tests for modulo of division of negative number [#20063](https://github.com/ClickHouse/ClickHouse/pull/20063) ([hexiaoting](https://github.com/hexiaoting)). +* detect unmarked long tests in flaky check [#20068](https://github.com/ClickHouse/ClickHouse/pull/20068) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix removing of filter column when split filter actions. [#20073](https://github.com/ClickHouse/ClickHouse/pull/20073) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove AddingConstColumn step and transform. [#20077](https://github.com/ClickHouse/ClickHouse/pull/20077) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* LimitReadBuffer: check that position always advances [#20078](https://github.com/ClickHouse/ClickHouse/pull/20078) ([Ivan](https://github.com/abyss7)). +* Add fuzzer for ColumnsDescription [#20094](https://github.com/ClickHouse/ClickHouse/pull/20094) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Build actions dag to evaluate missing defaults. [#20097](https://github.com/ClickHouse/ClickHouse/pull/20097) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove never existing insert_in_memory_parts_timeout [#20099](https://github.com/ClickHouse/ClickHouse/pull/20099) ([Azat Khuzhin](https://github.com/azat)). +* RANGE OFFSET window frame [#20111](https://github.com/ClickHouse/ClickHouse/pull/20111) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Allow to drop readonly tables [#20114](https://github.com/ClickHouse/ClickHouse/pull/20114) ([nvartolomei](https://github.com/nvartolomei)). +* Add logging if Poco cannot allocate thread in tcp server [#20120](https://github.com/ClickHouse/ClickHouse/pull/20120) ([alesapin](https://github.com/alesapin)). +* Use fixed version of confluent-kafka library in integration tests [#20124](https://github.com/ClickHouse/ClickHouse/pull/20124) ([alesapin](https://github.com/alesapin)). +* Useless changes [#20150](https://github.com/ClickHouse/ClickHouse/pull/20150) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* remove some useless code [#20152](https://github.com/ClickHouse/ClickHouse/pull/20152) ([flynn](https://github.com/ucasfl)). +* Allow using MergeTreeWhereOptimizer not only with MergeTree-based storages [#20153](https://github.com/ClickHouse/ClickHouse/pull/20153) ([Max Akhmedov](https://github.com/zlobober)). +* Fix build of utils [#20155](https://github.com/ClickHouse/ClickHouse/pull/20155) ([Ivan](https://github.com/abyss7)). +* Fix UBSan report in arrayCumSum [#20160](https://github.com/ClickHouse/ClickHouse/pull/20160) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add changelog for 21.2 [#20185](https://github.com/ClickHouse/ClickHouse/pull/20185) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ubsan error [#20204](https://github.com/ClickHouse/ClickHouse/pull/20204) ([Amos Bird](https://github.com/amosbird)). +* Suppress signed overflow in AggregateFunctionGroupArrayMoving [#20206](https://github.com/ClickHouse/ClickHouse/pull/20206) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove dead code [#20212](https://github.com/ClickHouse/ClickHouse/pull/20212) ([nvartolomei](https://github.com/nvartolomei)). +* Handle syntax error for ARRAY JOIN with no args [#20223](https://github.com/ClickHouse/ClickHouse/pull/20223) ([Vladimir C](https://github.com/vdimir)). +* Fix benign race in system.parts [#20226](https://github.com/ClickHouse/ClickHouse/pull/20226) ([alesapin](https://github.com/alesapin)). +* Add final to some classes [#20228](https://github.com/ClickHouse/ClickHouse/pull/20228) ([alesapin](https://github.com/alesapin)). +* refine code in MergeTreeData::loadDataParts to avoid parsing WAL file as data part [#20231](https://github.com/ClickHouse/ClickHouse/pull/20231) ([Fuwang Hu](https://github.com/fuwhu)). +* fix a problem in ArithmeticOperationsInAgrFuncOptimize [#20246](https://github.com/ClickHouse/ClickHouse/pull/20246) ([flynn](https://github.com/ucasfl)). +* Add retries to test_access_control_on_cluster [#20247](https://github.com/ClickHouse/ClickHouse/pull/20247) ([Ilya Yatsishin](https://github.com/qoega)). +* Stable sort for test cases in test_dictionaries_all_layouts_separate_sources [#20248](https://github.com/ClickHouse/ClickHouse/pull/20248) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix not closed ports in DelayedPortsProcessor [#20251](https://github.com/ClickHouse/ClickHouse/pull/20251) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* print changed settings in fuzzer when the server dies [#20281](https://github.com/ClickHouse/ClickHouse/pull/20281) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Put windows in such an order that we can sort less [#20284](https://github.com/ClickHouse/ClickHouse/pull/20284) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Aggregate function deltaSum use restrict keyword [#20287](https://github.com/ClickHouse/ClickHouse/pull/20287) ([Maksim Kita](https://github.com/kitaisreal)). +* make window functions faster [#20293](https://github.com/ClickHouse/ClickHouse/pull/20293) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Accept arbitrary numeric types for numbers() arguments (for scientific notation) [#20301](https://github.com/ClickHouse/ClickHouse/pull/20301) ([Azat Khuzhin](https://github.com/azat)). +* Fix 00738_lock_for_inner_table flakiness [#20305](https://github.com/ClickHouse/ClickHouse/pull/20305) ([Azat Khuzhin](https://github.com/azat)). +* Filter push down [#20341](https://github.com/ClickHouse/ClickHouse/pull/20341) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Common types template instantiations [#20348](https://github.com/ClickHouse/ClickHouse/pull/20348) ([Maksim Kita](https://github.com/kitaisreal)). +* Add test for sign column drop [#20398](https://github.com/ClickHouse/ClickHouse/pull/20398) ([alesapin](https://github.com/alesapin)). +* Merging [#19204](https://github.com/ClickHouse/ClickHouse/issues/19204) [#20399](https://github.com/ClickHouse/ClickHouse/pull/20399) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix 01700_system_zookeeper_path_in [#20449](https://github.com/ClickHouse/ClickHouse/pull/20449) ([alesapin](https://github.com/alesapin)). +* Print stack trace on SIGTRAP [#20453](https://github.com/ClickHouse/ClickHouse/pull/20453) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in arrayDifference [#20458](https://github.com/ClickHouse/ClickHouse/pull/20458) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Suppress UBSan report in Decimal comparison [#20459](https://github.com/ClickHouse/ClickHouse/pull/20459) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve backtrace catching on server failures in CI for stress tests [#20462](https://github.com/ClickHouse/ClickHouse/pull/20462) ([Azat Khuzhin](https://github.com/azat)). +* Fix abnormal server terminations due to write failures [#20465](https://github.com/ClickHouse/ClickHouse/pull/20465) ([Azat Khuzhin](https://github.com/azat)). +* Improve logging during reading from MergeTree [#20466](https://github.com/ClickHouse/ClickHouse/pull/20466) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in intDiv [#20475](https://github.com/ClickHouse/ClickHouse/pull/20475) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* (unrealistic case, never possible in prod) HashTable fix potential bug during resize with nonstandard grower [#20489](https://github.com/ClickHouse/ClickHouse/pull/20489) ([Maksim Kita](https://github.com/kitaisreal)). +* Add a test for [#8654](https://github.com/ClickHouse/ClickHouse/issues/8654) [#20492](https://github.com/ClickHouse/ClickHouse/pull/20492) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#10893](https://github.com/ClickHouse/ClickHouse/issues/10893) [#20493](https://github.com/ClickHouse/ClickHouse/pull/20493) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable testkeeper snapshots for tests [#20496](https://github.com/ClickHouse/ClickHouse/pull/20496) ([alesapin](https://github.com/alesapin)). +* Fix non-zero session reconnect in integration test [#20501](https://github.com/ClickHouse/ClickHouse/pull/20501) ([alesapin](https://github.com/alesapin)). +* Reduce test scale [#20511](https://github.com/ClickHouse/ClickHouse/pull/20511) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Quick build fix [#20513](https://github.com/ClickHouse/ClickHouse/pull/20513) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Replace null fields in tuple during parsing with default values [#20541](https://github.com/ClickHouse/ClickHouse/pull/20541) ([Maksim Kita](https://github.com/kitaisreal)). +* rabbitmq: add missing format factory settings [#20545](https://github.com/ClickHouse/ClickHouse/pull/20545) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Persistent coordination log storage [#20585](https://github.com/ClickHouse/ClickHouse/pull/20585) ([alesapin](https://github.com/alesapin)). +* Add hung check to stress test [#20588](https://github.com/ClickHouse/ClickHouse/pull/20588) ([Alexander Tokmakov](https://github.com/tavplubix)). +* ignore data store files [#20598](https://github.com/ClickHouse/ClickHouse/pull/20598) ([tison](https://github.com/tisonkun)). +* Dictionary create source with functions crash fix [#20623](https://github.com/ClickHouse/ClickHouse/pull/20623) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix fasttest retry for failed tests [#20627](https://github.com/ClickHouse/ClickHouse/pull/20627) ([alesapin](https://github.com/alesapin)). +* Don't backport base commit of branch in the same branch [#20628](https://github.com/ClickHouse/ClickHouse/pull/20628) ([Ivan](https://github.com/abyss7)). +* Add test for already fixed odbc Postgres date type conversion [#20712](https://github.com/ClickHouse/ClickHouse/pull/20712) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Improve backtrace catching on server failures in CI for fast-tests [#20864](https://github.com/ClickHouse/ClickHouse/pull/20864) ([Azat Khuzhin](https://github.com/azat)). +* Better postgres db engine numeric conversion [#20874](https://github.com/ClickHouse/ClickHouse/pull/20874) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix undefined-behavior in ReservoirSamplerDeterministic.h [#20879](https://github.com/ClickHouse/ClickHouse/pull/20879) ([Vitaly Baranov](https://github.com/vitlibar)). +* ccache 4.2+ does not requires any quirks for SOURCE_DATE_EPOCH [#20883](https://github.com/ClickHouse/ClickHouse/pull/20883) ([Azat Khuzhin](https://github.com/azat)). +* Check for EINTR in epoll_wait [#20958](https://github.com/ClickHouse/ClickHouse/pull/20958) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix push to dockerhub [#20960](https://github.com/ClickHouse/ClickHouse/pull/20960) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better tests for protobuf format. [#20968](https://github.com/ClickHouse/ClickHouse/pull/20968) ([Vitaly Baranov](https://github.com/vitlibar)). +* Function sumMap decimal fix [#20970](https://github.com/ClickHouse/ClickHouse/pull/20970) ([Maksim Kita](https://github.com/kitaisreal)). +* test for decimal ( p , s) in dictionaries [#20980](https://github.com/ClickHouse/ClickHouse/pull/20980) ([Denny Crane](https://github.com/den-crane)). +* Fix uncaught exception when HTTP client goes away [#20981](https://github.com/ClickHouse/ClickHouse/pull/20981) ([Azat Khuzhin](https://github.com/azat)). +* Increase buffer for uncaught exception / std::terminate [#20989](https://github.com/ClickHouse/ClickHouse/pull/20989) ([Azat Khuzhin](https://github.com/azat)). +* Constraints complex types support [#20990](https://github.com/ClickHouse/ClickHouse/pull/20990) ([Maksim Kita](https://github.com/kitaisreal)). +* Suppress signed overflow in AggregateFunctionGroupArrayMoving 2 [#20995](https://github.com/ClickHouse/ClickHouse/pull/20995) ([Amos Bird](https://github.com/amosbird)). +* Add log message when stacktrace cannot be obtained for thread [#20996](https://github.com/ClickHouse/ClickHouse/pull/20996) ([Azat Khuzhin](https://github.com/azat)). +* Preserve mysql logs in test_materialize_mysql_database [#21016](https://github.com/ClickHouse/ClickHouse/pull/21016) ([Azat Khuzhin](https://github.com/azat)). +* Minor changes in Decimal [#21017](https://github.com/ClickHouse/ClickHouse/pull/21017) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* yamllint style check [#21019](https://github.com/ClickHouse/ClickHouse/pull/21019) ([Azat Khuzhin](https://github.com/azat)). +* Fix buffer size for trace collection [#21020](https://github.com/ClickHouse/ClickHouse/pull/21020) ([Azat Khuzhin](https://github.com/azat)). +* Try fix MaterializeMySQL integration test [#21021](https://github.com/ClickHouse/ClickHouse/pull/21021) ([Winter Zhang](https://github.com/zhang2014)). +* Fix performance tests (by avoid sharing status file for right and left server) [#21022](https://github.com/ClickHouse/ClickHouse/pull/21022) ([Azat Khuzhin](https://github.com/azat)). +* Revert "optimize aggfunc column data copy ([#19407](https://github.com/ClickHouse/ClickHouse/issues/19407))" [#21024](https://github.com/ClickHouse/ClickHouse/pull/21024) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in Decimal arithmetic [#21025](https://github.com/ClickHouse/ClickHouse/pull/21025) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update lowcardinality.md [#21033](https://github.com/ClickHouse/ClickHouse/pull/21033) ([Michael Monashev](https://github.com/MichaelMonashev)). +* Fix DateTime64 from Float [#21050](https://github.com/ClickHouse/ClickHouse/pull/21050) ([Azat Khuzhin](https://github.com/azat)). +* Add test for [#19376](https://github.com/ClickHouse/ClickHouse/issues/19376) [#21051](https://github.com/ClickHouse/ClickHouse/pull/21051) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merging [#20620](https://github.com/ClickHouse/ClickHouse/issues/20620) [#21052](https://github.com/ClickHouse/ClickHouse/pull/21052) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#9781](https://github.com/ClickHouse/ClickHouse/issues/9781) [#21074](https://github.com/ClickHouse/ClickHouse/pull/21074) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix various issues in DDLWorker (SIGSEGV and others) [#21079](https://github.com/ClickHouse/ClickHouse/pull/21079) ([Azat Khuzhin](https://github.com/azat)). +* Documentation low cardinality fix [#21086](https://github.com/ClickHouse/ClickHouse/pull/21086) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix __has_feature macros under gcc [#21088](https://github.com/ClickHouse/ClickHouse/pull/21088) ([Azat Khuzhin](https://github.com/azat)). +* test for window functions [#21094](https://github.com/ClickHouse/ClickHouse/pull/21094) ([Denny Crane](https://github.com/den-crane)). +* Fix replace[All] functions so that they don't generate garbage to stderr [#21098](https://github.com/ClickHouse/ClickHouse/pull/21098) ([Amos Bird](https://github.com/amosbird)). +* Better tests for protobuf format #2. [#21148](https://github.com/ClickHouse/ClickHouse/pull/21148) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix ccache 4.2+ usage (RULE_LAUNCH_COMPILE/RULE_LAUNCH_LINK was not set) [#21158](https://github.com/ClickHouse/ClickHouse/pull/21158) ([Azat Khuzhin](https://github.com/azat)). +* Bump zookeeper version to 3.6.2 in tests [#21171](https://github.com/ClickHouse/ClickHouse/pull/21171) ([Azat Khuzhin](https://github.com/azat)). +* StorageRabbitMQ added UVLoop [#21193](https://github.com/ClickHouse/ClickHouse/pull/21193) ([Maksim Kita](https://github.com/kitaisreal)). +* Trying to make nukeeper better in single server mode [#21207](https://github.com/ClickHouse/ClickHouse/pull/21207) ([alesapin](https://github.com/alesapin)). +* A followup correction to [#19998](https://github.com/ClickHouse/ClickHouse/issues/19998) [#21221](https://github.com/ClickHouse/ClickHouse/pull/21221) ([Alexander Kazakov](https://github.com/Akazz)). +* Add tests for zstd and zlib http compression [#21279](https://github.com/ClickHouse/ClickHouse/pull/21279) ([Kseniia Sumarokova](https://github.com/kssenii)). +* CheckConstraintsBlockOutputStream optimize nullable column case [#21285](https://github.com/ClickHouse/ClickHouse/pull/21285) ([Maksim Kita](https://github.com/kitaisreal)). +* Rewrite extractTextFromHTML function [#21292](https://github.com/ClickHouse/ClickHouse/pull/21292) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix abnormal server termination for nested writers [#21305](https://github.com/ClickHouse/ClickHouse/pull/21305) ([Azat Khuzhin](https://github.com/azat)). +* [RFC] Remove unused writers [#21306](https://github.com/ClickHouse/ClickHouse/pull/21306) ([Azat Khuzhin](https://github.com/azat)). +* remove unused code in MergeTreeWriteAheadLog::restore [#21308](https://github.com/ClickHouse/ClickHouse/pull/21308) ([Fuwang Hu](https://github.com/fuwhu)). +* IColunm::hasEqualValues() [#21327](https://github.com/ClickHouse/ClickHouse/pull/21327) ([Amos Bird](https://github.com/amosbird)). +* AggregateFunctionSumMap better comment message [#21353](https://github.com/ClickHouse/ClickHouse/pull/21353) ([Maksim Kita](https://github.com/kitaisreal)). +* clickhouse stop: wait for the server to be killed (process exited) [#21365](https://github.com/ClickHouse/ClickHouse/pull/21365) ([Azat Khuzhin](https://github.com/azat)). +* [ClickHouse][LOG]correct shutdown timeout log [#21366](https://github.com/ClickHouse/ClickHouse/pull/21366) ([jasong](https://github.com/songenjie)). +* fix a rare false negative in perf tests [#21381](https://github.com/ClickHouse/ClickHouse/pull/21381) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for issue[#21369](https://github.com/ClickHouse/ClickHouse/issues/21369) [#21387](https://github.com/ClickHouse/ClickHouse/pull/21387) ([Denny Crane](https://github.com/den-crane)). +* Add a test for [#14740](https://github.com/ClickHouse/ClickHouse/issues/14740) [#21392](https://github.com/ClickHouse/ClickHouse/pull/21392) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#15469](https://github.com/ClickHouse/ClickHouse/issues/15469) [#21393](https://github.com/ClickHouse/ClickHouse/pull/21393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix typo in setting name [#21418](https://github.com/ClickHouse/ClickHouse/pull/21418) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix flaky tests with distributed queries [#21432](https://github.com/ClickHouse/ClickHouse/pull/21432) ([Azat Khuzhin](https://github.com/azat)). +* Fix ParsingException::displayText() [#21433](https://github.com/ClickHouse/ClickHouse/pull/21433) ([Azat Khuzhin](https://github.com/azat)). +* Use path as default prefix for coordination logs [#21439](https://github.com/ClickHouse/ClickHouse/pull/21439) ([alesapin](https://github.com/alesapin)). +* Try fix perftests. [#21447](https://github.com/ClickHouse/ClickHouse/pull/21447) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* avoid race in librdkafka [#21452](https://github.com/ClickHouse/ClickHouse/pull/21452) ([filimonov](https://github.com/filimonov)). +* test for [#21413](https://github.com/ClickHouse/ClickHouse/issues/21413) [#21455](https://github.com/ClickHouse/ClickHouse/pull/21455) ([Denny Crane](https://github.com/den-crane)). +* Tiny fix [#21458](https://github.com/ClickHouse/ClickHouse/pull/21458) ([Amos Bird](https://github.com/amosbird)). +* Just another fix for DDLWorker [#21461](https://github.com/ClickHouse/ClickHouse/pull/21461) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.3.10.1-lts.md b/docs/changelogs/v21.3.10.1-lts.md index 49ece009ad1..d43237428e2 100644 --- a/docs/changelogs/v21.3.10.1-lts.md +++ b/docs/changelogs/v21.3.10.1-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.10.1-lts FIXME as compared to v21.3.9.83-lts #### Bug Fix @@ -6,3 +13,7 @@ * Backported in [#23817](https://github.com/ClickHouse/ClickHouse/issues/23817): Fix crash when `PREWHERE` and row policy filter are both in effect with empty result. [#23763](https://github.com/ClickHouse/ClickHouse/pull/23763) ([Amos Bird](https://github.com/amosbird)). * Backported in [#23814](https://github.com/ClickHouse/ClickHouse/issues/23814): Fix `CLEAR COLUMN` does not work when it is referenced by materialized view. Close [#23764](https://github.com/ClickHouse/ClickHouse/issues/23764). [#23781](https://github.com/ClickHouse/ClickHouse/pull/23781) ([flynn](https://github.com/ucasfl)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Flat, Hashed dictionary include update field bytes into bytes_allocated [#23825](https://github.com/ClickHouse/ClickHouse/pull/23825) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.11.5-lts.md b/docs/changelogs/v21.3.11.5-lts.md index 61aa8a54688..2918ab860a6 100644 --- a/docs/changelogs/v21.3.11.5-lts.md +++ b/docs/changelogs/v21.3.11.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.11.5-lts FIXME as compared to v21.3.10.1-lts #### Improvement diff --git a/docs/changelogs/v21.3.12.2-lts.md b/docs/changelogs/v21.3.12.2-lts.md index cfddf8c9cdd..02c649284cc 100644 --- a/docs/changelogs/v21.3.12.2-lts.md +++ b/docs/changelogs/v21.3.12.2-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.12.2-lts FIXME as compared to v21.3.11.5-lts #### Bug Fix @@ -10,3 +17,9 @@ * Backported in [#24241](https://github.com/ClickHouse/ClickHouse/issues/24241): Fix wrong typo at StorageMemory, this bug was introduced at [#15127](https://github.com/ClickHouse/ClickHouse/issues/15127), now fixed, Closes [#24192](https://github.com/ClickHouse/ClickHouse/issues/24192). [#24193](https://github.com/ClickHouse/ClickHouse/pull/24193) ([张中南](https://github.com/plugine)). * Backported in [#24353](https://github.com/ClickHouse/ClickHouse/issues/24353): Fixed a bug in moving Materialized View from Ordinary to Atomic database (`RENAME TABLE` query). Now inner table is moved to new database together with Materialized View. Fixes [#23926](https://github.com/ClickHouse/ClickHouse/issues/23926). [#24309](https://github.com/ClickHouse/ClickHouse/pull/24309) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix segfault in TSan on _exit [#23616](https://github.com/ClickHouse/ClickHouse/pull/23616) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use memmove in PODArray::insert to handle memory overlapping. [#24271](https://github.com/ClickHouse/ClickHouse/pull/24271) ([Fu Zhe](https://github.com/fuzhe1989)). +* Fix cli argument in clickhouse-server.init [#24449](https://github.com/ClickHouse/ClickHouse/pull/24449) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.3.13.9-lts.md b/docs/changelogs/v21.3.13.9-lts.md index dccb9c53162..f78725e4408 100644 --- a/docs/changelogs/v21.3.13.9-lts.md +++ b/docs/changelogs/v21.3.13.9-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.13.9-lts FIXME as compared to v21.3.12.2-lts #### Improvement @@ -41,3 +48,8 @@ * NO CL ENTRY: 'Revert "Backport [#24721](https://github.com/ClickHouse/ClickHouse/issues/24721) to 21.3: Remove endless `wait` from ZooKeeper client"'. [#24799](https://github.com/ClickHouse/ClickHouse/pull/24799) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Try fix `test_consistent_parts_after_clone_replica` [#24968](https://github.com/ClickHouse/ClickHouse/pull/24968) ([Alexander Tokmakov](https://github.com/tavplubix)). +* DictionaryLoader unnecessary dictionary configuration creation fix [#25001](https://github.com/ClickHouse/ClickHouse/pull/25001) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.14.1-lts.md b/docs/changelogs/v21.3.14.1-lts.md index b2408471ccd..eb2ce73b01f 100644 --- a/docs/changelogs/v21.3.14.1-lts.md +++ b/docs/changelogs/v21.3.14.1-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.14.1-lts FIXME as compared to v21.3.13.9-lts #### Bug Fix @@ -8,3 +15,7 @@ * Backported in [#25716](https://github.com/ClickHouse/ClickHouse/issues/25716): `REPLACE PARTITION` might be ignored in rare cases if the source partition was empty. It's fixed. Fixes [#24869](https://github.com/ClickHouse/ClickHouse/issues/24869). [#25665](https://github.com/ClickHouse/ClickHouse/pull/25665) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#25712](https://github.com/ClickHouse/ClickHouse/issues/25712): Fixed `No such file or directory` error on moving `Distributed` table between databases. Fixes [#24971](https://github.com/ClickHouse/ClickHouse/issues/24971). [#25667](https://github.com/ClickHouse/ClickHouse/pull/25667) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Backport unrelated changes, which fixes aliases bug [#25681](https://github.com/ClickHouse/ClickHouse/pull/25681) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v21.3.15.4-stable.md b/docs/changelogs/v21.3.15.4-stable.md index c6b7a15aa4d..ca5a25850a1 100644 --- a/docs/changelogs/v21.3.15.4-stable.md +++ b/docs/changelogs/v21.3.15.4-stable.md @@ -1,6 +1,17 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.15.4-stable FIXME as compared to v21.3.14.1-lts #### Bug Fix * Backported in [#25956](https://github.com/ClickHouse/ClickHouse/issues/25956): Fix extremely long backoff for background tasks when the background pool is full. Fixes [#25836](https://github.com/ClickHouse/ClickHouse/issues/25836). [#25893](https://github.com/ClickHouse/ClickHouse/pull/25893) ([alesapin](https://github.com/alesapin)). * Backported in [#26141](https://github.com/ClickHouse/ClickHouse/issues/26141): Fix possible crash in `pointInPolygon` if the setting `validate_polygons` is turned off. [#26113](https://github.com/ClickHouse/ClickHouse/pull/26113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* 21.3 Docker unbundled fix [#26153](https://github.com/ClickHouse/ClickHouse/pull/26153) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.16.5-lts.md b/docs/changelogs/v21.3.16.5-lts.md index b5b31b64488..a1e73b7c019 100644 --- a/docs/changelogs/v21.3.16.5-lts.md +++ b/docs/changelogs/v21.3.16.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.16.5-lts FIXME as compared to v21.3.15.4-stable #### Bug Fix @@ -23,3 +30,15 @@ * Backported in [#28181](https://github.com/ClickHouse/ClickHouse/issues/28181): Fixed possible excessive number of conditions moved from `WHERE` to `PREWHERE` (optimization controlled by settings `optimize_move_to_prewhere`). [#28139](https://github.com/ClickHouse/ClickHouse/pull/28139) ([lthaooo](https://github.com/lthaooo)). * Backported in [#28293](https://github.com/ClickHouse/ClickHouse/issues/28293): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)). +* Fix mysql_kill_sync_thread_restore_test [#26673](https://github.com/ClickHouse/ClickHouse/pull/26673) ([Vladimir C](https://github.com/vdimir)). +* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.3.17.2-lts.md b/docs/changelogs/v21.3.17.2-lts.md index b6b48ddab61..41007a45634 100644 --- a/docs/changelogs/v21.3.17.2-lts.md +++ b/docs/changelogs/v21.3.17.2-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.17.2-lts FIXME as compared to v21.3.16.5-lts #### Bug Fix diff --git a/docs/changelogs/v21.3.18.4-lts.md b/docs/changelogs/v21.3.18.4-lts.md index 612b3660e3b..5bb29b08df8 100644 --- a/docs/changelogs/v21.3.18.4-lts.md +++ b/docs/changelogs/v21.3.18.4-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.18.4-lts FIXME as compared to v21.3.17.2-lts #### Improvement @@ -20,3 +27,10 @@ * Backported in [#30332](https://github.com/ClickHouse/ClickHouse/issues/30332): * Allow identifiers staring with numbers in multiple joins. [#30230](https://github.com/ClickHouse/ClickHouse/pull/30230) ([Vladimir C](https://github.com/vdimir)). * Backported in [#30379](https://github.com/ClickHouse/ClickHouse/issues/30379): fix replaceRegexpAll bug. [#30292](https://github.com/ClickHouse/ClickHouse/pull/30292) ([Memo](https://github.com/Joeywzr)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Don not add const group by key for query with only having. [#28975](https://github.com/ClickHouse/ClickHouse/pull/28975) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27963](https://github.com/ClickHouse/ClickHouse/issues/27963) [#29063](https://github.com/ClickHouse/ClickHouse/pull/29063) ([Maksim Kita](https://github.com/kitaisreal)). +* May be fix s3 tests [#29762](https://github.com/ClickHouse/ClickHouse/pull/29762) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix ca-bundle.crt in kerberized_hadoop/Dockerfile [#30358](https://github.com/ClickHouse/ClickHouse/pull/30358) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.3.19.1-lts.md b/docs/changelogs/v21.3.19.1-lts.md index 6a2be8d6dcb..fa8e94fa7a1 100644 --- a/docs/changelogs/v21.3.19.1-lts.md +++ b/docs/changelogs/v21.3.19.1-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.19.1-lts FIXME as compared to v21.3.18.4-lts #### Performance Improvement @@ -24,3 +31,8 @@ * Backported in [#31894](https://github.com/ClickHouse/ClickHouse/issues/31894): Fix possible assertion `../src/IO/ReadBuffer.h:58: bool DB::ReadBuffer::next(): Assertion '!hasPendingData()' failed.` in TSKV format. [#31804](https://github.com/ClickHouse/ClickHouse/pull/31804) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#32214](https://github.com/ClickHouse/ClickHouse/issues/32214): Number of active replicas might be determined incorrectly when inserting with quorum if setting `replicated_can_become_leader` is disabled on some replicas. It's fixed. [#32157](https://github.com/ClickHouse/ClickHouse/pull/32157) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.3.2.5-lts.md b/docs/changelogs/v21.3.2.5-lts.md index 5135f909b7f..0eb32439c6b 100644 --- a/docs/changelogs/v21.3.2.5-lts.md +++ b/docs/changelogs/v21.3.2.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.2.5-lts FIXME as compared to v21.2.1.5869-prestable #### Backward Incompatible Change @@ -159,3 +166,164 @@ * NO CL ENTRY: 'Revert "Fix access control manager destruction order"'. [#20394](https://github.com/ClickHouse/ClickHouse/pull/20394) ([alesapin](https://github.com/alesapin)). * NO CL ENTRY: 'Update argmax.md '. [#20625](https://github.com/ClickHouse/ClickHouse/pull/20625) ([Marvin Taschenberger](https://github.com/Taschenbergerm)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Update Pytest check [#18972](https://github.com/ClickHouse/ClickHouse/pull/18972) ([Ivan](https://github.com/abyss7)). +* [wip] support RANGE frame for window functions [#19299](https://github.com/ClickHouse/ClickHouse/pull/19299) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* In memory coordination inside ClickHouse [#19580](https://github.com/ClickHouse/ClickHouse/pull/19580) ([alesapin](https://github.com/alesapin)). +* client: more suggestions [#19584](https://github.com/ClickHouse/ClickHouse/pull/19584) ([Azat Khuzhin](https://github.com/azat)). +* Allow to run all style checks in one file [#19726](https://github.com/ClickHouse/ClickHouse/pull/19726) ([Anton Popov](https://github.com/CurtizJ)). +* Fix [#19371](https://github.com/ClickHouse/ClickHouse/issues/19371) [#19765](https://github.com/ClickHouse/ClickHouse/pull/19765) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor code improvements around ThreadStatus [#19771](https://github.com/ClickHouse/ClickHouse/pull/19771) ([Alexander Tokmakov](https://github.com/tavplubix)). +* continue of [#19487](https://github.com/ClickHouse/ClickHouse/issues/19487) [#19801](https://github.com/ClickHouse/ClickHouse/pull/19801) ([flynn](https://github.com/ucasfl)). +* Fix UBSan report in intDiv [#19876](https://github.com/ClickHouse/ClickHouse/pull/19876) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow writing partial buffer [#19886](https://github.com/ClickHouse/ClickHouse/pull/19886) ([Azat Khuzhin](https://github.com/azat)). +* Remove an always-false condition from query parser [#19919](https://github.com/ClickHouse/ClickHouse/pull/19919) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* UNBOUNDED FOLLOWING frame end [#19921](https://github.com/ClickHouse/ClickHouse/pull/19921) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix logical error in INSERT VALUES [#19925](https://github.com/ClickHouse/ClickHouse/pull/19925) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix test keeper integration tests [#19942](https://github.com/ClickHouse/ClickHouse/pull/19942) ([alesapin](https://github.com/alesapin)). +* Fix build [#19948](https://github.com/ClickHouse/ClickHouse/pull/19948) ([Kruglov Pavel](https://github.com/Avogar)). +* CURRENT ROW and offset for start of ROWS frame [#19951](https://github.com/ClickHouse/ClickHouse/pull/19951) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix UBSan report in geoHashesInBox [#19956](https://github.com/ClickHouse/ClickHouse/pull/19956) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add MSan annotation for system.stack_trace [#19957](https://github.com/ClickHouse/ClickHouse/pull/19957) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky integration tests [#19971](https://github.com/ClickHouse/ClickHouse/pull/19971) ([Azat Khuzhin](https://github.com/azat)). +* Do not use inputs which values are known constants in ActionsDAG. [#19991](https://github.com/ClickHouse/ClickHouse/pull/19991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* update perf tests [#20000](https://github.com/ClickHouse/ClickHouse/pull/20000) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* UBsan fixes [#20011](https://github.com/ClickHouse/ClickHouse/pull/20011) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless code [#20029](https://github.com/ClickHouse/ClickHouse/pull/20029) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* update changelog [#20033](https://github.com/ClickHouse/ClickHouse/pull/20033) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Close input ports in DelayedPortsProcessor as soon as all outputs are finished [#20035](https://github.com/ClickHouse/ClickHouse/pull/20035) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* ReadBuffer: check for unread data on next() [#20037](https://github.com/ClickHouse/ClickHouse/pull/20037) ([Ivan](https://github.com/abyss7)). +* ReadBuffer: check that buffer position is not set beyond end [#20039](https://github.com/ClickHouse/ClickHouse/pull/20039) ([Ivan](https://github.com/abyss7)). +* CURRENT ROW frame start for RANGE frame [#20041](https://github.com/ClickHouse/ClickHouse/pull/20041) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix log message with elapsed time while pushing to views [#20047](https://github.com/ClickHouse/ClickHouse/pull/20047) ([Azat Khuzhin](https://github.com/azat)). +* Avoid UBSan report in pointInPolygon [#20049](https://github.com/ClickHouse/ClickHouse/pull/20049) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix tests *.reference files [#20050](https://github.com/ClickHouse/ClickHouse/pull/20050) ([Azat Khuzhin](https://github.com/azat)). +* ROWS OFFSET frame end [#20060](https://github.com/ClickHouse/ClickHouse/pull/20060) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add more tests for modulo of division of negative number [#20063](https://github.com/ClickHouse/ClickHouse/pull/20063) ([hexiaoting](https://github.com/hexiaoting)). +* detect unmarked long tests in flaky check [#20068](https://github.com/ClickHouse/ClickHouse/pull/20068) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix removing of filter column when split filter actions. [#20073](https://github.com/ClickHouse/ClickHouse/pull/20073) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove AddingConstColumn step and transform. [#20077](https://github.com/ClickHouse/ClickHouse/pull/20077) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* LimitReadBuffer: check that position always advances [#20078](https://github.com/ClickHouse/ClickHouse/pull/20078) ([Ivan](https://github.com/abyss7)). +* Add fuzzer for ColumnsDescription [#20094](https://github.com/ClickHouse/ClickHouse/pull/20094) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Build actions dag to evaluate missing defaults. [#20097](https://github.com/ClickHouse/ClickHouse/pull/20097) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove never existing insert_in_memory_parts_timeout [#20099](https://github.com/ClickHouse/ClickHouse/pull/20099) ([Azat Khuzhin](https://github.com/azat)). +* RANGE OFFSET window frame [#20111](https://github.com/ClickHouse/ClickHouse/pull/20111) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Allow to drop readonly tables [#20114](https://github.com/ClickHouse/ClickHouse/pull/20114) ([nvartolomei](https://github.com/nvartolomei)). +* Add logging if Poco cannot allocate thread in tcp server [#20120](https://github.com/ClickHouse/ClickHouse/pull/20120) ([alesapin](https://github.com/alesapin)). +* Use fixed version of confluent-kafka library in integration tests [#20124](https://github.com/ClickHouse/ClickHouse/pull/20124) ([alesapin](https://github.com/alesapin)). +* Useless changes [#20150](https://github.com/ClickHouse/ClickHouse/pull/20150) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* remove some useless code [#20152](https://github.com/ClickHouse/ClickHouse/pull/20152) ([flynn](https://github.com/ucasfl)). +* Allow using MergeTreeWhereOptimizer not only with MergeTree-based storages [#20153](https://github.com/ClickHouse/ClickHouse/pull/20153) ([Max Akhmedov](https://github.com/zlobober)). +* Fix build of utils [#20155](https://github.com/ClickHouse/ClickHouse/pull/20155) ([Ivan](https://github.com/abyss7)). +* Fix UBSan report in arrayCumSum [#20160](https://github.com/ClickHouse/ClickHouse/pull/20160) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add changelog for 21.2 [#20185](https://github.com/ClickHouse/ClickHouse/pull/20185) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ubsan error [#20204](https://github.com/ClickHouse/ClickHouse/pull/20204) ([Amos Bird](https://github.com/amosbird)). +* Suppress signed overflow in AggregateFunctionGroupArrayMoving [#20206](https://github.com/ClickHouse/ClickHouse/pull/20206) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove dead code [#20212](https://github.com/ClickHouse/ClickHouse/pull/20212) ([nvartolomei](https://github.com/nvartolomei)). +* Handle syntax error for ARRAY JOIN with no args [#20223](https://github.com/ClickHouse/ClickHouse/pull/20223) ([Vladimir C](https://github.com/vdimir)). +* Fix benign race in system.parts [#20226](https://github.com/ClickHouse/ClickHouse/pull/20226) ([alesapin](https://github.com/alesapin)). +* Add final to some classes [#20228](https://github.com/ClickHouse/ClickHouse/pull/20228) ([alesapin](https://github.com/alesapin)). +* refine code in MergeTreeData::loadDataParts to avoid parsing WAL file as data part [#20231](https://github.com/ClickHouse/ClickHouse/pull/20231) ([Fuwang Hu](https://github.com/fuwhu)). +* fix a problem in ArithmeticOperationsInAgrFuncOptimize [#20246](https://github.com/ClickHouse/ClickHouse/pull/20246) ([flynn](https://github.com/ucasfl)). +* Add retries to test_access_control_on_cluster [#20247](https://github.com/ClickHouse/ClickHouse/pull/20247) ([Ilya Yatsishin](https://github.com/qoega)). +* Stable sort for test cases in test_dictionaries_all_layouts_separate_sources [#20248](https://github.com/ClickHouse/ClickHouse/pull/20248) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix not closed ports in DelayedPortsProcessor [#20251](https://github.com/ClickHouse/ClickHouse/pull/20251) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* print changed settings in fuzzer when the server dies [#20281](https://github.com/ClickHouse/ClickHouse/pull/20281) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Put windows in such an order that we can sort less [#20284](https://github.com/ClickHouse/ClickHouse/pull/20284) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Aggregate function deltaSum use restrict keyword [#20287](https://github.com/ClickHouse/ClickHouse/pull/20287) ([Maksim Kita](https://github.com/kitaisreal)). +* make window functions faster [#20293](https://github.com/ClickHouse/ClickHouse/pull/20293) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Accept arbitrary numeric types for numbers() arguments (for scientific notation) [#20301](https://github.com/ClickHouse/ClickHouse/pull/20301) ([Azat Khuzhin](https://github.com/azat)). +* Fix 00738_lock_for_inner_table flakiness [#20305](https://github.com/ClickHouse/ClickHouse/pull/20305) ([Azat Khuzhin](https://github.com/azat)). +* Filter push down [#20341](https://github.com/ClickHouse/ClickHouse/pull/20341) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Common types template instantiations [#20348](https://github.com/ClickHouse/ClickHouse/pull/20348) ([Maksim Kita](https://github.com/kitaisreal)). +* Add test for sign column drop [#20398](https://github.com/ClickHouse/ClickHouse/pull/20398) ([alesapin](https://github.com/alesapin)). +* Merging [#19204](https://github.com/ClickHouse/ClickHouse/issues/19204) [#20399](https://github.com/ClickHouse/ClickHouse/pull/20399) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix 01700_system_zookeeper_path_in [#20449](https://github.com/ClickHouse/ClickHouse/pull/20449) ([alesapin](https://github.com/alesapin)). +* Print stack trace on SIGTRAP [#20453](https://github.com/ClickHouse/ClickHouse/pull/20453) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in arrayDifference [#20458](https://github.com/ClickHouse/ClickHouse/pull/20458) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Suppress UBSan report in Decimal comparison [#20459](https://github.com/ClickHouse/ClickHouse/pull/20459) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve backtrace catching on server failures in CI for stress tests [#20462](https://github.com/ClickHouse/ClickHouse/pull/20462) ([Azat Khuzhin](https://github.com/azat)). +* Fix abnormal server terminations due to write failures [#20465](https://github.com/ClickHouse/ClickHouse/pull/20465) ([Azat Khuzhin](https://github.com/azat)). +* Improve logging during reading from MergeTree [#20466](https://github.com/ClickHouse/ClickHouse/pull/20466) ([Azat Khuzhin](https://github.com/azat)). +* Fix UBSan report in intDiv [#20475](https://github.com/ClickHouse/ClickHouse/pull/20475) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* (unrealistic case, never possible in prod) HashTable fix potential bug during resize with nonstandard grower [#20489](https://github.com/ClickHouse/ClickHouse/pull/20489) ([Maksim Kita](https://github.com/kitaisreal)). +* Add a test for [#8654](https://github.com/ClickHouse/ClickHouse/issues/8654) [#20492](https://github.com/ClickHouse/ClickHouse/pull/20492) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#10893](https://github.com/ClickHouse/ClickHouse/issues/10893) [#20493](https://github.com/ClickHouse/ClickHouse/pull/20493) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable testkeeper snapshots for tests [#20496](https://github.com/ClickHouse/ClickHouse/pull/20496) ([alesapin](https://github.com/alesapin)). +* Fix non-zero session reconnect in integration test [#20501](https://github.com/ClickHouse/ClickHouse/pull/20501) ([alesapin](https://github.com/alesapin)). +* Reduce test scale [#20511](https://github.com/ClickHouse/ClickHouse/pull/20511) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Quick build fix [#20513](https://github.com/ClickHouse/ClickHouse/pull/20513) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Replace null fields in tuple during parsing with default values [#20541](https://github.com/ClickHouse/ClickHouse/pull/20541) ([Maksim Kita](https://github.com/kitaisreal)). +* rabbitmq: add missing format factory settings [#20545](https://github.com/ClickHouse/ClickHouse/pull/20545) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Persistent coordination log storage [#20585](https://github.com/ClickHouse/ClickHouse/pull/20585) ([alesapin](https://github.com/alesapin)). +* Add hung check to stress test [#20588](https://github.com/ClickHouse/ClickHouse/pull/20588) ([Alexander Tokmakov](https://github.com/tavplubix)). +* ignore data store files [#20598](https://github.com/ClickHouse/ClickHouse/pull/20598) ([tison](https://github.com/tisonkun)). +* Dictionary create source with functions crash fix [#20623](https://github.com/ClickHouse/ClickHouse/pull/20623) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix fasttest retry for failed tests [#20627](https://github.com/ClickHouse/ClickHouse/pull/20627) ([alesapin](https://github.com/alesapin)). +* Don't backport base commit of branch in the same branch [#20628](https://github.com/ClickHouse/ClickHouse/pull/20628) ([Ivan](https://github.com/abyss7)). +* Add test for already fixed odbc Postgres date type conversion [#20712](https://github.com/ClickHouse/ClickHouse/pull/20712) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Improve backtrace catching on server failures in CI for fast-tests [#20864](https://github.com/ClickHouse/ClickHouse/pull/20864) ([Azat Khuzhin](https://github.com/azat)). +* Better postgres db engine numeric conversion [#20874](https://github.com/ClickHouse/ClickHouse/pull/20874) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix undefined-behavior in ReservoirSamplerDeterministic.h [#20879](https://github.com/ClickHouse/ClickHouse/pull/20879) ([Vitaly Baranov](https://github.com/vitlibar)). +* ccache 4.2+ does not requires any quirks for SOURCE_DATE_EPOCH [#20883](https://github.com/ClickHouse/ClickHouse/pull/20883) ([Azat Khuzhin](https://github.com/azat)). +* Check for EINTR in epoll_wait [#20958](https://github.com/ClickHouse/ClickHouse/pull/20958) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix push to dockerhub [#20960](https://github.com/ClickHouse/ClickHouse/pull/20960) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better tests for protobuf format. [#20968](https://github.com/ClickHouse/ClickHouse/pull/20968) ([Vitaly Baranov](https://github.com/vitlibar)). +* Function sumMap decimal fix [#20970](https://github.com/ClickHouse/ClickHouse/pull/20970) ([Maksim Kita](https://github.com/kitaisreal)). +* test for decimal ( p , s) in dictionaries [#20980](https://github.com/ClickHouse/ClickHouse/pull/20980) ([Denny Crane](https://github.com/den-crane)). +* Fix uncaught exception when HTTP client goes away [#20981](https://github.com/ClickHouse/ClickHouse/pull/20981) ([Azat Khuzhin](https://github.com/azat)). +* Increase buffer for uncaught exception / std::terminate [#20989](https://github.com/ClickHouse/ClickHouse/pull/20989) ([Azat Khuzhin](https://github.com/azat)). +* Constraints complex types support [#20990](https://github.com/ClickHouse/ClickHouse/pull/20990) ([Maksim Kita](https://github.com/kitaisreal)). +* Suppress signed overflow in AggregateFunctionGroupArrayMoving 2 [#20995](https://github.com/ClickHouse/ClickHouse/pull/20995) ([Amos Bird](https://github.com/amosbird)). +* Add log message when stacktrace cannot be obtained for thread [#20996](https://github.com/ClickHouse/ClickHouse/pull/20996) ([Azat Khuzhin](https://github.com/azat)). +* Preserve mysql logs in test_materialize_mysql_database [#21016](https://github.com/ClickHouse/ClickHouse/pull/21016) ([Azat Khuzhin](https://github.com/azat)). +* Minor changes in Decimal [#21017](https://github.com/ClickHouse/ClickHouse/pull/21017) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* yamllint style check [#21019](https://github.com/ClickHouse/ClickHouse/pull/21019) ([Azat Khuzhin](https://github.com/azat)). +* Fix buffer size for trace collection [#21020](https://github.com/ClickHouse/ClickHouse/pull/21020) ([Azat Khuzhin](https://github.com/azat)). +* Try fix MaterializeMySQL integration test [#21021](https://github.com/ClickHouse/ClickHouse/pull/21021) ([Winter Zhang](https://github.com/zhang2014)). +* Fix performance tests (by avoid sharing status file for right and left server) [#21022](https://github.com/ClickHouse/ClickHouse/pull/21022) ([Azat Khuzhin](https://github.com/azat)). +* Revert "optimize aggfunc column data copy ([#19407](https://github.com/ClickHouse/ClickHouse/issues/19407))" [#21024](https://github.com/ClickHouse/ClickHouse/pull/21024) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in Decimal arithmetic [#21025](https://github.com/ClickHouse/ClickHouse/pull/21025) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update lowcardinality.md [#21033](https://github.com/ClickHouse/ClickHouse/pull/21033) ([Michael Monashev](https://github.com/MichaelMonashev)). +* Fix DateTime64 from Float [#21050](https://github.com/ClickHouse/ClickHouse/pull/21050) ([Azat Khuzhin](https://github.com/azat)). +* Add test for [#19376](https://github.com/ClickHouse/ClickHouse/issues/19376) [#21051](https://github.com/ClickHouse/ClickHouse/pull/21051) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merging [#20620](https://github.com/ClickHouse/ClickHouse/issues/20620) [#21052](https://github.com/ClickHouse/ClickHouse/pull/21052) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#9781](https://github.com/ClickHouse/ClickHouse/issues/9781) [#21074](https://github.com/ClickHouse/ClickHouse/pull/21074) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix various issues in DDLWorker (SIGSEGV and others) [#21079](https://github.com/ClickHouse/ClickHouse/pull/21079) ([Azat Khuzhin](https://github.com/azat)). +* Documentation low cardinality fix [#21086](https://github.com/ClickHouse/ClickHouse/pull/21086) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix __has_feature macros under gcc [#21088](https://github.com/ClickHouse/ClickHouse/pull/21088) ([Azat Khuzhin](https://github.com/azat)). +* test for window functions [#21094](https://github.com/ClickHouse/ClickHouse/pull/21094) ([Denny Crane](https://github.com/den-crane)). +* Fix replace[All] functions so that they don't generate garbage to stderr [#21098](https://github.com/ClickHouse/ClickHouse/pull/21098) ([Amos Bird](https://github.com/amosbird)). +* Better tests for protobuf format #2. [#21148](https://github.com/ClickHouse/ClickHouse/pull/21148) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix ccache 4.2+ usage (RULE_LAUNCH_COMPILE/RULE_LAUNCH_LINK was not set) [#21158](https://github.com/ClickHouse/ClickHouse/pull/21158) ([Azat Khuzhin](https://github.com/azat)). +* Bump zookeeper version to 3.6.2 in tests [#21171](https://github.com/ClickHouse/ClickHouse/pull/21171) ([Azat Khuzhin](https://github.com/azat)). +* StorageRabbitMQ added UVLoop [#21193](https://github.com/ClickHouse/ClickHouse/pull/21193) ([Maksim Kita](https://github.com/kitaisreal)). +* Trying to make nukeeper better in single server mode [#21207](https://github.com/ClickHouse/ClickHouse/pull/21207) ([alesapin](https://github.com/alesapin)). +* A followup correction to [#19998](https://github.com/ClickHouse/ClickHouse/issues/19998) [#21221](https://github.com/ClickHouse/ClickHouse/pull/21221) ([Alexander Kazakov](https://github.com/Akazz)). +* Add tests for zstd and zlib http compression [#21279](https://github.com/ClickHouse/ClickHouse/pull/21279) ([Kseniia Sumarokova](https://github.com/kssenii)). +* CheckConstraintsBlockOutputStream optimize nullable column case [#21285](https://github.com/ClickHouse/ClickHouse/pull/21285) ([Maksim Kita](https://github.com/kitaisreal)). +* Rewrite extractTextFromHTML function [#21292](https://github.com/ClickHouse/ClickHouse/pull/21292) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix abnormal server termination for nested writers [#21305](https://github.com/ClickHouse/ClickHouse/pull/21305) ([Azat Khuzhin](https://github.com/azat)). +* [RFC] Remove unused writers [#21306](https://github.com/ClickHouse/ClickHouse/pull/21306) ([Azat Khuzhin](https://github.com/azat)). +* remove unused code in MergeTreeWriteAheadLog::restore [#21308](https://github.com/ClickHouse/ClickHouse/pull/21308) ([Fuwang Hu](https://github.com/fuwhu)). +* IColunm::hasEqualValues() [#21327](https://github.com/ClickHouse/ClickHouse/pull/21327) ([Amos Bird](https://github.com/amosbird)). +* AggregateFunctionSumMap better comment message [#21353](https://github.com/ClickHouse/ClickHouse/pull/21353) ([Maksim Kita](https://github.com/kitaisreal)). +* clickhouse stop: wait for the server to be killed (process exited) [#21365](https://github.com/ClickHouse/ClickHouse/pull/21365) ([Azat Khuzhin](https://github.com/azat)). +* [ClickHouse][LOG]correct shutdown timeout log [#21366](https://github.com/ClickHouse/ClickHouse/pull/21366) ([jasong](https://github.com/songenjie)). +* fix a rare false negative in perf tests [#21381](https://github.com/ClickHouse/ClickHouse/pull/21381) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for issue[#21369](https://github.com/ClickHouse/ClickHouse/issues/21369) [#21387](https://github.com/ClickHouse/ClickHouse/pull/21387) ([Denny Crane](https://github.com/den-crane)). +* Add a test for [#14740](https://github.com/ClickHouse/ClickHouse/issues/14740) [#21392](https://github.com/ClickHouse/ClickHouse/pull/21392) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#15469](https://github.com/ClickHouse/ClickHouse/issues/15469) [#21393](https://github.com/ClickHouse/ClickHouse/pull/21393) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix typo in setting name [#21418](https://github.com/ClickHouse/ClickHouse/pull/21418) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix flaky tests with distributed queries [#21432](https://github.com/ClickHouse/ClickHouse/pull/21432) ([Azat Khuzhin](https://github.com/azat)). +* Fix ParsingException::displayText() [#21433](https://github.com/ClickHouse/ClickHouse/pull/21433) ([Azat Khuzhin](https://github.com/azat)). +* Use path as default prefix for coordination logs [#21439](https://github.com/ClickHouse/ClickHouse/pull/21439) ([alesapin](https://github.com/alesapin)). +* Try fix perftests. [#21447](https://github.com/ClickHouse/ClickHouse/pull/21447) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* avoid race in librdkafka [#21452](https://github.com/ClickHouse/ClickHouse/pull/21452) ([filimonov](https://github.com/filimonov)). +* test for [#21413](https://github.com/ClickHouse/ClickHouse/issues/21413) [#21455](https://github.com/ClickHouse/ClickHouse/pull/21455) ([Denny Crane](https://github.com/den-crane)). +* Tiny fix [#21458](https://github.com/ClickHouse/ClickHouse/pull/21458) ([Amos Bird](https://github.com/amosbird)). +* Just another fix for DDLWorker [#21461](https://github.com/ClickHouse/ClickHouse/pull/21461) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable hedged requests for release 21.3 [#21534](https://github.com/ClickHouse/ClickHouse/pull/21534) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix a typo in window functions frame [#21572](https://github.com/ClickHouse/ClickHouse/pull/21572) ([Alexander Kuzmenkov](https://github.com/akuzm)). + diff --git a/docs/changelogs/v21.3.20.1-lts.md b/docs/changelogs/v21.3.20.1-lts.md index ac8c7d2ece2..19cca4babf4 100644 --- a/docs/changelogs/v21.3.20.1-lts.md +++ b/docs/changelogs/v21.3.20.1-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.20.1-lts FIXME as compared to v21.3.19.1-lts #### Bug Fix @@ -8,3 +15,13 @@ * Backported in [#32791](https://github.com/ClickHouse/ClickHouse/issues/32791): fix crash when used fuzzBits with multiply same FixedString, Close [#32737](https://github.com/ClickHouse/ClickHouse/issues/32737). [#32755](https://github.com/ClickHouse/ClickHouse/pull/32755) ([SuperDJY](https://github.com/cmsxbc)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Cherry pick [#33065](https://github.com/ClickHouse/ClickHouse/issues/33065) to 21.3: Merge [#33050](https://github.com/ClickHouse/ClickHouse/issues/33050) [#33079](https://github.com/ClickHouse/ClickHouse/issues/33079) [#33578](https://github.com/ClickHouse/ClickHouse/pull/33578) ([Heena Bansal](https://github.com/HeenaBansal2009)). + diff --git a/docs/changelogs/v21.3.3.14-lts.md b/docs/changelogs/v21.3.3.14-lts.md index 3dbfd7f0a04..9b99c6c1b5c 100644 --- a/docs/changelogs/v21.3.3.14-lts.md +++ b/docs/changelogs/v21.3.3.14-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.3.14-lts FIXME as compared to v21.3.2.5-lts #### Bug Fix @@ -9,3 +16,7 @@ * Backported in [#21883](https://github.com/ClickHouse/ClickHouse/issues/21883): Reverted S3 connection pools. [#21737](https://github.com/ClickHouse/ClickHouse/pull/21737) ([Vladimir Chebotarev](https://github.com/excitoon)). * Backported in [#21874](https://github.com/ClickHouse/ClickHouse/issues/21874): Fix incorrect query result (and possible crash) which could happen when `WHERE` or `HAVING` condition is pushed before `GROUP BY`. Fixes [#21773](https://github.com/ClickHouse/ClickHouse/issues/21773). [#21841](https://github.com/ClickHouse/ClickHouse/pull/21841) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* fix incorrect number of rows for Chunks with no columns in PartialSor… [#21761](https://github.com/ClickHouse/ClickHouse/pull/21761) ([Alexander Kuzmenkov](https://github.com/akuzm)). + diff --git a/docs/changelogs/v21.3.4.25-lts.md b/docs/changelogs/v21.3.4.25-lts.md index 431c498dbed..dee9e6a48b3 100644 --- a/docs/changelogs/v21.3.4.25-lts.md +++ b/docs/changelogs/v21.3.4.25-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.4.25-lts FIXME as compared to v21.3.3.14-lts #### Bug Fix @@ -16,3 +23,8 @@ * Backported in [#22185](https://github.com/ClickHouse/ClickHouse/issues/22185): Fix uncaught exception in InterserverIOHTTPHandler. [#22146](https://github.com/ClickHouse/ClickHouse/pull/22146) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#22205](https://github.com/ClickHouse/ClickHouse/issues/22205): Use finalize() over next() for nested writers. [#22147](https://github.com/ClickHouse/ClickHouse/pull/22147) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* LRUCache fix exception unsafe element insertion [#21891](https://github.com/ClickHouse/ClickHouse/pull/21891) ([Maksim Kita](https://github.com/kitaisreal)). +* 21.3: quick fix for broken resolution of apt.llvm.org on Yandex infra [#22047](https://github.com/ClickHouse/ClickHouse/pull/22047) ([Alexander Kuzmenkov](https://github.com/akuzm)). + diff --git a/docs/changelogs/v21.3.5.42-lts.md b/docs/changelogs/v21.3.5.42-lts.md index d8616efd0ff..bc68841062c 100644 --- a/docs/changelogs/v21.3.5.42-lts.md +++ b/docs/changelogs/v21.3.5.42-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.5.42-lts FIXME as compared to v21.3.4.25-lts #### Bug Fix @@ -25,3 +32,7 @@ * Backported in [#22653](https://github.com/ClickHouse/ClickHouse/issues/22653): Try flush write buffer only if it is initialized. Fixes segfault when client closes connection very early [#22579](https://github.com/ClickHouse/ClickHouse/issues/22579). [#22591](https://github.com/ClickHouse/ClickHouse/pull/22591) ([nvartolomei](https://github.com/nvartolomei)). * Backported in [#22681](https://github.com/ClickHouse/ClickHouse/issues/22681): Fix LOGICAL_ERROR for Log with nested types w/o columns in the SELECT clause. [#22654](https://github.com/ClickHouse/ClickHouse/pull/22654) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Manual backport of [#21429](https://github.com/ClickHouse/ClickHouse/issues/21429) in 21.3 [#22504](https://github.com/ClickHouse/ClickHouse/pull/22504) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.3.6.55-lts.md b/docs/changelogs/v21.3.6.55-lts.md index 68980af6264..3a517f7386f 100644 --- a/docs/changelogs/v21.3.6.55-lts.md +++ b/docs/changelogs/v21.3.6.55-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.6.55-lts FIXME as compared to v21.3.5.42-lts #### Improvement @@ -17,3 +24,7 @@ * Backported in [#22916](https://github.com/ClickHouse/ClickHouse/issues/22916): LIVE VIEW (experimental feature). Fix possible hanging in concurrent DROP/CREATE of TEMPORARY LIVE VIEW in `TemporaryLiveViewCleaner`, see https://gist.github.com/vzakaznikov/0c03195960fc86b56bfe2bc73a90019e. [#22858](https://github.com/ClickHouse/ClickHouse/pull/22858) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#22920](https://github.com/ClickHouse/ClickHouse/issues/22920): Fixed a crash when using `mannWhitneyUTest` and `rankCorr` with window functions. This fixes [#22728](https://github.com/ClickHouse/ClickHouse/issues/22728). [#22876](https://github.com/ClickHouse/ClickHouse/pull/22876) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* FileDictionarySource fix absolute file path [#22822](https://github.com/ClickHouse/ClickHouse/pull/22822) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.7.62-stable.md b/docs/changelogs/v21.3.7.62-stable.md index df919be0f42..24f759ca3c0 100644 --- a/docs/changelogs/v21.3.7.62-stable.md +++ b/docs/changelogs/v21.3.7.62-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.7.62-stable FIXME as compared to v21.3.6.55-lts #### Improvement @@ -10,3 +17,7 @@ * Backported in [#23075](https://github.com/ClickHouse/ClickHouse/issues/23075): Remove non-essential details from suggestions in clickhouse-client. This closes [#22158](https://github.com/ClickHouse/ClickHouse/issues/22158). [#23040](https://github.com/ClickHouse/ClickHouse/pull/23040) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#23170](https://github.com/ClickHouse/ClickHouse/issues/23170): Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* LibraryDictionarySource fix possible leak [#21686](https://github.com/ClickHouse/ClickHouse/pull/21686) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.3.8.76-lts.md b/docs/changelogs/v21.3.8.76-lts.md index 2002d9e3e1f..acb0b99bae5 100644 --- a/docs/changelogs/v21.3.8.76-lts.md +++ b/docs/changelogs/v21.3.8.76-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.8.76-lts FIXME as compared to v21.3.7.62-stable #### Improvement @@ -19,3 +26,7 @@ * Backported in [#23536](https://github.com/ClickHouse/ClickHouse/issues/23536): When modify column's default value without datatype, and this column is used as ReplacingMergeTree's parameter like column `b` in the below example, then the server will core dump: ``` CREATE TABLE alter_test (a Int32, b DateTime) ENGINE = ReplacingMergeTree(b) ORDER BY a; ALTER TABLE alter_test MODIFY COLUMN `b` DEFAULT now(); ``` the sever throw error: ``` 2021.04.22 09:48:00.685317 [ 2607 ] {} BaseDaemon: Received signal 11 2021.04.22 09:48:00.686110 [ 2705 ] {} BaseDaemon: ######################################## 2021.04.22 09:48:00.686336 [ 2705 ] {} BaseDaemon: (version 21.6.1.1, build id: 6459E84DFCF8E778546C5AD2FFE91B3AD71E1B1B) (from thread 2619) (no query) Received signal Segmentation fault (11) 2021.04.22 09:48:00.686572 [ 2705 ] {} BaseDaemon: Address: NULL pointer. Access: read. Address not mapped to object. 2021.04.22 09:48:00.686686 [ 2705 ] {} BaseDaemon: Stack trace: 0x1c2585d7 0x1c254f66 0x1bb7e403 0x1bb58923 0x1bb56a85 0x1c6840ef 0x1c691148 0x2061a05c 0x2061a8e4 0x20775a03 0x207722bd 0x20771048 0x7f6e5c25be25 0x7f6e5bd81bad 2021.04.22 09:48:02.283045 [ 2705 ] {} BaseDaemon: 4. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1449: DB::(anonymous namespace)::checkVersionColumnTypesConversion(DB::IDataType const*, DB::IDataType const*, std::__1::basic_string, std::__1::allocator >) @ 0x1c2585d7 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:03.714451 [ 2705 ] {} BaseDaemon: 5. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1582: DB::MergeTreeData::checkAlterIsPossible(DB::AlterCommands const&, std::__1::shared_ptr) const @ 0x1c254f66 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:04.692949 [ 2705 ] {} BaseDaemon: 6. /mnt/disk4/hewenting/ClickHouse/src/src/Interpreters/InterpreterAlterQuery.cpp:144: DB::InterpreterAlterQuery::execute() @ 0x1bb7e403 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server ```. [#23483](https://github.com/ClickHouse/ClickHouse/pull/23483) ([hexiaoting](https://github.com/hexiaoting)). * Backported in [#23533](https://github.com/ClickHouse/ClickHouse/issues/23533): Fix `columns` function when multiple joins in select query. Closes [#22736](https://github.com/ClickHouse/ClickHouse/issues/22736). [#23501](https://github.com/ClickHouse/ClickHouse/pull/23501) ([Maksim Kita](https://github.com/kitaisreal)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix logical error in stress tests [#23197](https://github.com/ClickHouse/ClickHouse/pull/23197) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + diff --git a/docs/changelogs/v21.3.9.83-lts.md b/docs/changelogs/v21.3.9.83-lts.md index e437ee4800f..450faeea3f2 100644 --- a/docs/changelogs/v21.3.9.83-lts.md +++ b/docs/changelogs/v21.3.9.83-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.3.9.83-lts FIXME as compared to v21.3.8.76-lts #### Improvement diff --git a/docs/changelogs/v21.4.1.6422-prestable.md b/docs/changelogs/v21.4.1.6422-prestable.md index 8cd10834fae..684fde2e3b7 100644 --- a/docs/changelogs/v21.4.1.6422-prestable.md +++ b/docs/changelogs/v21.4.1.6422-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.1.6422-prestable FIXME as compared to v21.3.1.6185-prestable #### Backward Incompatible Change @@ -167,6 +174,117 @@ * NO CL ENTRY: 'Flatten libcpuid PEERDIRs'. [#22078](https://github.com/ClickHouse/ClickHouse/pull/22078) ([Yuriy Chernyshov](https://github.com/georgthegreat)). * NO CL ENTRY: 'Revert "quick fix for broken resolution of apt.llvm.org on Yandex infra"'. [#22374](https://github.com/ClickHouse/ClickHouse/pull/22374) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add soft task timeout for Intergation tests [#16608](https://github.com/ClickHouse/ClickHouse/pull/16608) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging geometry functions [#19257](https://github.com/ClickHouse/ClickHouse/pull/19257) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove outdated suppressions, part 2 [#19496](https://github.com/ClickHouse/ClickHouse/pull/19496) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Move some CI-related scripts to github [#20946](https://github.com/ClickHouse/ClickHouse/pull/20946) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix [#21170](https://github.com/ClickHouse/ClickHouse/issues/21170) [#21182](https://github.com/ClickHouse/ClickHouse/pull/21182) ([Tachikoma](https://github.com/ikarishinjieva)). +* Add more tests for quota consumption by the SHOW statement [#21190](https://github.com/ClickHouse/ClickHouse/pull/21190) ([Vitaly Baranov](https://github.com/vitlibar)). +* Save packed keys for GROUP BY with multiple fixed size keys [#21196](https://github.com/ClickHouse/ClickHouse/pull/21196) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Refine MergeTreeData::loadDataParts to not parse format version file and detached directory [#21351](https://github.com/ClickHouse/ClickHouse/pull/21351) ([Fuwang Hu](https://github.com/fuwhu)). +* Persistent nukeeper snapshot storage [#21425](https://github.com/ClickHouse/ClickHouse/pull/21425) ([alesapin](https://github.com/alesapin)). +* Fix logging for optimize_aggregation_in_order=1 (with small max_block_size) [#21436](https://github.com/ClickHouse/ClickHouse/pull/21436) ([Azat Khuzhin](https://github.com/azat)). +* Adjust prewhere_with_row_level_filter performance test [#21442](https://github.com/ClickHouse/ClickHouse/pull/21442) ([Denis Glazachev](https://github.com/traceon)). +* Refactor actions dag [#21459](https://github.com/ClickHouse/ClickHouse/pull/21459) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* add query formatting idempotence check to fuzzer [#21466](https://github.com/ClickHouse/ClickHouse/pull/21466) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix heap-buffer-overflow in highlighting multi-line comments [#21492](https://github.com/ClickHouse/ClickHouse/pull/21492) ([Azat Khuzhin](https://github.com/azat)). +* Fix global stop merges in test [#21508](https://github.com/ClickHouse/ClickHouse/pull/21508) ([alesapin](https://github.com/alesapin)). +* DirectDictionary updated [#21513](https://github.com/ClickHouse/ClickHouse/pull/21513) ([Maksim Kita](https://github.com/kitaisreal)). +* Avoid processing optimize_skip_unused_shards twice [#21526](https://github.com/ClickHouse/ClickHouse/pull/21526) ([Azat Khuzhin](https://github.com/azat)). +* DOCSUP-7197: Escaped Unicode replaced with symbols [#21530](https://github.com/ClickHouse/ClickHouse/pull/21530) ([lehasm](https://github.com/lehasm)). +* Pod array left pad not multiple of element crash fix [#21532](https://github.com/ClickHouse/ClickHouse/pull/21532) ([Maksim Kita](https://github.com/kitaisreal)). +* ShellCommand waitpid eintr signal fix [#21546](https://github.com/ClickHouse/ClickHouse/pull/21546) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactoring of data types serialization [#21562](https://github.com/ClickHouse/ClickHouse/pull/21562) ([Anton Popov](https://github.com/CurtizJ)). +* fix a typo in window functions frame [#21572](https://github.com/ClickHouse/ClickHouse/pull/21572) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Added specialized CacheDictionaryStorage [#21573](https://github.com/ClickHouse/ClickHouse/pull/21573) ([Maksim Kita](https://github.com/kitaisreal)). +* [RFC] Union merge for arcadia_skip_list.txt to avoid frequent conflicts [#21580](https://github.com/ClickHouse/ClickHouse/pull/21580) ([Azat Khuzhin](https://github.com/azat)). +* Enable ipv6 in NuRaft [#21593](https://github.com/ClickHouse/ClickHouse/pull/21593) ([alesapin](https://github.com/alesapin)). +* add an article about ast-based fuzzer [#21608](https://github.com/ClickHouse/ClickHouse/pull/21608) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Changelog 21.3 [#21618](https://github.com/ClickHouse/ClickHouse/pull/21618) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* redefine some errorcode [#21629](https://github.com/ClickHouse/ClickHouse/pull/21629) ([flynn](https://github.com/ucasfl)). +* Fix ambigous column error in joins_in_memory [#21658](https://github.com/ClickHouse/ClickHouse/pull/21658) ([Vladimir C](https://github.com/vdimir)). +* Add test for path as a query parameter in system.zookeeper [#21661](https://github.com/ClickHouse/ClickHouse/pull/21661) ([Kruglov Pavel](https://github.com/Avogar)). +* ExecutablePool fix default max execution time setting [#21662](https://github.com/ClickHouse/ClickHouse/pull/21662) ([Maksim Kita](https://github.com/kitaisreal)). +* DictionaryStructure fix non unique attribute names [#21668](https://github.com/ClickHouse/ClickHouse/pull/21668) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix flaky test_replace_partition [#21674](https://github.com/ClickHouse/ClickHouse/pull/21674) ([Azat Khuzhin](https://github.com/azat)). +* DOC Fix ORDER BY syntax [#21675](https://github.com/ClickHouse/ClickHouse/pull/21675) ([Michael Monashev](https://github.com/MichaelMonashev)). +* PODArray swap fix [#21678](https://github.com/ClickHouse/ClickHouse/pull/21678) ([Maksim Kita](https://github.com/kitaisreal)). +* LibraryDictionarySource fix possible leak [#21686](https://github.com/ClickHouse/ClickHouse/pull/21686) ([Maksim Kita](https://github.com/kitaisreal)). +* Run three nodes with Replicated database and NuKeeper in CI [#21690](https://github.com/ClickHouse/ClickHouse/pull/21690) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix error message in clickhouse-test [#21691](https://github.com/ClickHouse/ClickHouse/pull/21691) ([Azat Khuzhin](https://github.com/azat)). +* Set SOCK_CLOEXEC for sockets (hardcoded via poco update) [#21695](https://github.com/ClickHouse/ClickHouse/pull/21695) ([Azat Khuzhin](https://github.com/azat)). +* Tests fixes (that was found by stress tests) [#21696](https://github.com/ClickHouse/ClickHouse/pull/21696) ([Azat Khuzhin](https://github.com/azat)). +* Fix log_comment for *.sh in clickhouse-test [#21700](https://github.com/ClickHouse/ClickHouse/pull/21700) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless CMake option [#21712](https://github.com/ClickHouse/ClickHouse/pull/21712) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in modulo by constant [#21713](https://github.com/ClickHouse/ClickHouse/pull/21713) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add more variants for memcpy benchmark [#21715](https://github.com/ClickHouse/ClickHouse/pull/21715) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not overlap zookeeper path for ReplicatedMergeTree in stateless *.sh tests [#21724](https://github.com/ClickHouse/ClickHouse/pull/21724) ([Azat Khuzhin](https://github.com/azat)). +* make the fuzzer use sources from the CI [#21754](https://github.com/ClickHouse/ClickHouse/pull/21754) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add one more variant to memcpy benchmark [#21759](https://github.com/ClickHouse/ClickHouse/pull/21759) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix incorrect number of rows for Chunks with no columns in PartialSor… [#21761](https://github.com/ClickHouse/ClickHouse/pull/21761) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* docs(fix): typo [#21775](https://github.com/ClickHouse/ClickHouse/pull/21775) ([Ali Demirci](https://github.com/depyronick)). +* DDLWorker.cpp: fixed exceeded amount of tries typo [#21807](https://github.com/ClickHouse/ClickHouse/pull/21807) ([Eldar Nasyrov](https://github.com/3ldar-nasyrov)). +* fix integration MaterializeMySQL test [#21819](https://github.com/ClickHouse/ClickHouse/pull/21819) ([TCeason](https://github.com/TCeason)). +* more robust error handling in perf test [#21846](https://github.com/ClickHouse/ClickHouse/pull/21846) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for [#17302](https://github.com/ClickHouse/ClickHouse/issues/17302) [#21848](https://github.com/ClickHouse/ClickHouse/pull/21848) ([Denny Crane](https://github.com/den-crane)). +* Add bash completion support for clickhouse utils [#21853](https://github.com/ClickHouse/ClickHouse/pull/21853) ([Azat Khuzhin](https://github.com/azat)). +* LRUCache fix exception unsafe element insertion [#21891](https://github.com/ClickHouse/ClickHouse/pull/21891) ([Maksim Kita](https://github.com/kitaisreal)). +* fix fuzzer failure in tupleElement formatting [#21896](https://github.com/ClickHouse/ClickHouse/pull/21896) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix possibly dangling reference to Context [#21913](https://github.com/ClickHouse/ClickHouse/pull/21913) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add stress test for distributed queries [#21944](https://github.com/ClickHouse/ClickHouse/pull/21944) ([Azat Khuzhin](https://github.com/azat)). +* Fix misleading log in WriteBufferFromS3 [#21954](https://github.com/ClickHouse/ClickHouse/pull/21954) ([flynn](https://github.com/ucasfl)). +* Add a test for [#21991](https://github.com/ClickHouse/ClickHouse/issues/21991) [#21995](https://github.com/ClickHouse/ClickHouse/pull/21995) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#11720](https://github.com/ClickHouse/ClickHouse/issues/11720) [#21997](https://github.com/ClickHouse/ClickHouse/pull/21997) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#15784](https://github.com/ClickHouse/ClickHouse/issues/15784) [#22002](https://github.com/ClickHouse/ClickHouse/pull/22002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* prevent accidental reinterpret_cast in Field::get<> [#22003](https://github.com/ClickHouse/ClickHouse/pull/22003) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix UBSan report in addMonths [#22006](https://github.com/ClickHouse/ClickHouse/pull/22006) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#7963](https://github.com/ClickHouse/ClickHouse/issues/7963) [#22007](https://github.com/ClickHouse/ClickHouse/pull/22007) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in intDiv [#21769](https://github.com/ClickHouse/ClickHouse/issues/21769) [#22009](https://github.com/ClickHouse/ClickHouse/pull/22009) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Cast to enum nullable fix [#22026](https://github.com/ClickHouse/ClickHouse/pull/22026) ([Maksim Kita](https://github.com/kitaisreal)). +* Small simplification in ExternalLoader. [#22027](https://github.com/ClickHouse/ClickHouse/pull/22027) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add test for [#21760](https://github.com/ClickHouse/ClickHouse/issues/21760) [#22036](https://github.com/ClickHouse/ClickHouse/pull/22036) ([Anton Popov](https://github.com/CurtizJ)). +* quick fix for broken resolution of apt.llvm.org on Yandex infra [#22055](https://github.com/ClickHouse/ClickHouse/pull/22055) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Darwin cmake disable memcpy benchmark [#22056](https://github.com/ClickHouse/ClickHouse/pull/22056) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix UBSan report in TransformDateTime64 [#22062](https://github.com/ClickHouse/ClickHouse/pull/22062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in mapPopulateSeries. [#22099](https://github.com/ClickHouse/ClickHouse/pull/22099) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bar with invalid float value [#22114](https://github.com/ClickHouse/ClickHouse/pull/22114) ([flynn](https://github.com/ucasfl)). +* remove useless code [#22117](https://github.com/ClickHouse/ClickHouse/pull/22117) ([flynn](https://github.com/ucasfl)). +* stable formatting for negate() [#22133](https://github.com/ClickHouse/ClickHouse/pull/22133) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* adjust perf test thresholds [#22148](https://github.com/ClickHouse/ClickHouse/pull/22148) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix sleep_in_send_tables_status_ms/sleep_in_send_data_ms in integration tests [#22151](https://github.com/ClickHouse/ClickHouse/pull/22151) ([Azat Khuzhin](https://github.com/azat)). +* Update requirements.txt [#22153](https://github.com/ClickHouse/ClickHouse/pull/22153) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix some flaky order dependent integration tests. [#22170](https://github.com/ClickHouse/ClickHouse/pull/22170) ([alesapin](https://github.com/alesapin)). +* Prevent busy waiting in hedged requests when async_socket_for_remote=0 [#22172](https://github.com/ClickHouse/ClickHouse/pull/22172) ([Kruglov Pavel](https://github.com/Avogar)). +* less flaky functional tests [#22181](https://github.com/ClickHouse/ClickHouse/pull/22181) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for [#10489](https://github.com/ClickHouse/ClickHouse/issues/10489) [#22219](https://github.com/ClickHouse/ClickHouse/pull/22219) ([Denny Crane](https://github.com/den-crane)). +* CachedCompressedReadBuffer fix cache usage [#22225](https://github.com/ClickHouse/ClickHouse/pull/22225) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix MSan report in `quantileDeterministic` [#22235](https://github.com/ClickHouse/ClickHouse/pull/22235) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove harmful default parameters [#22238](https://github.com/ClickHouse/ClickHouse/pull/22238) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix build error [#22243](https://github.com/ClickHouse/ClickHouse/pull/22243) ([hexiaoting](https://github.com/hexiaoting)). +* Rename NuKeeper and TestKeeper to Keeper in all places [#22274](https://github.com/ClickHouse/ClickHouse/pull/22274) ([alesapin](https://github.com/alesapin)). +* Update materialize-mysql.md [#22275](https://github.com/ClickHouse/ClickHouse/pull/22275) ([曲正鹏](https://github.com/quzhengpeng)). +* Fix native macOS build for ALL_BUILD (Xcode/AppleClang) [#22289](https://github.com/ClickHouse/ClickHouse/pull/22289) ([Denis Glazachev](https://github.com/traceon)). +* Add suffixes for dockerfile arguments [#22301](https://github.com/ClickHouse/ClickHouse/pull/22301) ([filimonov](https://github.com/filimonov)). +* More coarse test for DateLUT [#22320](https://github.com/ClickHouse/ClickHouse/pull/22320) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#22328](https://github.com/ClickHouse/ClickHouse/pull/22328) ([Anton Popov](https://github.com/CurtizJ)). +* Maybe fix false MSan report in GRPC [#22338](https://github.com/ClickHouse/ClickHouse/pull/22338) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old MSan suppressions (part 3) [#22357](https://github.com/ClickHouse/ClickHouse/pull/22357) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old MSan suppressions (part 4) [#22358](https://github.com/ClickHouse/ClickHouse/pull/22358) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky tests test_row_policy* and test_quota* [#22371](https://github.com/ClickHouse/ClickHouse/pull/22371) ([Vitaly Baranov](https://github.com/vitlibar)). +* Try fix flaky rabbitmq test [#22380](https://github.com/ClickHouse/ClickHouse/pull/22380) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix UBSan report in mapOp [#22389](https://github.com/ClickHouse/ClickHouse/pull/22389) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove recursive submodules from Arrow [#22390](https://github.com/ClickHouse/ClickHouse/pull/22390) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix some OOMs in stress tests [#22396](https://github.com/ClickHouse/ClickHouse/pull/22396) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Correctly place debug helpers [#22407](https://github.com/ClickHouse/ClickHouse/pull/22407) ([Amos Bird](https://github.com/amosbird)). +* fix error message for invalid window frame start [#22412](https://github.com/ClickHouse/ClickHouse/pull/22412) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix flapping test_s3_zero_copy_replication [#22440](https://github.com/ClickHouse/ClickHouse/pull/22440) ([ianton-ru](https://github.com/ianton-ru)). +* Lower scale of a test [#22446](https://github.com/ClickHouse/ClickHouse/pull/22446) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Adjust Keeper settings for CI [#22470](https://github.com/ClickHouse/ClickHouse/pull/22470) ([Alexander Tokmakov](https://github.com/tavplubix)). +* try clang 11 in fast test [#22472](https://github.com/ClickHouse/ClickHouse/pull/22472) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove TestFlows (2) [#22480](https://github.com/ClickHouse/ClickHouse/pull/22480) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + #### New Feature (datasketches support in clickhouse #14893) * Support ThetaSketch to do set operations. [#22207](https://github.com/ClickHouse/ClickHouse/pull/22207) ([Ping Yu](https://github.com/pingyu)). diff --git a/docs/changelogs/v21.4.2.10-prestable.md b/docs/changelogs/v21.4.2.10-prestable.md index 1bc440d126c..11767613551 100644 --- a/docs/changelogs/v21.4.2.10-prestable.md +++ b/docs/changelogs/v21.4.2.10-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.2.10-prestable FIXME as compared to v21.3.1.6185-prestable #### Backward Incompatible Change @@ -170,6 +177,114 @@ * NO CL ENTRY: 'Flatten libcpuid PEERDIRs'. [#22078](https://github.com/ClickHouse/ClickHouse/pull/22078) ([Yuriy Chernyshov](https://github.com/georgthegreat)). * NO CL ENTRY: 'Revert "quick fix for broken resolution of apt.llvm.org on Yandex infra"'. [#22374](https://github.com/ClickHouse/ClickHouse/pull/22374) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add soft task timeout for Intergation tests [#16608](https://github.com/ClickHouse/ClickHouse/pull/16608) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging geometry functions [#19257](https://github.com/ClickHouse/ClickHouse/pull/19257) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove outdated suppressions, part 2 [#19496](https://github.com/ClickHouse/ClickHouse/pull/19496) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Move some CI-related scripts to github [#20946](https://github.com/ClickHouse/ClickHouse/pull/20946) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix [#21170](https://github.com/ClickHouse/ClickHouse/issues/21170) [#21182](https://github.com/ClickHouse/ClickHouse/pull/21182) ([Tachikoma](https://github.com/ikarishinjieva)). +* Add more tests for quota consumption by the SHOW statement [#21190](https://github.com/ClickHouse/ClickHouse/pull/21190) ([Vitaly Baranov](https://github.com/vitlibar)). +* Save packed keys for GROUP BY with multiple fixed size keys [#21196](https://github.com/ClickHouse/ClickHouse/pull/21196) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Refine MergeTreeData::loadDataParts to not parse format version file and detached directory [#21351](https://github.com/ClickHouse/ClickHouse/pull/21351) ([Fuwang Hu](https://github.com/fuwhu)). +* Persistent nukeeper snapshot storage [#21425](https://github.com/ClickHouse/ClickHouse/pull/21425) ([alesapin](https://github.com/alesapin)). +* Fix logging for optimize_aggregation_in_order=1 (with small max_block_size) [#21436](https://github.com/ClickHouse/ClickHouse/pull/21436) ([Azat Khuzhin](https://github.com/azat)). +* Adjust prewhere_with_row_level_filter performance test [#21442](https://github.com/ClickHouse/ClickHouse/pull/21442) ([Denis Glazachev](https://github.com/traceon)). +* Refactor actions dag [#21459](https://github.com/ClickHouse/ClickHouse/pull/21459) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* add query formatting idempotence check to fuzzer [#21466](https://github.com/ClickHouse/ClickHouse/pull/21466) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix heap-buffer-overflow in highlighting multi-line comments [#21492](https://github.com/ClickHouse/ClickHouse/pull/21492) ([Azat Khuzhin](https://github.com/azat)). +* Fix global stop merges in test [#21508](https://github.com/ClickHouse/ClickHouse/pull/21508) ([alesapin](https://github.com/alesapin)). +* DirectDictionary updated [#21513](https://github.com/ClickHouse/ClickHouse/pull/21513) ([Maksim Kita](https://github.com/kitaisreal)). +* Avoid processing optimize_skip_unused_shards twice [#21526](https://github.com/ClickHouse/ClickHouse/pull/21526) ([Azat Khuzhin](https://github.com/azat)). +* DOCSUP-7197: Escaped Unicode replaced with symbols [#21530](https://github.com/ClickHouse/ClickHouse/pull/21530) ([lehasm](https://github.com/lehasm)). +* Pod array left pad not multiple of element crash fix [#21532](https://github.com/ClickHouse/ClickHouse/pull/21532) ([Maksim Kita](https://github.com/kitaisreal)). +* ShellCommand waitpid eintr signal fix [#21546](https://github.com/ClickHouse/ClickHouse/pull/21546) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactoring of data types serialization [#21562](https://github.com/ClickHouse/ClickHouse/pull/21562) ([Anton Popov](https://github.com/CurtizJ)). +* fix a typo in window functions frame [#21572](https://github.com/ClickHouse/ClickHouse/pull/21572) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Added specialized CacheDictionaryStorage [#21573](https://github.com/ClickHouse/ClickHouse/pull/21573) ([Maksim Kita](https://github.com/kitaisreal)). +* [RFC] Union merge for arcadia_skip_list.txt to avoid frequent conflicts [#21580](https://github.com/ClickHouse/ClickHouse/pull/21580) ([Azat Khuzhin](https://github.com/azat)). +* Enable ipv6 in NuRaft [#21593](https://github.com/ClickHouse/ClickHouse/pull/21593) ([alesapin](https://github.com/alesapin)). +* add an article about ast-based fuzzer [#21608](https://github.com/ClickHouse/ClickHouse/pull/21608) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Changelog 21.3 [#21618](https://github.com/ClickHouse/ClickHouse/pull/21618) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* redefine some errorcode [#21629](https://github.com/ClickHouse/ClickHouse/pull/21629) ([flynn](https://github.com/ucasfl)). +* Fix ambigous column error in joins_in_memory [#21658](https://github.com/ClickHouse/ClickHouse/pull/21658) ([Vladimir C](https://github.com/vdimir)). +* Add test for path as a query parameter in system.zookeeper [#21661](https://github.com/ClickHouse/ClickHouse/pull/21661) ([Kruglov Pavel](https://github.com/Avogar)). +* ExecutablePool fix default max execution time setting [#21662](https://github.com/ClickHouse/ClickHouse/pull/21662) ([Maksim Kita](https://github.com/kitaisreal)). +* DictionaryStructure fix non unique attribute names [#21668](https://github.com/ClickHouse/ClickHouse/pull/21668) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix flaky test_replace_partition [#21674](https://github.com/ClickHouse/ClickHouse/pull/21674) ([Azat Khuzhin](https://github.com/azat)). +* DOC Fix ORDER BY syntax [#21675](https://github.com/ClickHouse/ClickHouse/pull/21675) ([Michael Monashev](https://github.com/MichaelMonashev)). +* PODArray swap fix [#21678](https://github.com/ClickHouse/ClickHouse/pull/21678) ([Maksim Kita](https://github.com/kitaisreal)). +* LibraryDictionarySource fix possible leak [#21686](https://github.com/ClickHouse/ClickHouse/pull/21686) ([Maksim Kita](https://github.com/kitaisreal)). +* Run three nodes with Replicated database and NuKeeper in CI [#21690](https://github.com/ClickHouse/ClickHouse/pull/21690) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix error message in clickhouse-test [#21691](https://github.com/ClickHouse/ClickHouse/pull/21691) ([Azat Khuzhin](https://github.com/azat)). +* Set SOCK_CLOEXEC for sockets (hardcoded via poco update) [#21695](https://github.com/ClickHouse/ClickHouse/pull/21695) ([Azat Khuzhin](https://github.com/azat)). +* Tests fixes (that was found by stress tests) [#21696](https://github.com/ClickHouse/ClickHouse/pull/21696) ([Azat Khuzhin](https://github.com/azat)). +* Fix log_comment for *.sh in clickhouse-test [#21700](https://github.com/ClickHouse/ClickHouse/pull/21700) ([Azat Khuzhin](https://github.com/azat)). +* Remove useless CMake option [#21712](https://github.com/ClickHouse/ClickHouse/pull/21712) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in modulo by constant [#21713](https://github.com/ClickHouse/ClickHouse/pull/21713) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add more variants for memcpy benchmark [#21715](https://github.com/ClickHouse/ClickHouse/pull/21715) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Do not overlap zookeeper path for ReplicatedMergeTree in stateless *.sh tests [#21724](https://github.com/ClickHouse/ClickHouse/pull/21724) ([Azat Khuzhin](https://github.com/azat)). +* make the fuzzer use sources from the CI [#21754](https://github.com/ClickHouse/ClickHouse/pull/21754) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add one more variant to memcpy benchmark [#21759](https://github.com/ClickHouse/ClickHouse/pull/21759) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* fix incorrect number of rows for Chunks with no columns in PartialSor… [#21761](https://github.com/ClickHouse/ClickHouse/pull/21761) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* docs(fix): typo [#21775](https://github.com/ClickHouse/ClickHouse/pull/21775) ([Ali Demirci](https://github.com/depyronick)). +* DDLWorker.cpp: fixed exceeded amount of tries typo [#21807](https://github.com/ClickHouse/ClickHouse/pull/21807) ([Eldar Nasyrov](https://github.com/3ldar-nasyrov)). +* fix integration MaterializeMySQL test [#21819](https://github.com/ClickHouse/ClickHouse/pull/21819) ([TCeason](https://github.com/TCeason)). +* more robust error handling in perf test [#21846](https://github.com/ClickHouse/ClickHouse/pull/21846) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for [#17302](https://github.com/ClickHouse/ClickHouse/issues/17302) [#21848](https://github.com/ClickHouse/ClickHouse/pull/21848) ([Denny Crane](https://github.com/den-crane)). +* Add bash completion support for clickhouse utils [#21853](https://github.com/ClickHouse/ClickHouse/pull/21853) ([Azat Khuzhin](https://github.com/azat)). +* LRUCache fix exception unsafe element insertion [#21891](https://github.com/ClickHouse/ClickHouse/pull/21891) ([Maksim Kita](https://github.com/kitaisreal)). +* fix fuzzer failure in tupleElement formatting [#21896](https://github.com/ClickHouse/ClickHouse/pull/21896) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix possibly dangling reference to Context [#21913](https://github.com/ClickHouse/ClickHouse/pull/21913) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add stress test for distributed queries [#21944](https://github.com/ClickHouse/ClickHouse/pull/21944) ([Azat Khuzhin](https://github.com/azat)). +* Fix misleading log in WriteBufferFromS3 [#21954](https://github.com/ClickHouse/ClickHouse/pull/21954) ([flynn](https://github.com/ucasfl)). +* Add a test for [#21991](https://github.com/ClickHouse/ClickHouse/issues/21991) [#21995](https://github.com/ClickHouse/ClickHouse/pull/21995) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#11720](https://github.com/ClickHouse/ClickHouse/issues/11720) [#21997](https://github.com/ClickHouse/ClickHouse/pull/21997) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#15784](https://github.com/ClickHouse/ClickHouse/issues/15784) [#22002](https://github.com/ClickHouse/ClickHouse/pull/22002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* prevent accidental reinterpret_cast in Field::get<> [#22003](https://github.com/ClickHouse/ClickHouse/pull/22003) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix UBSan report in addMonths [#22006](https://github.com/ClickHouse/ClickHouse/pull/22006) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#7963](https://github.com/ClickHouse/ClickHouse/issues/7963) [#22007](https://github.com/ClickHouse/ClickHouse/pull/22007) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in intDiv [#21769](https://github.com/ClickHouse/ClickHouse/issues/21769) [#22009](https://github.com/ClickHouse/ClickHouse/pull/22009) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Cast to enum nullable fix [#22026](https://github.com/ClickHouse/ClickHouse/pull/22026) ([Maksim Kita](https://github.com/kitaisreal)). +* Small simplification in ExternalLoader. [#22027](https://github.com/ClickHouse/ClickHouse/pull/22027) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add test for [#21760](https://github.com/ClickHouse/ClickHouse/issues/21760) [#22036](https://github.com/ClickHouse/ClickHouse/pull/22036) ([Anton Popov](https://github.com/CurtizJ)). +* quick fix for broken resolution of apt.llvm.org on Yandex infra [#22055](https://github.com/ClickHouse/ClickHouse/pull/22055) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Darwin cmake disable memcpy benchmark [#22056](https://github.com/ClickHouse/ClickHouse/pull/22056) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix UBSan report in TransformDateTime64 [#22062](https://github.com/ClickHouse/ClickHouse/pull/22062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in mapPopulateSeries. [#22099](https://github.com/ClickHouse/ClickHouse/pull/22099) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bar with invalid float value [#22114](https://github.com/ClickHouse/ClickHouse/pull/22114) ([flynn](https://github.com/ucasfl)). +* remove useless code [#22117](https://github.com/ClickHouse/ClickHouse/pull/22117) ([flynn](https://github.com/ucasfl)). +* stable formatting for negate() [#22133](https://github.com/ClickHouse/ClickHouse/pull/22133) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* adjust perf test thresholds [#22148](https://github.com/ClickHouse/ClickHouse/pull/22148) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix sleep_in_send_tables_status_ms/sleep_in_send_data_ms in integration tests [#22151](https://github.com/ClickHouse/ClickHouse/pull/22151) ([Azat Khuzhin](https://github.com/azat)). +* Update requirements.txt [#22153](https://github.com/ClickHouse/ClickHouse/pull/22153) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix some flaky order dependent integration tests. [#22170](https://github.com/ClickHouse/ClickHouse/pull/22170) ([alesapin](https://github.com/alesapin)). +* Prevent busy waiting in hedged requests when async_socket_for_remote=0 [#22172](https://github.com/ClickHouse/ClickHouse/pull/22172) ([Kruglov Pavel](https://github.com/Avogar)). +* less flaky functional tests [#22181](https://github.com/ClickHouse/ClickHouse/pull/22181) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* test for [#10489](https://github.com/ClickHouse/ClickHouse/issues/10489) [#22219](https://github.com/ClickHouse/ClickHouse/pull/22219) ([Denny Crane](https://github.com/den-crane)). +* CachedCompressedReadBuffer fix cache usage [#22225](https://github.com/ClickHouse/ClickHouse/pull/22225) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix MSan report in `quantileDeterministic` [#22235](https://github.com/ClickHouse/ClickHouse/pull/22235) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove harmful default parameters [#22238](https://github.com/ClickHouse/ClickHouse/pull/22238) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix build error [#22243](https://github.com/ClickHouse/ClickHouse/pull/22243) ([hexiaoting](https://github.com/hexiaoting)). +* Rename NuKeeper and TestKeeper to Keeper in all places [#22274](https://github.com/ClickHouse/ClickHouse/pull/22274) ([alesapin](https://github.com/alesapin)). +* Update materialize-mysql.md [#22275](https://github.com/ClickHouse/ClickHouse/pull/22275) ([曲正鹏](https://github.com/quzhengpeng)). +* Fix native macOS build for ALL_BUILD (Xcode/AppleClang) [#22289](https://github.com/ClickHouse/ClickHouse/pull/22289) ([Denis Glazachev](https://github.com/traceon)). +* Add suffixes for dockerfile arguments [#22301](https://github.com/ClickHouse/ClickHouse/pull/22301) ([filimonov](https://github.com/filimonov)). +* More coarse test for DateLUT [#22320](https://github.com/ClickHouse/ClickHouse/pull/22320) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless code [#22328](https://github.com/ClickHouse/ClickHouse/pull/22328) ([Anton Popov](https://github.com/CurtizJ)). +* Maybe fix false MSan report in GRPC [#22338](https://github.com/ClickHouse/ClickHouse/pull/22338) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old MSan suppressions (part 3) [#22357](https://github.com/ClickHouse/ClickHouse/pull/22357) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old MSan suppressions (part 4) [#22358](https://github.com/ClickHouse/ClickHouse/pull/22358) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky tests test_row_policy* and test_quota* [#22371](https://github.com/ClickHouse/ClickHouse/pull/22371) ([Vitaly Baranov](https://github.com/vitlibar)). +* Try fix flaky rabbitmq test [#22380](https://github.com/ClickHouse/ClickHouse/pull/22380) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix UBSan report in mapOp [#22389](https://github.com/ClickHouse/ClickHouse/pull/22389) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove recursive submodules from Arrow [#22390](https://github.com/ClickHouse/ClickHouse/pull/22390) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix some OOMs in stress tests [#22396](https://github.com/ClickHouse/ClickHouse/pull/22396) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Correctly place debug helpers [#22407](https://github.com/ClickHouse/ClickHouse/pull/22407) ([Amos Bird](https://github.com/amosbird)). +* fix error message for invalid window frame start [#22412](https://github.com/ClickHouse/ClickHouse/pull/22412) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Lower scale of a test [#22446](https://github.com/ClickHouse/ClickHouse/pull/22446) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove TestFlows (2) [#22480](https://github.com/ClickHouse/ClickHouse/pull/22480) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + #### New Feature (datasketches support in clickhouse #14893) * Support ThetaSketch to do set operations. [#22207](https://github.com/ClickHouse/ClickHouse/pull/22207) ([Ping Yu](https://github.com/pingyu)). diff --git a/docs/changelogs/v21.4.3.21-stable.md b/docs/changelogs/v21.4.3.21-stable.md index dc3d7b7005b..a320e16f06a 100644 --- a/docs/changelogs/v21.4.3.21-stable.md +++ b/docs/changelogs/v21.4.3.21-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.3.21-stable FIXME as compared to v21.4.2.10-prestable #### Improvement @@ -16,3 +23,7 @@ * Backported in [#22921](https://github.com/ClickHouse/ClickHouse/issues/22921): Fixed a crash when using `mannWhitneyUTest` and `rankCorr` with window functions. This fixes [#22728](https://github.com/ClickHouse/ClickHouse/issues/22728). [#22876](https://github.com/ClickHouse/ClickHouse/pull/22876) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Backported in [#22957](https://github.com/ClickHouse/ClickHouse/issues/22957): Fix usage of constant columns of type `Map` with nullable values. [#22939](https://github.com/ClickHouse/ClickHouse/pull/22939) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* FileDictionarySource fix absolute file path [#22822](https://github.com/ClickHouse/ClickHouse/pull/22822) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.4.4.30-stable.md b/docs/changelogs/v21.4.4.30-stable.md index f029d334fc7..755bbede8f3 100644 --- a/docs/changelogs/v21.4.4.30-stable.md +++ b/docs/changelogs/v21.4.4.30-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.4.30-stable FIXME as compared to v21.4.3.21-stable #### Backward Incompatible Change diff --git a/docs/changelogs/v21.4.5.46-stable.md b/docs/changelogs/v21.4.5.46-stable.md index 664037ba596..9ca0553a154 100644 --- a/docs/changelogs/v21.4.5.46-stable.md +++ b/docs/changelogs/v21.4.5.46-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.5.46-stable FIXME as compared to v21.4.4.30-stable #### Improvement @@ -19,3 +26,9 @@ * Backported in [#23534](https://github.com/ClickHouse/ClickHouse/issues/23534): When modify column's default value without datatype, and this column is used as ReplacingMergeTree's parameter like column `b` in the below example, then the server will core dump: ``` CREATE TABLE alter_test (a Int32, b DateTime) ENGINE = ReplacingMergeTree(b) ORDER BY a; ALTER TABLE alter_test MODIFY COLUMN `b` DEFAULT now(); ``` the sever throw error: ``` 2021.04.22 09:48:00.685317 [ 2607 ] {} BaseDaemon: Received signal 11 2021.04.22 09:48:00.686110 [ 2705 ] {} BaseDaemon: ######################################## 2021.04.22 09:48:00.686336 [ 2705 ] {} BaseDaemon: (version 21.6.1.1, build id: 6459E84DFCF8E778546C5AD2FFE91B3AD71E1B1B) (from thread 2619) (no query) Received signal Segmentation fault (11) 2021.04.22 09:48:00.686572 [ 2705 ] {} BaseDaemon: Address: NULL pointer. Access: read. Address not mapped to object. 2021.04.22 09:48:00.686686 [ 2705 ] {} BaseDaemon: Stack trace: 0x1c2585d7 0x1c254f66 0x1bb7e403 0x1bb58923 0x1bb56a85 0x1c6840ef 0x1c691148 0x2061a05c 0x2061a8e4 0x20775a03 0x207722bd 0x20771048 0x7f6e5c25be25 0x7f6e5bd81bad 2021.04.22 09:48:02.283045 [ 2705 ] {} BaseDaemon: 4. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1449: DB::(anonymous namespace)::checkVersionColumnTypesConversion(DB::IDataType const*, DB::IDataType const*, std::__1::basic_string, std::__1::allocator >) @ 0x1c2585d7 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:03.714451 [ 2705 ] {} BaseDaemon: 5. /mnt/disk4/hewenting/ClickHouse/src/src/Storages/MergeTree/MergeTreeData.cpp:1582: DB::MergeTreeData::checkAlterIsPossible(DB::AlterCommands const&, std::__1::shared_ptr) const @ 0x1c254f66 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server 2021.04.22 09:48:04.692949 [ 2705 ] {} BaseDaemon: 6. /mnt/disk4/hewenting/ClickHouse/src/src/Interpreters/InterpreterAlterQuery.cpp:144: DB::InterpreterAlterQuery::execute() @ 0x1bb7e403 in /mnt/disk4/hewenting/ClickHouse/build-dbgsrc-clang-dev-nested/programs/clickhouse-server ```. [#23483](https://github.com/ClickHouse/ClickHouse/pull/23483) ([hexiaoting](https://github.com/hexiaoting)). * Backported in [#23531](https://github.com/ClickHouse/ClickHouse/issues/23531): Fix `columns` function when multiple joins in select query. Closes [#22736](https://github.com/ClickHouse/ClickHouse/issues/22736). [#23501](https://github.com/ClickHouse/ClickHouse/pull/23501) ([Maksim Kita](https://github.com/kitaisreal)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix logical error in stress tests [#23197](https://github.com/ClickHouse/ClickHouse/pull/23197) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix integration tests for Hedged requests [#23275](https://github.com/ClickHouse/ClickHouse/pull/23275) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* System dictionaries virtual key column [#23458](https://github.com/ClickHouse/ClickHouse/pull/23458) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.4.6.55-stable.md b/docs/changelogs/v21.4.6.55-stable.md index ea3e413ea0c..daca28d596c 100644 --- a/docs/changelogs/v21.4.6.55-stable.md +++ b/docs/changelogs/v21.4.6.55-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.6.55-stable FIXME as compared to v21.4.5.46-stable #### Improvement diff --git a/docs/changelogs/v21.4.7.3-stable.md b/docs/changelogs/v21.4.7.3-stable.md index 0dad6cfcb2b..612201a73ee 100644 --- a/docs/changelogs/v21.4.7.3-stable.md +++ b/docs/changelogs/v21.4.7.3-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.4.7.3-stable FIXME as compared to v21.4.6.55-stable #### Bug Fix @@ -15,3 +22,7 @@ * Backported in [#24215](https://github.com/ClickHouse/ClickHouse/issues/24215): Fix race condition which could happen in RBAC under a heavy load. This PR fixes [#24090](https://github.com/ClickHouse/ClickHouse/issues/24090), [#24134](https://github.com/ClickHouse/ClickHouse/issues/24134),. [#24176](https://github.com/ClickHouse/ClickHouse/pull/24176) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#24243](https://github.com/ClickHouse/ClickHouse/issues/24243): Fix abnormal server termination due to hdfs becoming not accessible during query execution. Closes [#24117](https://github.com/ClickHouse/ClickHouse/issues/24117). [#24191](https://github.com/ClickHouse/ClickHouse/pull/24191) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Flat, Hashed dictionary include update field bytes into bytes_allocated [#23825](https://github.com/ClickHouse/ClickHouse/pull/23825) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.5.1.6601-prestable.md b/docs/changelogs/v21.5.1.6601-prestable.md index d64936fefce..dbe75de93d4 100644 --- a/docs/changelogs/v21.5.1.6601-prestable.md +++ b/docs/changelogs/v21.5.1.6601-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.1.6601-prestable FIXME as compared to v21.4.1.6422-prestable #### Backward Incompatible Change @@ -34,7 +41,7 @@ * Allow to use CTE in VIEW definition. This closes [#22491](https://github.com/ClickHouse/ClickHouse/issues/22491). [#22657](https://github.com/ClickHouse/ClickHouse/pull/22657) ([Amos Bird](https://github.com/amosbird)). * Add metric to track how much time is spend during waiting for Buffer layer lock. [#22725](https://github.com/ClickHouse/ClickHouse/pull/22725) ([Azat Khuzhin](https://github.com/azat)). * Allow RBAC row policy via postgresql protocol. Closes [#22658](https://github.com/ClickHouse/ClickHouse/issues/22658). PostgreSQL protocol is enabled in configuration by default. [#22755](https://github.com/ClickHouse/ClickHouse/pull/22755) ([Kseniia Sumarokova](https://github.com/kssenii)). -* MaterializeMySQL (experimental feature). Make ClickHouse to be able to replicate MySQL databases containing views without failing. This is accomplished by ignoring the views. ... [#22760](https://github.com/ClickHouse/ClickHouse/pull/22760) ([Christian Frøystad](https://github.com/cfroystad)). +* MaterializeMySQL (experimental feature). Make Clickhouse to be able to replicate MySQL databases containing views without failing. This is accomplished by ignoring the views. ... [#22760](https://github.com/ClickHouse/ClickHouse/pull/22760) ([Christian Frøystad](https://github.com/cfroystad)). * `dateDiff` now works with `DateTime64` arguments (even for values outside of `DateTime` range) ... [#22931](https://github.com/ClickHouse/ClickHouse/pull/22931) ([Vasily Nemkov](https://github.com/Enmk)). * Set `background_fetches_pool_size` to 8 that is better for production usage with frequent small insertions or slow ZooKeeper cluster. [#22945](https://github.com/ClickHouse/ClickHouse/pull/22945) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Fix inactive_parts_to_throw_insert=0 with inactive_parts_to_delay_insert>0. [#22947](https://github.com/ClickHouse/ClickHouse/pull/22947) ([Azat Khuzhin](https://github.com/azat)). @@ -107,3 +114,96 @@ * NO CL ENTRY: 'Error message reads better'. [#22983](https://github.com/ClickHouse/ClickHouse/pull/22983) ([Igor O'sten](https://github.com/borodark)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix SIGSEGV by waiting servers thread pool [#21318](https://github.com/ClickHouse/ClickHouse/pull/21318) ([Azat Khuzhin](https://github.com/azat)). +* Better filter push down [#22087](https://github.com/ClickHouse/ClickHouse/pull/22087) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better tests for finalize in nested writers [#22110](https://github.com/ClickHouse/ClickHouse/pull/22110) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Replace all Context references with std::weak_ptr [#22297](https://github.com/ClickHouse/ClickHouse/pull/22297) ([Ivan](https://github.com/abyss7)). +* make some perf test queries more stable [#22324](https://github.com/ClickHouse/ClickHouse/pull/22324) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* fix ccache broken by prlimit [#22356](https://github.com/ClickHouse/ClickHouse/pull/22356) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove old MSan suppressions (part 5) [#22359](https://github.com/ClickHouse/ClickHouse/pull/22359) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for copier [#22441](https://github.com/ClickHouse/ClickHouse/pull/22441) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fix window frame offset check and add more tests [#22459](https://github.com/ClickHouse/ClickHouse/pull/22459) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* FormatSettings null_as_default default value fix [#22528](https://github.com/ClickHouse/ClickHouse/pull/22528) ([Maksim Kita](https://github.com/kitaisreal)). +* Minor fixes in tests for AArch64 [#22534](https://github.com/ClickHouse/ClickHouse/pull/22534) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Filter removed/renamed tests from ci-changed-files.txt for fuzzer [#22542](https://github.com/ClickHouse/ClickHouse/pull/22542) ([Azat Khuzhin](https://github.com/azat)). +* Try fix flaky test [#22543](https://github.com/ClickHouse/ClickHouse/pull/22543) ([Alexander Tokmakov](https://github.com/tavplubix)). +* AppleClang compilation fix [#22561](https://github.com/ClickHouse/ClickHouse/pull/22561) ([Denis Glazachev](https://github.com/traceon)). +* Fix assert in Arena when doing GROUP BY Array of Nothing of non-zero size. [#22565](https://github.com/ClickHouse/ClickHouse/pull/22565) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor improvement in index deserialization [#22586](https://github.com/ClickHouse/ClickHouse/pull/22586) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test after [#22427](https://github.com/ClickHouse/ClickHouse/issues/22427) [#22587](https://github.com/ClickHouse/ClickHouse/pull/22587) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix comments [#22589](https://github.com/ClickHouse/ClickHouse/pull/22589) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix some uncaught exceptions (in SCOPE_EXIT) under memory pressure [#22592](https://github.com/ClickHouse/ClickHouse/pull/22592) ([Azat Khuzhin](https://github.com/azat)). +* Introduce IStorage::distributedWrite method for distributed INSERT SELECTS [#22593](https://github.com/ClickHouse/ClickHouse/pull/22593) ([Max Akhmedov](https://github.com/zlobober)). +* Fix flaky test 00816_long_concurrent_alter_column [#22628](https://github.com/ClickHouse/ClickHouse/pull/22628) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* FlatDictionary fix perf test [#22629](https://github.com/ClickHouse/ClickHouse/pull/22629) ([Maksim Kita](https://github.com/kitaisreal)). +* DirectDictionary dictGet multiple columns optimization [#22630](https://github.com/ClickHouse/ClickHouse/pull/22630) ([Maksim Kita](https://github.com/kitaisreal)). +* Better retries on ZK errors in sh tests [#22633](https://github.com/ClickHouse/ClickHouse/pull/22633) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add log_comment setting for DROP/CREATE DATABASE in clickhouse-test [#22646](https://github.com/ClickHouse/ClickHouse/pull/22646) ([Azat Khuzhin](https://github.com/azat)). +* Add retires for docker-compose pull in integration tests [#22647](https://github.com/ClickHouse/ClickHouse/pull/22647) ([Azat Khuzhin](https://github.com/azat)). +* Fix impossible invalid-read for system.errors accounting [#22655](https://github.com/ClickHouse/ClickHouse/pull/22655) ([Azat Khuzhin](https://github.com/azat)). +* Skip compiling xz if we're using system xz (unbundled) [#22656](https://github.com/ClickHouse/ClickHouse/pull/22656) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Fix test 01294_create_settings_profile [#22662](https://github.com/ClickHouse/ClickHouse/pull/22662) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix test 01039_row_policy_dcl [#22663](https://github.com/ClickHouse/ClickHouse/pull/22663) ([Anton Popov](https://github.com/CurtizJ)). +* Another attempt to enable pytest [#22664](https://github.com/ClickHouse/ClickHouse/pull/22664) ([Ivan](https://github.com/abyss7)). +* Fix random failures of tests that are using query_log [#22666](https://github.com/ClickHouse/ClickHouse/pull/22666) ([Anton Popov](https://github.com/CurtizJ)). +* fix build error 'always_inline' function might not be inlinable [#22667](https://github.com/ClickHouse/ClickHouse/pull/22667) ([flynn](https://github.com/ucasfl)). +* Add bool type in postgres engine [#22668](https://github.com/ClickHouse/ClickHouse/pull/22668) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix mutation killers tests [#22670](https://github.com/ClickHouse/ClickHouse/pull/22670) ([alesapin](https://github.com/alesapin)). +* Change Aggregatingmergetree to AggregatingMergeTree in docs [#22687](https://github.com/ClickHouse/ClickHouse/pull/22687) ([Kruglov Pavel](https://github.com/Avogar)). +* fix window functions with multiple input streams and no sorting [#22704](https://github.com/ClickHouse/ClickHouse/pull/22704) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove redundant fsync on coordination logs rotation [#22707](https://github.com/ClickHouse/ClickHouse/pull/22707) ([alesapin](https://github.com/alesapin)). +* Fix test 01702_system_query_log [#22708](https://github.com/ClickHouse/ClickHouse/pull/22708) ([Anton Popov](https://github.com/CurtizJ)). +* MemoryStorage sync comments and code [#22721](https://github.com/ClickHouse/ClickHouse/pull/22721) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix potential segfault on Keeper startup [#22743](https://github.com/ClickHouse/ClickHouse/pull/22743) ([alesapin](https://github.com/alesapin)). +* Avoid using harmful function rand() [#22744](https://github.com/ClickHouse/ClickHouse/pull/22744) ([Amos Bird](https://github.com/amosbird)). +* Fix flacky hedged tests [#22746](https://github.com/ClickHouse/ClickHouse/pull/22746) ([Kruglov Pavel](https://github.com/Avogar)). +* add more messages when flushing the logs [#22761](https://github.com/ClickHouse/ClickHouse/pull/22761) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Moved BorrowedObjectPool to common [#22764](https://github.com/ClickHouse/ClickHouse/pull/22764) ([Maksim Kita](https://github.com/kitaisreal)). +* Functions ExternalDictionaries standardize exception throw [#22821](https://github.com/ClickHouse/ClickHouse/pull/22821) ([Maksim Kita](https://github.com/kitaisreal)). +* FileDictionarySource fix absolute file path [#22822](https://github.com/ClickHouse/ClickHouse/pull/22822) ([Maksim Kita](https://github.com/kitaisreal)). +* Small change in replicated database tests run [#22826](https://github.com/ClickHouse/ClickHouse/pull/22826) ([alesapin](https://github.com/alesapin)). +* Slightly improve logging messages for Distributed async sends [#22829](https://github.com/ClickHouse/ClickHouse/pull/22829) ([Azat Khuzhin](https://github.com/azat)). +* Fix what looks like a trivial mistake [#22833](https://github.com/ClickHouse/ClickHouse/pull/22833) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for already fixed issue [#22855](https://github.com/ClickHouse/ClickHouse/pull/22855) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* DataTypeLowCardinality format tsv parsing issue [#22863](https://github.com/ClickHouse/ClickHouse/pull/22863) ([Maksim Kita](https://github.com/kitaisreal)). +* Updated MariaDB connector fix cmake [#22865](https://github.com/ClickHouse/ClickHouse/pull/22865) ([Maksim Kita](https://github.com/kitaisreal)). +* Prettify logs of integration tests [#22868](https://github.com/ClickHouse/ClickHouse/pull/22868) ([Anton Popov](https://github.com/CurtizJ)). +* Util `memcpy-bench` is built only when position independent code is disabled [#22875](https://github.com/ClickHouse/ClickHouse/pull/22875) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix vanilla GCC compilation in macOS [#22885](https://github.com/ClickHouse/ClickHouse/pull/22885) ([Denis Glazachev](https://github.com/traceon)). +* Better diagnostics for OOM in stress test [#22894](https://github.com/ClickHouse/ClickHouse/pull/22894) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Dictionaries updated performance tests [#22925](https://github.com/ClickHouse/ClickHouse/pull/22925) ([Maksim Kita](https://github.com/kitaisreal)). +* IAggreagteFunction allocatesMemoryInArena removed default implementation [#22938](https://github.com/ClickHouse/ClickHouse/pull/22938) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix flapping test_merge_tree_s3 test [#22942](https://github.com/ClickHouse/ClickHouse/pull/22942) ([ianton-ru](https://github.com/ianton-ru)). +* less flaky test [#22944](https://github.com/ClickHouse/ClickHouse/pull/22944) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Dictionaries standardize exceptions [#22961](https://github.com/ClickHouse/ClickHouse/pull/22961) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageExternalDistributed arcadia fix [#22962](https://github.com/ClickHouse/ClickHouse/pull/22962) ([Maksim Kita](https://github.com/kitaisreal)). +* Check out of range values in FieldVisitorConverToNumber [#22964](https://github.com/ClickHouse/ClickHouse/pull/22964) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix combinators with common prefix name (State and SimpleState) with libstdc++ [#22977](https://github.com/ClickHouse/ClickHouse/pull/22977) ([Azat Khuzhin](https://github.com/azat)). +* Fix arcadia [#22982](https://github.com/ClickHouse/ClickHouse/pull/22982) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix excessive warning in StorageDistributed with cross-replication [#22990](https://github.com/ClickHouse/ClickHouse/pull/22990) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update MergeTreeData.cpp Better error message. [#22994](https://github.com/ClickHouse/ClickHouse/pull/22994) ([Denny Crane](https://github.com/den-crane)). +* Fix assertion when filtering tables in StorageMerge [#22998](https://github.com/ClickHouse/ClickHouse/pull/22998) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add a test for [#2719](https://github.com/ClickHouse/ClickHouse/issues/2719) [#23008](https://github.com/ClickHouse/ClickHouse/pull/23008) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix multi response in TestKeeper [#23041](https://github.com/ClickHouse/ClickHouse/pull/23041) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Improve hung check in Stress tests [#23043](https://github.com/ClickHouse/ClickHouse/pull/23043) ([Alexander Tokmakov](https://github.com/tavplubix)). +* blog article about code review [#23045](https://github.com/ClickHouse/ClickHouse/pull/23045) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* LibraryDictionary bridge library interface [#23048](https://github.com/ClickHouse/ClickHouse/pull/23048) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove useless files [#23049](https://github.com/ClickHouse/ClickHouse/pull/23049) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Upload keeper logs from stateless tests [#23077](https://github.com/ClickHouse/ClickHouse/pull/23077) ([alesapin](https://github.com/alesapin)). +* CI runner intergation tests logs update to tar.gz [#23078](https://github.com/ClickHouse/ClickHouse/pull/23078) ([Maksim Kita](https://github.com/kitaisreal)). +* Tiny logging improvements [#23083](https://github.com/ClickHouse/ClickHouse/pull/23083) ([Azat Khuzhin](https://github.com/azat)). +* Block all memory tracking limits in dtors/SCOPE_EXIT_*SAFE/tryLogCurrentException [#23086](https://github.com/ClickHouse/ClickHouse/pull/23086) ([Azat Khuzhin](https://github.com/azat)). +* jemalloc tuning [#23088](https://github.com/ClickHouse/ClickHouse/pull/23088) ([Azat Khuzhin](https://github.com/azat)). +* Rename strange tests [#23111](https://github.com/ClickHouse/ClickHouse/pull/23111) ([alesapin](https://github.com/alesapin)). +* Fix arcadia build S3 [#23114](https://github.com/ClickHouse/ClickHouse/pull/23114) ([Maksim Kita](https://github.com/kitaisreal)). +* Updated zlib [#23153](https://github.com/ClickHouse/ClickHouse/pull/23153) ([Maksim Kita](https://github.com/kitaisreal)). +* [RFC] Change logging from trace to debug for messages with rows/bytes [#23160](https://github.com/ClickHouse/ClickHouse/pull/23160) ([Azat Khuzhin](https://github.com/azat)). +* More verbose logs for debuging test failures with Replicated and Keeper [#23161](https://github.com/ClickHouse/ClickHouse/pull/23161) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix exception message for "parts_to_throw_insert" [#23177](https://github.com/ClickHouse/ClickHouse/pull/23177) ([madianjun](https://github.com/mdianjun)). +* Fix flapping tests test_s3_zero_copy_replication [#23184](https://github.com/ClickHouse/ClickHouse/pull/23184) ([ianton-ru](https://github.com/ianton-ru)). +* Add a test for [#14610](https://github.com/ClickHouse/ClickHouse/issues/14610) [#23209](https://github.com/ClickHouse/ClickHouse/pull/23209) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for fixed issue [#9363](https://github.com/ClickHouse/ClickHouse/issues/9363) [#23216](https://github.com/ClickHouse/ClickHouse/pull/23216) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Disable postgresql_port in perf tests [#23218](https://github.com/ClickHouse/ClickHouse/pull/23218) ([Azat Khuzhin](https://github.com/azat)). + diff --git a/docs/changelogs/v21.5.2.25-prestable.md b/docs/changelogs/v21.5.2.25-prestable.md index 45e784218da..1d7418a30c2 100644 --- a/docs/changelogs/v21.5.2.25-prestable.md +++ b/docs/changelogs/v21.5.2.25-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.2.25-prestable FIXME as compared to v21.5.1.6601-prestable #### Improvement @@ -38,3 +45,10 @@ * Backported in [#23347](https://github.com/ClickHouse/ClickHouse/issues/23347):. [#23334](https://github.com/ClickHouse/ClickHouse/pull/23334) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix logical error in stress tests [#23197](https://github.com/ClickHouse/ClickHouse/pull/23197) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Zlib use attribute constructor for functable initialization [#23266](https://github.com/ClickHouse/ClickHouse/pull/23266) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix integration tests for Hedged requests [#23275](https://github.com/ClickHouse/ClickHouse/pull/23275) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* System dictionaries virtual key column [#23458](https://github.com/ClickHouse/ClickHouse/pull/23458) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.5.3.1-prestable.md b/docs/changelogs/v21.5.3.1-prestable.md index 24b29d1495e..2e52d8ba6a8 100644 --- a/docs/changelogs/v21.5.3.1-prestable.md +++ b/docs/changelogs/v21.5.3.1-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.3.1-prestable FIXME as compared to v21.5.2.25-prestable #### Bug Fix @@ -7,3 +14,7 @@ * Backported in [#23832](https://github.com/ClickHouse/ClickHouse/issues/23832): Fix error `Can't initialize pipeline with empty pipe` for queries with `GLOBAL IN/JOIN` and `use_hedged_requests`. Fixes [#23431](https://github.com/ClickHouse/ClickHouse/issues/23431). [#23805](https://github.com/ClickHouse/ClickHouse/pull/23805) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#23927](https://github.com/ClickHouse/ClickHouse/issues/23927): HashedDictionary complex key update field initial load fix. Closes [#23800](https://github.com/ClickHouse/ClickHouse/issues/23800). [#23824](https://github.com/ClickHouse/ClickHouse/pull/23824) ([Maksim Kita](https://github.com/kitaisreal)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Flat, Hashed dictionary include update field bytes into bytes_allocated [#23825](https://github.com/ClickHouse/ClickHouse/pull/23825) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.5.4.6-prestable.md b/docs/changelogs/v21.5.4.6-prestable.md index c9e040309cc..37de3225996 100644 --- a/docs/changelogs/v21.5.4.6-prestable.md +++ b/docs/changelogs/v21.5.4.6-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.4.6-prestable FIXME as compared to v21.5.3.1-prestable #### Bug Fix diff --git a/docs/changelogs/v21.5.5.12-stable.md b/docs/changelogs/v21.5.5.12-stable.md index b49e2c60b08..71467dd2ffa 100644 --- a/docs/changelogs/v21.5.5.12-stable.md +++ b/docs/changelogs/v21.5.5.12-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.5.12-stable FIXME as compared to v21.5.4.6-prestable #### Bug Fix diff --git a/docs/changelogs/v21.5.6.6-stable.md b/docs/changelogs/v21.5.6.6-stable.md index e6160dfa784..04fe3ec4010 100644 --- a/docs/changelogs/v21.5.6.6-stable.md +++ b/docs/changelogs/v21.5.6.6-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.6.6-stable FIXME as compared to v21.5.5.12-stable #### Bug Fix @@ -15,3 +22,9 @@ * Backported in [#24750](https://github.com/ClickHouse/ClickHouse/issues/24750):. [#23276](https://github.com/ClickHouse/ClickHouse/pull/23276) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix segfault in TSan on _exit [#23616](https://github.com/ClickHouse/ClickHouse/pull/23616) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use memmove in PODArray::insert to handle memory overlapping. [#24271](https://github.com/ClickHouse/ClickHouse/pull/24271) ([Fu Zhe](https://github.com/fuzhe1989)). +* Fix cli argument in clickhouse-server.init [#24449](https://github.com/ClickHouse/ClickHouse/pull/24449) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.5.7.9-stable.md b/docs/changelogs/v21.5.7.9-stable.md index 9586e398547..eaf0cb6255b 100644 --- a/docs/changelogs/v21.5.7.9-stable.md +++ b/docs/changelogs/v21.5.7.9-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.7.9-stable FIXME as compared to v21.5.6.6-stable #### Improvement @@ -40,3 +47,9 @@ * NO CL ENTRY: 'Revert "Backport [#24721](https://github.com/ClickHouse/ClickHouse/issues/24721) to 21.5: Remove endless `wait` from ZooKeeper client"'. [#24798](https://github.com/ClickHouse/ClickHouse/pull/24798) ([alesapin](https://github.com/alesapin)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Try fix `test_consistent_parts_after_clone_replica` [#24968](https://github.com/ClickHouse/ClickHouse/pull/24968) ([Alexander Tokmakov](https://github.com/tavplubix)). +* DictionaryLoader unnecessary dictionary configuration creation fix [#25001](https://github.com/ClickHouse/ClickHouse/pull/25001) ([Maksim Kita](https://github.com/kitaisreal)). +* odbc fix [#25045](https://github.com/ClickHouse/ClickHouse/pull/25045) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.5.8.21-stable.md b/docs/changelogs/v21.5.8.21-stable.md index 4d9ffd687eb..fe59efd2255 100644 --- a/docs/changelogs/v21.5.8.21-stable.md +++ b/docs/changelogs/v21.5.8.21-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.8.21-stable FIXME as compared to v21.5.7.9-stable #### Bug Fix @@ -11,3 +18,8 @@ * Backported in [#25695](https://github.com/ClickHouse/ClickHouse/issues/25695): Fixed `No such file or directory` error on moving `Distributed` table between databases. Fixes [#24971](https://github.com/ClickHouse/ClickHouse/issues/24971). [#25667](https://github.com/ClickHouse/ClickHouse/pull/25667) ([Alexander Tokmakov](https://github.com/tavplubix)). * Backported in [#25754](https://github.com/ClickHouse/ClickHouse/issues/25754): Fix data race when querying `system.clusters` while reloading the cluster configuration at the same time. [#25737](https://github.com/ClickHouse/ClickHouse/pull/25737) ([Amos Bird](https://github.com/amosbird)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Backport unrelated changes, which fixes aliases bug [#25680](https://github.com/ClickHouse/ClickHouse/pull/25680) ([alesapin](https://github.com/alesapin)). +* ExpressionCache destruction fix [#25835](https://github.com/ClickHouse/ClickHouse/pull/25835) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.5.9.4-stable.md b/docs/changelogs/v21.5.9.4-stable.md index 17ef067194b..b9d643dda09 100644 --- a/docs/changelogs/v21.5.9.4-stable.md +++ b/docs/changelogs/v21.5.9.4-stable.md @@ -1,6 +1,18 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.5.9.4-stable FIXME as compared to v21.5.8.21-stable #### Bug Fix * Backported in [#25958](https://github.com/ClickHouse/ClickHouse/issues/25958): Fix extremely long backoff for background tasks when the background pool is full. Fixes [#25836](https://github.com/ClickHouse/ClickHouse/issues/25836). [#25893](https://github.com/ClickHouse/ClickHouse/pull/25893) ([alesapin](https://github.com/alesapin)). * Backported in [#26144](https://github.com/ClickHouse/ClickHouse/issues/26144): Fix possible crash in `pointInPolygon` if the setting `validate_polygons` is turned off. [#26113](https://github.com/ClickHouse/ClickHouse/pull/26113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* 21.5 manual backport of [#25970](https://github.com/ClickHouse/ClickHouse/issues/25970) [#26139](https://github.com/ClickHouse/ClickHouse/pull/26139) ([Maksim Kita](https://github.com/kitaisreal)). +* 21.5 Docker unbundled fix [#26154](https://github.com/ClickHouse/ClickHouse/pull/26154) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.6.1.6891-prestable.md b/docs/changelogs/v21.6.1.6891-prestable.md index 7750cac32eb..11fcd854989 100644 --- a/docs/changelogs/v21.6.1.6891-prestable.md +++ b/docs/changelogs/v21.6.1.6891-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.1.6891-prestable FIXME as compared to v21.5.1.6601-prestable #### New Feature @@ -152,6 +159,169 @@ * NO CL ENTRY: 'Revert "Fix CI build for gcc-10"'. [#23772](https://github.com/ClickHouse/ClickHouse/pull/23772) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * NO CL ENTRY: 'Update syntax.md'. [#24267](https://github.com/ClickHouse/ClickHouse/pull/24267) ([lulichao](https://github.com/lulichao)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Apply idle_connnection_timeout/poll_interval after each query [#21938](https://github.com/ClickHouse/ClickHouse/pull/21938) ([Azat Khuzhin](https://github.com/azat)). +* Do not silently catch errors for writing to S3 [#22208](https://github.com/ClickHouse/ClickHouse/pull/22208) ([Azat Khuzhin](https://github.com/azat)). +* Add dockerhub-proxy to runner [#23138](https://github.com/ClickHouse/ClickHouse/pull/23138) ([Ilya Yatsishin](https://github.com/qoega)). +* merging sumCount fusion PR [#21337](https://github.com/ClickHouse/ClickHouse/issues/21337) [#23159](https://github.com/ClickHouse/ClickHouse/pull/23159) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Minor fixes in ATTACH query [#23189](https://github.com/ClickHouse/ClickHouse/pull/23189) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merging [#22503](https://github.com/ClickHouse/ClickHouse/issues/22503) [#23195](https://github.com/ClickHouse/ClickHouse/pull/23195) ([Anton Popov](https://github.com/CurtizJ)). +* Fix logical error in stress tests [#23197](https://github.com/ClickHouse/ClickHouse/pull/23197) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* more stable formatting for negate() [#23201](https://github.com/ClickHouse/ClickHouse/pull/23201) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Use double quote identifier in odbc as default in case of error [#23217](https://github.com/ClickHouse/ClickHouse/pull/23217) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add -Wundef for gcc builds [#23258](https://github.com/ClickHouse/ClickHouse/pull/23258) ([Azat Khuzhin](https://github.com/azat)). +* Report an error if jemalloc.background_thread was requested [#23259](https://github.com/ClickHouse/ClickHouse/pull/23259) ([Azat Khuzhin](https://github.com/azat)). +* Add trace_log into stateless/stress test artifacts [#23264](https://github.com/ClickHouse/ClickHouse/pull/23264) ([Azat Khuzhin](https://github.com/azat)). +* Zlib use attribute constructor for functable initialization [#23266](https://github.com/ClickHouse/ClickHouse/pull/23266) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix integration tests for Hedged requests [#23275](https://github.com/ClickHouse/ClickHouse/pull/23275) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Function toDateTime decimal overflow ubsan fix [#23278](https://github.com/ClickHouse/ClickHouse/pull/23278) ([Maksim Kita](https://github.com/kitaisreal)). +* Link keeper-bench to clickhouse_common_zookeeper [#23302](https://github.com/ClickHouse/ClickHouse/pull/23302) ([Raúl Marín](https://github.com/Algunenano)). +* Print errors on db creation in clickhouse-test [#23304](https://github.com/ClickHouse/ClickHouse/pull/23304) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix broken perf test [#23308](https://github.com/ClickHouse/ClickHouse/pull/23308) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix bad test 01602_max_distributed_connections [#23317](https://github.com/ClickHouse/ClickHouse/pull/23317) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix hdfs reading from files with spaces [#23318](https://github.com/ClickHouse/ClickHouse/pull/23318) ([Kseniia Sumarokova](https://github.com/kssenii)). +* add check that p.pruning works [#23321](https://github.com/ClickHouse/ClickHouse/pull/23321) ([Denny Crane](https://github.com/den-crane)). +* Add test for [#7815](https://github.com/ClickHouse/ClickHouse/issues/7815) [#23332](https://github.com/ClickHouse/ClickHouse/pull/23332) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test 01666_merge_tree_max_query_limit.sh [#23335](https://github.com/ClickHouse/ClickHouse/pull/23335) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for [#12077](https://github.com/ClickHouse/ClickHouse/issues/12077) [#23352](https://github.com/ClickHouse/ClickHouse/pull/23352) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Disable clickhouse-odbc-bridge build when ODBC is disabled [#23357](https://github.com/ClickHouse/ClickHouse/pull/23357) ([Denis Glazachev](https://github.com/traceon)). +* Fix AppleClang build [#23358](https://github.com/ClickHouse/ClickHouse/pull/23358) ([Denis Glazachev](https://github.com/traceon)). +* Use Atomic database for development environment [#23377](https://github.com/ClickHouse/ClickHouse/pull/23377) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for [#2582](https://github.com/ClickHouse/ClickHouse/issues/2582) [#23378](https://github.com/ClickHouse/ClickHouse/pull/23378) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for [#1647](https://github.com/ClickHouse/ClickHouse/issues/1647) [#23379](https://github.com/ClickHouse/ClickHouse/pull/23379) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow rabbitmq vhost in table settings [#23452](https://github.com/ClickHouse/ClickHouse/pull/23452) ([Kseniia Sumarokova](https://github.com/kssenii)). +* System dictionaries virtual key column [#23458](https://github.com/ClickHouse/ClickHouse/pull/23458) ([Maksim Kita](https://github.com/kitaisreal)). +* ISSUES-23310 Try fix MySQL 8.0 address already in use [#23462](https://github.com/ClickHouse/ClickHouse/pull/23462) ([Winter Zhang](https://github.com/zhang2014)). +* Fix error in perf test [#23469](https://github.com/ClickHouse/ClickHouse/pull/23469) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* clickhouse-test: print database name on failures [#23486](https://github.com/ClickHouse/ClickHouse/pull/23486) ([Azat Khuzhin](https://github.com/azat)). +* upload cpu model to perf test db [#23514](https://github.com/ClickHouse/ClickHouse/pull/23514) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix function tests flaps [#23517](https://github.com/ClickHouse/ClickHouse/pull/23517) ([Azat Khuzhin](https://github.com/azat)). +* fix pvs warnings [#23520](https://github.com/ClickHouse/ClickHouse/pull/23520) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* ignore empty input chunks generated by joins [#23542](https://github.com/ClickHouse/ClickHouse/pull/23542) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* fix window functions for Distributed tables [#23546](https://github.com/ClickHouse/ClickHouse/pull/23546) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* add more info to perf test report [#23550](https://github.com/ClickHouse/ClickHouse/pull/23550) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* export trace log from stateless tests in flamegraph-friendly format [#23553](https://github.com/ClickHouse/ClickHouse/pull/23553) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* add numactl info to perf test run attributes [#23554](https://github.com/ClickHouse/ClickHouse/pull/23554) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Less chance of OOM in stress tests [#23561](https://github.com/ClickHouse/ClickHouse/pull/23561) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix inconsistent formatting for tupleElement (for fuzzer) [#23595](https://github.com/ClickHouse/ClickHouse/pull/23595) ([Azat Khuzhin](https://github.com/azat)). +* Remove unneeded code from CMakeLists [#23597](https://github.com/ClickHouse/ClickHouse/pull/23597) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Somewhat better handling of paths in CMake (incomplete) [#23598](https://github.com/ClickHouse/ClickHouse/pull/23598) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old trash (a little) [#23599](https://github.com/ClickHouse/ClickHouse/pull/23599) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove some garbage from RocksDB CMakeLists [#23601](https://github.com/ClickHouse/ClickHouse/pull/23601) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add warning about gcc usage [#23603](https://github.com/ClickHouse/ClickHouse/pull/23603) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove garbage from CMakeLists (2) [#23604](https://github.com/ClickHouse/ClickHouse/pull/23604) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove rotten parts of release script [#23605](https://github.com/ClickHouse/ClickHouse/pull/23605) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove useless file [#23606](https://github.com/ClickHouse/ClickHouse/pull/23606) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Skip CatBoost tests under MSan [#23614](https://github.com/ClickHouse/ClickHouse/pull/23614) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix segfault in TSan on _exit [#23616](https://github.com/ClickHouse/ClickHouse/pull/23616) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bad test 01641_memory_tracking_insert_optimize_long [#23617](https://github.com/ClickHouse/ClickHouse/pull/23617) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove pbuilder [#23618](https://github.com/ClickHouse/ClickHouse/pull/23618) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Aggregator remove unused code [#23635](https://github.com/ClickHouse/ClickHouse/pull/23635) ([Maksim Kita](https://github.com/kitaisreal)). +* Merging [#22984](https://github.com/ClickHouse/ClickHouse/issues/22984) [#23637](https://github.com/ClickHouse/ClickHouse/pull/23637) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Move non gtest unit tests to /examples folder [#23644](https://github.com/ClickHouse/ClickHouse/pull/23644) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Kerberized HDFS test is fluky - retries and extra output if failure [#23650](https://github.com/ClickHouse/ClickHouse/pull/23650) ([Ilya Golshtein](https://github.com/ilejn)). +* Fix documentation for DETACH ON CLUSTER PERMANENTLY [#23653](https://github.com/ClickHouse/ClickHouse/pull/23653) ([Azat Khuzhin](https://github.com/azat)). +* Skip integration test for library bridge under MSan [#23662](https://github.com/ClickHouse/ClickHouse/pull/23662) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix a few PVS-Studio warnings [#23663](https://github.com/ClickHouse/ClickHouse/pull/23663) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enable use-after-destruction detection in MSan [#23664](https://github.com/ClickHouse/ClickHouse/pull/23664) ([Azat Khuzhin](https://github.com/azat)). +* Fix cyrus-sasl msan warning [#23672](https://github.com/ClickHouse/ClickHouse/pull/23672) ([Ilya Yatsishin](https://github.com/qoega)). +* A little bit faster merge of aggregating states. [#23681](https://github.com/ClickHouse/ClickHouse/pull/23681) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* stable query indexes in perf test [#23707](https://github.com/ClickHouse/ClickHouse/pull/23707) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Minor changes in code [#14254](https://github.com/ClickHouse/ClickHouse/issues/14254) [#23709](https://github.com/ClickHouse/ClickHouse/pull/23709) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix illiterate error message [#23700](https://github.com/ClickHouse/ClickHouse/issues/23700) [#23710](https://github.com/ClickHouse/ClickHouse/pull/23710) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove obsolete compilers [#23711](https://github.com/ClickHouse/ClickHouse/pull/23711) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Follow-up for [#23644](https://github.com/ClickHouse/ClickHouse/issues/23644) [#23712](https://github.com/ClickHouse/ClickHouse/pull/23712) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable test_odbc_interaction for MSan build [#23722](https://github.com/ClickHouse/ClickHouse/pull/23722) ([Vladimir C](https://github.com/vdimir)). +* fix a typo in query formatting check in fuzzer [#23726](https://github.com/ClickHouse/ClickHouse/pull/23726) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* ExternalLoaderRepository fix arcadia [#23732](https://github.com/ClickHouse/ClickHouse/pull/23732) ([Maksim Kita](https://github.com/kitaisreal)). +* Ignore cmake-in-clickhouse [#23740](https://github.com/ClickHouse/ClickHouse/pull/23740) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Refactor join step [#23743](https://github.com/ClickHouse/ClickHouse/pull/23743) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* clickhouse-test: send TERM to all childs (to avoid hung check triggering) [#23750](https://github.com/ClickHouse/ClickHouse/pull/23750) ([Azat Khuzhin](https://github.com/azat)). +* Improve test_insert_into_distributed [#23751](https://github.com/ClickHouse/ClickHouse/pull/23751) ([Azat Khuzhin](https://github.com/azat)). +* Fix CI build for gcc-10 [#23760](https://github.com/ClickHouse/ClickHouse/pull/23760) ([Maksim Kita](https://github.com/kitaisreal)). +* Update array-functions.md [#23762](https://github.com/ClickHouse/ClickHouse/pull/23762) ([fancno](https://github.com/fancno)). +* Remove unused compilers (fixed for the troublesome "unbundled" build) [#23766](https://github.com/ClickHouse/ClickHouse/pull/23766) ([Maksim Kita](https://github.com/kitaisreal)). +* Slightly better hardening for intersecting parts [#23767](https://github.com/ClickHouse/ClickHouse/pull/23767) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Workaround for PVS-Studio [#23776](https://github.com/ClickHouse/ClickHouse/pull/23776) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add examples folder filter to ya.make.in [#23786](https://github.com/ClickHouse/ClickHouse/pull/23786) ([Maksim Kita](https://github.com/kitaisreal)). +* Function default implementation for nulls small optimization [#23799](https://github.com/ClickHouse/ClickHouse/pull/23799) ([Maksim Kita](https://github.com/kitaisreal)). +* autodetect arch of gosu in server dockerfile [#23802](https://github.com/ClickHouse/ClickHouse/pull/23802) ([filimonov](https://github.com/filimonov)). +* Add test for [#18170](https://github.com/ClickHouse/ClickHouse/issues/18170) [#23813](https://github.com/ClickHouse/ClickHouse/pull/23813) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* clickhouse-test: add missing whitespace before printing database on error [#23820](https://github.com/ClickHouse/ClickHouse/pull/23820) ([Azat Khuzhin](https://github.com/azat)). +* Improve 00840_long_concurrent_select_and_drop_deadlock [#23823](https://github.com/ClickHouse/ClickHouse/pull/23823) ([Azat Khuzhin](https://github.com/azat)). +* Flat, Hashed dictionary include update field bytes into bytes_allocated [#23825](https://github.com/ClickHouse/ClickHouse/pull/23825) ([Maksim Kita](https://github.com/kitaisreal)). +* XDBCBridgeHelper use global context [#23836](https://github.com/ClickHouse/ClickHouse/pull/23836) ([Maksim Kita](https://github.com/kitaisreal)). +* gcc-10 installation no-install-reccomends option fix [#23840](https://github.com/ClickHouse/ClickHouse/pull/23840) ([Maksim Kita](https://github.com/kitaisreal)). +* replxx readline compatibility [#23855](https://github.com/ClickHouse/ClickHouse/pull/23855) ([Azat Khuzhin](https://github.com/azat)). +* Add file paths into logs on failed distributed async sends [#23856](https://github.com/ClickHouse/ClickHouse/pull/23856) ([Azat Khuzhin](https://github.com/azat)). +* Reduce the amount of logs that StorageMergeTree::selectPartsToMutate outputs in busy systems. [#23863](https://github.com/ClickHouse/ClickHouse/pull/23863) ([Raúl Marín](https://github.com/Algunenano)). +* Add DiskRestartProxy.cpp to ya.make [#23868](https://github.com/ClickHouse/ClickHouse/pull/23868) ([Vladimir C](https://github.com/vdimir)). +* Fix some warnings by PVS-Studio [#23877](https://github.com/ClickHouse/ClickHouse/pull/23877) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add Read/WriteBufferFromFileDecorator.cpp to ya.make [#23879](https://github.com/ClickHouse/ClickHouse/pull/23879) ([Vladimir C](https://github.com/vdimir)). +* @CurtizJ convinced me that this test has to be deleted [#23883](https://github.com/ClickHouse/ClickHouse/pull/23883) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* bash completion improvements [#23884](https://github.com/ClickHouse/ClickHouse/pull/23884) ([Azat Khuzhin](https://github.com/azat)). +* Remove obsolete code [#23886](https://github.com/ClickHouse/ClickHouse/pull/23886) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor code simplification (implemented TODO) [#23896](https://github.com/ClickHouse/ClickHouse/pull/23896) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add run options with default config path according to issues/23875 [#23898](https://github.com/ClickHouse/ClickHouse/pull/23898) ([ice1x](https://github.com/ice1x)). +* Fix test_insert_into_distributed flaps [#23903](https://github.com/ClickHouse/ClickHouse/pull/23903) ([Azat Khuzhin](https://github.com/azat)). +* Fix bad test about compression codecs [#23908](https://github.com/ClickHouse/ClickHouse/pull/23908) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Cleanup IDatabase.h from extra headers [#23912](https://github.com/ClickHouse/ClickHouse/pull/23912) ([Azat Khuzhin](https://github.com/azat)). +* Minor simplification [#23923](https://github.com/ClickHouse/ClickHouse/pull/23923) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Drop unnecessary ports where applicable [#23928](https://github.com/ClickHouse/ClickHouse/pull/23928) ([Ernest Zaslavsky](https://github.com/kreuzerkrieg)). +* Check MIN/MAX attributes in the list of dictionary attributes [#23948](https://github.com/ClickHouse/ClickHouse/pull/23948) ([Azat Khuzhin](https://github.com/azat)). +* typo: fix a typo in Compression/CodecT64 [#23952](https://github.com/ClickHouse/ClickHouse/pull/23952) ([mwish](https://github.com/mapleFU)). +* Don't try GLIBC_COMPATIBILITY for i686 Linux [#23959](https://github.com/ClickHouse/ClickHouse/pull/23959) ([divanorama](https://github.com/divanorama)). +* Function arrayDifference decimal math overflow [#23961](https://github.com/ClickHouse/ClickHouse/pull/23961) ([Maksim Kita](https://github.com/kitaisreal)). +* Use 0 over nan for hit_rate in case of 0 queries to the cache dictionary [#23963](https://github.com/ClickHouse/ClickHouse/pull/23963) ([Azat Khuzhin](https://github.com/azat)). +* Round floats in Aggregator log messages [#23965](https://github.com/ClickHouse/ClickHouse/pull/23965) ([Azat Khuzhin](https://github.com/azat)). +* support longer query ids in trace log for perf tests [#23969](https://github.com/ClickHouse/ClickHouse/pull/23969) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* PVS-Studio fixes, part 6 [#23970](https://github.com/ClickHouse/ClickHouse/pull/23970) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Functional stateless tests fix numbers [#23974](https://github.com/ClickHouse/ClickHouse/pull/23974) ([Maksim Kita](https://github.com/kitaisreal)). +* use LowCardinality for AsynchronousMetricLog name column [#23981](https://github.com/ClickHouse/ClickHouse/pull/23981) ([flynn](https://github.com/ucasfl)). +* Function dictGetOrNull handle empty rows execute [#23990](https://github.com/ClickHouse/ClickHouse/pull/23990) ([Maksim Kita](https://github.com/kitaisreal)). +* CompileDAG fix Sip hash [#24004](https://github.com/ClickHouse/ClickHouse/pull/24004) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix bad code [#24007](https://github.com/ClickHouse/ClickHouse/pull/24007) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better error codes in Keeper when no leader alive [#24017](https://github.com/ClickHouse/ClickHouse/pull/24017) ([alesapin](https://github.com/alesapin)). +* Fix ArenaWithFreeLists test [#24021](https://github.com/ClickHouse/ClickHouse/pull/24021) ([Maksim Kita](https://github.com/kitaisreal)). +* Run check_*_compiler_flag earlier [#24037](https://github.com/ClickHouse/ClickHouse/pull/24037) ([Amos Bird](https://github.com/amosbird)). +* Performance tests disable compile expressions [#24043](https://github.com/ClickHouse/ClickHouse/pull/24043) ([Maksim Kita](https://github.com/kitaisreal)). +* for trivial INSERT SELECT, adjust block size in bytes as well [#24048](https://github.com/ClickHouse/ClickHouse/pull/24048) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Enable thread_local in Arcadia build [#24051](https://github.com/ClickHouse/ClickHouse/pull/24051) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* lower two-level aggregation threshold for uniq test to avoid jitter [#24058](https://github.com/ClickHouse/ClickHouse/pull/24058) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix empty key projection query analysis [#24062](https://github.com/ClickHouse/ClickHouse/pull/24062) ([Amos Bird](https://github.com/amosbird)). +* Generate ya.make for missing UUID.cpp [#24064](https://github.com/ClickHouse/ClickHouse/pull/24064) ([Alexander Gololobov](https://github.com/davenger)). +* bash-completion: complete available formats [#24065](https://github.com/ClickHouse/ClickHouse/pull/24065) ([Azat Khuzhin](https://github.com/azat)). +* Fix concurrent snapshot read/write [#24073](https://github.com/ClickHouse/ClickHouse/pull/24073) ([alesapin](https://github.com/alesapin)). +* Also retry database creation in `clickhouse-test` [#24088](https://github.com/ClickHouse/ClickHouse/pull/24088) ([alesapin](https://github.com/alesapin)). +* Calculate header from ActionsDAG [#24108](https://github.com/ClickHouse/ClickHouse/pull/24108) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix arcadia [#24133](https://github.com/ClickHouse/ClickHouse/pull/24133) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* TestReadAfterAIO: Use the current path instead of /tmp for temporal files [#24139](https://github.com/ClickHouse/ClickHouse/pull/24139) ([Raúl Marín](https://github.com/Algunenano)). +* Fix a few trailing whitespaces in output [#24150](https://github.com/ClickHouse/ClickHouse/pull/24150) ([Azat Khuzhin](https://github.com/azat)). +* IFunction refactoring [#24155](https://github.com/ClickHouse/ClickHouse/pull/24155) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix add projection to replicated mergetree [#24162](https://github.com/ClickHouse/ClickHouse/pull/24162) ([Amos Bird](https://github.com/amosbird)). +* Fix distributed processing when using projection [#24168](https://github.com/ClickHouse/ClickHouse/pull/24168) ([Amos Bird](https://github.com/amosbird)). +* fix tiny code style [#24169](https://github.com/ClickHouse/ClickHouse/pull/24169) ([flynn](https://github.com/ucasfl)). +* fix reinterpretAsFixedString for UUID [#24177](https://github.com/ClickHouse/ClickHouse/pull/24177) ([flynn](https://github.com/ucasfl)). +* Function move file [#24178](https://github.com/ClickHouse/ClickHouse/pull/24178) ([Maksim Kita](https://github.com/kitaisreal)). +* Updated LRUHashMap benchmarks [#24182](https://github.com/ClickHouse/ClickHouse/pull/24182) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove temporary files [#24186](https://github.com/ClickHouse/ClickHouse/pull/24186) ([Kruglov Pavel](https://github.com/Avogar)). +* Try mute grpc msan [#24197](https://github.com/ClickHouse/ClickHouse/pull/24197) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update slack link [#24200](https://github.com/ClickHouse/ClickHouse/pull/24200) ([Ilya Yatsishin](https://github.com/qoega)). +* Simplier isLocalAddress [#24203](https://github.com/ClickHouse/ClickHouse/pull/24203) ([alesapin](https://github.com/alesapin)). +* Increase timeout for server restart in integration tests. [#24205](https://github.com/ClickHouse/ClickHouse/pull/24205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Addition to [#23997](https://github.com/ClickHouse/ClickHouse/issues/23997) [#24208](https://github.com/ClickHouse/ClickHouse/pull/24208) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Small code simplification [#24210](https://github.com/ClickHouse/ClickHouse/pull/24210) ([Maksim Kita](https://github.com/kitaisreal)). +* Speedup test [test_jbod_balancer/test.py] [#24247](https://github.com/ClickHouse/ClickHouse/pull/24247) ([alesapin](https://github.com/alesapin)). +* remove useless code [#24248](https://github.com/ClickHouse/ClickHouse/pull/24248) ([flynn](https://github.com/ucasfl)). +* One more attempt to fix retries in clickhouse-test [#24249](https://github.com/ClickHouse/ClickHouse/pull/24249) ([alesapin](https://github.com/alesapin)). +* Log exception in Allocator::free [#24256](https://github.com/ClickHouse/ClickHouse/pull/24256) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix broken HTML markup on website [#24265](https://github.com/ClickHouse/ClickHouse/pull/24265) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix arcadia [#24282](https://github.com/ClickHouse/ClickHouse/pull/24282) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fixed Arcadia after IFunctionOverloadResolver interface refactoring [#24284](https://github.com/ClickHouse/ClickHouse/pull/24284) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix isLocalAddress() (ifa_addr maybe NULL) [#24308](https://github.com/ClickHouse/ClickHouse/pull/24308) ([Azat Khuzhin](https://github.com/azat)). +* Fix bad test for Enum hints [#24313](https://github.com/ClickHouse/ClickHouse/pull/24313) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + #### New Feature #14893 * - Add uniqThetaSketch to support [Theta Sketch](https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html) in ClickHouse. [#22609](https://github.com/ClickHouse/ClickHouse/pull/22609) ([Ping Yu](https://github.com/pingyu)). diff --git a/docs/changelogs/v21.6.2.7-prestable.md b/docs/changelogs/v21.6.2.7-prestable.md index c5dec251786..55032e0258e 100644 --- a/docs/changelogs/v21.6.2.7-prestable.md +++ b/docs/changelogs/v21.6.2.7-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.2.7-prestable FIXME as compared to v21.6.1.6891-prestable #### Bug Fix @@ -11,3 +18,9 @@ * Backported in [#24597](https://github.com/ClickHouse/ClickHouse/issues/24597): Fix usage of tuples in `CREATE .. AS SELECT` queries. [#24464](https://github.com/ClickHouse/ClickHouse/pull/24464) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#24600](https://github.com/ClickHouse/ClickHouse/issues/24600): Enable reading of subcolumns for distributed tables. [#24472](https://github.com/ClickHouse/ClickHouse/pull/24472) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Use memmove in PODArray::insert to handle memory overlapping. [#24271](https://github.com/ClickHouse/ClickHouse/pull/24271) ([Fu Zhe](https://github.com/fuzhe1989)). +* Fix cli argument in clickhouse-server.init [#24449](https://github.com/ClickHouse/ClickHouse/pull/24449) ([Vladimir C](https://github.com/vdimir)). +* Fix several cases in cast operator [#24471](https://github.com/ClickHouse/ClickHouse/pull/24471) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v21.6.3.14-stable.md b/docs/changelogs/v21.6.3.14-stable.md index 8d2e2a03320..551f959aefd 100644 --- a/docs/changelogs/v21.6.3.14-stable.md +++ b/docs/changelogs/v21.6.3.14-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.3.14-stable FIXME as compared to v21.6.2.7-prestable #### Improvement diff --git a/docs/changelogs/v21.6.4.26-stable.md b/docs/changelogs/v21.6.4.26-stable.md index 9bf7d0e68cb..dc235d91e6d 100644 --- a/docs/changelogs/v21.6.4.26-stable.md +++ b/docs/changelogs/v21.6.4.26-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.4.26-stable FIXME as compared to v21.6.3.14-stable #### Bug Fix @@ -12,3 +19,8 @@ * Backported in [#25157](https://github.com/ClickHouse/ClickHouse/issues/25157): Fix possible parts loss after updating up to 21.5 in case table used `UUID` in partition key. (It is not recommended to use `UUID` in partition key). Fixes [#25070](https://github.com/ClickHouse/ClickHouse/issues/25070). [#25127](https://github.com/ClickHouse/ClickHouse/pull/25127) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#25182](https://github.com/ClickHouse/ClickHouse/issues/25182): Do not use table's projection for `SELECT` with `FINAL`. It is not supported yet. [#25163](https://github.com/ClickHouse/ClickHouse/pull/25163) ([Amos Bird](https://github.com/amosbird)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Try fix `test_consistent_parts_after_clone_replica` [#24968](https://github.com/ClickHouse/ClickHouse/pull/24968) ([Alexander Tokmakov](https://github.com/tavplubix)). +* DictionaryLoader unnecessary dictionary configuration creation fix [#25001](https://github.com/ClickHouse/ClickHouse/pull/25001) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.6.5.37-stable.md b/docs/changelogs/v21.6.5.37-stable.md index 3d03c31c7e8..45a43199c28 100644 --- a/docs/changelogs/v21.6.5.37-stable.md +++ b/docs/changelogs/v21.6.5.37-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.5.37-stable FIXME as compared to v21.6.4.26-stable #### Improvement @@ -19,3 +26,8 @@ * Backported in [#25448](https://github.com/ClickHouse/ClickHouse/issues/25448): Fix lost `WHERE` condition in expression-push-down optimization of query plan (setting `query_plan_filter_push_down = 1` by default). Fixes [#25368](https://github.com/ClickHouse/ClickHouse/issues/25368). [#25370](https://github.com/ClickHouse/ClickHouse/pull/25370) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#25407](https://github.com/ClickHouse/ClickHouse/issues/25407): Fix `REPLACE` column transformer when used in DDL by correctly quoting the formated query. This fixes [#23925](https://github.com/ClickHouse/ClickHouse/issues/23925). [#25391](https://github.com/ClickHouse/ClickHouse/pull/25391) ([Amos Bird](https://github.com/amosbird)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* ExpressionActions compile only necessary places [#24273](https://github.com/ClickHouse/ClickHouse/pull/24273) ([Maksim Kita](https://github.com/kitaisreal)). +* odbc fix [#25045](https://github.com/ClickHouse/ClickHouse/pull/25045) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.6.6.51-stable.md b/docs/changelogs/v21.6.6.51-stable.md index 55f1fd46119..39441adaa9e 100644 --- a/docs/changelogs/v21.6.6.51-stable.md +++ b/docs/changelogs/v21.6.6.51-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.6.51-stable FIXME as compared to v21.6.5.37-stable #### Bug Fix @@ -17,3 +24,7 @@ * NO CL ENTRY: 'Partial backport [#24061](https://github.com/ClickHouse/ClickHouse/issues/24061) to 21.6'. [#25621](https://github.com/ClickHouse/ClickHouse/pull/25621) ([Vladimir C](https://github.com/vdimir)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* ExpressionCache destruction fix [#25835](https://github.com/ClickHouse/ClickHouse/pull/25835) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.6.7.57-stable.md b/docs/changelogs/v21.6.7.57-stable.md index 5ef9026794b..26887fac120 100644 --- a/docs/changelogs/v21.6.7.57-stable.md +++ b/docs/changelogs/v21.6.7.57-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.7.57-stable FIXME as compared to v21.6.6.51-stable #### Bug Fix @@ -6,3 +13,7 @@ * Backported in [#26096](https://github.com/ClickHouse/ClickHouse/issues/26096): Fix wrong thread estimation for right subquery join in some cases. Close [#24075](https://github.com/ClickHouse/ClickHouse/issues/24075). [#26052](https://github.com/ClickHouse/ClickHouse/pull/26052) ([Vladimir C](https://github.com/vdimir)). * Backported in [#26143](https://github.com/ClickHouse/ClickHouse/issues/26143): Fix possible crash in `pointInPolygon` if the setting `validate_polygons` is turned off. [#26113](https://github.com/ClickHouse/ClickHouse/pull/26113) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* 21.6 manual backport of [#25970](https://github.com/ClickHouse/ClickHouse/issues/25970) [#26138](https://github.com/ClickHouse/ClickHouse/pull/26138) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.6.8.62-stable.md b/docs/changelogs/v21.6.8.62-stable.md index 1357c762181..0dc35d41344 100644 --- a/docs/changelogs/v21.6.8.62-stable.md +++ b/docs/changelogs/v21.6.8.62-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.8.62-stable FIXME as compared to v21.6.7.57-stable #### Bug Fix @@ -7,3 +14,7 @@ * Backported in [#26205](https://github.com/ClickHouse/ClickHouse/issues/26205): Fix potential crash if more than one `untuple` expression is used. [#26179](https://github.com/ClickHouse/ClickHouse/pull/26179) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#26227](https://github.com/ClickHouse/ClickHouse/issues/26227): Remove excessive newline in `thread_name` column in `system.stack_trace` table. This fixes [#24124](https://github.com/ClickHouse/ClickHouse/issues/24124). [#26210](https://github.com/ClickHouse/ClickHouse/pull/26210) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Manual backport of [#25531](https://github.com/ClickHouse/ClickHouse/issues/25531) in 21.6 [#25776](https://github.com/ClickHouse/ClickHouse/pull/25776) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.6.9.7-stable.md b/docs/changelogs/v21.6.9.7-stable.md index ecac9dd75ef..fae1c8d2fef 100644 --- a/docs/changelogs/v21.6.9.7-stable.md +++ b/docs/changelogs/v21.6.9.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.6.9.7-stable FIXME as compared to v21.6.8.62-stable #### Improvement @@ -41,3 +48,19 @@ * Backported in [#28264](https://github.com/ClickHouse/ClickHouse/issues/28264): Fix possible read of uninitialized memory for queries with `Nullable(LowCardinality)` type and extremes. Fixes [#28165](https://github.com/ClickHouse/ClickHouse/issues/28165). [#28205](https://github.com/ClickHouse/ClickHouse/pull/28205) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#28291](https://github.com/ClickHouse/ClickHouse/issues/28291): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)). +* Fix mysql_kill_sync_thread_restore_test [#26673](https://github.com/ClickHouse/ClickHouse/pull/26673) ([Vladimir C](https://github.com/vdimir)). +* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* One more library bridge fix [#26873](https://github.com/ClickHouse/ClickHouse/pull/26873) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* ODBC connection holder fix dangling reference [#28298](https://github.com/ClickHouse/ClickHouse/pull/28298) ([Maksim Kita](https://github.com/kitaisreal)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.7.1.7283-prestable.md b/docs/changelogs/v21.7.1.7283-prestable.md index 52de493c4ab..a862e6c4750 100644 --- a/docs/changelogs/v21.7.1.7283-prestable.md +++ b/docs/changelogs/v21.7.1.7283-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.1.7283-prestable FIXME as compared to v21.6.1.6891-prestable #### Backward Incompatible Change @@ -172,6 +179,236 @@ * NO CL ENTRY: 'Revert "Add run-id option to integration tests"'. [#25526](https://github.com/ClickHouse/ClickHouse/pull/25526) ([alesapin](https://github.com/alesapin)). * NO CL ENTRY: 'Revert "Implement h3ToGeo function"'. [#25593](https://github.com/ClickHouse/ClickHouse/pull/25593) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Part movement between shards [#17871](https://github.com/ClickHouse/ClickHouse/pull/17871) ([nvartolomei](https://github.com/nvartolomei)). +* Pass Settings to aggregate function creator [#22762](https://github.com/ClickHouse/ClickHouse/pull/22762) ([Vladimir C](https://github.com/vdimir)). +* Add uniqTheta in performance test [#23311](https://github.com/ClickHouse/ClickHouse/pull/23311) ([Kruglov Pavel](https://github.com/Avogar)). +* More pytest fixes [#23538](https://github.com/ClickHouse/ClickHouse/pull/23538) ([Ivan](https://github.com/abyss7)). +* Improved `test_storage_s3_get_unstable` [#23976](https://github.com/ClickHouse/ClickHouse/pull/23976) ([Vladimir Chebotarev](https://github.com/excitoon)). +* CompileExpressions comparison function constant case fix [#24023](https://github.com/ClickHouse/ClickHouse/pull/24023) ([Maksim Kita](https://github.com/kitaisreal)). +* Added llvm-project submodule [#24030](https://github.com/ClickHouse/ClickHouse/pull/24030) ([Maksim Kita](https://github.com/kitaisreal)). +* complain about unstable perf test queries [#24049](https://github.com/ClickHouse/ClickHouse/pull/24049) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Changelog 21.5 [#24098](https://github.com/ClickHouse/ClickHouse/pull/24098) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Implement antlr parser for projections [#24245](https://github.com/ClickHouse/ClickHouse/pull/24245) ([Amos Bird](https://github.com/amosbird)). +* Try to fix GROUP BY _shard_num in a different way [#24252](https://github.com/ClickHouse/ClickHouse/pull/24252) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Use memmove in PODArray::insert to handle memory overlapping. [#24271](https://github.com/ClickHouse/ClickHouse/pull/24271) ([Fu Zhe](https://github.com/fuzhe1989)). +* ExpressionActions compile only necessary places [#24273](https://github.com/ClickHouse/ClickHouse/pull/24273) ([Maksim Kita](https://github.com/kitaisreal)). +* consolidate connection loss handling in fuzzer [#24290](https://github.com/ClickHouse/ClickHouse/pull/24290) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Skip data finalization when doing projection materialization. [#24296](https://github.com/ClickHouse/ClickHouse/pull/24296) ([Amos Bird](https://github.com/amosbird)). +* One more error to retry in clickhouse-test [#24307](https://github.com/ClickHouse/ClickHouse/pull/24307) ([alesapin](https://github.com/alesapin)). +* Merging [#23461](https://github.com/ClickHouse/ClickHouse/issues/23461) [#24311](https://github.com/ClickHouse/ClickHouse/pull/24311) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove AutoArray [#24320](https://github.com/ClickHouse/ClickHouse/pull/24320) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Compile expressions added partition by test [#24323](https://github.com/ClickHouse/ClickHouse/pull/24323) ([Maksim Kita](https://github.com/kitaisreal)). +* PODArray insert in the middle tests [#24333](https://github.com/ClickHouse/ClickHouse/pull/24333) ([Maksim Kita](https://github.com/kitaisreal)). +* remove retries from fast test [#24338](https://github.com/ClickHouse/ClickHouse/pull/24338) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Better parallelism for functional tests [#24349](https://github.com/ClickHouse/ClickHouse/pull/24349) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* AARCH64 hash tables benchmark [#24364](https://github.com/ClickHouse/ClickHouse/pull/24364) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix diagnostics in documentation script [#24379](https://github.com/ClickHouse/ClickHouse/pull/24379) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky 01033_quota_dcl [#24393](https://github.com/ClickHouse/ClickHouse/pull/24393) ([alesapin](https://github.com/alesapin)). +* Fix logical error AggregateFunctionFactory returned nullptr [#24398](https://github.com/ClickHouse/ClickHouse/pull/24398) ([Kruglov Pavel](https://github.com/Avogar)). +* Add log record for removed from AWS S3 keys [#24400](https://github.com/ClickHouse/ClickHouse/pull/24400) ([ianton-ru](https://github.com/ianton-ru)). +* Added AggregateFunctionSegmentLengthSum to ya.make [#24408](https://github.com/ClickHouse/ClickHouse/pull/24408) ([Maksim Kita](https://github.com/kitaisreal)). +* Stateless tests fixes [#24411](https://github.com/ClickHouse/ClickHouse/pull/24411) ([Azat Khuzhin](https://github.com/azat)). +* Function constant result with nonconstant arguments [#24417](https://github.com/ClickHouse/ClickHouse/pull/24417) ([Maksim Kita](https://github.com/kitaisreal)). +* Libunwind update version [#24419](https://github.com/ClickHouse/ClickHouse/pull/24419) ([Maksim Kita](https://github.com/kitaisreal)). +* Do not built clickhouse-keeper w/o NuRaft [#24421](https://github.com/ClickHouse/ClickHouse/pull/24421) ([Azat Khuzhin](https://github.com/azat)). +* Switch message level to WARNING for FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION=OFF [#24423](https://github.com/ClickHouse/ClickHouse/pull/24423) ([Azat Khuzhin](https://github.com/azat)). +* Merging "experimental compression codecs" [#24424](https://github.com/ClickHouse/ClickHouse/pull/24424) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix formatting of negative values [#24427](https://github.com/ClickHouse/ClickHouse/pull/24427) ([Azat Khuzhin](https://github.com/azat)). +* Fix dictionary functions documentation [#24432](https://github.com/ClickHouse/ClickHouse/pull/24432) ([Maksim Kita](https://github.com/kitaisreal)). +* Mark false positives for PVS-Studio [#24434](https://github.com/ClickHouse/ClickHouse/pull/24434) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* ActionsDAG compile expressions update [#24442](https://github.com/ClickHouse/ClickHouse/pull/24442) ([Maksim Kita](https://github.com/kitaisreal)). +* LLVM remove orc jit library [#24443](https://github.com/ClickHouse/ClickHouse/pull/24443) ([Maksim Kita](https://github.com/kitaisreal)). +* Add YAMLParser to ya.make [#24447](https://github.com/ClickHouse/ClickHouse/pull/24447) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix cli argument in clickhouse-server.init [#24449](https://github.com/ClickHouse/ClickHouse/pull/24449) ([Vladimir C](https://github.com/vdimir)). +* Fix fast test [#24454](https://github.com/ClickHouse/ClickHouse/pull/24454) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix flaky test 01085_max_distributed_connections [#24455](https://github.com/ClickHouse/ClickHouse/pull/24455) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Manually disable LLVM parts submodule update [#24460](https://github.com/ClickHouse/ClickHouse/pull/24460) ([Maksim Kita](https://github.com/kitaisreal)). +* Account total_rows_approx as soon as possible [#24462](https://github.com/ClickHouse/ClickHouse/pull/24462) ([Azat Khuzhin](https://github.com/azat)). +* Fix header mismatch for UNION. [#24463](https://github.com/ClickHouse/ClickHouse/pull/24463) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Try to improve kafka flaky test [#24465](https://github.com/ClickHouse/ClickHouse/pull/24465) ([filimonov](https://github.com/filimonov)). +* CompileExpression cached functions with context fix [#24468](https://github.com/ClickHouse/ClickHouse/pull/24468) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed IFunction then typos [#24469](https://github.com/ClickHouse/ClickHouse/pull/24469) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix several cases in cast operator [#24471](https://github.com/ClickHouse/ClickHouse/pull/24471) ([Anton Popov](https://github.com/CurtizJ)). +* MemoryTracker new no throw [#24483](https://github.com/ClickHouse/ClickHouse/pull/24483) ([Maksim Kita](https://github.com/kitaisreal)). +* Revent libunwind [#24485](https://github.com/ClickHouse/ClickHouse/pull/24485) ([Maksim Kita](https://github.com/kitaisreal)). +* Log file name on error in FileChecker ctor [#24489](https://github.com/ClickHouse/ClickHouse/pull/24489) ([Vladimir C](https://github.com/vdimir)). +* Remove containers in intergational runner entrypoint [#24492](https://github.com/ClickHouse/ClickHouse/pull/24492) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix benign race (detected by clang-12) in Keeper snapshots [#24499](https://github.com/ClickHouse/ClickHouse/pull/24499) ([alesapin](https://github.com/alesapin)). +* [backport] Find only opened PRs by label [#24501](https://github.com/ClickHouse/ClickHouse/pull/24501) ([Ivan](https://github.com/abyss7)). +* shellcheck fix [#24512](https://github.com/ClickHouse/ClickHouse/pull/24512) ([Ilya Yatsishin](https://github.com/qoega)). +* Do not peerdir enabled-by-default libcxxabi-parts [#24518](https://github.com/ClickHouse/ClickHouse/pull/24518) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Merging [#22762](https://github.com/ClickHouse/ClickHouse/issues/22762) [#24525](https://github.com/ClickHouse/ClickHouse/pull/24525) ([Vladimir C](https://github.com/vdimir)). +* Fix Arcadia [#24527](https://github.com/ClickHouse/ClickHouse/pull/24527) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Make `get/check/read/list` requests always read in Keeper [#24533](https://github.com/ClickHouse/ClickHouse/pull/24533) ([alesapin](https://github.com/alesapin)). +* calculate perf test precision thresholds from historical data [#24534](https://github.com/ClickHouse/ClickHouse/pull/24534) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Update jit_example [#24550](https://github.com/ClickHouse/ClickHouse/pull/24550) ([Maksim Kita](https://github.com/kitaisreal)). +* Update test_multiple_disks [#24560](https://github.com/ClickHouse/ClickHouse/pull/24560) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix flaky test [#24561](https://github.com/ClickHouse/ClickHouse/pull/24561) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix arcadia [#24563](https://github.com/ClickHouse/ClickHouse/pull/24563) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix avro [#24569](https://github.com/ClickHouse/ClickHouse/pull/24569) ([Ilya Yatsishin](https://github.com/qoega)). +* Refactor MergeTreeDataSelectExecutor [#24574](https://github.com/ClickHouse/ClickHouse/pull/24574) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* allow inheriting from a named window in window definition [#24576](https://github.com/ClickHouse/ClickHouse/pull/24576) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* do not auto-apply -OrNull combinator to pure window functions [#24579](https://github.com/ClickHouse/ClickHouse/pull/24579) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* fix ORDER BY after window fuctions over Distributed [#24580](https://github.com/ClickHouse/ClickHouse/pull/24580) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* ast fuzzer: determine server death more robustly [#24584](https://github.com/ClickHouse/ClickHouse/pull/24584) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Enable tests, utils and examples in builds with clang-tidy [#24587](https://github.com/ClickHouse/ClickHouse/pull/24587) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* more fuzzer fixes [#24690](https://github.com/ClickHouse/ClickHouse/pull/24690) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix flaky integration tests [#24691](https://github.com/ClickHouse/ClickHouse/pull/24691) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix alter table drop projection if exists [#24692](https://github.com/ClickHouse/ClickHouse/pull/24692) ([Amos Bird](https://github.com/amosbird)). +* Block memory tracker earlier in `tryLogCurrentException` [#24722](https://github.com/ClickHouse/ClickHouse/pull/24722) ([alesapin](https://github.com/alesapin)). +* Minor fixes in AggregateFunctionSegmentLengthSumData [#24729](https://github.com/ClickHouse/ClickHouse/pull/24729) ([Vladimir C](https://github.com/vdimir)). +* remove mutable references to Context from IFunction interface [#24732](https://github.com/ClickHouse/ClickHouse/pull/24732) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* MemoryTracker enable throw logical error [#24733](https://github.com/ClickHouse/ClickHouse/pull/24733) ([Maksim Kita](https://github.com/kitaisreal)). +* support expressions in window frame [#24734](https://github.com/ClickHouse/ClickHouse/pull/24734) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Updated libunwind [#24735](https://github.com/ClickHouse/ClickHouse/pull/24735) ([Maksim Kita](https://github.com/kitaisreal)). +* ExecuteScalarSubqueriesVisitor fix error code style check [#24736](https://github.com/ClickHouse/ClickHouse/pull/24736) ([Maksim Kita](https://github.com/kitaisreal)). +* Process config w/o extensions as XML format [#24763](https://github.com/ClickHouse/ClickHouse/pull/24763) ([Azat Khuzhin](https://github.com/azat)). +* Update NuRaft [#24775](https://github.com/ClickHouse/ClickHouse/pull/24775) ([alesapin](https://github.com/alesapin)). +* Fix empty part set with force_use_projection = 1 [#24782](https://github.com/ClickHouse/ClickHouse/pull/24782) ([Amos Bird](https://github.com/amosbird)). +* Better exception for invalid projection creation [#24785](https://github.com/ClickHouse/ClickHouse/pull/24785) ([Amos Bird](https://github.com/amosbird)). +* Improve "Cannot schedule a task" error message [#24786](https://github.com/ClickHouse/ClickHouse/pull/24786) ([Azat Khuzhin](https://github.com/azat)). +* Fix 00953_zookeeper_suetin_deduplication_bug [#24801](https://github.com/ClickHouse/ClickHouse/pull/24801) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Don't capture temporary references in ZooKeeper client callbacks. [#24803](https://github.com/ClickHouse/ClickHouse/pull/24803) ([alesapin](https://github.com/alesapin)). +* Small improvement for StorageMaterializedView::getActionLock(...) [#24806](https://github.com/ClickHouse/ClickHouse/pull/24806) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Make ContextPtr const by default. [#24808](https://github.com/ClickHouse/ClickHouse/pull/24808) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix typo. Remove the "allow_experimental_bigint_types" setting. [#24812](https://github.com/ClickHouse/ClickHouse/pull/24812) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* ExpressionJIT remove unncecessary logging [#24813](https://github.com/ClickHouse/ClickHouse/pull/24813) ([Maksim Kita](https://github.com/kitaisreal)). +* ExpressionJIT simplify loop [#24814](https://github.com/ClickHouse/ClickHouse/pull/24814) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed clang tidy [#24821](https://github.com/ClickHouse/ClickHouse/pull/24821) ([Maksim Kita](https://github.com/kitaisreal)). +* Merging [#23260](https://github.com/ClickHouse/ClickHouse/issues/23260) [#24822](https://github.com/ClickHouse/ClickHouse/pull/24822) ([Anton Popov](https://github.com/CurtizJ)). +* Simplify code around TraceCollector, alternative to [#24829](https://github.com/ClickHouse/ClickHouse/issues/24829) [#24833](https://github.com/ClickHouse/ClickHouse/pull/24833) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Some improvements in stress test [#24835](https://github.com/ClickHouse/ClickHouse/pull/24835) ([alesapin](https://github.com/alesapin)). +* Rename ContextConstPtr to ContextPtr. [#24855](https://github.com/ClickHouse/ClickHouse/pull/24855) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Reverted libunwind from upstream [#24860](https://github.com/ClickHouse/ClickHouse/pull/24860) ([Maksim Kita](https://github.com/kitaisreal)). +* Testflows Extended Precision Data Type testing - Adding snapshots for output comparison. [#24861](https://github.com/ClickHouse/ClickHouse/pull/24861) ([MyroTk](https://github.com/MyroTk)). +* clickhouse-client: echo hint improvements [#24863](https://github.com/ClickHouse/ClickHouse/pull/24863) ([Azat Khuzhin](https://github.com/azat)). +* docs: update requests (to fix conflicts with urllib3) [#24865](https://github.com/ClickHouse/ClickHouse/pull/24865) ([Azat Khuzhin](https://github.com/azat)). +* Trying to resurrect woboq [#24875](https://github.com/ClickHouse/ClickHouse/pull/24875) ([alesapin](https://github.com/alesapin)). +* Get rid of std::stringstream in Suggest [#24882](https://github.com/ClickHouse/ClickHouse/pull/24882) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Suppress RocksDB error [#24886](https://github.com/ClickHouse/ClickHouse/pull/24886) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Remove some outdated integration tests [#24911](https://github.com/ClickHouse/ClickHouse/pull/24911) ([alesapin](https://github.com/alesapin)). +* Mute test_memory_consumption before [#24784](https://github.com/ClickHouse/ClickHouse/issues/24784) [#24919](https://github.com/ClickHouse/ClickHouse/pull/24919) ([Ilya Yatsishin](https://github.com/qoega)). +* Update something in KeyCondition [#24920](https://github.com/ClickHouse/ClickHouse/pull/24920) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Relax hung check [#24929](https://github.com/ClickHouse/ClickHouse/pull/24929) ([alesapin](https://github.com/alesapin)). +* some perf test script improvements [#24938](https://github.com/ClickHouse/ClickHouse/pull/24938) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* RFC: clickhouse-test: do not substitude db name in stderr by default [#24939](https://github.com/ClickHouse/ClickHouse/pull/24939) ([Azat Khuzhin](https://github.com/azat)). +* Add test issue [#23430](https://github.com/ClickHouse/ClickHouse/issues/23430) [#24941](https://github.com/ClickHouse/ClickHouse/pull/24941) ([filimonov](https://github.com/filimonov)). +* Followup fixes for integration tests [#24954](https://github.com/ClickHouse/ClickHouse/pull/24954) ([alesapin](https://github.com/alesapin)). +* Fix bad error message in docker entrypoint [#24955](https://github.com/ClickHouse/ClickHouse/pull/24955) ([filimonov](https://github.com/filimonov)). +* Fix endless wait in replica clone [#24957](https://github.com/ClickHouse/ClickHouse/pull/24957) ([alesapin](https://github.com/alesapin)). +* Remove subprocess_call from cluster.py [#24959](https://github.com/ClickHouse/ClickHouse/pull/24959) ([Ilya Yatsishin](https://github.com/qoega)). +* Delete support for waiting on queue- entries, is this dead code? [#24960](https://github.com/ClickHouse/ClickHouse/pull/24960) ([nvartolomei](https://github.com/nvartolomei)). +* Fix the test after [#20393](https://github.com/ClickHouse/ClickHouse/issues/20393) [#24967](https://github.com/ClickHouse/ClickHouse/pull/24967) ([filimonov](https://github.com/filimonov)). +* Try fix `test_consistent_parts_after_clone_replica` [#24968](https://github.com/ClickHouse/ClickHouse/pull/24968) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix typo in usage section in s3 documentation [#24970](https://github.com/ClickHouse/ClickHouse/pull/24970) ([presto53](https://github.com/presto53)). +* Cleanup iptables and containers on session start [#24973](https://github.com/ClickHouse/ClickHouse/pull/24973) ([Ilya Yatsishin](https://github.com/qoega)). +* Part movement between shards ACL [#24979](https://github.com/ClickHouse/ClickHouse/pull/24979) ([nvartolomei](https://github.com/nvartolomei)). +* Cleanup changelog script [#24987](https://github.com/ClickHouse/ClickHouse/pull/24987) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add changelog for 21.6 [#24989](https://github.com/ClickHouse/ClickHouse/pull/24989) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* clickhouse-test: fix log_comment for .sql tests [#24999](https://github.com/ClickHouse/ClickHouse/pull/24999) ([Azat Khuzhin](https://github.com/azat)). +* DictionaryLoader unnecessary dictionary configuration creation fix [#25001](https://github.com/ClickHouse/ClickHouse/pull/25001) ([Maksim Kita](https://github.com/kitaisreal)). +* Compression codecs refactoring [#25002](https://github.com/ClickHouse/ClickHouse/pull/25002) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Maybe Minio starts for too long in tests [#25007](https://github.com/ClickHouse/ClickHouse/pull/25007) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bad test [#25012](https://github.com/ClickHouse/ClickHouse/pull/25012) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disks fix ya.make [#25031](https://github.com/ClickHouse/ClickHouse/pull/25031) ([Maksim Kita](https://github.com/kitaisreal)). +* tests: disable suggestions for expect tests that does not requires it [#25033](https://github.com/ClickHouse/ClickHouse/pull/25033) ([Azat Khuzhin](https://github.com/azat)). +* fix jemalloc build on OS_DARWIN [#25034](https://github.com/ClickHouse/ClickHouse/pull/25034) ([sdk2](https://github.com/sdk2)). +* Update waitForTableReplicaToProcessLogEntry comments [#25037](https://github.com/ClickHouse/ClickHouse/pull/25037) ([nvartolomei](https://github.com/nvartolomei)). +* Remove database before model name in docs [#25038](https://github.com/ClickHouse/ClickHouse/pull/25038) ([Kruglov Pavel](https://github.com/Avogar)). +* odbc fix [#25045](https://github.com/ClickHouse/ClickHouse/pull/25045) ([Kseniia Sumarokova](https://github.com/kssenii)). +* test for attach partition from [#25060](https://github.com/ClickHouse/ClickHouse/pull/25060) ([Denny Crane](https://github.com/den-crane)). +* AggregateFunctionAnyHeavyData use fixed size type [#25066](https://github.com/ClickHouse/ClickHouse/pull/25066) ([Maksim Kita](https://github.com/kitaisreal)). +* Update Dockerfile [#25078](https://github.com/ClickHouse/ClickHouse/pull/25078) ([Ivan](https://github.com/abyss7)). +* Trying to debug fetches bandwith test failures [#25079](https://github.com/ClickHouse/ClickHouse/pull/25079) ([alesapin](https://github.com/alesapin)). +* Remove strange timeout in test [#25084](https://github.com/ClickHouse/ClickHouse/pull/25084) ([alesapin](https://github.com/alesapin)). +* Remove copypaste from StorageReplicatedMergeTree [#25087](https://github.com/ClickHouse/ClickHouse/pull/25087) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Less timeouts in integration tests [#25111](https://github.com/ClickHouse/ClickHouse/pull/25111) ([alesapin](https://github.com/alesapin)). +* Better odbc integration test [#25113](https://github.com/ClickHouse/ClickHouse/pull/25113) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix flaky check report in CI runner [#25126](https://github.com/ClickHouse/ClickHouse/pull/25126) ([alesapin](https://github.com/alesapin)). +* Better hdfs tests [#25128](https://github.com/ClickHouse/ClickHouse/pull/25128) ([Ilya Yatsishin](https://github.com/qoega)). +* Use zstd/include in ya.make [#25145](https://github.com/ClickHouse/ClickHouse/pull/25145) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Adds a better way to include binary resources [#25146](https://github.com/ClickHouse/ClickHouse/pull/25146) ([bnaecker](https://github.com/bnaecker)). +* check for row_policy defined using user() function [#25147](https://github.com/ClickHouse/ClickHouse/pull/25147) ([Denny Crane](https://github.com/den-crane)). +* Update Roaring Bitmaps just in case [#25151](https://github.com/ClickHouse/ClickHouse/pull/25151) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Move code to more appropriate place [#25152](https://github.com/ClickHouse/ClickHouse/pull/25152) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Dictionary sources add update lag option [#25161](https://github.com/ClickHouse/ClickHouse/pull/25161) ([Maksim Kita](https://github.com/kitaisreal)). +* RewriteFunctionToSubcolumnVisitor add into ya.make [#25162](https://github.com/ClickHouse/ClickHouse/pull/25162) ([Maksim Kita](https://github.com/kitaisreal)). +* Enable back the ANTLR in .sql [#25170](https://github.com/ClickHouse/ClickHouse/pull/25170) ([Ivan](https://github.com/abyss7)). +* Added ExecutablePool documentation [#25196](https://github.com/ClickHouse/ClickHouse/pull/25196) ([Maksim Kita](https://github.com/kitaisreal)). +* Do not optimize query plan for mutations. [#25197](https://github.com/ClickHouse/ClickHouse/pull/25197) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Dictionary added update field documentation [#25198](https://github.com/ClickHouse/ClickHouse/pull/25198) ([Maksim Kita](https://github.com/kitaisreal)). +* Dictionaries attribute support default nullable type [#25203](https://github.com/ClickHouse/ClickHouse/pull/25203) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix tests [#25204](https://github.com/ClickHouse/ClickHouse/pull/25204) ([Ivan](https://github.com/abyss7)). +* Try run tests with Replicated database in parallel [#25210](https://github.com/ClickHouse/ClickHouse/pull/25210) ([Alexander Tokmakov](https://github.com/tavplubix)). +* More integration tests improvements [#25212](https://github.com/ClickHouse/ClickHouse/pull/25212) ([Ilya Yatsishin](https://github.com/qoega)). +* Add function toJSONString to ya.make [#25246](https://github.com/ClickHouse/ClickHouse/pull/25246) ([Maksim Kita](https://github.com/kitaisreal)). +* Minor change [#25247](https://github.com/ClickHouse/ClickHouse/pull/25247) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add missed #include [#25248](https://github.com/ClickHouse/ClickHouse/pull/25248) ([Matwey V. Kornilov](https://github.com/matwey)). +* Add test for [#4113](https://github.com/ClickHouse/ClickHouse/issues/4113) [#25249](https://github.com/ClickHouse/ClickHouse/pull/25249) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add test for [#11535](https://github.com/ClickHouse/ClickHouse/issues/11535) [#25250](https://github.com/ClickHouse/ClickHouse/pull/25250) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#9932](https://github.com/ClickHouse/ClickHouse/issues/9932) [#25253](https://github.com/ClickHouse/ClickHouse/pull/25253) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Copy-paste some code [#25264](https://github.com/ClickHouse/ClickHouse/pull/25264) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor change [#25265](https://github.com/ClickHouse/ClickHouse/pull/25265) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#20315](https://github.com/ClickHouse/ClickHouse/issues/20315) [#25266](https://github.com/ClickHouse/ClickHouse/pull/25266) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add missed #include [#25267](https://github.com/ClickHouse/ClickHouse/pull/25267) ([Matwey V. Kornilov](https://github.com/matwey)). +* test for `PARTITION BY 0 * id` [#25274](https://github.com/ClickHouse/ClickHouse/pull/25274) ([Denny Crane](https://github.com/den-crane)). +* Enable TestFlows LDAP tests [#25278](https://github.com/ClickHouse/ClickHouse/pull/25278) ([vzakaznikov](https://github.com/vzakaznikov)). +* Add a test for [#17964](https://github.com/ClickHouse/ClickHouse/issues/17964) [#25285](https://github.com/ClickHouse/ClickHouse/pull/25285) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#17367](https://github.com/ClickHouse/ClickHouse/issues/17367) [#25286](https://github.com/ClickHouse/ClickHouse/pull/25286) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove some code from KeyCondition. [#25295](https://github.com/ClickHouse/ClickHouse/pull/25295) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* CMake dictionaries disable debug info fix [#25301](https://github.com/ClickHouse/ClickHouse/pull/25301) ([Maksim Kita](https://github.com/kitaisreal)). +* Merge ext into common [#25303](https://github.com/ClickHouse/ClickHouse/pull/25303) ([Maksim Kita](https://github.com/kitaisreal)). +* test for null array orc load [#25304](https://github.com/ClickHouse/ClickHouse/pull/25304) ([Denny Crane](https://github.com/den-crane)). +* Improve Replicated database tests [#25305](https://github.com/ClickHouse/ClickHouse/pull/25305) ([Alexander Tokmakov](https://github.com/tavplubix)). +* SimpleCache key constructor improvement [#25307](https://github.com/ClickHouse/ClickHouse/pull/25307) ([Maksim Kita](https://github.com/kitaisreal)). +* Catch ErrnoException during parts cleaning [#25309](https://github.com/ClickHouse/ClickHouse/pull/25309) ([Azat Khuzhin](https://github.com/azat)). +* Fix flaky test 01520_client_print_query_id and others. [#25311](https://github.com/ClickHouse/ClickHouse/pull/25311) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix UBSan report in quantileTiming [#25314](https://github.com/ClickHouse/ClickHouse/pull/25314) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update test_ttl_replicated [#25317](https://github.com/ClickHouse/ClickHouse/pull/25317) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix `00601_kill_running_query` [#25318](https://github.com/ClickHouse/ClickHouse/pull/25318) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Use signal 15 in restart_with_latest_version [#25323](https://github.com/ClickHouse/ClickHouse/pull/25323) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Setting min_count_to_compile_expression fix [#25332](https://github.com/ClickHouse/ClickHouse/pull/25332) ([Maksim Kita](https://github.com/kitaisreal)). +* Function formatDateTime fix code comments [#25334](https://github.com/ClickHouse/ClickHouse/pull/25334) ([Maksim Kita](https://github.com/kitaisreal)). +* Improve Replicated database tests 2 [#25373](https://github.com/ClickHouse/ClickHouse/pull/25373) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add a test for [#23163](https://github.com/ClickHouse/ClickHouse/issues/23163) [#25390](https://github.com/ClickHouse/ClickHouse/pull/25390) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Rename & reimport murmurhash sources from smhasher repo [#25400](https://github.com/ClickHouse/ClickHouse/pull/25400) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Add test from issue [#20624](https://github.com/ClickHouse/ClickHouse/issues/20624) [#25409](https://github.com/ClickHouse/ClickHouse/pull/25409) ([Vladimir C](https://github.com/vdimir)). +* Turn off WITH_COVERAGE in build with clang-tidy [#25417](https://github.com/ClickHouse/ClickHouse/pull/25417) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix container-overflow in replxx during incremental search (Ctrl-R) [#25427](https://github.com/ClickHouse/ClickHouse/pull/25427) ([Azat Khuzhin](https://github.com/azat)). +* DatabaseMySQL rename [#25430](https://github.com/ClickHouse/ClickHouse/pull/25430) ([Maksim Kita](https://github.com/kitaisreal)). +* Support REPLACE DICTIONARY, CREATE OR REPLACE DICTIONARY queries [#25444](https://github.com/ClickHouse/ClickHouse/pull/25444) ([Maksim Kita](https://github.com/kitaisreal)). +* Add a test for [#8417](https://github.com/ClickHouse/ClickHouse/issues/8417) [#25446](https://github.com/ClickHouse/ClickHouse/pull/25446) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove assumeMutable from removeColumnNullability [#25454](https://github.com/ClickHouse/ClickHouse/pull/25454) ([Vladimir C](https://github.com/vdimir)). +* DataTypeLowCardinality support DataTypeInterval tests [#25458](https://github.com/ClickHouse/ClickHouse/pull/25458) ([Maksim Kita](https://github.com/kitaisreal)). +* Add run-id option to integration tests [#25459](https://github.com/ClickHouse/ClickHouse/pull/25459) ([alesapin](https://github.com/alesapin)). +* Stable NOT chain formatting [#25494](https://github.com/ClickHouse/ClickHouse/pull/25494) ([Azat Khuzhin](https://github.com/azat)). +* Fix alternative stack for SIGSEGV handling [#25509](https://github.com/ClickHouse/ClickHouse/pull/25509) ([Azat Khuzhin](https://github.com/azat)). +* Catch "Maximum parse depth" error in fuzzer [#25510](https://github.com/ClickHouse/ClickHouse/pull/25510) ([Azat Khuzhin](https://github.com/azat)). +* Fix NOT parsing [#25520](https://github.com/ClickHouse/ClickHouse/pull/25520) ([Azat Khuzhin](https://github.com/azat)). +* Catch TOO_DEEP_RECURSION in fuzzer for formatted query too [#25521](https://github.com/ClickHouse/ClickHouse/pull/25521) ([Azat Khuzhin](https://github.com/azat)). +* Fix some bugs in integration tests [#25525](https://github.com/ClickHouse/ClickHouse/pull/25525) ([alesapin](https://github.com/alesapin)). +* Fix query progress [#25545](https://github.com/ClickHouse/ClickHouse/pull/25545) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add test for progress bar [#25551](https://github.com/ClickHouse/ClickHouse/pull/25551) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix some tests [#25564](https://github.com/ClickHouse/ClickHouse/pull/25564) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix flaky test 01249_flush_interactive.sh [#25565](https://github.com/ClickHouse/ClickHouse/pull/25565) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* METR-41529 [#25568](https://github.com/ClickHouse/ClickHouse/pull/25568) ([egatov](https://github.com/egatov)). +* clickhouse-client: fix NULL dereference for --param w/o value [#25579](https://github.com/ClickHouse/ClickHouse/pull/25579) ([Azat Khuzhin](https://github.com/azat)). +* Remove only symlinks during force_restore_data of Atomic engine [#25582](https://github.com/ClickHouse/ClickHouse/pull/25582) ([Azat Khuzhin](https://github.com/azat)). +* clickhouse-test: use basename (instead of full path) for log_comment [#25583](https://github.com/ClickHouse/ClickHouse/pull/25583) ([Azat Khuzhin](https://github.com/azat)). +* Fix typo in hardware failure error message [#25584](https://github.com/ClickHouse/ClickHouse/pull/25584) ([Stas Kelvich](https://github.com/kelvich)). +* Small change in Roaring Bitmaps [#25604](https://github.com/ClickHouse/ClickHouse/pull/25604) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ANTLR parser and enable it back in CI [#25638](https://github.com/ClickHouse/ClickHouse/pull/25638) ([Ivan](https://github.com/abyss7)). +* Fix alternative stack (MINSIGSTKSZ) and stack size check under osx [#25654](https://github.com/ClickHouse/ClickHouse/pull/25654) ([Azat Khuzhin](https://github.com/azat)). +* Enable MurmurHash in ArcadiaBuild [#25666](https://github.com/ClickHouse/ClickHouse/pull/25666) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* Add working test [#25720](https://github.com/ClickHouse/ClickHouse/pull/25720) ([alesapin](https://github.com/alesapin)). +* Compile expressions updated documentation [#25725](https://github.com/ClickHouse/ClickHouse/pull/25725) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix some more missed includes [#25732](https://github.com/ClickHouse/ClickHouse/pull/25732) ([Matwey V. Kornilov](https://github.com/matwey)). +* Fix native macOS (Xcode) builds [#25736](https://github.com/ClickHouse/ClickHouse/pull/25736) ([Denis Glazachev](https://github.com/traceon)). +* Fix arcadia [#25738](https://github.com/ClickHouse/ClickHouse/pull/25738) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Change error code in LIVE VIEW [#25739](https://github.com/ClickHouse/ClickHouse/pull/25739) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + #### Testing Improvement * * Add join related options to stress tests. [#25200](https://github.com/ClickHouse/ClickHouse/pull/25200) ([Vladimir C](https://github.com/vdimir)). diff --git a/docs/changelogs/v21.7.10.4-stable.md b/docs/changelogs/v21.7.10.4-stable.md index 9056da8ac89..963ac6d6ac7 100644 --- a/docs/changelogs/v21.7.10.4-stable.md +++ b/docs/changelogs/v21.7.10.4-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.10.4-stable FIXME as compared to v21.7.9.7-stable #### Improvement @@ -17,3 +24,11 @@ * Backported in [#28947](https://github.com/ClickHouse/ClickHouse/issues/28947): Fix reading of subcolumns from compact parts. [#28873](https://github.com/ClickHouse/ClickHouse/pull/28873) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#28931](https://github.com/ClickHouse/ClickHouse/issues/28931): Fix higher-order array functions (`SIGSEGV` for `arrayCompact`/`ILLEGAL_COLUMN` for `arrayDifference`/`arrayCumSumNonNegative`) with consts. [#28904](https://github.com/ClickHouse/ClickHouse/pull/28904) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add more checks for LC in native protocol. [#27827](https://github.com/ClickHouse/ClickHouse/pull/27827) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* CHJIT custom memory manager [#28236](https://github.com/ClickHouse/ClickHouse/pull/28236) ([Maksim Kita](https://github.com/kitaisreal)). +* Function dictGet default implementation for nulls [#28530](https://github.com/ClickHouse/ClickHouse/pull/28530) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix usage of nested columns with non-array columns with the same prefix [2] [#28762](https://github.com/ClickHouse/ClickHouse/pull/28762) ([Anton Popov](https://github.com/CurtizJ)). +* Lower compiled_expression_cache_size to 128MB [#28816](https://github.com/ClickHouse/ClickHouse/pull/28816) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.7.11.3-stable.md b/docs/changelogs/v21.7.11.3-stable.md index 66672204713..e130550fb9c 100644 --- a/docs/changelogs/v21.7.11.3-stable.md +++ b/docs/changelogs/v21.7.11.3-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.11.3-stable FIXME as compared to v21.7.10.4-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) @@ -5,3 +12,9 @@ * Backported in [#29024](https://github.com/ClickHouse/ClickHouse/issues/29024): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#29195](https://github.com/ClickHouse/ClickHouse/issues/29195): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Don not add const group by key for query with only having. [#28975](https://github.com/ClickHouse/ClickHouse/pull/28975) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27963](https://github.com/ClickHouse/ClickHouse/issues/27963) [#29063](https://github.com/ClickHouse/ClickHouse/pull/29063) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix terminate on uncaught exception [#29216](https://github.com/ClickHouse/ClickHouse/pull/29216) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.7.2.7-stable.md b/docs/changelogs/v21.7.2.7-stable.md index a7af0fed667..5c772eab664 100644 --- a/docs/changelogs/v21.7.2.7-stable.md +++ b/docs/changelogs/v21.7.2.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.2.7-stable FIXME as compared to v21.7.1.7283-prestable #### Improvement @@ -15,3 +22,8 @@ * Backported in [#26010](https://github.com/ClickHouse/ClickHouse/issues/26010): Fix formatting of type `Map` with integer keys to `JSON`. [#25982](https://github.com/ClickHouse/ClickHouse/pull/25982) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#26097](https://github.com/ClickHouse/ClickHouse/issues/26097): Fix wrong thread estimation for right subquery join in some cases. Close [#24075](https://github.com/ClickHouse/ClickHouse/issues/24075). [#26052](https://github.com/ClickHouse/ClickHouse/pull/26052) ([Vladimir C](https://github.com/vdimir)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* ExpressionCache destruction fix [#25835](https://github.com/ClickHouse/ClickHouse/pull/25835) ([Maksim Kita](https://github.com/kitaisreal)). +* Proper fix of serialization of type Map to JSON [#26048](https://github.com/ClickHouse/ClickHouse/pull/26048) ([Anton Popov](https://github.com/CurtizJ)). + diff --git a/docs/changelogs/v21.7.3.14-stable.md b/docs/changelogs/v21.7.3.14-stable.md index d24b7bcbf39..0ad15f765a1 100644 --- a/docs/changelogs/v21.7.3.14-stable.md +++ b/docs/changelogs/v21.7.3.14-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.3.14-stable FIXME as compared to v21.7.2.7-stable #### Bug Fix @@ -9,3 +16,7 @@ * Backported in [#26229](https://github.com/ClickHouse/ClickHouse/issues/26229): Remove excessive newline in `thread_name` column in `system.stack_trace` table. This fixes [#24124](https://github.com/ClickHouse/ClickHouse/issues/24124). [#26210](https://github.com/ClickHouse/ClickHouse/pull/26210) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Fix throwing exception when iterate over non existing remote directory. [#26296](https://github.com/ClickHouse/ClickHouse/pull/26296) ([ianton-ru](https://github.com/ianton-ru)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Remove misleading stderr output [#26155](https://github.com/ClickHouse/ClickHouse/pull/26155) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.7.4.18-stable.md b/docs/changelogs/v21.7.4.18-stable.md index 7bc08e2a0e3..34d457015cb 100644 --- a/docs/changelogs/v21.7.4.18-stable.md +++ b/docs/changelogs/v21.7.4.18-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.4.18-stable FIXME as compared to v21.7.3.14-stable #### Bug Fix diff --git a/docs/changelogs/v21.7.5.29-stable.md b/docs/changelogs/v21.7.5.29-stable.md index 3f24e3eded9..6b01d912e17 100644 --- a/docs/changelogs/v21.7.5.29-stable.md +++ b/docs/changelogs/v21.7.5.29-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.5.29-stable FIXME as compared to v21.7.4.18-stable #### Performance Improvement @@ -14,3 +21,10 @@ * Backported in [#26772](https://github.com/ClickHouse/ClickHouse/issues/26772): Sometimes SET ROLE could work incorrectly, this PR fixes that. [#26707](https://github.com/ClickHouse/ClickHouse/pull/26707) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#26907](https://github.com/ClickHouse/ClickHouse/issues/26907): Fix library-bridge ids load. [#26834](https://github.com/ClickHouse/ClickHouse/pull/26834) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Remove MySQLWireContext [#26429](https://github.com/ClickHouse/ClickHouse/pull/26429) ([Vitaly Baranov](https://github.com/vitlibar)). +* Update link to dpkg-deb in dockerfiles [#26497](https://github.com/ClickHouse/ClickHouse/pull/26497) ([Vladimir C](https://github.com/vdimir)). +* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* One more library bridge fix [#26873](https://github.com/ClickHouse/ClickHouse/pull/26873) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.7.6.39-stable.md b/docs/changelogs/v21.7.6.39-stable.md index a7913aca193..03afc95515e 100644 --- a/docs/changelogs/v21.7.6.39-stable.md +++ b/docs/changelogs/v21.7.6.39-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.6.39-stable FIXME as compared to v21.7.5.29-stable #### Bug Fix @@ -12,3 +19,7 @@ * Backported in [#27156](https://github.com/ClickHouse/ClickHouse/issues/27156): Fix synchronization in GRPCServer This PR fixes [#27024](https://github.com/ClickHouse/ClickHouse/issues/27024). [#27064](https://github.com/ClickHouse/ClickHouse/pull/27064) ([Vitaly Baranov](https://github.com/vitlibar)). * Backported in [#27261](https://github.com/ClickHouse/ClickHouse/issues/27261): In rare cases `system.detached_parts` table might contain incorrect information for some parts, it's fixed. Fixes [#27114](https://github.com/ClickHouse/ClickHouse/issues/27114). [#27183](https://github.com/ClickHouse/ClickHouse/pull/27183) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* library bridge fixes [#27060](https://github.com/ClickHouse/ClickHouse/pull/27060) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v21.7.7.47-stable.md b/docs/changelogs/v21.7.7.47-stable.md index f81abca2600..3c7bf09433e 100644 --- a/docs/changelogs/v21.7.7.47-stable.md +++ b/docs/changelogs/v21.7.7.47-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.7.47-stable FIXME as compared to v21.7.6.39-stable #### Bug Fix @@ -6,3 +13,12 @@ * Backported in [#27418](https://github.com/ClickHouse/ClickHouse/issues/27418): Fix `Cannot find column` error for queries with sampling. Was introduced in [#24574](https://github.com/ClickHouse/ClickHouse/issues/24574). Fixes [#26522](https://github.com/ClickHouse/ClickHouse/issues/26522). [#27301](https://github.com/ClickHouse/ClickHouse/pull/27301) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#27416](https://github.com/ClickHouse/ClickHouse/issues/27416): Fixed incorrect validation of partition id for MergeTree tables that created with old syntax. [#27328](https://github.com/ClickHouse/ClickHouse/pull/27328) ([Alexander Tokmakov](https://github.com/tavplubix)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix parallel execution of integration tests [#25986](https://github.com/ClickHouse/ClickHouse/pull/25986) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix mysql_kill_sync_thread_restore_test [#26673](https://github.com/ClickHouse/ClickHouse/pull/26673) ([Vladimir C](https://github.com/vdimir)). +* Wait for self datasource to be initialized in test_jdbc_bridge [#26827](https://github.com/ClickHouse/ClickHouse/pull/26827) ([Ilya Yatsishin](https://github.com/qoega)). +* Better integration tests [#26894](https://github.com/ClickHouse/ClickHouse/pull/26894) ([Ilya Yatsishin](https://github.com/qoega)). +* Implement `legacy_column_name_of_tuple_literal` in a less intrusive way [#27153](https://github.com/ClickHouse/ClickHouse/pull/27153) ([Anton Popov](https://github.com/CurtizJ)). +* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.7.8.58-stable.md b/docs/changelogs/v21.7.8.58-stable.md index aea1ae083f0..4f99e5439a4 100644 --- a/docs/changelogs/v21.7.8.58-stable.md +++ b/docs/changelogs/v21.7.8.58-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.8.58-stable FIXME as compared to v21.7.7.47-stable #### Bug Fix @@ -10,3 +17,8 @@ * Backported in [#27697](https://github.com/ClickHouse/ClickHouse/issues/27697): Fix bad type cast when functions like `arrayHas` are applied to arrays of LowCardinality of Nullable of different non-numeric types like `DateTime` and `DateTime64`. In previous versions bad cast occurs. In new version it will lead to exception. This closes [#26330](https://github.com/ClickHouse/ClickHouse/issues/26330). [#27682](https://github.com/ClickHouse/ClickHouse/pull/27682) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Backported in [#27748](https://github.com/ClickHouse/ClickHouse/issues/27748): Remove duplicated source files in CMakeLists.txt in arrow-cmake. [#27736](https://github.com/ClickHouse/ClickHouse/pull/27736) ([李扬](https://github.com/taiyang-li)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix killing unstopped containers in integration tests. [#26818](https://github.com/ClickHouse/ClickHouse/pull/26818) ([Vitaly Baranov](https://github.com/vitlibar)). +* Revert [#24095](https://github.com/ClickHouse/ClickHouse/issues/24095). User-level settings will affect queries from view. [#27227](https://github.com/ClickHouse/ClickHouse/pull/27227) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.7.9.7-stable.md b/docs/changelogs/v21.7.9.7-stable.md index 0d1a2a521e7..1d859b08da5 100644 --- a/docs/changelogs/v21.7.9.7-stable.md +++ b/docs/changelogs/v21.7.9.7-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.7.9.7-stable FIXME as compared to v21.7.8.58-stable #### Improvement @@ -25,3 +32,15 @@ * Backported in [#28288](https://github.com/ClickHouse/ClickHouse/issues/28288): Fix reading of custom TLD w/o new line at EOF. [#28213](https://github.com/ClickHouse/ClickHouse/pull/28213) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#28295](https://github.com/ClickHouse/ClickHouse/issues/28295): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)). +* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* ODBC connection holder fix dangling reference [#28298](https://github.com/ClickHouse/ClickHouse/pull/28298) ([Maksim Kita](https://github.com/kitaisreal)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.8.1.7409-prestable.md b/docs/changelogs/v21.8.1.7409-prestable.md index e703d227603..96f13713984 100644 --- a/docs/changelogs/v21.8.1.7409-prestable.md +++ b/docs/changelogs/v21.8.1.7409-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.1.7409-prestable FIXME as compared to v21.7.1.7283-prestable #### Backward Incompatible Change @@ -86,3 +93,62 @@ * NO CL ENTRY: '[ImgBot] Optimize images'. [#26054](https://github.com/ClickHouse/ClickHouse/pull/26054) ([imgbot[bot]](https://github.com/apps/imgbot)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix hang and incorrect exit code returned from clickhouse-test [#25537](https://github.com/ClickHouse/ClickHouse/pull/25537) ([nvartolomei](https://github.com/nvartolomei)). +* Remove PrewhereDAGInfo. [#25719](https://github.com/ClickHouse/ClickHouse/pull/25719) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix 01641_memory_tracking_insert_optimize [#25731](https://github.com/ClickHouse/ClickHouse/pull/25731) ([Azat Khuzhin](https://github.com/azat)). +* Separate log files for separate runs in stress test [#25741](https://github.com/ClickHouse/ClickHouse/pull/25741) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix slow performance test [#25742](https://github.com/ClickHouse/ClickHouse/pull/25742) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* DatabaseAtomic EXCHANGE DICTIONARIES fix test [#25753](https://github.com/ClickHouse/ClickHouse/pull/25753) ([Maksim Kita](https://github.com/kitaisreal)). +* Try fix flacky rabbitmq test [#25756](https://github.com/ClickHouse/ClickHouse/pull/25756) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add a test for [#13993](https://github.com/ClickHouse/ClickHouse/issues/13993) [#25758](https://github.com/ClickHouse/ClickHouse/pull/25758) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Set follow-fork-mode child for gdb in stress/fasttest/fuzzer [#25769](https://github.com/ClickHouse/ClickHouse/pull/25769) ([Azat Khuzhin](https://github.com/azat)). +* Ignore TOO_DEEP_RECURSION server exception during fuzzing [#25770](https://github.com/ClickHouse/ClickHouse/pull/25770) ([Azat Khuzhin](https://github.com/azat)). +* Add comments for VERSION_REVISION vs DBMS_TCP_PROTOCOL_VERSION [#25771](https://github.com/ClickHouse/ClickHouse/pull/25771) ([Azat Khuzhin](https://github.com/azat)). +* Fix flaky test and wrong message [#25772](https://github.com/ClickHouse/ClickHouse/pull/25772) ([alesapin](https://github.com/alesapin)). +* MaterializeMySQL: Improved column comments support [#25781](https://github.com/ClickHouse/ClickHouse/pull/25781) ([Storozhuk Kostiantyn](https://github.com/sand6255)). +* Fix ANTRL merge_prewhere_table test [#25782](https://github.com/ClickHouse/ClickHouse/pull/25782) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove dead code from VirtualColumnUtils.cpp [#25787](https://github.com/ClickHouse/ClickHouse/pull/25787) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix some typos in Storage classes [#25796](https://github.com/ClickHouse/ClickHouse/pull/25796) ([Raúl Marín](https://github.com/Algunenano)). +* Fix DateLUT on Darwin [#25803](https://github.com/ClickHouse/ClickHouse/pull/25803) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better test_version_update_after_mutation [#25810](https://github.com/ClickHouse/ClickHouse/pull/25810) ([alesapin](https://github.com/alesapin)). +* Collect stderr.log and stdout.log in all integration tests by default. [#25816](https://github.com/ClickHouse/ClickHouse/pull/25816) ([Vitaly Baranov](https://github.com/vitlibar)). +* METR-41529 [#25819](https://github.com/ClickHouse/ClickHouse/pull/25819) ([egatov](https://github.com/egatov)). +* tests/integration: use iptables --wait [#25823](https://github.com/ClickHouse/ClickHouse/pull/25823) ([Azat Khuzhin](https://github.com/azat)). +* Add a test for [#25611](https://github.com/ClickHouse/ClickHouse/issues/25611) [#25831](https://github.com/ClickHouse/ClickHouse/pull/25831) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix odbc test [#25834](https://github.com/ClickHouse/ClickHouse/pull/25834) ([Kseniia Sumarokova](https://github.com/kssenii)). +* ExpressionCache destruction fix [#25835](https://github.com/ClickHouse/ClickHouse/pull/25835) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix performance tests after converting ProfileEvents to Map type [#25837](https://github.com/ClickHouse/ClickHouse/pull/25837) ([Azat Khuzhin](https://github.com/azat)). +* Correct messages in integration tests. [#25861](https://github.com/ClickHouse/ClickHouse/pull/25861) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix typo in the test 00900_orc_arrow_parquet_maps [#25874](https://github.com/ClickHouse/ClickHouse/pull/25874) ([Kruglov Pavel](https://github.com/Avogar)). +* Improve logging in integration tests. [#25899](https://github.com/ClickHouse/ClickHouse/pull/25899) ([Vitaly Baranov](https://github.com/vitlibar)). +* kerberized HDFS test fix if run in parallel [#25908](https://github.com/ClickHouse/ClickHouse/pull/25908) ([Ilya Golshtein](https://github.com/ilejn)). +* fix special build on clang 11 [#25912](https://github.com/ClickHouse/ClickHouse/pull/25912) ([flynn](https://github.com/ucasfl)). +* Remove obsolete code from init script [#25920](https://github.com/ClickHouse/ClickHouse/pull/25920) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* FunctionInitializeAggregation build fix [#25922](https://github.com/ClickHouse/ClickHouse/pull/25922) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix clang tidy build check [#25939](https://github.com/ClickHouse/ClickHouse/pull/25939) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Build subquery in replaceJoinedTable without parsing [#25941](https://github.com/ClickHouse/ClickHouse/pull/25941) ([Vladimir C](https://github.com/vdimir)). +* Remove experimental ANTLR parser [#25942](https://github.com/ClickHouse/ClickHouse/pull/25942) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Follow-up for [#20470](https://github.com/ClickHouse/ClickHouse/issues/20470) [#25975](https://github.com/ClickHouse/ClickHouse/pull/25975) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Correct test [#25984](https://github.com/ClickHouse/ClickHouse/pull/25984) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix parallel execution of integration tests [#25986](https://github.com/ClickHouse/ClickHouse/pull/25986) ([Vitaly Baranov](https://github.com/vitlibar)). +* Compile aggregate functions perf tests fix [#25989](https://github.com/ClickHouse/ClickHouse/pull/25989) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix alter of settings in `MergeTree` tables [#25995](https://github.com/ClickHouse/ClickHouse/pull/25995) ([Anton Popov](https://github.com/CurtizJ)). +* Fix arcadia [#26002](https://github.com/ClickHouse/ClickHouse/pull/26002) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove old code [#26014](https://github.com/ClickHouse/ClickHouse/pull/26014) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* FunctionsLogical const result for non const arguments fix [#26018](https://github.com/ClickHouse/ClickHouse/pull/26018) ([Maksim Kita](https://github.com/kitaisreal)). +* FunctionSQLJSON ContextPtr build fix [#26022](https://github.com/ClickHouse/ClickHouse/pull/26022) ([Maksim Kita](https://github.com/kitaisreal)). +* Replace print() with logging.debug() in integration tests. [#26023](https://github.com/ClickHouse/ClickHouse/pull/26023) ([Vitaly Baranov](https://github.com/vitlibar)). +* Try to fix some flaky tests [#26032](https://github.com/ClickHouse/ClickHouse/pull/26032) ([Anton Popov](https://github.com/CurtizJ)). +* Fix for ZK watch metric drift in rare conditions [#26034](https://github.com/ClickHouse/ClickHouse/pull/26034) ([nvartolomei](https://github.com/nvartolomei)). +* AsynchronousMetrics: Don't assume temperature is always positive [#26045](https://github.com/ClickHouse/ClickHouse/pull/26045) ([Raúl Marín](https://github.com/Algunenano)). +* Proper fix of serialization of type Map to JSON [#26048](https://github.com/ClickHouse/ClickHouse/pull/26048) ([Anton Popov](https://github.com/CurtizJ)). +* ClickHouse dictionary source secure setting added documentation [#26055](https://github.com/ClickHouse/ClickHouse/pull/26055) ([Maksim Kita](https://github.com/kitaisreal)). +* Add changelog for 21.7 [#26057](https://github.com/ClickHouse/ClickHouse/pull/26057) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix 01791_dist_INSERT_block_structure_mismatch flakiness [#26058](https://github.com/ClickHouse/ClickHouse/pull/26058) ([Azat Khuzhin](https://github.com/azat)). +* Fix logical error with signed and unsigned offset in WindowFrame::checkValid [#26072](https://github.com/ClickHouse/ClickHouse/pull/26072) ([Vladimir C](https://github.com/vdimir)). +* Remove unused code [#26077](https://github.com/ClickHouse/ClickHouse/pull/26077) ([Anton Popov](https://github.com/CurtizJ)). +* Disabling annoying copier tests [#26099](https://github.com/ClickHouse/ClickHouse/pull/26099) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Make graph pipeline rendering compatible with Dagre.JS [#26114](https://github.com/ClickHouse/ClickHouse/pull/26114) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.8.10.19-lts.md b/docs/changelogs/v21.8.10.19-lts.md index 92347642ffa..3e73e7892ec 100644 --- a/docs/changelogs/v21.8.10.19-lts.md +++ b/docs/changelogs/v21.8.10.19-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.10.19-lts FIXME as compared to v21.8.9.13-lts #### Improvement @@ -10,3 +17,7 @@ * Backported in [#30333](https://github.com/ClickHouse/ClickHouse/issues/30333): * Allow identifiers staring with numbers in multiple joins. [#30230](https://github.com/ClickHouse/ClickHouse/pull/30230) ([Vladimir C](https://github.com/vdimir)). * Backported in [#30377](https://github.com/ClickHouse/ClickHouse/issues/30377): fix replaceRegexpAll bug. [#30292](https://github.com/ClickHouse/ClickHouse/pull/30292) ([Memo](https://github.com/Joeywzr)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix ca-bundle.crt in kerberized_hadoop/Dockerfile [#30358](https://github.com/ClickHouse/ClickHouse/pull/30358) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.8.11.4-lts.md b/docs/changelogs/v21.8.11.4-lts.md index e36d75c32ea..644a9ece633 100644 --- a/docs/changelogs/v21.8.11.4-lts.md +++ b/docs/changelogs/v21.8.11.4-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.11.4-lts FIXME as compared to v21.8.10.19-lts #### New Feature @@ -38,3 +45,10 @@ * Backported in [#31132](https://github.com/ClickHouse/ClickHouse/issues/31132): Fix JSONValue/Query with quoted identifiers. This allows to have spaces in json path. Closes [#30971](https://github.com/ClickHouse/ClickHouse/issues/30971). [#31003](https://github.com/ClickHouse/ClickHouse/pull/31003) ([Kseniia Sumarokova](https://github.com/kssenii)). * Backported in [#31372](https://github.com/ClickHouse/ClickHouse/issues/31372): Fix StorageMerge with aliases and where (it did not work before at all). Closes [#28802](https://github.com/ClickHouse/ClickHouse/issues/28802). [#31044](https://github.com/ClickHouse/ClickHouse/pull/31044) ([Kseniia Sumarokova](https://github.com/kssenii)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix assert in table function `merge` with database regexp [#29355](https://github.com/ClickHouse/ClickHouse/pull/29355) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* AddDefaultDatabaseVisitor support dictGet [#29650](https://github.com/ClickHouse/ClickHouse/pull/29650) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageDictionary fix potential configuration race [#30502](https://github.com/ClickHouse/ClickHouse/pull/30502) ([Maksim Kita](https://github.com/kitaisreal)). +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.8.12.29-lts.md b/docs/changelogs/v21.8.12.29-lts.md index 63b34c367f3..3dea113485d 100644 --- a/docs/changelogs/v21.8.12.29-lts.md +++ b/docs/changelogs/v21.8.12.29-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.12.29-lts FIXME as compared to v21.8.11.4-lts #### Performance Improvement @@ -22,3 +29,7 @@ * Backported in [#31890](https://github.com/ClickHouse/ClickHouse/issues/31890): Fix possible assertion `../src/IO/ReadBuffer.h:58: bool DB::ReadBuffer::next(): Assertion '!hasPendingData()' failed.` in TSKV format. [#31804](https://github.com/ClickHouse/ClickHouse/pull/31804) ([Kruglov Pavel](https://github.com/Avogar)). * Backported in [#31910](https://github.com/ClickHouse/ClickHouse/issues/31910): Fix functions `empty` and `notEmpty` with arguments of `UUID` type. Fixes [#31819](https://github.com/ClickHouse/ClickHouse/issues/31819). [#31883](https://github.com/ClickHouse/ClickHouse/pull/31883) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.8.13.6-lts.md b/docs/changelogs/v21.8.13.6-lts.md index 06e8c366ff2..1911de83009 100644 --- a/docs/changelogs/v21.8.13.6-lts.md +++ b/docs/changelogs/v21.8.13.6-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.13.6-lts FIXME as compared to v21.8.12.29-lts #### Bug Fix @@ -27,3 +34,15 @@ * NO CL ENTRY: 'fix json error after downgrade'. [#33166](https://github.com/ClickHouse/ClickHouse/pull/33166) ([bullet1337](https://github.com/bullet1337)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix data race in ProtobufSchemas [#27822](https://github.com/ClickHouse/ClickHouse/pull/27822) ([filimonov](https://github.com/filimonov)). +* Column default dictGet identifier fix [#28863](https://github.com/ClickHouse/ClickHouse/pull/28863) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix arraySlice with null args. [#32456](https://github.com/ClickHouse/ClickHouse/pull/32456) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* 21.8 ExternalDictionariesLoader fix build [#32501](https://github.com/ClickHouse/ClickHouse/pull/32501) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix queries with hasColumnInTable constant condition and non existing column [#32506](https://github.com/ClickHouse/ClickHouse/pull/32506) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* 21.8 ExternalDictionariesLoader fix [#32752](https://github.com/ClickHouse/ClickHouse/pull/32752) ([Maksim Kita](https://github.com/kitaisreal)). +* Merge [#33024](https://github.com/ClickHouse/ClickHouse/issues/33024) [#33061](https://github.com/ClickHouse/ClickHouse/pull/33061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Merge [#33022](https://github.com/ClickHouse/ClickHouse/issues/33022) [#33062](https://github.com/ClickHouse/ClickHouse/pull/33062) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + diff --git a/docs/changelogs/v21.8.14.5-lts.md b/docs/changelogs/v21.8.14.5-lts.md index 481327a35c9..8310573f94c 100644 --- a/docs/changelogs/v21.8.14.5-lts.md +++ b/docs/changelogs/v21.8.14.5-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.14.5-lts FIXME as compared to v21.8.13.6-lts #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v21.8.15.7-lts.md b/docs/changelogs/v21.8.15.7-lts.md index 7411fbff9ae..64da1824bcf 100644 --- a/docs/changelogs/v21.8.15.7-lts.md +++ b/docs/changelogs/v21.8.15.7-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.15.7-lts FIXME as compared to v21.8.14.5-lts #### Bug Fix (user-visible misbehaviour in official stable or prestable release) diff --git a/docs/changelogs/v21.8.2.19-prestable.md b/docs/changelogs/v21.8.2.19-prestable.md index 15726747e65..8b2bdd4e185 100644 --- a/docs/changelogs/v21.8.2.19-prestable.md +++ b/docs/changelogs/v21.8.2.19-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.2.19-prestable FIXME as compared to v21.8.1.7409-prestable #### Performance Improvement @@ -27,3 +34,10 @@ * Backported in [#26705](https://github.com/ClickHouse/ClickHouse/issues/26705): Fix potential nullptr dereference in window functions. This fixes [#25276](https://github.com/ClickHouse/ClickHouse/issues/25276). [#26668](https://github.com/ClickHouse/ClickHouse/pull/26668) ([Alexander Kuzmenkov](https://github.com/akuzm)). * Backported in [#26771](https://github.com/ClickHouse/ClickHouse/issues/26771): Sometimes SET ROLE could work incorrectly, this PR fixes that. [#26707](https://github.com/ClickHouse/ClickHouse/pull/26707) ([Vitaly Baranov](https://github.com/vitlibar)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Remove misleading stderr output [#26155](https://github.com/ClickHouse/ClickHouse/pull/26155) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Less logging in AsynchronousMetrics [#26391](https://github.com/ClickHouse/ClickHouse/pull/26391) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove MySQLWireContext [#26429](https://github.com/ClickHouse/ClickHouse/pull/26429) ([Vitaly Baranov](https://github.com/vitlibar)). +* Update link to dpkg-deb in dockerfiles [#26497](https://github.com/ClickHouse/ClickHouse/pull/26497) ([Vladimir C](https://github.com/vdimir)). + diff --git a/docs/changelogs/v21.8.3.44-lts.md b/docs/changelogs/v21.8.3.44-lts.md index 21fe655870a..12ecd521c15 100644 --- a/docs/changelogs/v21.8.3.44-lts.md +++ b/docs/changelogs/v21.8.3.44-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.3.44-lts FIXME as compared to v21.8.2.19-prestable #### Improvement @@ -26,3 +33,17 @@ * Backported in [#27419](https://github.com/ClickHouse/ClickHouse/issues/27419): /proc/info contains metrics like. [#27361](https://github.com/ClickHouse/ClickHouse/pull/27361) ([Mike Kot](https://github.com/myrrc)). * Backported in [#27472](https://github.com/ClickHouse/ClickHouse/issues/27472): fix metric BackgroundMessageBrokerSchedulePoolTask, maybe mistyped。. [#27452](https://github.com/ClickHouse/ClickHouse/pull/27452) ([Ben](https://github.com/benbiti)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix mysql_kill_sync_thread_restore_test [#26673](https://github.com/ClickHouse/ClickHouse/pull/26673) ([Vladimir C](https://github.com/vdimir)). +* Fix killing unstopped containers in integration tests. [#26818](https://github.com/ClickHouse/ClickHouse/pull/26818) ([Vitaly Baranov](https://github.com/vitlibar)). +* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Wait for self datasource to be initialized in test_jdbc_bridge [#26827](https://github.com/ClickHouse/ClickHouse/pull/26827) ([Ilya Yatsishin](https://github.com/qoega)). +* One more library bridge fix [#26873](https://github.com/ClickHouse/ClickHouse/pull/26873) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Better integration tests [#26894](https://github.com/ClickHouse/ClickHouse/pull/26894) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix assertions in Replicated database [#27033](https://github.com/ClickHouse/ClickHouse/pull/27033) ([Alexander Tokmakov](https://github.com/tavplubix)). +* library bridge fixes [#27060](https://github.com/ClickHouse/ClickHouse/pull/27060) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Implement `legacy_column_name_of_tuple_literal` in a less intrusive way [#27153](https://github.com/ClickHouse/ClickHouse/pull/27153) ([Anton Popov](https://github.com/CurtizJ)). +* Revert [#24095](https://github.com/ClickHouse/ClickHouse/issues/24095). User-level settings will affect queries from view. [#27227](https://github.com/ClickHouse/ClickHouse/pull/27227) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.8.4.51-lts.md b/docs/changelogs/v21.8.4.51-lts.md index a8494ebb1d1..9b0dba786e5 100644 --- a/docs/changelogs/v21.8.4.51-lts.md +++ b/docs/changelogs/v21.8.4.51-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.4.51-lts FIXME as compared to v21.8.3.44-lts #### Bug Fix diff --git a/docs/changelogs/v21.8.5.7-lts.md b/docs/changelogs/v21.8.5.7-lts.md index d78eb98b472..782c93ce9b1 100644 --- a/docs/changelogs/v21.8.5.7-lts.md +++ b/docs/changelogs/v21.8.5.7-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.5.7-lts FIXME as compared to v21.8.4.51-lts #### Improvement @@ -29,3 +36,16 @@ * Backported in [#28292](https://github.com/ClickHouse/ClickHouse/issues/28292): Fix inconsistent result in queries with `ORDER BY` and `Merge` tables with enabled setting `optimize_read_in_order`. [#28266](https://github.com/ClickHouse/ClickHouse/pull/28266) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#28402](https://github.com/ClickHouse/ClickHouse/issues/28402): Fix intersecting parts due to new part had been replaced with an empty part. [#28310](https://github.com/ClickHouse/ClickHouse/pull/28310) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix several bugs in ZooKeeper snapshots deserialization [#26127](https://github.com/ClickHouse/ClickHouse/pull/26127) ([alesapin](https://github.com/alesapin)). +* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)). +* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* ODBC connection holder fix dangling reference [#28298](https://github.com/ClickHouse/ClickHouse/pull/28298) ([Maksim Kita](https://github.com/kitaisreal)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + diff --git a/docs/changelogs/v21.8.6.15-lts.md b/docs/changelogs/v21.8.6.15-lts.md index de38572d94a..95d349dcacb 100644 --- a/docs/changelogs/v21.8.6.15-lts.md +++ b/docs/changelogs/v21.8.6.15-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.6.15-lts FIXME as compared to v21.8.5.7-lts #### Improvement @@ -23,3 +30,12 @@ * Backported in [#28948](https://github.com/ClickHouse/ClickHouse/issues/28948): Fix reading of subcolumns from compact parts. [#28873](https://github.com/ClickHouse/ClickHouse/pull/28873) ([Anton Popov](https://github.com/CurtizJ)). * Backported in [#28930](https://github.com/ClickHouse/ClickHouse/issues/28930): Fix higher-order array functions (`SIGSEGV` for `arrayCompact`/`ILLEGAL_COLUMN` for `arrayDifference`/`arrayCumSumNonNegative`) with consts. [#28904](https://github.com/ClickHouse/ClickHouse/pull/28904) ([Azat Khuzhin](https://github.com/azat)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add more checks for LC in native protocol. [#27827](https://github.com/ClickHouse/ClickHouse/pull/27827) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* CHJIT custom memory manager [#28236](https://github.com/ClickHouse/ClickHouse/pull/28236) ([Maksim Kita](https://github.com/kitaisreal)). +* Function dictGet default implementation for nulls [#28530](https://github.com/ClickHouse/ClickHouse/pull/28530) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix usage of nested columns with non-array columns with the same prefix [2] [#28762](https://github.com/ClickHouse/ClickHouse/pull/28762) ([Anton Popov](https://github.com/CurtizJ)). +* Lower compiled_expression_cache_size to 128MB [#28816](https://github.com/ClickHouse/ClickHouse/pull/28816) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix some images in release PRs [#29059](https://github.com/ClickHouse/ClickHouse/pull/29059) ([alesapin](https://github.com/alesapin)). + diff --git a/docs/changelogs/v21.8.7.22-lts.md b/docs/changelogs/v21.8.7.22-lts.md index 92ba59f13cd..31d9c15debb 100644 --- a/docs/changelogs/v21.8.7.22-lts.md +++ b/docs/changelogs/v21.8.7.22-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.7.22-lts FIXME as compared to v21.8.6.15-lts #### Bug Fix (user-visible misbehaviour in official stable or prestable release) @@ -6,3 +13,9 @@ * Backported in [#29027](https://github.com/ClickHouse/ClickHouse/issues/29027): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Backported in [#29193](https://github.com/ClickHouse/ClickHouse/issues/29193): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Don not add const group by key for query with only having. [#28975](https://github.com/ClickHouse/ClickHouse/pull/28975) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27963](https://github.com/ClickHouse/ClickHouse/issues/27963) [#29063](https://github.com/ClickHouse/ClickHouse/pull/29063) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix terminate on uncaught exception [#29216](https://github.com/ClickHouse/ClickHouse/pull/29216) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.8.8.29-lts.md b/docs/changelogs/v21.8.8.29-lts.md index 199be63424c..b6645362139 100644 --- a/docs/changelogs/v21.8.8.29-lts.md +++ b/docs/changelogs/v21.8.8.29-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.8.29-lts FIXME as compared to v21.8.7.22-lts #### Bug Fix diff --git a/docs/changelogs/v21.8.9.13-lts.md b/docs/changelogs/v21.8.9.13-lts.md index e4cd5f45b9b..1d9b436f302 100644 --- a/docs/changelogs/v21.8.9.13-lts.md +++ b/docs/changelogs/v21.8.9.13-lts.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.8.9.13-lts FIXME as compared to v21.8.8.29-lts #### Improvement @@ -29,3 +36,9 @@ * Avoid deadlocks when reading and writting on JOIN Engine tables at the same time. [#30187](https://github.com/ClickHouse/ClickHouse/pull/30187) ([Raúl Marín](https://github.com/Algunenano)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Maybe fix livelock in ZooKeeper client [#28195](https://github.com/ClickHouse/ClickHouse/pull/28195) ([Alexander Tokmakov](https://github.com/tavplubix)). +* May be fix s3 tests [#29762](https://github.com/ClickHouse/ClickHouse/pull/29762) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update BoringSSL [#29998](https://github.com/ClickHouse/ClickHouse/pull/29998) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + diff --git a/docs/changelogs/v21.9.1.8000-prestable.md b/docs/changelogs/v21.9.1.8000-prestable.md index cee357658d2..31fb71948d0 100644 --- a/docs/changelogs/v21.9.1.8000-prestable.md +++ b/docs/changelogs/v21.9.1.8000-prestable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.1.8000-prestable FIXME as compared to v21.8.1.7409-prestable #### Backward Incompatible Change @@ -188,3 +195,209 @@ * NO CL ENTRY: 'Revert "less sys calls #2: make vdso work again"'. [#27829](https://github.com/ClickHouse/ClickHouse/pull/27829) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * NO CL ENTRY: 'Revert "Do not miss exceptions from the ThreadPool"'. [#27844](https://github.com/ClickHouse/ClickHouse/pull/27844) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)). +* Add comments for the implementations of the pad functions [#26147](https://github.com/ClickHouse/ClickHouse/pull/26147) ([Vitaly Baranov](https://github.com/vitlibar)). +* Change color in client for double colon [#26152](https://github.com/ClickHouse/ClickHouse/pull/26152) ([Anton Popov](https://github.com/CurtizJ)). +* Remove misleading stderr output [#26155](https://github.com/ClickHouse/ClickHouse/pull/26155) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Merging [#26041](https://github.com/ClickHouse/ClickHouse/issues/26041). [#26180](https://github.com/ClickHouse/ClickHouse/pull/26180) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix bad code (default function argument) [#26213](https://github.com/ClickHouse/ClickHouse/pull/26213) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support for `pread` in `ReadBufferFromFile` [#26232](https://github.com/ClickHouse/ClickHouse/pull/26232) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ReadBufferFromS3 [#26249](https://github.com/ClickHouse/ClickHouse/pull/26249) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix output of TSV in integration tests [#26257](https://github.com/ClickHouse/ClickHouse/pull/26257) ([Anton Popov](https://github.com/CurtizJ)). +* Enum type additional support for compilation [#26258](https://github.com/ClickHouse/ClickHouse/pull/26258) ([Maksim Kita](https://github.com/kitaisreal)). +* Bump poco (now poco fork has CI via github actions) [#26262](https://github.com/ClickHouse/ClickHouse/pull/26262) ([Azat Khuzhin](https://github.com/azat)). +* Add check for sqlite database path [#26266](https://github.com/ClickHouse/ClickHouse/pull/26266) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix arcadia [#26285](https://github.com/ClickHouse/ClickHouse/pull/26285) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* yandex/clickhouse-test-base Dockerfile fix (mysql connector moved) [#26294](https://github.com/ClickHouse/ClickHouse/pull/26294) ([Ilya Golshtein](https://github.com/ilejn)). +* Improve read_file_to_stringcolumn test compatibility [#26317](https://github.com/ClickHouse/ClickHouse/pull/26317) ([Raúl Marín](https://github.com/Algunenano)). +* Make socket poll() 7x faster (by replacing epoll() with poll()) [#26323](https://github.com/ClickHouse/ClickHouse/pull/26323) ([Azat Khuzhin](https://github.com/azat)). +* Modifications to an obscure Yandex TSKV format [#26326](https://github.com/ClickHouse/ClickHouse/pull/26326) ([egatov](https://github.com/egatov)). +* Fixing RBAC sample by tests in TestFlows. [#26329](https://github.com/ClickHouse/ClickHouse/pull/26329) ([vzakaznikov](https://github.com/vzakaznikov)). +* Fix error in stress test script [#26349](https://github.com/ClickHouse/ClickHouse/pull/26349) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky integration test about "replicated max parallel fetches". [#26362](https://github.com/ClickHouse/ClickHouse/pull/26362) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Continuation of [#25774](https://github.com/ClickHouse/ClickHouse/issues/25774) [#26364](https://github.com/ClickHouse/ClickHouse/pull/26364) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Enabling all TestFlows modules except LDAP after Kerberos merge. [#26366](https://github.com/ClickHouse/ClickHouse/pull/26366) ([vzakaznikov](https://github.com/vzakaznikov)). +* Fix flaky test 01293_client_interactive_vertical_multiline_long [#26367](https://github.com/ClickHouse/ClickHouse/pull/26367) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Small bugfix in Block [#26373](https://github.com/ClickHouse/ClickHouse/pull/26373) ([Anton Popov](https://github.com/CurtizJ)). +* More integration tests improvements. [#26375](https://github.com/ClickHouse/ClickHouse/pull/26375) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix arcadia [#26378](https://github.com/ClickHouse/ClickHouse/pull/26378) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add separate step to read from remote. [#26381](https://github.com/ClickHouse/ClickHouse/pull/26381) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix calculating of intersection of access rights. [#26383](https://github.com/ClickHouse/ClickHouse/pull/26383) ([Vitaly Baranov](https://github.com/vitlibar)). +* Less logging in AsynchronousMetrics [#26391](https://github.com/ClickHouse/ClickHouse/pull/26391) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* stress tests report improvements [#26393](https://github.com/ClickHouse/ClickHouse/pull/26393) ([Azat Khuzhin](https://github.com/azat)). +* Fix failed assertion in RocksDB in case of bad_alloc exception during batch write [#26394](https://github.com/ClickHouse/ClickHouse/pull/26394) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add integrity for loaded scripts in play.html [#26409](https://github.com/ClickHouse/ClickHouse/pull/26409) ([Vladimir C](https://github.com/vdimir)). +* Relax condition in flaky test [#26425](https://github.com/ClickHouse/ClickHouse/pull/26425) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Split FunctionsCoding into several files [#26426](https://github.com/ClickHouse/ClickHouse/pull/26426) ([Vladimir C](https://github.com/vdimir)). +* Remove MySQLWireContext [#26429](https://github.com/ClickHouse/ClickHouse/pull/26429) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix "While sending batch" (on Distributed async send) [#26430](https://github.com/ClickHouse/ClickHouse/pull/26430) ([Azat Khuzhin](https://github.com/azat)). +* Tests fixes v21.9.1.7477 [#26431](https://github.com/ClickHouse/ClickHouse/pull/26431) ([Azat Khuzhin](https://github.com/azat)). +* Fix flaky test_replicated_mutations (due to lack of threads in pool) [#26461](https://github.com/ClickHouse/ClickHouse/pull/26461) ([Azat Khuzhin](https://github.com/azat)). +* Fix undefined-behavior in DirectoryMonitor (for exponential back off) [#26464](https://github.com/ClickHouse/ClickHouse/pull/26464) ([Azat Khuzhin](https://github.com/azat)). +* Remove some code [#26468](https://github.com/ClickHouse/ClickHouse/pull/26468) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Rewrite distributed DDL to Processors [#26469](https://github.com/ClickHouse/ClickHouse/pull/26469) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test `distributed_ddl_output_mode` [#26470](https://github.com/ClickHouse/ClickHouse/pull/26470) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update link to dpkg-deb in dockerfiles [#26497](https://github.com/ClickHouse/ClickHouse/pull/26497) ([Vladimir C](https://github.com/vdimir)). +* SELECT String from ClickHouse as Avro string - PartialMatch [#26499](https://github.com/ClickHouse/ClickHouse/pull/26499) ([Ilya Golshtein](https://github.com/ilejn)). +* Fix arcadia [#26505](https://github.com/ClickHouse/ClickHouse/pull/26505) ([Anton Popov](https://github.com/CurtizJ)). +* Fix build under AppleClang 12 [#26509](https://github.com/ClickHouse/ClickHouse/pull/26509) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* fix lagInFrame for nullable types [#26521](https://github.com/ClickHouse/ClickHouse/pull/26521) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Handle empty testset in 'Functional stateless tests flaky check' [#26552](https://github.com/ClickHouse/ClickHouse/pull/26552) ([Vladimir C](https://github.com/vdimir)). +* Remove some streams. [#26590](https://github.com/ClickHouse/ClickHouse/pull/26590) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix flaky test 01622_defaults_for_url_engine [#26617](https://github.com/ClickHouse/ClickHouse/pull/26617) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky test 01509_check_many_parallel_quorum_inserts [#26618](https://github.com/ClickHouse/ClickHouse/pull/26618) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix one possible cause of tests flakiness [#26619](https://github.com/ClickHouse/ClickHouse/pull/26619) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove some code, more C++ way [#26620](https://github.com/ClickHouse/ClickHouse/pull/26620) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix mysql_kill_sync_thread_restore_test [#26673](https://github.com/ClickHouse/ClickHouse/pull/26673) ([Vladimir C](https://github.com/vdimir)). +* Minor bugfix [#26678](https://github.com/ClickHouse/ClickHouse/pull/26678) ([Anton Popov](https://github.com/CurtizJ)). +* copypaste error [#26679](https://github.com/ClickHouse/ClickHouse/pull/26679) ([Denny Crane](https://github.com/den-crane)). +* Fix flaky test `mutation_stuck_after_replace_partition` [#26684](https://github.com/ClickHouse/ClickHouse/pull/26684) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Log errors on integration test listing error in ci-runner [#26691](https://github.com/ClickHouse/ClickHouse/pull/26691) ([Vladimir C](https://github.com/vdimir)). +* more debug checks for window functions [#26701](https://github.com/ClickHouse/ClickHouse/pull/26701) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* record server exit code in fuzzer [#26706](https://github.com/ClickHouse/ClickHouse/pull/26706) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Remove more streams. [#26713](https://github.com/ClickHouse/ClickHouse/pull/26713) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* 01946_test_wrong_host_name_access: Clear DNS in the end [#26715](https://github.com/ClickHouse/ClickHouse/pull/26715) ([Raúl Marín](https://github.com/Algunenano)). +* Setting min_count_to_compile_aggregate_expression fix [#26718](https://github.com/ClickHouse/ClickHouse/pull/26718) ([Maksim Kita](https://github.com/kitaisreal)). +* Compile aggregate functions profile events fix [#26719](https://github.com/ClickHouse/ClickHouse/pull/26719) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix use after free in AsyncDrain connection from S3Cluster [#26731](https://github.com/ClickHouse/ClickHouse/pull/26731) ([Vladimir C](https://github.com/vdimir)). +* Fixed wrong error message in `S3Common` [#26738](https://github.com/ClickHouse/ClickHouse/pull/26738) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Lock mutex before access to std::cerr in clickhouse-benchmark [#26742](https://github.com/ClickHouse/ClickHouse/pull/26742) ([Vladimir C](https://github.com/vdimir)). +* Remove some output streams [#26758](https://github.com/ClickHouse/ClickHouse/pull/26758) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#25960](https://github.com/ClickHouse/ClickHouse/issues/25960) (Bolonini/read_from_file) [#26777](https://github.com/ClickHouse/ClickHouse/pull/26777) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Heredoc updated tests [#26784](https://github.com/ClickHouse/ClickHouse/pull/26784) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix double unlock in RocksDB [#26786](https://github.com/ClickHouse/ClickHouse/pull/26786) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix sqlite engine attach [#26795](https://github.com/ClickHouse/ClickHouse/pull/26795) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove unneeded mutex during connection draining [#26807](https://github.com/ClickHouse/ClickHouse/pull/26807) ([Amos Bird](https://github.com/amosbird)). +* Make sure table is readonly when restarting fails. [#26808](https://github.com/ClickHouse/ClickHouse/pull/26808) ([Amos Bird](https://github.com/amosbird)). +* Check stdout for messages to retry in clickhouse-test [#26817](https://github.com/ClickHouse/ClickHouse/pull/26817) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix killing unstopped containers in integration tests. [#26818](https://github.com/ClickHouse/ClickHouse/pull/26818) ([Vitaly Baranov](https://github.com/vitlibar)). +* Do not start new hedged connection if query was already canceled. [#26820](https://github.com/ClickHouse/ClickHouse/pull/26820) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Wait for self datasource to be initialized in test_jdbc_bridge [#26827](https://github.com/ClickHouse/ClickHouse/pull/26827) ([Ilya Yatsishin](https://github.com/qoega)). +* Flush LazyOutputFormat on query cancel. [#26828](https://github.com/ClickHouse/ClickHouse/pull/26828) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Try fix timeout in functional tests with pytest [#26830](https://github.com/ClickHouse/ClickHouse/pull/26830) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Compile aggregate functions without key [#26845](https://github.com/ClickHouse/ClickHouse/pull/26845) ([Maksim Kita](https://github.com/kitaisreal)). +* Introduce sessions [#26864](https://github.com/ClickHouse/ClickHouse/pull/26864) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix rabbitmq sink [#26871](https://github.com/ClickHouse/ClickHouse/pull/26871) ([Kseniia Sumarokova](https://github.com/kssenii)). +* One more library bridge fix [#26873](https://github.com/ClickHouse/ClickHouse/pull/26873) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix keeper bench compilation [#26874](https://github.com/ClickHouse/ClickHouse/pull/26874) ([Raúl Marín](https://github.com/Algunenano)). +* Better integration tests [#26894](https://github.com/ClickHouse/ClickHouse/pull/26894) ([Ilya Yatsishin](https://github.com/qoega)). +* Maybe fix extremely rare `intersecting parts`. [#26896](https://github.com/ClickHouse/ClickHouse/pull/26896) ([alesapin](https://github.com/alesapin)). +* Try increase diff upper bound [#26897](https://github.com/ClickHouse/ClickHouse/pull/26897) ([Ilya Yatsishin](https://github.com/qoega)). +* Enable Arrow format in Arcadia [#26898](https://github.com/ClickHouse/ClickHouse/pull/26898) ([Vitaly Stoyan](https://github.com/vitstn)). +* Improve test compatibility (00646_url_engine and 01854_HTTP_dict_decompression) [#26915](https://github.com/ClickHouse/ClickHouse/pull/26915) ([Raúl Marín](https://github.com/Algunenano)). +* Update NuRaft [#26916](https://github.com/ClickHouse/ClickHouse/pull/26916) ([alesapin](https://github.com/alesapin)). +* 01921_datatype_date32: Adapt it to work under Pacific/Fiji [#26918](https://github.com/ClickHouse/ClickHouse/pull/26918) ([Raúl Marín](https://github.com/Algunenano)). +* Remove unused files that confuse developers [#26913](https://github.com/ClickHouse/ClickHouse/issues/26913) [#26947](https://github.com/ClickHouse/ClickHouse/pull/26947) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Set allow_remote_fs_zero_copy_replication to true by default [#26951](https://github.com/ClickHouse/ClickHouse/pull/26951) ([ianton-ru](https://github.com/ianton-ru)). +* Hold context in HedgedConnections to prevent use-after-free on settings. [#26953](https://github.com/ClickHouse/ClickHouse/pull/26953) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix client options in stress test [#26959](https://github.com/ClickHouse/ClickHouse/pull/26959) ([Alexander Tokmakov](https://github.com/tavplubix)). +* fix window function partition boundary search [#26960](https://github.com/ClickHouse/ClickHouse/pull/26960) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Print trace from std::terminate exception line-by-line to make it grep easier [#26962](https://github.com/ClickHouse/ClickHouse/pull/26962) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix system.zookeeper_log initialization [#26972](https://github.com/ClickHouse/ClickHouse/pull/26972) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Benchmark script fix [#26974](https://github.com/ClickHouse/ClickHouse/pull/26974) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix 01600_quota_by_forwarded_ip [#26975](https://github.com/ClickHouse/ClickHouse/pull/26975) ([Raúl Marín](https://github.com/Algunenano)). +* 01674_executable_dictionary_implicit_key: executable_dictionary: Use printf [#26978](https://github.com/ClickHouse/ClickHouse/pull/26978) ([Raúl Marín](https://github.com/Algunenano)). +* Remove test_keeper_server usage (for {operation/session}_timeout_ms) [#26984](https://github.com/ClickHouse/ClickHouse/pull/26984) ([Azat Khuzhin](https://github.com/azat)). +* Set insert_quorum_timeout to 1 minute for tests. [#27007](https://github.com/ClickHouse/ClickHouse/pull/27007) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Adjust 00537_quarters to be timezone independent [#27008](https://github.com/ClickHouse/ClickHouse/pull/27008) ([Raúl Marín](https://github.com/Algunenano)). +* Help with [#26424](https://github.com/ClickHouse/ClickHouse/issues/26424) [#27009](https://github.com/ClickHouse/ClickHouse/pull/27009) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Attempt to fix flaky 00705_drop_create_merge_tree [#27023](https://github.com/ClickHouse/ClickHouse/pull/27023) ([Raúl Marín](https://github.com/Algunenano)). +* Improved `runner` to use `pytest` keyword expressions [#27026](https://github.com/ClickHouse/ClickHouse/pull/27026) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Improve 01006_simpod_empty_part_single_column_write [#27028](https://github.com/ClickHouse/ClickHouse/pull/27028) ([Raúl Marín](https://github.com/Algunenano)). +* Improved logging of `hwmon` sensor errors in `AsynchronousMetrics` [#27031](https://github.com/ClickHouse/ClickHouse/pull/27031) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fix assertions in Replicated database [#27033](https://github.com/ClickHouse/ClickHouse/pull/27033) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Make test 01852_cast_operator independent of timezone [#27037](https://github.com/ClickHouse/ClickHouse/pull/27037) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Moving to TestFlows 1.7.20 that has native support for parallel tests. [#27040](https://github.com/ClickHouse/ClickHouse/pull/27040) ([vzakaznikov](https://github.com/vzakaznikov)). +* library bridge fixes [#27060](https://github.com/ClickHouse/ClickHouse/pull/27060) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix synchronization while updating from the config of an encrypted disk. [#27065](https://github.com/ClickHouse/ClickHouse/pull/27065) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix excessive logging in NuRaft on server shutdown [#27081](https://github.com/ClickHouse/ClickHouse/pull/27081) ([alesapin](https://github.com/alesapin)). +* Fix test_merge_tree_s3_failover with debug build [#27089](https://github.com/ClickHouse/ClickHouse/pull/27089) ([ianton-ru](https://github.com/ianton-ru)). +* Stateless tests: Keep an DNS error free log [#27092](https://github.com/ClickHouse/ClickHouse/pull/27092) ([Raúl Marín](https://github.com/Algunenano)). +* Normalize hostname in stateless tests [#27093](https://github.com/ClickHouse/ClickHouse/pull/27093) ([Amos Bird](https://github.com/amosbird)). +* Try update AMQP-CPP [#27095](https://github.com/ClickHouse/ClickHouse/pull/27095) ([Ilya Yatsishin](https://github.com/qoega)). +* Try update arrow [#27097](https://github.com/ClickHouse/ClickHouse/pull/27097) ([Ilya Yatsishin](https://github.com/qoega)). +* GlobalSubqueriesVisitor external storage check fix [#27131](https://github.com/ClickHouse/ClickHouse/pull/27131) ([Maksim Kita](https://github.com/kitaisreal)). +* Better code around decompression [#27136](https://github.com/ClickHouse/ClickHouse/pull/27136) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add test for parsing maps with integer keys [#27146](https://github.com/ClickHouse/ClickHouse/pull/27146) ([Anton Popov](https://github.com/CurtizJ)). +* Fix arcadia src/Access gtest [#27152](https://github.com/ClickHouse/ClickHouse/pull/27152) ([Ilya Yatsishin](https://github.com/qoega)). +* Implement `legacy_column_name_of_tuple_literal` in a less intrusive way [#27153](https://github.com/ClickHouse/ClickHouse/pull/27153) ([Anton Popov](https://github.com/CurtizJ)). +* Updated readIntTextUnsafe [#27155](https://github.com/ClickHouse/ClickHouse/pull/27155) ([Maksim Kita](https://github.com/kitaisreal)). +* Safer `ReadBufferFromS3` for merges and backports [#27168](https://github.com/ClickHouse/ClickHouse/pull/27168) ([Vladimir Chebotarev](https://github.com/excitoon)). +* properly check the settings in perf test [#27190](https://github.com/ClickHouse/ClickHouse/pull/27190) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Save information about used functions/tables/... into query_log on error [#27194](https://github.com/ClickHouse/ClickHouse/pull/27194) ([Azat Khuzhin](https://github.com/azat)). +* Fix polling of /sys/block [#27195](https://github.com/ClickHouse/ClickHouse/pull/27195) ([Azat Khuzhin](https://github.com/azat)). +* Allow parallel execution of *.sql tests with ReplicatedMergeTree (by using {database} macro) [#27214](https://github.com/ClickHouse/ClickHouse/pull/27214) ([Azat Khuzhin](https://github.com/azat)). +* Fix NLP performance test [#27219](https://github.com/ClickHouse/ClickHouse/pull/27219) ([Nikolay Degterinsky](https://github.com/evillique)). +* Update changelog/README.md [#27221](https://github.com/ClickHouse/ClickHouse/pull/27221) ([filimonov](https://github.com/filimonov)). +* more careful handling of reconnects in fuzzer [#27222](https://github.com/ClickHouse/ClickHouse/pull/27222) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* ADDINCL proper fast_float directory [#27224](https://github.com/ClickHouse/ClickHouse/pull/27224) ([Yuriy Chernyshov](https://github.com/georgthegreat)). +* DatabaseReplicatedWorker logs_to_keep race fix [#27225](https://github.com/ClickHouse/ClickHouse/pull/27225) ([Maksim Kita](https://github.com/kitaisreal)). +* Revert [#24095](https://github.com/ClickHouse/ClickHouse/issues/24095). User-level settings will affect queries from view. [#27227](https://github.com/ClickHouse/ClickHouse/pull/27227) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Using formatted string literals in clickhouse-test, extracted sort key functions and stacktraces printer [#27228](https://github.com/ClickHouse/ClickHouse/pull/27228) ([Mike Kot](https://github.com/myrrc)). +* Fix polling of /sys/block in case of block devices reopened on error [#27266](https://github.com/ClickHouse/ClickHouse/pull/27266) ([Azat Khuzhin](https://github.com/azat)). +* Remove streams from dicts [#27273](https://github.com/ClickHouse/ClickHouse/pull/27273) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* 01099_operators_date_and_timestamp: Use dates that work with all available timezones [#27275](https://github.com/ClickHouse/ClickHouse/pull/27275) ([Raúl Marín](https://github.com/Algunenano)). +* Improve kafka integration test error messages [#27294](https://github.com/ClickHouse/ClickHouse/pull/27294) ([Raúl Marín](https://github.com/Algunenano)). +* Improve 00738_lock_for_inner_table stability [#27300](https://github.com/ClickHouse/ClickHouse/pull/27300) ([Raúl Marín](https://github.com/Algunenano)). +* Add and check system.projection_parts for database filter [#27303](https://github.com/ClickHouse/ClickHouse/pull/27303) ([Azat Khuzhin](https://github.com/azat)). +* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 01300_client_save_history_when_terminated_long [#27324](https://github.com/ClickHouse/ClickHouse/pull/27324) ([Raúl Marín](https://github.com/Algunenano)). +* Try update contrib/zlib-ng [#27327](https://github.com/ClickHouse/ClickHouse/pull/27327) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add and check system.mutations for database filter [#27384](https://github.com/ClickHouse/ClickHouse/pull/27384) ([Azat Khuzhin](https://github.com/azat)). +* Correct the key data type used in mapContains [#27423](https://github.com/ClickHouse/ClickHouse/pull/27423) ([Fuwang Hu](https://github.com/fuwhu)). +* Fix tests for WithMergeableStateAfterAggregationAndLimit [#27424](https://github.com/ClickHouse/ClickHouse/pull/27424) ([Azat Khuzhin](https://github.com/azat)). +* Do not miss exceptions from the ThreadPool [#27428](https://github.com/ClickHouse/ClickHouse/pull/27428) ([Azat Khuzhin](https://github.com/azat)). +* Accept error code by error name in client test hints [#27430](https://github.com/ClickHouse/ClickHouse/pull/27430) ([Azat Khuzhin](https://github.com/azat)). +* Added reserve method to Block [#27483](https://github.com/ClickHouse/ClickHouse/pull/27483) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* make it possible to cancel window functions on ctrl+c [#27487](https://github.com/ClickHouse/ClickHouse/pull/27487) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix Nullable const columns in JOIN [#27516](https://github.com/ClickHouse/ClickHouse/pull/27516) ([Vladimir C](https://github.com/vdimir)). +* Fix Logical error: 'Table UUID is not specified in DDL log' [#27521](https://github.com/ClickHouse/ClickHouse/pull/27521) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 01236_graphite_mt for random timezones [#27525](https://github.com/ClickHouse/ClickHouse/pull/27525) ([Raúl Marín](https://github.com/Algunenano)). +* Add timeout for integration tests runner [#27535](https://github.com/ClickHouse/ClickHouse/pull/27535) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix polling of /sys/class on errors [#27554](https://github.com/ClickHouse/ClickHouse/pull/27554) ([Azat Khuzhin](https://github.com/azat)). +* fix recalculate QueryMemoryLimitExceeded event error [#27556](https://github.com/ClickHouse/ClickHouse/pull/27556) ([Ben](https://github.com/benbiti)). +* Fix 01961_roaring_memory_tracking for split builds [#27557](https://github.com/ClickHouse/ClickHouse/pull/27557) ([Raúl Marín](https://github.com/Algunenano)). +* Improve the experience of running stateless tests locally [#27561](https://github.com/ClickHouse/ClickHouse/pull/27561) ([Raúl Marín](https://github.com/Algunenano)). +* Fix integration tests [#27562](https://github.com/ClickHouse/ClickHouse/pull/27562) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Dictionaries refactor [#27566](https://github.com/ClickHouse/ClickHouse/pull/27566) ([Maksim Kita](https://github.com/kitaisreal)). +* Less Stopwatch.h [#27569](https://github.com/ClickHouse/ClickHouse/pull/27569) ([filimonov](https://github.com/filimonov)). +* Aggregation temporary disable compilation without key [#27574](https://github.com/ClickHouse/ClickHouse/pull/27574) ([Maksim Kita](https://github.com/kitaisreal)). +* Removed some data streams [#27575](https://github.com/ClickHouse/ClickHouse/pull/27575) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove streams from lv [#27577](https://github.com/ClickHouse/ClickHouse/pull/27577) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* fix ProfileEvents::CompileFunction [#27606](https://github.com/ClickHouse/ClickHouse/pull/27606) ([Ben](https://github.com/benbiti)). +* Refactor mysql format check [#27609](https://github.com/ClickHouse/ClickHouse/pull/27609) ([Raúl Marín](https://github.com/Algunenano)). +* enable part_log by default [#27631](https://github.com/ClickHouse/ClickHouse/pull/27631) ([Denny Crane](https://github.com/den-crane)). +* Remove the remains of ANTLR in the tests [#27637](https://github.com/ClickHouse/ClickHouse/pull/27637) ([Raúl Marín](https://github.com/Algunenano)). +* MV: Improve text logs when doing parallel processing [#27639](https://github.com/ClickHouse/ClickHouse/pull/27639) ([Raúl Marín](https://github.com/Algunenano)). +* Fix test_sqlite_odbc_hashed_dictionary [#27647](https://github.com/ClickHouse/ClickHouse/pull/27647) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add a test for [#10735](https://github.com/ClickHouse/ClickHouse/issues/10735) [#27677](https://github.com/ClickHouse/ClickHouse/pull/27677) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable memory tracking for roaring bitmaps on Mac OS [#27681](https://github.com/ClickHouse/ClickHouse/pull/27681) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Use only SSE2 in "unbundled" build [#27683](https://github.com/ClickHouse/ClickHouse/pull/27683) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove trash [#27685](https://github.com/ClickHouse/ClickHouse/pull/27685) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix stress test in `~CompressedWriteBuffer` [#27686](https://github.com/ClickHouse/ClickHouse/pull/27686) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Mark tests for `DatabaseReplicated` as green [#27688](https://github.com/ClickHouse/ClickHouse/pull/27688) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Removed DenseHashMap, DenseHashSet [#27690](https://github.com/ClickHouse/ClickHouse/pull/27690) ([Maksim Kita](https://github.com/kitaisreal)). +* Map data type parsing tests [#27692](https://github.com/ClickHouse/ClickHouse/pull/27692) ([Maksim Kita](https://github.com/kitaisreal)). +* Refactor arrayJoin check on partition expressions [#27733](https://github.com/ClickHouse/ClickHouse/pull/27733) ([Raúl Marín](https://github.com/Algunenano)). +* Fix test 01014_lazy_database_concurrent_recreate_reattach_and_show_tables [#27734](https://github.com/ClickHouse/ClickHouse/pull/27734) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable jemalloc under OSX [#27751](https://github.com/ClickHouse/ClickHouse/pull/27751) ([Raúl Marín](https://github.com/Algunenano)). +* Fix jemalloc under osx (zone_register() had been optimized out again) [#27753](https://github.com/ClickHouse/ClickHouse/pull/27753) ([Azat Khuzhin](https://github.com/azat)). +* Fix intersect/except with limit [#27757](https://github.com/ClickHouse/ClickHouse/pull/27757) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add HTTP string parsing test [#27762](https://github.com/ClickHouse/ClickHouse/pull/27762) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix some tests [#27785](https://github.com/ClickHouse/ClickHouse/pull/27785) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set function divide as suitable for short-circuit in case of Nullable(Decimal) [#27788](https://github.com/ClickHouse/ClickHouse/pull/27788) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove unnecessary files [#27789](https://github.com/ClickHouse/ClickHouse/pull/27789) ([Kruglov Pavel](https://github.com/Avogar)). +* Revert "Mark tests for `DatabaseReplicated` as green" [#27791](https://github.com/ClickHouse/ClickHouse/pull/27791) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove hardening for watches in DDLWorker [#27792](https://github.com/ClickHouse/ClickHouse/pull/27792) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Stateless test: Cleanup leftovers [#27793](https://github.com/ClickHouse/ClickHouse/pull/27793) ([Raúl Marín](https://github.com/Algunenano)). +* Dictionaries key types refactoring [#27795](https://github.com/ClickHouse/ClickHouse/pull/27795) ([Maksim Kita](https://github.com/kitaisreal)). +* Update 01822_short_circuit.reference (after merging [#27680](https://github.com/ClickHouse/ClickHouse/issues/27680)) [#27802](https://github.com/ClickHouse/ClickHouse/pull/27802) ([Azat Khuzhin](https://github.com/azat)). +* Proper shutdown global context [#27804](https://github.com/ClickHouse/ClickHouse/pull/27804) ([Amos Bird](https://github.com/amosbird)). +* 01766_todatetime64_no_timezone_arg: Use a date without timezone changes [#27810](https://github.com/ClickHouse/ClickHouse/pull/27810) ([Raúl Marín](https://github.com/Algunenano)). +* Fix setting name "allow_experimental_database_materialized_postgresql" in the error message [#27824](https://github.com/ClickHouse/ClickHouse/pull/27824) ([Denny Crane](https://github.com/den-crane)). +* Fix bug in short-circuit found by fuzzer [#27826](https://github.com/ClickHouse/ClickHouse/pull/27826) ([Kruglov Pavel](https://github.com/Avogar)). + diff --git a/docs/changelogs/v21.9.2.17-stable.md b/docs/changelogs/v21.9.2.17-stable.md index 3f132b983c0..c125fab9ed6 100644 --- a/docs/changelogs/v21.9.2.17-stable.md +++ b/docs/changelogs/v21.9.2.17-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.2.17-stable FIXME as compared to v21.9.1.8000-prestable #### Improvement @@ -43,3 +50,15 @@ * Backported in [#28715](https://github.com/ClickHouse/ClickHouse/issues/28715): Add Settings.Names, Settings.Values aliases for system.processes table. [#28685](https://github.com/ClickHouse/ClickHouse/pull/28685) ([Vitaly Orlov](https://github.com/orloffv)). * Backported in [#28744](https://github.com/ClickHouse/ClickHouse/issues/28744): Fix the coredump in the creation of distributed tables, when the parameters passed in are wrong. [#28686](https://github.com/ClickHouse/ClickHouse/pull/28686) ([Zhiyong Wang](https://github.com/ljcui)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Set version of tzlocal to 2.1 [#28063](https://github.com/ClickHouse/ClickHouse/pull/28063) ([Vitaly Baranov](https://github.com/vitlibar)). +* CHJIT custom memory manager [#28236](https://github.com/ClickHouse/ClickHouse/pull/28236) ([Maksim Kita](https://github.com/kitaisreal)). +* ODBC connection holder fix dangling reference [#28298](https://github.com/ClickHouse/ClickHouse/pull/28298) ([Maksim Kita](https://github.com/kitaisreal)). +* Another try to fix BackgroundPoolTask decrement. [#28353](https://github.com/ClickHouse/ClickHouse/pull/28353) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* More accurate check that zk root exists. [#28412](https://github.com/ClickHouse/ClickHouse/pull/28412) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Function dictGet default implementation for nulls [#28530](https://github.com/ClickHouse/ClickHouse/pull/28530) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.9.3.30-stable.md b/docs/changelogs/v21.9.3.30-stable.md index 3b665365668..5318f654f88 100644 --- a/docs/changelogs/v21.9.3.30-stable.md +++ b/docs/changelogs/v21.9.3.30-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.3.30-stable FIXME as compared to v21.9.2.17-stable #### Improvement @@ -14,3 +21,10 @@ * Backported in [#28927](https://github.com/ClickHouse/ClickHouse/issues/28927): Fix higher-order array functions (`SIGSEGV` for `arrayCompact`/`ILLEGAL_COLUMN` for `arrayDifference`/`arrayCumSumNonNegative`) with consts. [#28904](https://github.com/ClickHouse/ClickHouse/pull/28904) ([Azat Khuzhin](https://github.com/azat)). * Backported in [#29025](https://github.com/ClickHouse/ClickHouse/issues/29025): Fix the number of threads used in `GLOBAL IN` subquery (it was executed in single threads since [#19414](https://github.com/ClickHouse/ClickHouse/issues/19414) bugfix). [#28997](https://github.com/ClickHouse/ClickHouse/pull/28997) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Add more checks for LC in native protocol. [#27827](https://github.com/ClickHouse/ClickHouse/pull/27827) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix usage of nested columns with non-array columns with the same prefix [2] [#28762](https://github.com/ClickHouse/ClickHouse/pull/28762) ([Anton Popov](https://github.com/CurtizJ)). +* Lower compiled_expression_cache_size to 128MB [#28816](https://github.com/ClickHouse/ClickHouse/pull/28816) ([Maksim Kita](https://github.com/kitaisreal)). +* Column default dictGet identifier fix [#28863](https://github.com/ClickHouse/ClickHouse/pull/28863) ([Maksim Kita](https://github.com/kitaisreal)). + diff --git a/docs/changelogs/v21.9.4.35-stable.md b/docs/changelogs/v21.9.4.35-stable.md index 8b919ecb268..65ba54de6e8 100644 --- a/docs/changelogs/v21.9.4.35-stable.md +++ b/docs/changelogs/v21.9.4.35-stable.md @@ -1,6 +1,19 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.4.35-stable FIXME as compared to v21.9.3.30-stable #### Bug Fix (user-visible misbehaviour in official stable or prestable release) * Backported in [#29191](https://github.com/ClickHouse/ClickHouse/issues/29191): Fix segfault while inserting into column with type LowCardinality(Nullable) in Avro input format. [#29132](https://github.com/ClickHouse/ClickHouse/pull/29132) ([Kruglov Pavel](https://github.com/Avogar)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Don not add const group by key for query with only having. [#28975](https://github.com/ClickHouse/ClickHouse/pull/28975) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Merging [#27963](https://github.com/ClickHouse/ClickHouse/issues/27963) [#29063](https://github.com/ClickHouse/ClickHouse/pull/29063) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix terminate on uncaught exception [#29216](https://github.com/ClickHouse/ClickHouse/pull/29216) ([Alexander Tokmakov](https://github.com/tavplubix)). + diff --git a/docs/changelogs/v21.9.5.16-stable.md b/docs/changelogs/v21.9.5.16-stable.md index 7287c58064d..305e49a8633 100644 --- a/docs/changelogs/v21.9.5.16-stable.md +++ b/docs/changelogs/v21.9.5.16-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.5.16-stable FIXME as compared to v21.9.4.35-stable #### Improvement @@ -46,3 +53,9 @@ * Avoid deadlocks when reading and writting on JOIN Engine tables at the same time. [#30185](https://github.com/ClickHouse/ClickHouse/pull/30185) ([Raúl Marín](https://github.com/Algunenano)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Maybe fix livelock in ZooKeeper client [#28195](https://github.com/ClickHouse/ClickHouse/pull/28195) ([Alexander Tokmakov](https://github.com/tavplubix)). +* May be fix s3 tests [#29762](https://github.com/ClickHouse/ClickHouse/pull/29762) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update BoringSSL [#29998](https://github.com/ClickHouse/ClickHouse/pull/29998) ([Filatenkov Artur](https://github.com/FArthur-cmd)). + diff --git a/docs/changelogs/v21.9.6.24-stable.md b/docs/changelogs/v21.9.6.24-stable.md index f1d097ab646..dbecd4b3634 100644 --- a/docs/changelogs/v21.9.6.24-stable.md +++ b/docs/changelogs/v21.9.6.24-stable.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + ### ClickHouse release v21.9.6.24-stable FIXME as compared to v21.9.5.16-stable #### New Feature @@ -55,3 +62,10 @@ * Backported in [#32078](https://github.com/ClickHouse/ClickHouse/issues/32078): Fix a bug about function transform with decimal args. [#31839](https://github.com/ClickHouse/ClickHouse/pull/31839) ([Shuai li](https://github.com/loneylee)). * Backported in [#31907](https://github.com/ClickHouse/ClickHouse/issues/31907): Fix functions `empty` and `notEmpty` with arguments of `UUID` type. Fixes [#31819](https://github.com/ClickHouse/ClickHouse/issues/31819). [#31883](https://github.com/ClickHouse/ClickHouse/pull/31883) ([Anton Popov](https://github.com/CurtizJ)). +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* AddDefaultDatabaseVisitor support dictGet [#29650](https://github.com/ClickHouse/ClickHouse/pull/29650) ([Maksim Kita](https://github.com/kitaisreal)). +* StorageDictionary fix potential configuration race [#30502](https://github.com/ClickHouse/ClickHouse/pull/30502) ([Maksim Kita](https://github.com/kitaisreal)). +* BloomFilter index check fix [#31334](https://github.com/ClickHouse/ClickHouse/pull/31334) ([Maksim Kita](https://github.com/kitaisreal)). +* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + From 267025c35bcd48070bfe4f924c9761ca0e1bf0a4 Mon Sep 17 00:00:00 2001 From: zvonand Date: Thu, 23 Jun 2022 16:51:15 +0500 Subject: [PATCH 056/123] update base-x cmakelists --- contrib/base-x-cmake/CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/contrib/base-x-cmake/CMakeLists.txt b/contrib/base-x-cmake/CMakeLists.txt index ab5696c9fb6..b0c11a176c6 100644 --- a/contrib/base-x-cmake/CMakeLists.txt +++ b/contrib/base-x-cmake/CMakeLists.txt @@ -12,9 +12,8 @@ set (SRCS ${LIBRARY_DIR}/uinteger_t.hh ) -add_library(_base-x ${SRCS}) - -target_include_directories(_base-x SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}) +add_library(_base-x INTERFACE) +target_include_directories(_base-x SYSTEM BEFORE INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/base-x") if (XCODE OR XCODE_VERSION) # https://gitlab.kitware.com/cmake/cmake/issues/17457 From e33324bd358533aeafad16e7f1675a1d148b1060 Mon Sep 17 00:00:00 2001 From: zvonand Date: Thu, 23 Jun 2022 16:56:28 +0500 Subject: [PATCH 057/123] fix docs --- docs/en/sql-reference/functions/string-functions.md | 2 +- docs/ru/sql-reference/functions/string-functions.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 7c6ae903acf..61efc2cfcdb 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -509,7 +509,7 @@ base58Decode(encoded[, alphabet_name]) - `decoded` — [String](../../sql-reference/data-types/string.md) column or constant. - `encoded` — [String](../../sql-reference/data-types/string.md) column or constant. If the string is not a valid base58-encoded value, an exception is thrown. -- `alphabet_name` — String constant. Specifies alphabet used for encoding. Possible values: `gmp`, `bitcoin`, `ripple`, `flickr`. Default: `gmp`. +- `alphabet_name` — String constant. Specifies alphabet used for encoding. Possible values: `gmp`, `bitcoin`, `ripple`, `flickr`. Default: `bitcoin`. **Returned value** diff --git a/docs/ru/sql-reference/functions/string-functions.md b/docs/ru/sql-reference/functions/string-functions.md index e85a97e0099..1acf5ec58b2 100644 --- a/docs/ru/sql-reference/functions/string-functions.md +++ b/docs/ru/sql-reference/functions/string-functions.md @@ -505,7 +505,7 @@ base58Decode(encoded[, alphabet_name]) - `decoded` — Колонка или строка типа [String](../../sql-reference/data-types/string.md). - `encoded` — Колонка или строка типа [String](../../sql-reference/data-types/string.md). Если входная строка не является корректным кодом для какой-либо другой строки, возникнет исключение `1001`. -- `alphabet_name` — Строковая константа. Указывает алфавит, для которого необходимо получить код. Может принимать одно из следующих значений: `gmp`, `bitcoin`, `ripple`, `flickr`. По умолчанию: `gmp`. +- `alphabet_name` — Строковая константа. Указывает алфавит, для которого необходимо получить код. Может принимать одно из следующих значений: `gmp`, `bitcoin`, `ripple`, `flickr`. По умолчанию: `bitcoin`. **Возвращаемое значение** From cb748cd8ec46b1962bfd28c41eeaab4b53e90c75 Mon Sep 17 00:00:00 2001 From: Roman Vasin Date: Thu, 23 Jun 2022 16:11:48 +0300 Subject: [PATCH 058/123] Fix code style in KerberosInit --- src/Access/KerberosInit.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Access/KerberosInit.cpp b/src/Access/KerberosInit.cpp index 9ca45b12531..ace03a5e0b5 100644 --- a/src/Access/KerberosInit.cpp +++ b/src/Access/KerberosInit.cpp @@ -47,12 +47,12 @@ private: krb5_creds my_creds; krb5_keytab keytab = nullptr; krb5_principal defcache_princ = nullptr; - String fmtError(krb5_error_code code); + String fmtError(krb5_error_code code) const; }; } -String KerberosInit::fmtError(krb5_error_code code) +String KerberosInit::fmtError(krb5_error_code code) const { const char *msg; msg = krb5_get_error_message(k5.ctx, code); @@ -75,7 +75,7 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co ret = krb5_init_context(&k5.ctx); if (ret) - throw Exception(fmt::format("Error while initializing Kerberos 5 library ({})", ret), ErrorCodes::KERBEROS_ERROR); + throw Exception(ErrorCodes::KERBEROS_ERROR, "Error while initializing Kerberos 5 library ({})", ret); if (!cache_name.empty()) { @@ -160,7 +160,8 @@ void KerberosInit::init(const String & keytab_file, const String & principal, co ret = krb5_get_renewed_creds(k5.ctx, &my_creds, k5.me, k5.out_cc, nullptr); if (ret) { - LOG_TRACE(log,"Renew failed ({}). Trying to get initial credentials", ret); + LOG_TRACE(log,"Renew failed {}", fmtError(ret)); + LOG_TRACE(log,"Trying to get initial credentials"); // Request KDC for an initial credentials using keytab. ret = krb5_get_init_creds_keytab(k5.ctx, &my_creds, k5.me, keytab, 0, nullptr, options); if (ret) From 7f7d082fb30e120ddd0cf94ec8fccd01b2442ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 23 Jun 2022 15:23:37 +0200 Subject: [PATCH 059/123] Add implicit_transaction setting --- src/Core/Settings.h | 1 + .../InterpreterTransactionControlQuery.h | 1 - src/Interpreters/executeQuery.cpp | 94 ++++++++++++++----- .../02345_implicit_transaction.reference | 14 +++ .../02345_implicit_transaction.sql | 92 ++++++++++++++++++ 5 files changed, 178 insertions(+), 24 deletions(-) create mode 100644 tests/queries/0_stateless/02345_implicit_transaction.reference create mode 100644 tests/queries/0_stateless/02345_implicit_transaction.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index f1fd9d20f00..72ba2c0c13b 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -601,6 +601,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, count_distinct_optimization, false, "Rewrite count distinct to subquery of group by", 0) \ M(Bool, throw_on_unsupported_query_inside_transaction, true, "Throw exception if unsupported query is used inside transaction", 0) \ M(TransactionsWaitCSNMode, wait_changes_become_visible_after_commit_mode, TransactionsWaitCSNMode::WAIT_UNKNOWN, "Wait for committed changes to become actually visible in the latest snapshot", 0) \ + M(Bool, implicit_transaction, false, "If enabled and not already inside a transaction, wraps the query inside a full transaction (begin + commit or rollback)", 0) \ M(Bool, throw_if_no_data_to_insert, true, "Enables or disables empty INSERTs, enabled by default", 0) \ M(Bool, compatibility_ignore_auto_increment_in_create_table, false, "Ignore AUTO_INCREMENT keyword in column declaration if true, otherwise return error. It simplifies migration from MySQL", 0) \ M(Bool, multiple_joins_try_to_keep_original_names, false, "Do not add aliases to top level expression list on multiple joins rewrite", 0) \ diff --git a/src/Interpreters/InterpreterTransactionControlQuery.h b/src/Interpreters/InterpreterTransactionControlQuery.h index bf2dc7891a7..a66a740ce0c 100644 --- a/src/Interpreters/InterpreterTransactionControlQuery.h +++ b/src/Interpreters/InterpreterTransactionControlQuery.h @@ -20,7 +20,6 @@ public: bool ignoreLimits() const override { return true; } bool supportsTransactions() const override { return true; } -private: BlockIO executeBegin(ContextMutablePtr session_context); BlockIO executeCommit(ContextMutablePtr session_context); static BlockIO executeRollback(ContextMutablePtr session_context); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 24649128cee..ae622e5e1f0 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -42,18 +42,19 @@ #include #include #include -#include #include +#include +#include #include #include #include -#include #include +#include #include -#include -#include -#include #include +#include +#include +#include #include #include @@ -68,6 +69,7 @@ #include #include +#include #include @@ -416,7 +418,9 @@ static std::tuple executeQueryImpl( chassert(txn->getState() != MergeTreeTransaction::COMMITTING); chassert(txn->getState() != MergeTreeTransaction::COMMITTED); if (txn->getState() == MergeTreeTransaction::ROLLED_BACK && !ast->as() && !ast->as()) - throw Exception(ErrorCodes::INVALID_TRANSACTION, "Cannot execute query because current transaction failed. Expecting ROLLBACK statement."); + throw Exception( + ErrorCodes::INVALID_TRANSACTION, + "Cannot execute query because current transaction failed. Expecting ROLLBACK statement"); } /// Interpret SETTINGS clauses as early as possible (before invoking the corresponding interpreter), @@ -498,6 +502,7 @@ static std::tuple executeQueryImpl( BlockIO res; String query_for_logging; + std::shared_ptr implicit_txn_control{}; try { @@ -626,6 +631,27 @@ static std::tuple executeQueryImpl( } else { + /// We need to start the (implicit) transaction before getting the interpreter as this will get links to the latest snapshots + if (!context->getCurrentTransaction() && settings.implicit_transaction && !ast->as()) + { + try + { + /// If there is no session (which is the default for the HTTP Handler), set up one just for this as it is necessary + /// to control the transaction lifetime + if (!context->hasSessionContext()) + context->makeSessionContext(); + + auto tc = std::make_shared(ast, context); + tc->executeBegin(context->getSessionContext()); + implicit_txn_control = std::move(tc); + } + catch (Exception & e) + { + e.addMessage("while starting a transaction with 'implicit_transaction'"); + throw; + } + } + interpreter = InterpreterFactory::get(ast, context, SelectQueryOptions(stage).setInternal(internal)); if (context->getCurrentTransaction() && !interpreter->supportsTransactions() && @@ -813,15 +839,16 @@ static std::tuple executeQueryImpl( }; /// Also make possible for caller to log successful query finish and exception during execution. - auto finish_callback = [elem, context, ast, - log_queries, - log_queries_min_type = settings.log_queries_min_type, - log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(), - log_processors_profiles = settings.log_processors_profiles, - status_info_to_query_log, - pulling_pipeline = pipeline.pulling() - ] - (QueryPipeline & query_pipeline) mutable + auto finish_callback = [elem, + context, + ast, + log_queries, + log_queries_min_type = settings.log_queries_min_type, + log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(), + log_processors_profiles = settings.log_processors_profiles, + status_info_to_query_log, + implicit_txn_control, + pulling_pipeline = pipeline.pulling()](QueryPipeline & query_pipeline) mutable { QueryStatus * process_list_elem = context->getProcessListElement(); @@ -942,15 +969,30 @@ static std::tuple executeQueryImpl( opentelemetry_span_log->add(span); } + + if (implicit_txn_control) + { + implicit_txn_control->executeCommit(context->getSessionContext()); + implicit_txn_control.reset(); + } }; - auto exception_callback = [elem, context, ast, - log_queries, - log_queries_min_type = settings.log_queries_min_type, - log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(), - quota(quota), status_info_to_query_log] () mutable + auto exception_callback = [elem, + context, + ast, + log_queries, + log_queries_min_type = settings.log_queries_min_type, + log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(), + quota(quota), + status_info_to_query_log, + implicit_txn_control]() mutable { - if (auto txn = context->getCurrentTransaction()) + if (implicit_txn_control) + { + implicit_txn_control->executeRollback(context->getSessionContext()); + implicit_txn_control.reset(); + } + else if (auto txn = context->getCurrentTransaction()) txn->onException(); if (quota) @@ -1000,7 +1042,6 @@ static std::tuple executeQueryImpl( { ProfileEvents::increment(ProfileEvents::FailedInsertQuery); } - }; res.finish_callback = std::move(finish_callback); @@ -1009,8 +1050,15 @@ static std::tuple executeQueryImpl( } catch (...) { - if (auto txn = context->getCurrentTransaction()) + if (implicit_txn_control) + { + implicit_txn_control->executeRollback(context->getSessionContext()); + implicit_txn_control.reset(); + } + else if (auto txn = context->getCurrentTransaction()) + { txn->onException(); + } if (!internal) { diff --git a/tests/queries/0_stateless/02345_implicit_transaction.reference b/tests/queries/0_stateless/02345_implicit_transaction.reference new file mode 100644 index 00000000000..e4dd35600f7 --- /dev/null +++ b/tests/queries/0_stateless/02345_implicit_transaction.reference @@ -0,0 +1,14 @@ +no_transaction_landing 10000 +no_transaction_target 0 +after_transaction_landing 0 +after_transaction_target 0 +after_implicit_txn_in_query_settings_landing 0 +after_implicit_txn_in_query_settings_target 0 +after_implicit_txn_in_session_landing 0 +after_implicit_txn_in_session_target 0 +inside_txn_and_implicit 1 +inside_txn_and_implicit 1 +in_transaction 10000 +out_transaction 0 +{"'implicit_True'":"implicit_True","all":"2","is_empty":0} +{"'implicit_False'":"implicit_False","all":"2","is_empty":1} diff --git a/tests/queries/0_stateless/02345_implicit_transaction.sql b/tests/queries/0_stateless/02345_implicit_transaction.sql new file mode 100644 index 00000000000..677affeec39 --- /dev/null +++ b/tests/queries/0_stateless/02345_implicit_transaction.sql @@ -0,0 +1,92 @@ +CREATE TABLE landing (n Int64) engine=MergeTree order by n; +CREATE TABLE target (n Int64) engine=MergeTree order by n; +CREATE MATERIALIZED VIEW landing_to_target TO target AS + SELECT n + throwIf(n == 3333) + FROM landing; + +INSERT INTO landing SELECT * FROM numbers(10000); -- { serverError 395 } +SELECT 'no_transaction_landing', count() FROM landing; +SELECT 'no_transaction_target', count() FROM target; + +TRUNCATE TABLE landing; +TRUNCATE TABLE target; + + +BEGIN TRANSACTION; +INSERT INTO landing SELECT * FROM numbers(10000); -- { serverError 395 } +ROLLBACK; +SELECT 'after_transaction_landing', count() FROM landing; +SELECT 'after_transaction_target', count() FROM target; + +-- Same but using implicit_transaction +INSERT INTO landing SETTINGS implicit_transaction=True SELECT * FROM numbers(10000); -- { serverError 395 } +SELECT 'after_implicit_txn_in_query_settings_landing', count() FROM landing; +SELECT 'after_implicit_txn_in_query_settings_target', count() FROM target; + +-- Same but using implicit_transaction in a session +SET implicit_transaction=True; +INSERT INTO landing SELECT * FROM numbers(10000); -- { serverError 395 } +SET implicit_transaction=False; +SELECT 'after_implicit_txn_in_session_landing', count() FROM landing; +SELECT 'after_implicit_txn_in_session_target', count() FROM target; + +-- Reading from incompatible sources with implicit_transaction works the same way as with normal transactions: +-- Currently reading from system tables inside a transaction is Not implemented: +SELECT name, value, changed FROM system.settings where name = 'implicit_transaction' SETTINGS implicit_transaction=True; -- { serverError 48 } + + +-- Verify that you don't have to manually close transactions with implicit_transaction +SET implicit_transaction=True; +SELECT throwIf(number == 0) FROM numbers(100); -- { serverError 395 } +SELECT throwIf(number == 0) FROM numbers(100); -- { serverError 395 } +SELECT throwIf(number == 0) FROM numbers(100); -- { serverError 395 } +SELECT throwIf(number == 0) FROM numbers(100); -- { serverError 395 } +SET implicit_transaction=False; + +-- implicit_transaction is ignored when inside a transaction (no recursive transaction error) +BEGIN TRANSACTION; +SELECT 'inside_txn_and_implicit', 1 SETTINGS implicit_transaction=True; +SELECT throwIf(number == 0) FROM numbers(100) SETTINGS implicit_transaction=True; -- { serverError 395 } +ROLLBACK; + +SELECT 'inside_txn_and_implicit', 1 SETTINGS implicit_transaction=True; + +-- You can work with transactions even if `implicit_transaction=True` is set +SET implicit_transaction=True; +BEGIN TRANSACTION; +INSERT INTO target SELECT * FROM numbers(10000); +SELECT 'in_transaction', count() FROM target; +ROLLBACK; +SELECT 'out_transaction', count() FROM target; +SET implicit_transaction=False; + + +-- Verify that the transaction_id column is populated correctly +SELECT 'Looking_at_transaction_id_True' FORMAT Null SETTINGS implicit_transaction=1; +-- Verify that the transaction_id column is NOT populated without transaction +SELECT 'Looking_at_transaction_id_False' FORMAT Null SETTINGS implicit_transaction=0; +SYSTEM FLUSH LOGS; + +SELECT + 'implicit_True', + count() as all, + transaction_id = (0,0,'00000000-0000-0000-0000-000000000000') as is_empty +FROM system.query_log +WHERE + current_database = currentDatabase() AND + event_date >= yesterday() AND + query LIKE '-- Verify that the transaction_id column is populated correctly%' +GROUP BY transaction_id +FORMAT JSONEachRow; + +SELECT + 'implicit_False', + count() as all, + transaction_id = (0,0,'00000000-0000-0000-0000-000000000000') as is_empty +FROM system.query_log +WHERE + current_database = currentDatabase() AND + event_date >= yesterday() AND + query LIKE '-- Verify that the transaction_id column is NOT populated without transaction%' +GROUP BY transaction_id +FORMAT JSONEachRow; From dfae6a5634f868d37889f671e2470950bb2a916f Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 23 Jun 2022 14:53:39 +0200 Subject: [PATCH 060/123] Upload logs for getting all tests command --- tests/integration/ci-runner.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index e4bd1be9027..562497fe8b0 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -367,7 +367,7 @@ class ClickhouseIntegrationTestsRunner: def _get_all_tests(self, repo_path): image_cmd = self._get_runner_image_cmd(repo_path) out_file = "all_tests.txt" - out_file_full = "all_tests_full.txt" + out_file_full = os.path.join(self.result_path, "runner_get_all_tests.log") cmd = ( "cd {repo_path}/tests/integration && " "timeout -s 9 1h ./runner {runner_opts} {image_cmd} ' --setup-plan' " @@ -393,21 +393,16 @@ class ClickhouseIntegrationTestsRunner: not os.path.isfile(all_tests_file_path) or os.path.getsize(all_tests_file_path) == 0 ): - all_tests_full_file_path = ( - "{repo_path}/tests/integration/{out_file}".format( - repo_path=repo_path, out_file=out_file_full - ) - ) - if os.path.isfile(all_tests_full_file_path): + if os.path.isfile(out_file_full): # log runner output logging.info("runner output:") - with open(all_tests_full_file_path, "r") as all_tests_full_file: + with open(out_file_full, "r") as all_tests_full_file: for line in all_tests_full_file: line = line.rstrip() if line: logging.info("runner output: %s", line) else: - logging.info("runner output '%s' is empty", all_tests_full_file_path) + logging.info("runner output '%s' is empty", out_file_full) raise Exception( "There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format( From dd8203038f249080bbcf4e937e81a9c651c882d1 Mon Sep 17 00:00:00 2001 From: zvonand Date: Fri, 24 Jun 2022 00:36:57 +0500 Subject: [PATCH 061/123] updated exception handling --- src/Functions/FunctionBase58Conversion.h | 26 ++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionBase58Conversion.h b/src/Functions/FunctionBase58Conversion.h index ed4667aa63b..6979ce849f4 100644 --- a/src/Functions/FunctionBase58Conversion.h +++ b/src/Functions/FunctionBase58Conversion.h @@ -61,7 +61,18 @@ struct Base58Encode /// This way we do exponential resizes and one final resize after whole operation is complete encoded.clear(); if (srclen) - encoder.encode(encoded, source, srclen); + try + { + encoder.encode(encoded, source, srclen); + } + catch (const std::invalid_argument& e) + { + throw Exception(e.what(), ErrorCodes::BAD_ARGUMENTS); + } + catch (const std::domain_error& e) + { + throw Exception(e.what(), ErrorCodes::BAD_ARGUMENTS); + } size_t outlen = encoded.size(); if (processed_size + outlen >= current_allocated_size) @@ -126,7 +137,18 @@ struct Base58Decode /// This way we do exponential resizes and one final resize after whole operation is complete decoded.clear(); if (srclen) - decoder.decode(decoded, source, srclen); + try + { + decoder.decode(decoded, source, srclen); + } + catch (const std::invalid_argument& e) + { + throw Exception(e.what(), ErrorCodes::BAD_ARGUMENTS); + } + catch (const std::domain_error& e) + { + throw Exception(e.what(), ErrorCodes::BAD_ARGUMENTS); + } size_t outlen = decoded.size(); if (processed_size + outlen >= current_allocated_size) From a3823c67eee57597c22e5af7411fc8f2078beab4 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 24 Jun 2022 02:44:22 +0200 Subject: [PATCH 062/123] Fix --- .../PostgreSQL/fetchPostgreSQLTableStructure.cpp | 12 +++++++----- tests/integration/test_storage_postgresql/test.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp index 9f136efa1ff..08a7e78d0e9 100644 --- a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp +++ b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp @@ -263,10 +263,11 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure( "WHERE attrelid = (SELECT oid FROM pg_class WHERE {}) " "AND NOT attisdropped AND attnum > 0", where); - table.physical_columns = readNamesAndTypesList(tx, postgres_table, query, use_nulls, false); + auto postgres_table_with_schema = postgres_schema.empty() ? postgres_table : doubleQuoteString(postgres_schema) + '.' + doubleQuoteString(postgres_table); + table.physical_columns = readNamesAndTypesList(tx, postgres_table_with_schema, query, use_nulls, false); if (!table.physical_columns) - throw Exception(ErrorCodes::UNKNOWN_TABLE, "PostgreSQL table {} does not exist", postgres_table); + throw Exception(ErrorCodes::UNKNOWN_TABLE, "PostgreSQL table {} does not exist", postgres_table_with_schema); if (with_primary_key) { @@ -278,7 +279,7 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure( "AND a.attnum = ANY(i.indkey) " "WHERE attrelid = (SELECT oid FROM pg_class WHERE {}) AND i.indisprimary", where); - table.primary_key_columns = readNamesAndTypesList(tx, postgres_table, query, use_nulls, true); + table.primary_key_columns = readNamesAndTypesList(tx, postgres_table_with_schema, query, use_nulls, true); } if (with_replica_identity_index && !table.primary_key_columns) @@ -299,11 +300,12 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure( "and a.attnum = ANY(ix.indkey) " "and t.relkind in ('r', 'p') " /// simple tables "and t.relname = {} " /// Connection is already done to a needed database, only table name is needed. + "and t.relnamespace = {} " "and ix.indisreplident = 't' " /// index is is replica identity index "ORDER BY a.attname", /// column names - quoteString(postgres_table)); + quoteString(postgres_table), quoteString(postgres_schema.empty() ? "public" : postgres_schema)); - table.replica_identity_columns = readNamesAndTypesList(tx, postgres_table, query, use_nulls, true); + table.replica_identity_columns = readNamesAndTypesList(tx, postgres_table_with_schema, query, use_nulls, true); } return table; diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index 8366ca5dc25..1fc0475419c 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -433,6 +433,7 @@ def test_datetime_with_timezone(started_cluster): def test_postgres_ndim(started_cluster): cursor = started_cluster.postgres_conn.cursor() + cursor.execute("DROP TABLE IF EXISTS arr1, arr2") cursor.execute("CREATE TABLE arr1 (a Integer[])") @@ -452,6 +453,20 @@ def test_postgres_ndim(started_cluster): assert result.strip() == "Array(Array(Nullable(Int32)))" cursor.execute("DROP TABLE arr1, arr2") + cursor.execute("DROP SCHEMA IF EXISTS ndim_schema CASCADE") + cursor.execute("CREATE SCHEMA ndim_schema") + cursor.execute("CREATE TABLE ndim_schema.arr1 (a integer[])") + cursor.execute("INSERT INTO ndim_schema.arr1 SELECT '{{1}, {2}}'") + # The point is in creating a table via 'as select *', in postgres att_ndim will not be correct in this case. + cursor.execute("CREATE TABLE ndim_schema.arr2 AS SELECT * FROM ndim_schema.arr1") + result = node1.query( + """SELECT toTypeName(a) FROM postgresql(postgres1, schema='ndim_schema', table='arr2')""" + ) + assert result.strip() == "Array(Array(Nullable(Int32)))" + + cursor.execute("DROP TABLE ndim_schema.arr1, ndim_schema.arr2") + cursor.execute("DROP SCHEMA ndim_schema CASCADE") + def test_postgres_on_conflict(started_cluster): cursor = started_cluster.postgres_conn.cursor() From 6cb1d60883b6baa5e7ef72813b25c7eec359b697 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 24 Jun 2022 03:24:54 +0200 Subject: [PATCH 063/123] Fix --- src/Databases/DatabaseFactory.cpp | 8 +++++++- .../test_postgresql_database_engine/test.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 5cc334eaad4..82a7dff7125 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -335,7 +335,13 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String configuration.password = safeGetLiteralValue(engine_args[3], engine_name); if (engine_args.size() >= 5) - configuration.schema = safeGetLiteralValue(engine_args[4], engine_name); + { + auto arg_value = engine_args[4]->as()->value; + if (arg_value.getType() == Field::Types::Which::String) + configuration.schema = safeGetLiteralValue(engine_args[4], engine_name); + else + use_table_cache = safeGetLiteralValue(engine_args[4], engine_name); + } } if (engine_args.size() >= 6) diff --git a/tests/integration/test_postgresql_database_engine/test.py b/tests/integration/test_postgresql_database_engine/test.py index dd5b3a09ca5..aabf3507d8f 100644 --- a/tests/integration/test_postgresql_database_engine/test.py +++ b/tests/integration/test_postgresql_database_engine/test.py @@ -320,6 +320,20 @@ def test_predefined_connection_configuration(started_cluster): cursor.execute("DROP SCHEMA IF EXISTS test_schema CASCADE") +def test_postgres_database_old_syntax(started_cluster): + conn = get_postgres_conn(started_cluster, True) + cursor = conn.cursor() + + node1.query( + """ + DROP DATABASE IF EXISTS test_database; + CREATE DATABASE test_database ENGINE = PostgreSQL('postgres1:5432', 'test_database', 'postgres', 'mysecretpassword', 1); + """ + ) + create_postgres_table(cursor, "test_table") + assert "test_table" in node1.query("SHOW TABLES FROM test_database") + + if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") From 2c828338f409d88f860fe4236045a3cab0956aad Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 17 Jun 2022 12:15:19 +0200 Subject: [PATCH 064/123] Replace hyperscan by vectorscan This commit migrates ClickHouse to Vectorscan. The first 10 min of [0] explain the reasons for it. (*) Addresses (but does not resolve) #38046 (*) Config parameter names (e.g. "max_hyperscan_regexp_length") are preserved for compatibility. Likewise, error codes (e.g. "ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT") and function/class names (e.g. "HyperscanDeleter") are preserved as vectorscan aims to be a drop-in replacement. [0] https://www.youtube.com/watch?v=KlZWmmflW6M --- .gitmodules | 6 +- contrib/CMakeLists.txt | 2 +- contrib/hyperscan | 1 - contrib/vectorscan | 1 + .../CMakeLists.txt | 180 +- contrib/vectorscan-cmake/aarch64/config.h | 142 + .../common/hs_version.h | 5 +- .../vectorscan-cmake/rageled_files/Parser.cpp | 5605 +++++++++++++++++ .../rageled_files/control_verbs.cpp | 443 ++ .../x86_64/config.h | 38 +- docs/en/development/contrib.md | 2 +- src/Functions/CMakeLists.txt | 4 +- src/Functions/MultiMatchAllIndicesImpl.h | 10 +- src/Functions/MultiMatchAnyImpl.h | 16 +- src/Functions/Regexps.h | 6 +- src/Functions/URL/CMakeLists.txt | 4 +- src/Functions/config_functions.h.in | 2 +- src/Functions/configure_config.cmake | 4 +- ...StorageSystemBuildOptions.generated.cpp.in | 2 +- .../queries/0_stateless/00926_multimatch.sql | 2 +- .../00929_multi_match_edit_distance.sql | 2 +- .../02004_max_hyperscan_regex_length.sql | 2 +- .../1_stateful/00095_hyperscan_profiler.sql | 2 +- .../aspell-ignore/en/aspell-dict.txt | 1 + 24 files changed, 6383 insertions(+), 99 deletions(-) delete mode 160000 contrib/hyperscan create mode 160000 contrib/vectorscan rename contrib/{hyperscan-cmake => vectorscan-cmake}/CMakeLists.txt (73%) create mode 100644 contrib/vectorscan-cmake/aarch64/config.h rename contrib/{hyperscan-cmake => vectorscan-cmake}/common/hs_version.h (94%) create mode 100644 contrib/vectorscan-cmake/rageled_files/Parser.cpp create mode 100644 contrib/vectorscan-cmake/rageled_files/control_verbs.cpp rename contrib/{hyperscan-cmake => vectorscan-cmake}/x86_64/config.h (73%) diff --git a/.gitmodules b/.gitmodules index b102267c7aa..16099c63368 100644 --- a/.gitmodules +++ b/.gitmodules @@ -86,9 +86,6 @@ [submodule "contrib/h3"] path = contrib/h3 url = https://github.com/ClickHouse/h3 -[submodule "contrib/hyperscan"] - path = contrib/hyperscan - url = https://github.com/ClickHouse/hyperscan.git [submodule "contrib/libunwind"] path = contrib/libunwind url = https://github.com/ClickHouse/libunwind.git @@ -268,6 +265,9 @@ [submodule "contrib/hashidsxx"] path = contrib/hashidsxx url = https://github.com/schoentoon/hashidsxx.git +[submodule "contrib/vectorscan"] + path = contrib/vectorscan + url = https://github.com/VectorCamp/vectorscan.git [submodule "contrib/liburing"] path = contrib/liburing url = https://github.com/axboe/liburing.git diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index c2ffd0131da..1bb53669cfa 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -58,7 +58,7 @@ add_contrib (boost-cmake boost) add_contrib (cctz-cmake cctz) add_contrib (consistent-hashing) add_contrib (dragonbox-cmake dragonbox) -add_contrib (hyperscan-cmake hyperscan) +add_contrib (vectorscan-cmake vectorscan) add_contrib (jemalloc-cmake jemalloc) add_contrib (libcpuid-cmake libcpuid) add_contrib (libdivide) diff --git a/contrib/hyperscan b/contrib/hyperscan deleted file mode 160000 index 5edc68c5ac6..00000000000 --- a/contrib/hyperscan +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5edc68c5ac68d2d4f876159e9ee84def6d3dc87c diff --git a/contrib/vectorscan b/contrib/vectorscan new file mode 160000 index 00000000000..73695e419c2 --- /dev/null +++ b/contrib/vectorscan @@ -0,0 +1 @@ +Subproject commit 73695e419c27af7fe2a099c7aa57931cc02aea5d diff --git a/contrib/hyperscan-cmake/CMakeLists.txt b/contrib/vectorscan-cmake/CMakeLists.txt similarity index 73% rename from contrib/hyperscan-cmake/CMakeLists.txt rename to contrib/vectorscan-cmake/CMakeLists.txt index 02c823a3a42..140c174cd73 100644 --- a/contrib/hyperscan-cmake/CMakeLists.txt +++ b/contrib/vectorscan-cmake/CMakeLists.txt @@ -1,54 +1,65 @@ -if (HAVE_SSSE3) - option (ENABLE_HYPERSCAN "Enable hyperscan library" ${ENABLE_LIBRARIES}) -elseif(ENABLE_HYPERSCAN) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use hyperscan without SSSE3") - set (ENABLE_HYPERSCAN OFF) -endif () +# We use vectorscan, a portable and API/ABI-compatible drop-in replacement for hyperscan. -if (NOT ENABLE_HYPERSCAN) - message (STATUS "Not using hyperscan") +if (ARCH_AMD64 OR ARCH_AARCH64) + option (ENABLE_VECTORSCAN "Enable vectorscan library" ${ENABLE_LIBRARIES}) +endif() + +# TODO PPC should generally work but needs manual generation of ppc/config.h file on a PPC machine + +if (NOT ENABLE_VECTORSCAN) + message (STATUS "Not using vectorscan") return() endif() -set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/hyperscan") +set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/vectorscan") +# Gobble up all c/cpp files in vectorscan/src/, omit *dump*.c/cpp files as we don't use the dump feature (see x86/config.h) set (SRCS "${LIBRARY_DIR}/src/alloc.c" + "${LIBRARY_DIR}/src/crc32.c" + "${LIBRARY_DIR}/src/database.c" + # "${LIBRARY_DIR}/src/dispatcher.c" # the linker's wrath be upon those who include dispatcher.c. + "${LIBRARY_DIR}/src/grey.cpp" + "${LIBRARY_DIR}/src/hs.cpp" + "${LIBRARY_DIR}/src/hs_valid_platform.c" + "${LIBRARY_DIR}/src/hs_version.c" + "${LIBRARY_DIR}/src/runtime.c" + "${LIBRARY_DIR}/src/scratch.c" + "${LIBRARY_DIR}/src/stream_compress.c" + "${LIBRARY_DIR}/src/compiler/asserts.cpp" "${LIBRARY_DIR}/src/compiler/compiler.cpp" "${LIBRARY_DIR}/src/compiler/error.cpp" - "${LIBRARY_DIR}/src/crc32.c" - "${LIBRARY_DIR}/src/database.c" + "${LIBRARY_DIR}/src/fdr/engine_description.cpp" - "${LIBRARY_DIR}/src/fdr/fdr_compile_util.cpp" + "${LIBRARY_DIR}/src/fdr/fdr.c" "${LIBRARY_DIR}/src/fdr/fdr_compile.cpp" + "${LIBRARY_DIR}/src/fdr/fdr_compile_util.cpp" "${LIBRARY_DIR}/src/fdr/fdr_confirm_compile.cpp" "${LIBRARY_DIR}/src/fdr/fdr_engine_description.cpp" - "${LIBRARY_DIR}/src/fdr/fdr.c" "${LIBRARY_DIR}/src/fdr/flood_compile.cpp" + "${LIBRARY_DIR}/src/fdr/teddy.c" + "${LIBRARY_DIR}/src/fdr/teddy_avx2.c" "${LIBRARY_DIR}/src/fdr/teddy_compile.cpp" "${LIBRARY_DIR}/src/fdr/teddy_engine_description.cpp" - "${LIBRARY_DIR}/src/fdr/teddy.c" - "${LIBRARY_DIR}/src/grey.cpp" - "${LIBRARY_DIR}/src/hs_valid_platform.c" - "${LIBRARY_DIR}/src/hs_version.c" - "${LIBRARY_DIR}/src/hs.cpp" + + "${LIBRARY_DIR}/src/hwlm/hwlm.c" "${LIBRARY_DIR}/src/hwlm/hwlm_build.cpp" "${LIBRARY_DIR}/src/hwlm/hwlm_literal.cpp" - "${LIBRARY_DIR}/src/hwlm/hwlm.c" "${LIBRARY_DIR}/src/hwlm/noodle_build.cpp" - "${LIBRARY_DIR}/src/hwlm/noodle_engine.c" - "${LIBRARY_DIR}/src/nfa/accel_dfa_build_strat.cpp" + "${LIBRARY_DIR}/src/hwlm/noodle_engine.cpp" + "${LIBRARY_DIR}/src/nfa/accel.c" + "${LIBRARY_DIR}/src/nfa/accel_dfa_build_strat.cpp" "${LIBRARY_DIR}/src/nfa/accelcompile.cpp" "${LIBRARY_DIR}/src/nfa/castle.c" "${LIBRARY_DIR}/src/nfa/castlecompile.cpp" "${LIBRARY_DIR}/src/nfa/dfa_build_strat.cpp" "${LIBRARY_DIR}/src/nfa/dfa_min.cpp" "${LIBRARY_DIR}/src/nfa/gough.c" + "${LIBRARY_DIR}/src/nfa/goughcompile.cpp" "${LIBRARY_DIR}/src/nfa/goughcompile_accel.cpp" "${LIBRARY_DIR}/src/nfa/goughcompile_reg.cpp" - "${LIBRARY_DIR}/src/nfa/goughcompile.cpp" "${LIBRARY_DIR}/src/nfa/lbr.c" "${LIBRARY_DIR}/src/nfa/limex_64.c" "${LIBRARY_DIR}/src/nfa/limex_accel.c" @@ -59,28 +70,32 @@ set (SRCS "${LIBRARY_DIR}/src/nfa/limex_simd384.c" "${LIBRARY_DIR}/src/nfa/limex_simd512.c" "${LIBRARY_DIR}/src/nfa/mcclellan.c" - "${LIBRARY_DIR}/src/nfa/mcclellancompile_util.cpp" "${LIBRARY_DIR}/src/nfa/mcclellancompile.cpp" + "${LIBRARY_DIR}/src/nfa/mcclellancompile_util.cpp" + "${LIBRARY_DIR}/src/nfa/mcsheng.c" "${LIBRARY_DIR}/src/nfa/mcsheng_compile.cpp" "${LIBRARY_DIR}/src/nfa/mcsheng_data.c" - "${LIBRARY_DIR}/src/nfa/mcsheng.c" "${LIBRARY_DIR}/src/nfa/mpv.c" "${LIBRARY_DIR}/src/nfa/mpvcompile.cpp" "${LIBRARY_DIR}/src/nfa/nfa_api_dispatch.c" "${LIBRARY_DIR}/src/nfa/nfa_build_util.cpp" + "${LIBRARY_DIR}/src/nfa/rdfa.cpp" "${LIBRARY_DIR}/src/nfa/rdfa_graph.cpp" "${LIBRARY_DIR}/src/nfa/rdfa_merge.cpp" - "${LIBRARY_DIR}/src/nfa/rdfa.cpp" "${LIBRARY_DIR}/src/nfa/repeat.c" "${LIBRARY_DIR}/src/nfa/repeatcompile.cpp" "${LIBRARY_DIR}/src/nfa/sheng.c" "${LIBRARY_DIR}/src/nfa/shengcompile.cpp" - "${LIBRARY_DIR}/src/nfa/shufti.c" + "${LIBRARY_DIR}/src/nfa/shufti.cpp" "${LIBRARY_DIR}/src/nfa/shufticompile.cpp" "${LIBRARY_DIR}/src/nfa/tamarama.c" "${LIBRARY_DIR}/src/nfa/tamaramacompile.cpp" - "${LIBRARY_DIR}/src/nfa/truffle.c" + "${LIBRARY_DIR}/src/nfa/truffle.cpp" "${LIBRARY_DIR}/src/nfa/trufflecompile.cpp" + "${LIBRARY_DIR}/src/nfa/vermicelli_simd.cpp" + "${LIBRARY_DIR}/src/nfa/vermicellicompile.cpp" + + "${LIBRARY_DIR}/src/nfagraph/ng.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_anchored_acyclic.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_anchored_dots.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_asserts.cpp" @@ -100,8 +115,8 @@ set (SRCS "${LIBRARY_DIR}/src/nfagraph/ng_holder.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_is_equal.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_lbr.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_limex_accel.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_limex.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_limex_accel.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_literal_analysis.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_literal_component.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_literal_decorated.cpp" @@ -112,17 +127,17 @@ set (SRCS "${LIBRARY_DIR}/src/nfagraph/ng_prune.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_puff.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_redundancy.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_region_redundancy.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_region.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_region_redundancy.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_repeat.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_reports.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_restructuring.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_revacc.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_sep.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_small_literal_set.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_som.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_som_add_redundancy.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_som_util.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_som.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_split.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_squash.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_stop.cpp" @@ -132,10 +147,8 @@ set (SRCS "${LIBRARY_DIR}/src/nfagraph/ng_vacuous.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_violet.cpp" "${LIBRARY_DIR}/src/nfagraph/ng_width.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng.cpp" + "${LIBRARY_DIR}/src/parser/AsciiComponentClass.cpp" - "${LIBRARY_DIR}/src/parser/buildstate.cpp" - "${LIBRARY_DIR}/src/parser/check_refs.cpp" "${LIBRARY_DIR}/src/parser/Component.cpp" "${LIBRARY_DIR}/src/parser/ComponentAlternation.cpp" "${LIBRARY_DIR}/src/parser/ComponentAssertion.cpp" @@ -145,31 +158,34 @@ set (SRCS "${LIBRARY_DIR}/src/parser/ComponentByte.cpp" "${LIBRARY_DIR}/src/parser/ComponentClass.cpp" "${LIBRARY_DIR}/src/parser/ComponentCondReference.cpp" - "${LIBRARY_DIR}/src/parser/ComponentEmpty.cpp" "${LIBRARY_DIR}/src/parser/ComponentEUS.cpp" + "${LIBRARY_DIR}/src/parser/ComponentEmpty.cpp" "${LIBRARY_DIR}/src/parser/ComponentRepeat.cpp" "${LIBRARY_DIR}/src/parser/ComponentSequence.cpp" "${LIBRARY_DIR}/src/parser/ComponentVisitor.cpp" "${LIBRARY_DIR}/src/parser/ComponentWordBoundary.cpp" "${LIBRARY_DIR}/src/parser/ConstComponentVisitor.cpp" - "${LIBRARY_DIR}/src/parser/control_verbs.cpp" + "${LIBRARY_DIR}/src/parser/Utf8ComponentClass.cpp" + "${LIBRARY_DIR}/src/parser/buildstate.cpp" + "${LIBRARY_DIR}/src/parser/buildstate.cpp" + "${LIBRARY_DIR}/src/parser/check_refs.cpp" + "${LIBRARY_DIR}/src/parser/check_refs.cpp" "${LIBRARY_DIR}/src/parser/logical_combination.cpp" "${LIBRARY_DIR}/src/parser/parse_error.cpp" "${LIBRARY_DIR}/src/parser/parser_util.cpp" - "${LIBRARY_DIR}/src/parser/Parser.cpp" "${LIBRARY_DIR}/src/parser/prefilter.cpp" "${LIBRARY_DIR}/src/parser/shortcut_literal.cpp" "${LIBRARY_DIR}/src/parser/ucp_table.cpp" "${LIBRARY_DIR}/src/parser/unsupported.cpp" "${LIBRARY_DIR}/src/parser/utf8_validate.cpp" - "${LIBRARY_DIR}/src/parser/Utf8ComponentClass.cpp" + "${LIBRARY_DIR}/src/rose/block.c" "${LIBRARY_DIR}/src/rose/catchup.c" "${LIBRARY_DIR}/src/rose/init.c" "${LIBRARY_DIR}/src/rose/match.c" "${LIBRARY_DIR}/src/rose/program_runtime.c" - "${LIBRARY_DIR}/src/rose/rose_build_add_mask.cpp" "${LIBRARY_DIR}/src/rose/rose_build_add.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_add_mask.cpp" "${LIBRARY_DIR}/src/rose/rose_build_anchored.cpp" "${LIBRARY_DIR}/src/rose/rose_build_bytecode.cpp" "${LIBRARY_DIR}/src/rose/rose_build_castle.cpp" @@ -187,53 +203,95 @@ set (SRCS "${LIBRARY_DIR}/src/rose/rose_build_matchers.cpp" "${LIBRARY_DIR}/src/rose/rose_build_merge.cpp" "${LIBRARY_DIR}/src/rose/rose_build_misc.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_misc.cpp" "${LIBRARY_DIR}/src/rose/rose_build_program.cpp" "${LIBRARY_DIR}/src/rose/rose_build_role_aliasing.cpp" "${LIBRARY_DIR}/src/rose/rose_build_scatter.cpp" "${LIBRARY_DIR}/src/rose/rose_build_width.cpp" "${LIBRARY_DIR}/src/rose/rose_in_util.cpp" "${LIBRARY_DIR}/src/rose/stream.c" - "${LIBRARY_DIR}/src/runtime.c" - "${LIBRARY_DIR}/src/scratch.c" + "${LIBRARY_DIR}/src/smallwrite/smallwrite_build.cpp" + "${LIBRARY_DIR}/src/som/slot_manager.cpp" "${LIBRARY_DIR}/src/som/som_runtime.c" "${LIBRARY_DIR}/src/som/som_stream.c" - "${LIBRARY_DIR}/src/stream_compress.c" + "${LIBRARY_DIR}/src/util/alloc.cpp" "${LIBRARY_DIR}/src/util/charreach.cpp" "${LIBRARY_DIR}/src/util/clique.cpp" "${LIBRARY_DIR}/src/util/compile_context.cpp" "${LIBRARY_DIR}/src/util/compile_error.cpp" - "${LIBRARY_DIR}/src/util/cpuid_flags.c" "${LIBRARY_DIR}/src/util/depth.cpp" "${LIBRARY_DIR}/src/util/fatbit_build.cpp" - "${LIBRARY_DIR}/src/util/multibit_build.cpp" "${LIBRARY_DIR}/src/util/multibit.c" + "${LIBRARY_DIR}/src/util/multibit_build.cpp" "${LIBRARY_DIR}/src/util/report_manager.cpp" - "${LIBRARY_DIR}/src/util/simd_utils.c" "${LIBRARY_DIR}/src/util/state_compress.c" "${LIBRARY_DIR}/src/util/target_info.cpp" "${LIBRARY_DIR}/src/util/ue2string.cpp" ) -add_library (_hyperscan ${SRCS}) +# The original build system invokes ragel on src/parser/{Parser|control_verbs}.rl (+ a few more .rl files which are unneeded). To avoid a +# build-time dependency on ragel (via contrib/ or find_program()), add the manually generated output of ragel to the sources. +# Please regenerate these files if you update vectorscan. +list (APPEND SRCS + "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/Parser.cpp" + "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/control_verbs.cpp" +) -target_compile_options (_hyperscan - PRIVATE -g0 # Library has too much debug information - -mno-avx -mno-avx2 # The library is using dynamic dispatch and is confused if AVX is enabled globally - -march=corei7 -O2 -fno-strict-aliasing -fno-omit-frame-pointer -fvisibility=hidden # The options from original build system - -fno-sanitize=undefined # Assume the library takes care of itself -) -target_include_directories (_hyperscan - PRIVATE - common - "${LIBRARY_DIR}/include" -) -target_include_directories (_hyperscan SYSTEM PUBLIC "${LIBRARY_DIR}/src") +# Platform-dependent files if (ARCH_AMD64) - target_include_directories (_hyperscan PRIVATE x86_64) -endif () -target_link_libraries (_hyperscan PRIVATE boost::headers_only) + list(APPEND SRCS + "${LIBRARY_DIR}/src/util/arch/x86/cpuid_flags.c" + "${LIBRARY_DIR}/src/util/arch/x86/masked_move.c" + "${LIBRARY_DIR}/src/util/supervector/arch/x86/impl.cpp" + ) +endif() -add_library (ch_contrib::hyperscan ALIAS _hyperscan) +if (ARCH_AARCH64) + list(APPEND SRCS + "${LIBRARY_DIR}/src/util/arch/arm/cpuid_flags.c" + "${LIBRARY_DIR}/src/util/supervector/arch/arm/impl.cpp" + ) +endif() + +# TODO +# if (ARCH_PPC64LE) +# list(APPEND SRCS +# "${LIBRARY_DIR}/src/util/supervector/arch/ppc64el/impl.cpp" +# ) +# endif() + +add_library (_vectorscan ${SRCS}) + +target_compile_options (_vectorscan PRIVATE + -g0 # library has too much debug information + -fno-sanitize=undefined # assume the library takes care of itself + -O2 -fno-strict-aliasing -fno-omit-frame-pointer -fvisibility=hidden # options from original build system +) + +# Include version header manually generated by running the original build system +target_include_directories (_vectorscan SYSTEM PRIVATE common) + +# vectorscan inherited some patched in-source versions of boost headers to fix a bug in +# boost 1.69. This bug has been solved long ago but vectorscan's source code still +# points to the patched versions, so include it here. +target_include_directories (_vectorscan SYSTEM PRIVATE "${LIBRARY_DIR}/include") + +target_include_directories (_vectorscan SYSTEM PUBLIC "${LIBRARY_DIR}/src") + +# Include platform-specific config header generated by manually running the original build system +# Please regenerate these files if you update vectorscan. + +if (ARCH_AMD64) + target_include_directories (_vectorscan PRIVATE x86_64) +endif () + +if (ARCH_AARCH64) + target_include_directories (_vectorscan PRIVATE aarch64) +endif () + +target_link_libraries (_vectorscan PRIVATE boost::headers_only) + +add_library (ch_contrib::vectorscan ALIAS _vectorscan) diff --git a/contrib/vectorscan-cmake/aarch64/config.h b/contrib/vectorscan-cmake/aarch64/config.h new file mode 100644 index 00000000000..78da1c8ad00 --- /dev/null +++ b/contrib/vectorscan-cmake/aarch64/config.h @@ -0,0 +1,142 @@ +/* used by cmake */ + +#ifndef CONFIG_H_ +#define CONFIG_H_ + +/* "Define if the build is 32 bit" */ +/* #undef ARCH_32_BIT */ + +/* "Define if the build is 64 bit" */ +#define ARCH_64_BIT + +/* "Define if building for IA32" */ +/* #undef ARCH_IA32 */ + +/* "Define if building for EM64T" */ +/* #undef ARCH_X86_64 */ + +/* "Define if building for ARM32" */ +/* #undef ARCH_ARM32 */ + +/* "Define if building for AARCH64" */ +#define ARCH_AARCH64 + +/* "Define if building for PPC64EL" */ +/* #undef ARCH_PPC64EL */ + +/* "Define if cross compiling for AARCH64" */ +/* #undef CROSS_COMPILE_AARCH64 */ + +/* Define if building SVE for AARCH64. */ +/* #undef BUILD_SVE */ + +/* Define if building SVE2 for AARCH64. */ +/* #undef BUILD_SVE2 */ + +/* Define if building SVE2+BITPERM for AARCH64. */ +/* #undef BUILD_SVE2_BITPERM */ + +/* internal build, switch on dump support. */ +/* #undef DUMP_SUPPORT */ + +/* Define if building "fat" runtime. */ +/* #undef FAT_RUNTIME */ + +/* Define if building AVX2 in the fat runtime. */ +/* #undef BUILD_AVX2 */ + +/* Define if building AVX-512 in the fat runtime. */ +/* #undef BUILD_AVX512 */ + +/* Define if building AVX512VBMI in the fat runtime. */ +/* #undef BUILD_AVX512VBMI */ + +/* Define to 1 if `backtrace' works. */ +#define HAVE_BACKTRACE + +/* C compiler has __builtin_assume_aligned */ +#define HAVE_CC_BUILTIN_ASSUME_ALIGNED + +/* C++ compiler has __builtin_assume_aligned */ +#define HAVE_CXX_BUILTIN_ASSUME_ALIGNED + +/* C++ compiler has x86intrin.h */ +/* #undef HAVE_CXX_X86INTRIN_H */ + +/* C compiler has x86intrin.h */ +/* #undef HAVE_C_X86INTRIN_H */ + +/* C++ compiler has intrin.h */ +/* #undef HAVE_CXX_INTRIN_H */ + +/* C compiler has intrin.h */ +/* #undef HAVE_C_INTRIN_H */ + +/* C compiler has arm_neon.h */ +#define HAVE_C_ARM_NEON_H + +/* C compiler has arm_sve.h */ +/* #undef HAVE_C_ARM_SVE_H */ + +/* C compiler has arm_neon.h */ +/* #undef HAVE_C_PPC64EL_ALTIVEC_H */ + +/* Define to 1 if you have the declaration of `pthread_setaffinity_np', and to + 0 if you don't. */ +/* #undef HAVE_DECL_PTHREAD_SETAFFINITY_NP */ + +/* #undef HAVE_PTHREAD_NP_H */ + +/* Define to 1 if you have the `malloc_info' function. */ +/* #undef HAVE_MALLOC_INFO */ + +/* Define to 1 if you have the `memmem' function. */ +/* #undef HAVE_MEMMEM */ + +/* Define to 1 if you have a working `mmap' system call. */ +#define HAVE_MMAP + +/* Define to 1 if `posix_memalign' works. */ +#define HAVE_POSIX_MEMALIGN + +/* Define to 1 if you have the `setrlimit' function. */ +#define HAVE_SETRLIMIT + +/* Define to 1 if you have the `shmget' function. */ +/* #undef HAVE_SHMGET */ + +/* Define to 1 if you have the `sigaction' function. */ +#define HAVE_SIGACTION + +/* Define to 1 if you have the `sigaltstack' function. */ +#define HAVE_SIGALTSTACK + +/* Define if the sqlite3_open_v2 call is available */ +/* #undef HAVE_SQLITE3_OPEN_V2 */ + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H + +/* Define to 1 if you have the `_aligned_malloc' function. */ +/* #undef HAVE__ALIGNED_MALLOC */ + +/* Define if compiler has __builtin_constant_p */ +/* #undef HAVE__BUILTIN_CONSTANT_P */ + +/* Optimize, inline critical functions */ +#define HS_OPTIMIZE + +#define HS_VERSION +#define HS_MAJOR_VERSION +#define HS_MINOR_VERSION +#define HS_PATCH_VERSION + +#define BUILD_DATE + +/* define if this is a release build. */ +#define RELEASE_BUILD + +/* define if reverse_graph requires patch for boost 1.62.0 */ +/* #undef BOOST_REVGRAPH_PATCH */ + +#endif /* CONFIG_H_ */ diff --git a/contrib/hyperscan-cmake/common/hs_version.h b/contrib/vectorscan-cmake/common/hs_version.h similarity index 94% rename from contrib/hyperscan-cmake/common/hs_version.h rename to contrib/vectorscan-cmake/common/hs_version.h index f6fa8cb209f..8315b44fb2a 100644 --- a/contrib/hyperscan-cmake/common/hs_version.h +++ b/contrib/vectorscan-cmake/common/hs_version.h @@ -32,9 +32,8 @@ /** * A version string to identify this release of Hyperscan. */ -#define HS_VERSION_STRING "5.1.1 2000-01-01" +#define HS_VERSION_STRING "5.4.7 2022-06-20" -#define HS_VERSION_32BIT ((5 << 24) | (1 << 16) | (1 << 8) | 0) +#define HS_VERSION_32BIT ((5 << 24) | (1 << 16) | (7 << 8) | 0) #endif /* HS_VERSION_H_C6428FAF8E3713 */ - diff --git a/contrib/vectorscan-cmake/rageled_files/Parser.cpp b/contrib/vectorscan-cmake/rageled_files/Parser.cpp new file mode 100644 index 00000000000..aebbd7ace1e --- /dev/null +++ b/contrib/vectorscan-cmake/rageled_files/Parser.cpp @@ -0,0 +1,5605 @@ + +#line 1 "Parser.rl" +/* + * Copyright (c) 2015-2017, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * \brief Parser code (generated with Ragel from Parser.rl). + */ + +#include "config.h" + +/* Parser.cpp is a built source, may not be in same dir as parser files */ +#include "parser/check_refs.h" +#include "parser/control_verbs.h" +#include "parser/ComponentAlternation.h" +#include "parser/ComponentAssertion.h" +#include "parser/ComponentAtomicGroup.h" +#include "parser/ComponentBackReference.h" +#include "parser/ComponentBoundary.h" +#include "parser/ComponentByte.h" +#include "parser/ComponentClass.h" +#include "parser/ComponentCondReference.h" +#include "parser/ComponentEmpty.h" +#include "parser/ComponentEUS.h" +#include "parser/Component.h" +#include "parser/ComponentRepeat.h" +#include "parser/ComponentSequence.h" +#include "parser/ComponentWordBoundary.h" +#include "parser/parse_error.h" +#include "parser/Parser.h" +#include "ue2common.h" +#include "util/compare.h" +#include "util/flat_containers.h" +#include "util/unicode_def.h" +#include "util/verify_types.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +namespace ue2 { + +#define PUSH_SEQUENCE do {\ + sequences.push_back(ExprState(currentSeq, (size_t)(ts - ptr), \ + mode)); \ + } while(0) +#define POP_SEQUENCE do {\ + currentSeq = sequences.back().seq; \ + mode = sequences.back().mode; \ + sequences.pop_back(); \ + } while(0) + +namespace { + +/** \brief Structure representing current state as we're parsing (current + * sequence, current options). Stored in the 'sequences' vector. */ +struct ExprState { + ExprState(ComponentSequence *seq_in, size_t offset, + const ParseMode &mode_in) : + seq(seq_in), seqOffset(offset), mode(mode_in) {} + + ComponentSequence *seq; //!< current sequence + size_t seqOffset; //!< offset seq was entered, for error reporting + ParseMode mode; //!< current mode flags +}; + +} // namespace + +static +unsigned parseAsDecimal(unsigned oct) { + // The input was parsed as octal, but should have been parsed as decimal. + // Deconstruct the octal number and reconstruct into decimal + unsigned ret = 0; + unsigned multiplier = 1; + while (oct) { + ret += (oct & 0x7) * multiplier; + oct >>= 3; + multiplier *= 10; + } + return ret; +} + +/** \brief Maximum value for a positive integer. We use INT_MAX, as that's what + * PCRE uses. */ +static constexpr u32 MAX_NUMBER = INT_MAX; + +static +void pushDec(u32 *acc, char raw_digit) { + assert(raw_digit >= '0' && raw_digit <= '9'); + u32 digit_val = raw_digit - '0'; + + // Ensure that we don't overflow. + u64a val = ((u64a)*acc * 10) + digit_val; + if (val > MAX_NUMBER) { + throw LocatedParseError("Number is too big"); + } + + *acc = verify_u32(val); +} + +static +void pushOct(u32 *acc, char raw_digit) { + assert(raw_digit >= '0' && raw_digit <= '7'); + u32 digit_val = raw_digit - '0'; + + // Ensure that we don't overflow. + u64a val = ((u64a)*acc * 8) + digit_val; + if (val > MAX_NUMBER) { + throw LocatedParseError("Number is too big"); + } + + *acc = verify_u32(val); +} + +static +void throwInvalidRepeat(void) { + throw LocatedParseError("Invalid repeat"); +} + +static +void throwInvalidUtf8(void) { + throw ParseError("Expression is not valid UTF-8."); +} + +/** + * Adds the given child component to the parent sequence, returning a pointer + * to the new (child) "current sequence". + */ +static +ComponentSequence *enterSequence(ComponentSequence *parent, + unique_ptr child) { + assert(parent); + assert(child); + + ComponentSequence *seq = child.get(); + parent->addComponent(move(child)); + return seq; +} + +static +void addLiteral(ComponentSequence *currentSeq, char c, const ParseMode &mode) { + if (mode.utf8 && mode.caseless) { + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + assert(cc); + cc->add(c); + cc->finalize(); + currentSeq->addComponent(move(cc)); + } else { + currentSeq->addComponent(getLiteralComponentClass(c, mode.caseless)); + } +} + +static +void addEscaped(ComponentSequence *currentSeq, unichar accum, + const ParseMode &mode, const char *err_msg) { + if (mode.utf8) { + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + assert(cc); + cc->add(accum); + cc->finalize(); + currentSeq->addComponent(move(cc)); + } else { + if (accum > 255) { + throw LocatedParseError(err_msg); + } + addLiteral(currentSeq, (char)accum, mode); + } +} + +static +void addEscapedOctal(ComponentSequence *currentSeq, unichar accum, + const ParseMode &mode) { + addEscaped(currentSeq, accum, mode, "Octal value is greater than \\377"); +} + +static +void addEscapedHex(ComponentSequence *currentSeq, unichar accum, + const ParseMode &mode) { + addEscaped(currentSeq, accum, mode, + "Hexadecimal value is greater than \\xFF"); +} + +#define SLASH_C_ERROR "\\c must be followed by an ASCII character" + +static +u8 decodeCtrl(char raw) { + if (raw & 0x80) { + throw LocatedParseError(SLASH_C_ERROR); + } + return mytoupper(raw) ^ 0x40; +} + +static +unichar readUtf8CodePoint2c(const char *s) { + auto *ts = (const u8 *)s; + assert(ts[0] >= 0xc0 && ts[0] < 0xe0); + assert(ts[1] >= 0x80 && ts[1] < 0xc0); + unichar val = ts[0] & 0x1f; + val <<= 6; + val |= ts[1] & 0x3f; + DEBUG_PRINTF("utf8 %02hhx %02hhx ->\\x{%x}\n", ts[0], + ts[1], val); + return val; +} + +static +unichar readUtf8CodePoint3c(const char *s) { + auto *ts = (const u8 *)s; + assert(ts[0] >= 0xe0 && ts[0] < 0xf0); + assert(ts[1] >= 0x80 && ts[1] < 0xc0); + assert(ts[2] >= 0x80 && ts[2] < 0xc0); + unichar val = ts[0] & 0x0f; + val <<= 6; + val |= ts[1] & 0x3f; + val <<= 6; + val |= ts[2] & 0x3f; + DEBUG_PRINTF("utf8 %02hhx %02hhx %02hhx ->\\x{%x}\n", ts[0], + ts[1], ts[2], val); + return val; +} + +static +unichar readUtf8CodePoint4c(const char *s) { + auto *ts = (const u8 *)s; + assert(ts[0] >= 0xf0 && ts[0] < 0xf8); + assert(ts[1] >= 0x80 && ts[1] < 0xc0); + assert(ts[2] >= 0x80 && ts[2] < 0xc0); + assert(ts[3] >= 0x80 && ts[3] < 0xc0); + unichar val = ts[0] & 0x07; + val <<= 6; + val |= ts[1] & 0x3f; + val <<= 6; + val |= ts[2] & 0x3f; + val <<= 6; + val |= ts[3] & 0x3f; + DEBUG_PRINTF("utf8 %02hhx %02hhx %02hhx %02hhx ->\\x{%x}\n", ts[0], + ts[1], ts[2], ts[3], val); + return val; +} + + +#line 1909 "Parser.rl" + + + +#line 281 "Parser.cpp" +static const short _regex_actions[] = { + 0, 1, 0, 1, 1, 1, 2, 1, + 3, 1, 4, 1, 7, 1, 8, 1, + 9, 1, 10, 1, 11, 1, 12, 1, + 13, 1, 15, 1, 16, 1, 17, 1, + 18, 1, 19, 1, 20, 1, 21, 1, + 22, 1, 23, 1, 24, 1, 25, 1, + 26, 1, 27, 1, 28, 1, 29, 1, + 30, 1, 31, 1, 32, 1, 33, 1, + 34, 1, 35, 1, 36, 1, 37, 1, + 38, 1, 39, 1, 40, 1, 41, 1, + 42, 1, 43, 1, 44, 1, 45, 1, + 46, 1, 47, 1, 48, 1, 49, 1, + 50, 1, 51, 1, 52, 1, 53, 1, + 54, 1, 55, 1, 56, 1, 57, 1, + 58, 1, 59, 1, 60, 1, 61, 1, + 62, 1, 63, 1, 64, 1, 65, 1, + 66, 1, 67, 1, 68, 1, 69, 1, + 70, 1, 71, 1, 72, 1, 73, 1, + 74, 1, 75, 1, 76, 1, 77, 1, + 78, 1, 79, 1, 80, 1, 81, 1, + 82, 1, 83, 1, 84, 1, 85, 1, + 86, 1, 87, 1, 88, 1, 89, 1, + 90, 1, 91, 1, 92, 1, 93, 1, + 94, 1, 95, 1, 96, 1, 97, 1, + 98, 1, 99, 1, 100, 1, 101, 1, + 102, 1, 103, 1, 104, 1, 105, 1, + 106, 1, 107, 1, 108, 1, 109, 1, + 110, 1, 111, 1, 112, 1, 113, 1, + 114, 1, 115, 1, 116, 1, 117, 1, + 118, 1, 119, 1, 120, 1, 121, 1, + 122, 1, 123, 1, 124, 1, 125, 1, + 126, 1, 127, 1, 128, 1, 129, 1, + 130, 1, 131, 1, 132, 1, 133, 1, + 134, 1, 135, 1, 136, 1, 137, 1, + 138, 1, 139, 1, 140, 1, 141, 1, + 142, 1, 143, 1, 144, 1, 145, 1, + 146, 1, 147, 1, 148, 1, 149, 1, + 150, 1, 151, 1, 152, 1, 153, 1, + 154, 1, 155, 1, 156, 1, 157, 1, + 158, 1, 159, 1, 160, 1, 161, 1, + 162, 1, 163, 1, 164, 1, 165, 1, + 166, 1, 167, 1, 168, 1, 169, 1, + 170, 1, 171, 1, 172, 1, 173, 1, + 174, 1, 175, 1, 176, 1, 177, 1, + 178, 1, 179, 1, 180, 1, 181, 1, + 182, 1, 183, 1, 184, 1, 185, 1, + 186, 1, 187, 1, 188, 1, 189, 1, + 190, 1, 191, 1, 192, 1, 193, 1, + 194, 1, 195, 1, 196, 1, 197, 1, + 198, 1, 199, 1, 200, 1, 201, 1, + 202, 1, 203, 1, 204, 1, 205, 1, + 206, 1, 207, 1, 208, 1, 209, 1, + 210, 1, 211, 1, 212, 1, 213, 1, + 214, 1, 215, 1, 216, 1, 217, 1, + 218, 1, 219, 1, 220, 1, 221, 1, + 222, 1, 223, 1, 224, 1, 225, 1, + 226, 1, 227, 1, 228, 1, 229, 1, + 230, 1, 231, 1, 232, 1, 233, 1, + 234, 1, 235, 1, 236, 1, 237, 1, + 240, 1, 242, 1, 243, 1, 244, 1, + 245, 1, 246, 1, 247, 1, 248, 1, + 249, 1, 250, 1, 251, 1, 252, 1, + 253, 1, 254, 1, 255, 1, 256, 1, + 257, 1, 258, 1, 259, 1, 260, 1, + 261, 1, 262, 1, 263, 1, 264, 1, + 265, 1, 266, 1, 267, 1, 268, 1, + 269, 1, 270, 1, 271, 1, 272, 1, + 273, 1, 274, 1, 275, 1, 276, 1, + 277, 1, 278, 1, 279, 1, 280, 1, + 281, 1, 282, 1, 283, 1, 284, 1, + 285, 1, 286, 1, 287, 1, 288, 1, + 289, 1, 290, 1, 291, 1, 292, 1, + 293, 1, 294, 1, 295, 1, 296, 1, + 297, 1, 298, 1, 299, 1, 300, 1, + 301, 1, 302, 1, 303, 1, 307, 1, + 308, 1, 309, 1, 310, 1, 311, 1, + 312, 1, 313, 1, 314, 1, 315, 1, + 316, 1, 317, 1, 318, 1, 319, 1, + 320, 1, 321, 1, 322, 1, 323, 1, + 324, 1, 325, 1, 326, 1, 327, 1, + 328, 1, 329, 1, 330, 1, 331, 1, + 332, 1, 333, 1, 334, 1, 335, 1, + 336, 1, 337, 1, 338, 1, 342, 1, + 343, 1, 344, 1, 345, 1, 346, 1, + 347, 1, 348, 1, 349, 1, 350, 1, + 352, 1, 353, 1, 354, 1, 355, 1, + 356, 1, 357, 1, 358, 1, 359, 1, + 360, 1, 361, 1, 362, 1, 363, 1, + 364, 1, 365, 1, 366, 1, 367, 1, + 368, 1, 369, 1, 370, 1, 371, 1, + 372, 1, 373, 1, 374, 1, 375, 1, + 376, 1, 377, 1, 378, 1, 379, 1, + 380, 1, 381, 1, 382, 1, 383, 1, + 384, 1, 385, 1, 386, 1, 387, 1, + 388, 1, 389, 1, 390, 1, 391, 1, + 392, 1, 393, 1, 394, 1, 395, 1, + 396, 1, 397, 1, 398, 1, 399, 1, + 400, 1, 401, 1, 402, 1, 403, 1, + 404, 1, 405, 1, 406, 1, 407, 1, + 408, 1, 409, 1, 410, 1, 411, 1, + 412, 1, 413, 1, 414, 1, 415, 1, + 416, 1, 417, 1, 418, 1, 419, 1, + 420, 1, 421, 1, 422, 1, 423, 1, + 424, 1, 425, 1, 426, 1, 427, 1, + 428, 1, 429, 1, 430, 1, 431, 1, + 432, 1, 433, 1, 434, 1, 435, 1, + 436, 2, 3, 0, 2, 4, 5, 2, + 5, 1, 2, 9, 10, 2, 9, 238, + 2, 9, 239, 2, 9, 339, 2, 10, + 1, 2, 10, 340, 2, 10, 341, 2, + 11, 241, 2, 11, 351, 2, 12, 241, + 2, 12, 351, 2, 13, 241, 2, 13, + 351, 2, 14, 375, 2, 14, 376, 2, + 25, 0, 2, 25, 3, 2, 25, 6, + 2, 25, 14, 3, 25, 5, 306, 3, + 25, 10, 305, 3, 25, 14, 15, 4, + 25, 9, 304, 10 +}; + +static const char _regex_cond_offsets[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, + 2, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, + 18, 19, 20, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 27, 28, 29, 31, 31, 36, + 36, 37, 38, 39, 44, 44, 45, 46, + 47, 47 +}; + +static const char _regex_cond_lengths[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 6, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 5, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 1, 2, 0, 5, 0, + 1, 1, 1, 5, 0, 1, 1, 1, + 0, 0 +}; + +static const short _regex_cond_keys[] = { + -128, -65, -128, -65, -128, -65, -128, -65, + -128, -65, -128, -65, -128, -65, -128, -65, + -128, -65, -128, -65, -128, -65, -128, -65, + -128, -65, -64, -33, -32, -17, -16, -9, + -8, -1, 35, 35, -128, -65, -128, -65, + -128, -65, -128, -65, -64, -33, -32, -17, + -16, -9, -8, -1, -128, -65, -128, -65, + -128, -65, 93, 93, 94, 94, -128, -65, + -64, -33, -32, -17, -16, -9, -8, -1, + -128, -65, -128, -65, -128, -65, -128, -65, + -64, -33, -32, -17, -16, -9, -8, -1, + -128, -65, -128, -65, -128, -65, 0 +}; + +static const char _regex_cond_spaces[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 2, 2, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 +}; + +static const short _regex_key_offsets[] = { + 0, 0, 1, 23, 31, 39, 46, 54, + 55, 63, 71, 79, 86, 94, 97, 99, + 108, 115, 123, 131, 134, 140, 148, 151, + 158, 165, 173, 180, 184, 191, 194, 197, + 199, 202, 205, 207, 210, 213, 215, 216, + 218, 219, 227, 229, 232, 235, 236, 244, + 252, 260, 268, 275, 283, 290, 298, 305, + 313, 315, 318, 325, 329, 332, 335, 337, + 339, 341, 342, 344, 345, 347, 349, 350, + 351, 353, 354, 355, 356, 357, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 369, + 370, 371, 372, 373, 374, 375, 376, 377, + 378, 379, 380, 381, 382, 383, 384, 385, + 386, 387, 388, 389, 390, 392, 393, 394, + 395, 396, 397, 399, 400, 401, 402, 403, + 404, 405, 406, 408, 409, 410, 411, 412, + 413, 414, 415, 416, 417, 418, 419, 420, + 421, 422, 423, 424, 425, 426, 427, 429, + 430, 431, 432, 433, 434, 435, 436, 437, + 438, 439, 440, 441, 442, 443, 444, 445, + 446, 447, 448, 450, 451, 452, 453, 454, + 455, 456, 457, 458, 459, 461, 462, 463, + 464, 465, 466, 467, 468, 469, 470, 471, + 472, 473, 474, 475, 476, 477, 478, 479, + 480, 481, 482, 483, 484, 485, 486, 487, + 488, 489, 490, 491, 492, 493, 494, 495, + 496, 497, 498, 499, 500, 501, 502, 503, + 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 516, 517, 519, 520, + 521, 522, 523, 524, 525, 526, 527, 528, + 529, 530, 531, 532, 533, 534, 535, 536, + 537, 538, 539, 540, 541, 542, 543, 544, + 545, 546, 547, 548, 549, 550, 551, 552, + 553, 554, 555, 556, 557, 558, 559, 561, + 562, 563, 564, 565, 566, 567, 568, 569, + 570, 571, 572, 573, 574, 575, 576, 577, + 578, 579, 580, 582, 583, 584, 585, 586, + 587, 588, 589, 590, 591, 592, 593, 594, + 595, 596, 597, 601, 602, 603, 604, 605, + 606, 607, 608, 609, 610, 611, 612, 613, + 614, 615, 616, 617, 618, 620, 621, 622, + 623, 624, 625, 626, 627, 628, 629, 631, + 632, 633, 634, 635, 636, 637, 640, 641, + 642, 643, 644, 645, 646, 647, 648, 650, + 651, 652, 653, 654, 655, 656, 658, 659, + 660, 661, 662, 663, 664, 665, 666, 667, + 668, 669, 670, 671, 672, 673, 674, 675, + 676, 677, 678, 679, 680, 681, 682, 683, + 684, 685, 686, 687, 688, 689, 690, 691, + 692, 693, 694, 695, 696, 697, 698, 699, + 700, 701, 702, 704, 705, 706, 707, 708, + 709, 710, 714, 715, 716, 717, 718, 719, + 720, 721, 722, 723, 724, 725, 726, 727, + 728, 729, 730, 731, 732, 733, 734, 735, + 736, 737, 738, 739, 740, 741, 742, 743, + 744, 745, 746, 747, 748, 749, 750, 752, + 753, 754, 755, 756, 757, 758, 759, 760, + 761, 762, 763, 764, 765, 766, 767, 768, + 769, 770, 771, 773, 774, 775, 776, 777, + 778, 779, 780, 781, 782, 783, 784, 785, + 786, 787, 788, 789, 790, 791, 792, 793, + 794, 795, 796, 797, 798, 799, 800, 801, + 802, 803, 805, 806, 807, 808, 809, 810, + 811, 812, 813, 814, 815, 816, 817, 820, + 822, 823, 824, 825, 826, 827, 828, 829, + 830, 833, 834, 835, 836, 837, 838, 839, + 840, 841, 842, 843, 844, 845, 846, 847, + 849, 850, 851, 853, 854, 855, 856, 857, + 858, 859, 860, 861, 862, 863, 864, 865, + 866, 867, 868, 869, 870, 871, 872, 873, + 874, 875, 876, 877, 880, 883, 885, 900, + 903, 906, 908, 922, 927, 932, 936, 940, + 943, 946, 950, 954, 957, 960, 964, 968, + 972, 975, 978, 982, 986, 990, 994, 997, + 1000, 1004, 1008, 1012, 1016, 1019, 1022, 1026, + 1030, 1034, 1038, 1041, 1044, 1048, 1052, 1056, + 1060, 1063, 1066, 1070, 1074, 1078, 1082, 1085, + 1088, 1093, 1097, 1101, 1105, 1108, 1111, 1115, + 1119, 1123, 1126, 1129, 1133, 1137, 1141, 1145, + 1148, 1151, 1155, 1159, 1163, 1167, 1170, 1173, + 1177, 1181, 1185, 1188, 1191, 1195, 1199, 1203, + 1207, 1211, 1214, 1217, 1222, 1227, 1231, 1235, + 1238, 1241, 1245, 1249, 1252, 1255, 1259, 1263, + 1267, 1270, 1273, 1277, 1281, 1285, 1289, 1292, + 1295, 1299, 1303, 1307, 1311, 1314, 1317, 1321, + 1325, 1329, 1333, 1336, 1339, 1343, 1347, 1351, + 1355, 1358, 1361, 1365, 1369, 1373, 1377, 1380, + 1383, 1388, 1392, 1396, 1400, 1403, 1406, 1410, + 1414, 1418, 1421, 1424, 1428, 1432, 1436, 1440, + 1443, 1446, 1450, 1454, 1458, 1462, 1465, 1468, + 1472, 1476, 1480, 1483, 1486, 1490, 1494, 1498, + 1502, 1506, 1509, 1512, 1515, 1518, 1520, 1522, + 1525, 1532, 1534, 1536, 1538, 1540, 1542, 1544, + 1546, 1548, 1550, 1584, 1586, 1593, 1600, 1614, + 1616, 1622, 1625, 1634, 1635, 1638, 1641, 1648, + 1650, 1652, 1654, 1657, 1702, 1704, 1706, 1710, + 1714, 1716, 1717, 1717, 1723, 1725, 1727, 1729, + 1731, 1734, 1735, 1736, 1743, 1749, 1755, 1757, + 1759, 1761, 1763, 1765, 1767, 1768, 1771, 1794, + 1797, 1802, 1811, 1813, 1814, 1816, 1821, 1824, + 1826, 1828, 1829, 1831, 1841, 1847, 1848, 1853, + 1857, 1865, 1867, 1876, 1880, 1881, 1882, 1886, + 1887, 1890, 1890, 1897, 1913, 1916, 1955, 1957, + 1959, 1961, 1963, 1964, 1964, 1965, 1966, 1973, + 1979, 1985, 1987, 1989, 1991, 2000, 2002, 2015, + 2016, 2018, 2020, 2022, 2035, 2036, 2038, 2040, + 2042, 2043 +}; + +static const short _regex_trans_keys[] = { + 41, 33, 35, 38, 39, 40, 41, 43, + 45, 58, 60, 61, 62, 63, 67, 80, + 105, 109, 115, 120, 123, 48, 57, 41, + 95, 48, 57, 65, 90, 97, 122, 39, + 95, 48, 57, 65, 90, 97, 122, 95, + 48, 57, 65, 90, 97, 122, 39, 95, + 48, 57, 65, 90, 97, 122, 41, 41, + 95, 48, 57, 65, 90, 97, 122, 41, + 95, 48, 57, 65, 90, 97, 122, 41, + 95, 48, 57, 65, 90, 97, 122, 95, + 48, 57, 65, 90, 97, 122, 62, 95, + 48, 57, 65, 90, 97, 122, 33, 60, + 61, 33, 61, 38, 41, 95, 48, 57, + 65, 90, 97, 122, 95, 48, 57, 65, + 90, 97, 122, 41, 95, 48, 57, 65, + 90, 97, 122, 41, 95, 48, 57, 65, + 90, 97, 122, 41, 48, 57, 41, 58, + 105, 109, 115, 120, 62, 95, 48, 57, + 65, 90, 97, 122, 41, 48, 57, 95, + 48, 57, 65, 90, 97, 122, 95, 48, + 57, 65, 90, 97, 122, 41, 95, 48, + 57, 65, 90, 97, 122, 95, 48, 57, + 65, 90, 97, 122, 105, 109, 115, 120, + 41, 45, 58, 105, 109, 115, 120, 46, + 92, 93, 46, 92, 93, 46, 92, 58, + 92, 93, 58, 92, 93, 58, 92, 61, + 92, 93, 61, 92, 93, 61, 92, 39, + 48, 57, 62, 45, 95, 48, 57, 65, + 90, 97, 122, 48, 57, 125, 48, 57, + 125, 48, 57, 125, 95, 125, 48, 57, + 65, 90, 97, 122, 95, 125, 48, 57, + 65, 90, 97, 122, 95, 125, 48, 57, + 65, 90, 97, 122, 95, 125, 48, 57, + 65, 90, 97, 122, 95, 48, 57, 65, + 90, 97, 122, 39, 95, 48, 57, 65, + 90, 97, 122, 95, 48, 57, 65, 90, + 97, 122, 62, 95, 48, 57, 65, 90, + 97, 122, 95, 48, 57, 65, 90, 97, + 122, 95, 125, 48, 57, 65, 90, 97, + 122, 48, 55, 125, 48, 55, 125, 48, + 57, 65, 70, 97, 102, 44, 125, 48, + 57, 125, 48, 57, 125, 48, 57, 384, + 447, 384, 447, 384, 447, 41, 41, 80, + 41, 41, 70, 41, 56, 41, 121, 97, + 109, 98, 105, 99, 101, 110, 105, 97, + 110, 101, 115, 116, 97, 110, 108, 109, + 116, 105, 110, 101, 115, 101, 117, 109, + 97, 107, 110, 103, 97, 108, 105, 112, + 111, 109, 111, 102, 111, 97, 104, 105, + 109, 105, 108, 108, 101, 103, 104, 105, + 110, 101, 115, 101, 105, 100, 110, 114, + 97, 100, 105, 97, 110, 95, 65, 98, + 111, 114, 105, 103, 105, 110, 97, 108, + 105, 97, 110, 97, 101, 109, 114, 111, + 107, 101, 101, 109, 111, 110, 116, 105, + 99, 110, 101, 105, 102, 111, 114, 109, + 112, 114, 114, 105, 111, 116, 105, 108, + 108, 105, 99, 115, 118, 101, 114, 101, + 116, 97, 110, 97, 103, 97, 114, 105, + 121, 112, 116, 105, 97, 110, 95, 72, + 105, 101, 114, 111, 103, 108, 121, 112, + 104, 115, 104, 105, 111, 112, 105, 99, + 111, 114, 103, 105, 97, 110, 97, 103, + 111, 108, 105, 116, 105, 99, 116, 104, + 105, 99, 101, 101, 107, 106, 114, 97, + 114, 97, 116, 105, 109, 117, 107, 104, + 105, 110, 117, 108, 110, 111, 111, 98, + 114, 101, 119, 114, 97, 103, 97, 110, + 97, 112, 101, 114, 105, 97, 108, 95, + 65, 114, 97, 109, 97, 105, 99, 104, + 115, 101, 114, 105, 116, 101, 100, 99, + 114, 105, 112, 116, 105, 111, 110, 97, + 108, 95, 80, 97, 104, 114, 108, 97, + 118, 105, 116, 104, 105, 97, 110, 118, + 97, 110, 101, 115, 101, 105, 110, 116, + 121, 116, 104, 105, 110, 97, 100, 97, + 97, 107, 97, 110, 97, 97, 104, 95, + 76, 105, 97, 109, 114, 111, 115, 104, + 116, 104, 105, 101, 114, 111, 116, 105, + 110, 112, 99, 104, 97, 109, 110, 115, + 98, 117, 101, 97, 114, 95, 66, 117, + 99, 100, 105, 97, 110, 105, 97, 110, + 108, 110, 97, 121, 97, 108, 97, 109, + 100, 97, 105, 99, 116, 101, 105, 95, + 77, 97, 121, 101, 107, 110, 103, 111, + 108, 105, 97, 110, 97, 110, 109, 97, + 114, 119, 95, 84, 97, 105, 95, 76, + 117, 101, 111, 104, 97, 109, 95, 100, + 67, 104, 105, 107, 105, 95, 73, 80, + 83, 84, 116, 97, 108, 105, 99, 101, + 114, 115, 105, 97, 110, 111, 117, 116, + 104, 95, 65, 114, 97, 98, 105, 97, + 110, 117, 114, 107, 105, 99, 105, 121, + 97, 109, 97, 110, 121, 97, 97, 111, + 103, 115, 95, 80, 97, 101, 110, 105, + 99, 105, 97, 110, 106, 97, 110, 103, + 110, 105, 99, 109, 117, 97, 114, 105, + 116, 97, 110, 114, 97, 115, 104, 116, + 114, 97, 97, 118, 105, 97, 110, 110, + 104, 97, 108, 97, 110, 100, 97, 110, + 101, 115, 101, 108, 114, 111, 116, 105, + 95, 78, 97, 103, 114, 105, 105, 97, + 99, 103, 105, 109, 97, 98, 108, 111, + 103, 97, 110, 119, 97, 95, 76, 84, + 86, 101, 104, 97, 109, 105, 101, 116, + 105, 108, 108, 117, 103, 117, 97, 97, + 105, 110, 97, 98, 102, 101, 116, 97, + 110, 105, 110, 97, 103, 104, 97, 114, + 105, 116, 105, 99, 105, 110, 115, 112, + 100, 123, 94, 125, 94, 46, 92, 93, + 46, 92, 93, 46, 92, 58, 92, 93, + 94, 97, 98, 99, 100, 103, 108, 112, + 115, 117, 119, 120, 58, 92, 93, 58, + 92, 93, 58, 92, 58, 92, 93, 97, + 98, 99, 100, 103, 108, 112, 115, 117, + 119, 120, 58, 92, 93, 108, 115, 58, + 92, 93, 110, 112, 58, 92, 93, 117, + 58, 92, 93, 109, 58, 92, 93, 58, + 92, 93, 58, 92, 93, 104, 58, 92, + 93, 97, 58, 92, 93, 58, 92, 93, + 58, 92, 93, 99, 58, 92, 93, 105, + 58, 92, 93, 105, 58, 92, 93, 58, + 92, 93, 58, 92, 93, 108, 58, 92, + 93, 97, 58, 92, 93, 110, 58, 92, + 93, 107, 58, 92, 93, 58, 92, 93, + 58, 92, 93, 110, 58, 92, 93, 116, + 58, 92, 93, 114, 58, 92, 93, 108, + 58, 92, 93, 58, 92, 93, 58, 92, + 93, 105, 58, 92, 93, 103, 58, 92, + 93, 105, 58, 92, 93, 116, 58, 92, + 93, 58, 92, 93, 58, 92, 93, 114, + 58, 92, 93, 97, 58, 92, 93, 112, + 58, 92, 93, 104, 58, 92, 93, 58, + 92, 93, 58, 92, 93, 111, 58, 92, + 93, 119, 58, 92, 93, 101, 58, 92, + 93, 114, 58, 92, 93, 58, 92, 93, + 58, 92, 93, 114, 117, 58, 92, 93, + 105, 58, 92, 93, 110, 58, 92, 93, + 116, 58, 92, 93, 58, 92, 93, 58, + 92, 93, 110, 58, 92, 93, 99, 58, + 92, 93, 116, 58, 92, 93, 58, 92, + 93, 58, 92, 93, 112, 58, 92, 93, + 97, 58, 92, 93, 99, 58, 92, 93, + 101, 58, 92, 93, 58, 92, 93, 58, + 92, 93, 112, 58, 92, 93, 112, 58, + 92, 93, 101, 58, 92, 93, 114, 58, + 92, 93, 58, 92, 93, 58, 92, 93, + 111, 58, 92, 93, 114, 58, 92, 93, + 100, 58, 92, 93, 58, 92, 93, 58, + 92, 93, 100, 58, 92, 93, 105, 58, + 92, 93, 103, 58, 92, 93, 105, 58, + 92, 93, 116, 58, 92, 93, 58, 92, + 93, 58, 92, 93, 108, 115, 58, 92, + 93, 110, 112, 58, 92, 93, 117, 58, + 92, 93, 109, 58, 92, 93, 58, 92, + 93, 58, 92, 93, 104, 58, 92, 93, + 97, 58, 92, 93, 58, 92, 93, 58, + 92, 93, 99, 58, 92, 93, 105, 58, + 92, 93, 105, 58, 92, 93, 58, 92, + 93, 58, 92, 93, 108, 58, 92, 93, + 97, 58, 92, 93, 110, 58, 92, 93, + 107, 58, 92, 93, 58, 92, 93, 58, + 92, 93, 110, 58, 92, 93, 116, 58, + 92, 93, 114, 58, 92, 93, 108, 58, + 92, 93, 58, 92, 93, 58, 92, 93, + 105, 58, 92, 93, 103, 58, 92, 93, + 105, 58, 92, 93, 116, 58, 92, 93, + 58, 92, 93, 58, 92, 93, 114, 58, + 92, 93, 97, 58, 92, 93, 112, 58, + 92, 93, 104, 58, 92, 93, 58, 92, + 93, 58, 92, 93, 111, 58, 92, 93, + 119, 58, 92, 93, 101, 58, 92, 93, + 114, 58, 92, 93, 58, 92, 93, 58, + 92, 93, 114, 117, 58, 92, 93, 105, + 58, 92, 93, 110, 58, 92, 93, 116, + 58, 92, 93, 58, 92, 93, 58, 92, + 93, 110, 58, 92, 93, 99, 58, 92, + 93, 116, 58, 92, 93, 58, 92, 93, + 58, 92, 93, 112, 58, 92, 93, 97, + 58, 92, 93, 99, 58, 92, 93, 101, + 58, 92, 93, 58, 92, 93, 58, 92, + 93, 112, 58, 92, 93, 112, 58, 92, + 93, 101, 58, 92, 93, 114, 58, 92, + 93, 58, 92, 93, 58, 92, 93, 111, + 58, 92, 93, 114, 58, 92, 93, 100, + 58, 92, 93, 58, 92, 93, 58, 92, + 93, 100, 58, 92, 93, 105, 58, 92, + 93, 103, 58, 92, 93, 105, 58, 92, + 93, 116, 58, 92, 93, 58, 92, 93, + 61, 92, 93, 61, 92, 93, 61, 92, + 48, 55, 125, 48, 55, 125, 48, 57, + 65, 70, 97, 102, 384, 447, 384, 447, + 384, 447, 384, 447, 384, 447, 384, 447, + 384, 447, 384, 447, 384, 447, 0, 32, + 36, 40, 41, 42, 43, 46, 63, 91, + 92, 94, 123, 124, 1315, 1571, 1, 8, + 9, 13, 14, 34, 37, 255, 384, 447, + 448, 479, 480, 495, 496, 503, 504, 511, + 42, 63, 95, 48, 57, 65, 90, 97, + 122, 95, 48, 57, 65, 90, 97, 122, + 39, 48, 60, 63, 82, 95, 49, 55, + 56, 57, 65, 90, 97, 122, 48, 57, + 105, 109, 115, 120, 48, 57, 41, 48, + 57, 33, 61, 95, 48, 57, 65, 90, + 97, 122, 123, 41, 48, 57, 60, 61, + 62, 41, 45, 58, 105, 109, 115, 120, + 43, 63, 43, 63, 43, 63, 46, 58, + 61, 48, 65, 66, 67, 68, 69, 71, + 72, 75, 76, 78, 80, 81, 82, 83, + 85, 86, 87, 88, 90, 97, 98, 99, + 100, 101, 102, 103, 104, 107, 108, 110, + 111, 112, 114, 115, 116, 117, 118, 119, + 120, 122, 49, 55, 56, 57, 48, 55, + 48, 55, 48, 55, 56, 57, 48, 55, + 56, 57, 48, 57, 123, 39, 45, 60, + 123, 48, 57, 48, 57, 48, 57, 48, + 57, 48, 57, 39, 60, 123, 123, 123, + 123, 48, 57, 65, 70, 97, 102, 48, + 57, 65, 70, 97, 102, 48, 57, 65, + 70, 97, 102, 48, 57, 43, 63, 384, + 447, 384, 447, 384, 447, 41, 85, 41, + 41, 67, 84, 65, 66, 67, 68, 69, + 71, 72, 73, 74, 75, 76, 77, 78, + 79, 80, 82, 83, 84, 85, 86, 88, + 89, 90, 110, 114, 118, 97, 101, 111, + 114, 117, 97, 99, 102, 104, 110, 111, + 115, 117, 121, 109, 112, 101, 103, 116, + 101, 108, 111, 114, 117, 97, 101, 105, + 103, 117, 109, 110, 97, 97, 104, 38, + 97, 101, 105, 108, 109, 111, 116, 117, + 121, 97, 99, 101, 110, 111, 121, 101, + 100, 101, 107, 108, 111, 103, 108, 114, + 115, 99, 100, 101, 102, 104, 105, 111, + 115, 101, 117, 97, 99, 104, 105, 107, + 109, 111, 117, 121, 97, 101, 104, 105, + 103, 97, 97, 112, 115, 119, 105, 108, + 112, 115, 67, 76, 77, 78, 80, 83, + 90, 45, 91, 92, 93, 0, 255, 384, + 447, 448, 479, 480, 495, 496, 503, 504, + 511, 46, 58, 61, 48, 68, 69, 72, + 76, 78, 80, 81, 83, 85, 86, 87, + 97, 98, 99, 100, 101, 102, 103, 104, + 108, 110, 111, 112, 114, 115, 116, 117, + 118, 119, 120, 49, 55, 56, 57, 65, + 90, 105, 122, 48, 55, 48, 55, 48, + 55, 48, 55, 123, 123, 123, 123, 48, + 57, 65, 70, 97, 102, 48, 57, 65, + 70, 97, 102, 48, 57, 65, 70, 97, + 102, 384, 447, 384, 447, 384, 447, 92, + 1117, 1118, -128, 91, 95, 127, 861, 862, + 69, 81, 92, 0, 255, 384, 447, 448, + 479, 480, 495, 496, 503, 504, 511, 69, + 384, 447, 384, 447, 384, 447, 92, 0, + 255, 384, 447, 448, 479, 480, 495, 496, + 503, 504, 511, 69, 384, 447, 384, 447, + 384, 447, 41, 10, 0 +}; + +static const char _regex_single_lengths[] = { + 0, 1, 20, 2, 2, 1, 2, 1, + 2, 2, 2, 1, 2, 3, 2, 3, + 1, 2, 2, 1, 6, 2, 1, 1, + 1, 2, 1, 4, 7, 3, 3, 2, + 3, 3, 2, 3, 3, 2, 1, 0, + 1, 2, 0, 1, 1, 1, 2, 2, + 2, 2, 1, 2, 1, 2, 1, 2, + 0, 1, 1, 2, 1, 1, 0, 0, + 0, 1, 2, 1, 2, 2, 1, 1, + 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 3, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 1, 1, 1, + 1, 1, 2, 1, 1, 1, 1, 1, + 1, 1, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 2, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 4, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 1, 1, 1, 1, 1, 3, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 1, 1, 1, 1, 1, 2, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 2, 1, 1, 1, 1, 1, + 1, 4, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 3, 2, + 1, 1, 1, 1, 1, 1, 1, 1, + 3, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 2, + 1, 1, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 3, 3, 2, 15, 3, + 3, 2, 14, 5, 5, 4, 4, 3, + 3, 4, 4, 3, 3, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 4, 4, 4, 4, 3, 3, 4, 4, + 4, 4, 3, 3, 4, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 5, 4, 4, 4, 3, 3, 4, 4, + 4, 3, 3, 4, 4, 4, 4, 3, + 3, 4, 4, 4, 4, 3, 3, 4, + 4, 4, 3, 3, 4, 4, 4, 4, + 4, 3, 3, 5, 5, 4, 4, 3, + 3, 4, 4, 3, 3, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 4, 4, 4, 4, 3, 3, 4, 4, + 4, 4, 3, 3, 4, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 5, 4, 4, 4, 3, 3, 4, 4, + 4, 3, 3, 4, 4, 4, 4, 3, + 3, 4, 4, 4, 4, 3, 3, 4, + 4, 4, 3, 3, 4, 4, 4, 4, + 4, 3, 3, 3, 3, 2, 0, 1, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 16, 2, 1, 1, 6, 0, + 4, 1, 3, 1, 1, 3, 7, 2, + 2, 2, 3, 41, 0, 0, 0, 0, + 0, 1, 0, 4, 0, 0, 0, 0, + 3, 1, 1, 1, 0, 0, 0, 2, + 0, 0, 0, 2, 1, 3, 23, 3, + 5, 9, 2, 1, 2, 5, 3, 2, + 2, 1, 2, 10, 6, 1, 5, 4, + 8, 2, 9, 4, 1, 1, 4, 1, + 3, 0, 7, 4, 3, 31, 0, 0, + 0, 0, 1, 0, 1, 1, 1, 0, + 0, 0, 0, 0, 3, 2, 1, 1, + 0, 0, 0, 1, 1, 0, 0, 0, + 1, 1 +}; + +static const char _regex_range_lengths[] = { + 0, 0, 1, 3, 3, 3, 3, 0, + 3, 3, 3, 3, 3, 0, 0, 3, + 3, 3, 3, 1, 0, 3, 1, 3, + 3, 3, 3, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, + 0, 3, 1, 1, 1, 0, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 1, 1, 3, 1, 1, 1, 1, 1, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, + 3, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 9, 0, 3, 3, 4, 1, + 1, 1, 3, 0, 1, 0, 0, 0, + 0, 0, 0, 2, 1, 1, 2, 2, + 1, 0, 0, 1, 1, 1, 1, 1, + 0, 0, 0, 3, 3, 3, 1, 0, + 1, 1, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 6, 0, 4, 1, 1, + 1, 1, 0, 0, 0, 0, 3, 3, + 3, 1, 1, 1, 3, 0, 6, 0, + 1, 1, 1, 6, 0, 1, 1, 1, + 0, 0 +}; + +static const short _regex_index_offsets[] = { + 0, 0, 2, 24, 30, 36, 41, 47, + 49, 55, 61, 67, 72, 78, 82, 85, + 92, 97, 103, 109, 112, 119, 125, 128, + 133, 138, 144, 149, 154, 162, 166, 170, + 173, 177, 181, 184, 188, 192, 195, 197, + 199, 201, 207, 209, 212, 215, 217, 223, + 229, 235, 241, 246, 252, 257, 263, 268, + 274, 276, 279, 284, 288, 291, 294, 296, + 298, 300, 302, 305, 307, 310, 313, 315, + 317, 320, 322, 324, 326, 328, 330, 332, + 334, 336, 338, 340, 342, 344, 346, 350, + 352, 354, 356, 358, 360, 362, 364, 366, + 368, 370, 372, 374, 376, 378, 380, 382, + 384, 386, 388, 390, 392, 395, 397, 399, + 401, 403, 405, 408, 410, 412, 414, 416, + 418, 420, 422, 425, 427, 429, 431, 433, + 435, 437, 439, 441, 443, 445, 447, 449, + 451, 453, 455, 457, 459, 461, 463, 466, + 468, 470, 472, 474, 476, 478, 480, 482, + 484, 486, 488, 490, 492, 494, 496, 498, + 500, 502, 504, 507, 509, 511, 513, 515, + 517, 519, 521, 523, 525, 528, 530, 532, + 534, 536, 538, 540, 542, 544, 546, 548, + 550, 552, 554, 556, 558, 560, 562, 564, + 566, 568, 570, 572, 574, 576, 578, 580, + 582, 584, 586, 588, 590, 592, 594, 596, + 598, 600, 602, 604, 606, 608, 610, 612, + 614, 616, 618, 620, 622, 624, 626, 628, + 630, 632, 634, 636, 638, 640, 643, 645, + 647, 649, 651, 653, 655, 657, 659, 661, + 663, 665, 667, 669, 671, 673, 675, 677, + 679, 681, 683, 685, 687, 689, 691, 693, + 695, 697, 699, 701, 703, 705, 707, 709, + 711, 713, 715, 717, 719, 721, 723, 726, + 728, 730, 732, 734, 736, 738, 740, 742, + 744, 746, 748, 750, 752, 754, 756, 758, + 760, 762, 764, 767, 769, 771, 773, 775, + 777, 779, 781, 783, 785, 787, 789, 791, + 793, 795, 797, 802, 804, 806, 808, 810, + 812, 814, 816, 818, 820, 822, 824, 826, + 828, 830, 832, 834, 836, 839, 841, 843, + 845, 847, 849, 851, 853, 855, 857, 860, + 862, 864, 866, 868, 870, 872, 876, 878, + 880, 882, 884, 886, 888, 890, 892, 895, + 897, 899, 901, 903, 905, 907, 910, 912, + 914, 916, 918, 920, 922, 924, 926, 928, + 930, 932, 934, 936, 938, 940, 942, 944, + 946, 948, 950, 952, 954, 956, 958, 960, + 962, 964, 966, 968, 970, 972, 974, 976, + 978, 980, 982, 984, 986, 988, 990, 992, + 994, 996, 998, 1001, 1003, 1005, 1007, 1009, + 1011, 1013, 1018, 1020, 1022, 1024, 1026, 1028, + 1030, 1032, 1034, 1036, 1038, 1040, 1042, 1044, + 1046, 1048, 1050, 1052, 1054, 1056, 1058, 1060, + 1062, 1064, 1066, 1068, 1070, 1072, 1074, 1076, + 1078, 1080, 1082, 1084, 1086, 1088, 1090, 1093, + 1095, 1097, 1099, 1101, 1103, 1105, 1107, 1109, + 1111, 1113, 1115, 1117, 1119, 1121, 1123, 1125, + 1127, 1129, 1131, 1134, 1136, 1138, 1140, 1142, + 1144, 1146, 1148, 1150, 1152, 1154, 1156, 1158, + 1160, 1162, 1164, 1166, 1168, 1170, 1172, 1174, + 1176, 1178, 1180, 1182, 1184, 1186, 1188, 1190, + 1192, 1194, 1197, 1199, 1201, 1203, 1205, 1207, + 1209, 1211, 1213, 1215, 1217, 1219, 1221, 1225, + 1228, 1230, 1232, 1234, 1236, 1238, 1240, 1242, + 1244, 1248, 1250, 1252, 1254, 1256, 1258, 1260, + 1262, 1264, 1266, 1268, 1270, 1272, 1274, 1276, + 1279, 1281, 1283, 1286, 1288, 1290, 1292, 1294, + 1296, 1298, 1300, 1302, 1304, 1306, 1308, 1310, + 1312, 1314, 1316, 1318, 1320, 1322, 1324, 1326, + 1328, 1330, 1332, 1334, 1338, 1342, 1345, 1361, + 1365, 1369, 1372, 1387, 1393, 1399, 1404, 1409, + 1413, 1417, 1422, 1427, 1431, 1435, 1440, 1445, + 1450, 1454, 1458, 1463, 1468, 1473, 1478, 1482, + 1486, 1491, 1496, 1501, 1506, 1510, 1514, 1519, + 1524, 1529, 1534, 1538, 1542, 1547, 1552, 1557, + 1562, 1566, 1570, 1575, 1580, 1585, 1590, 1594, + 1598, 1604, 1609, 1614, 1619, 1623, 1627, 1632, + 1637, 1642, 1646, 1650, 1655, 1660, 1665, 1670, + 1674, 1678, 1683, 1688, 1693, 1698, 1702, 1706, + 1711, 1716, 1721, 1725, 1729, 1734, 1739, 1744, + 1749, 1754, 1758, 1762, 1768, 1774, 1779, 1784, + 1788, 1792, 1797, 1802, 1806, 1810, 1815, 1820, + 1825, 1829, 1833, 1838, 1843, 1848, 1853, 1857, + 1861, 1866, 1871, 1876, 1881, 1885, 1889, 1894, + 1899, 1904, 1909, 1913, 1917, 1922, 1927, 1932, + 1937, 1941, 1945, 1950, 1955, 1960, 1965, 1969, + 1973, 1979, 1984, 1989, 1994, 1998, 2002, 2007, + 2012, 2017, 2021, 2025, 2030, 2035, 2040, 2045, + 2049, 2053, 2058, 2063, 2068, 2073, 2077, 2081, + 2086, 2091, 2096, 2100, 2104, 2109, 2114, 2119, + 2124, 2129, 2133, 2137, 2141, 2145, 2148, 2150, + 2153, 2158, 2160, 2162, 2164, 2166, 2168, 2170, + 2172, 2174, 2176, 2202, 2205, 2210, 2215, 2226, + 2228, 2234, 2237, 2244, 2246, 2249, 2253, 2261, + 2264, 2267, 2270, 2274, 2318, 2320, 2322, 2325, + 2328, 2330, 2332, 2333, 2339, 2341, 2343, 2345, + 2347, 2351, 2353, 2355, 2360, 2364, 2368, 2370, + 2373, 2375, 2377, 2379, 2382, 2384, 2388, 2412, + 2416, 2422, 2432, 2435, 2437, 2440, 2446, 2450, + 2453, 2456, 2458, 2461, 2472, 2479, 2481, 2487, + 2492, 2501, 2504, 2514, 2519, 2521, 2523, 2528, + 2530, 2534, 2535, 2543, 2554, 2558, 2594, 2596, + 2598, 2600, 2602, 2604, 2605, 2607, 2609, 2614, + 2618, 2622, 2624, 2626, 2628, 2635, 2638, 2646, + 2648, 2650, 2652, 2654, 2662, 2664, 2666, 2668, + 2670, 2672 +}; + +static const short _regex_indicies[] = { + 0, 1, 3, 4, 5, 6, 7, 8, + 9, 10, 12, 13, 14, 15, 16, 17, + 18, 19, 19, 19, 19, 20, 11, 2, + 22, 23, 23, 23, 23, 21, 24, 25, + 25, 25, 25, 21, 27, 27, 27, 27, + 26, 28, 27, 27, 27, 27, 26, 29, + 26, 29, 30, 30, 30, 30, 26, 31, + 30, 32, 30, 30, 26, 29, 30, 32, + 30, 30, 26, 33, 33, 33, 33, 26, + 28, 33, 33, 33, 33, 26, 34, 35, + 36, 26, 37, 38, 26, 39, 40, 30, + 41, 30, 30, 26, 42, 42, 42, 42, + 26, 40, 42, 42, 42, 42, 26, 40, + 30, 41, 30, 30, 26, 43, 44, 21, + 45, 46, 47, 47, 47, 47, 21, 24, + 48, 48, 48, 48, 21, 49, 50, 21, + 48, 48, 48, 48, 21, 51, 51, 51, + 51, 21, 52, 51, 51, 51, 51, 21, + 23, 23, 23, 23, 21, 47, 47, 47, + 47, 21, 45, 53, 46, 54, 54, 54, + 54, 21, 57, 58, 55, 56, 57, 58, + 59, 56, 57, 58, 56, 61, 62, 55, + 60, 61, 62, 63, 60, 61, 62, 60, + 65, 66, 55, 64, 65, 66, 59, 64, + 65, 66, 64, 69, 68, 70, 67, 69, + 71, 72, 74, 73, 74, 74, 67, 75, + 67, 77, 76, 67, 77, 78, 67, 77, + 67, 74, 80, 79, 74, 74, 67, 74, + 80, 81, 74, 74, 67, 74, 80, 74, + 74, 74, 67, 74, 82, 74, 74, 74, + 67, 84, 84, 84, 84, 83, 85, 84, + 84, 84, 84, 83, 86, 86, 86, 86, + 83, 87, 86, 86, 86, 86, 83, 88, + 88, 88, 88, 83, 88, 89, 88, 88, + 88, 83, 91, 90, 92, 91, 90, 95, + 94, 94, 94, 93, 97, 99, 98, 96, + 101, 100, 96, 102, 100, 96, 104, 103, + 105, 103, 106, 103, 109, 108, 109, 110, + 108, 111, 108, 109, 112, 108, 113, 114, + 108, 115, 108, 117, 116, 118, 119, 116, + 120, 116, 121, 116, 122, 116, 123, 116, + 124, 116, 125, 116, 126, 116, 127, 116, + 128, 116, 129, 116, 130, 116, 131, 116, + 132, 116, 133, 134, 135, 116, 136, 116, + 137, 116, 138, 116, 139, 116, 140, 116, + 141, 116, 142, 116, 143, 116, 144, 116, + 145, 116, 146, 116, 147, 116, 148, 116, + 149, 116, 150, 116, 151, 116, 152, 116, + 153, 116, 154, 116, 155, 116, 156, 116, + 157, 158, 116, 159, 116, 160, 116, 161, + 116, 162, 116, 163, 116, 164, 165, 116, + 166, 116, 167, 116, 168, 116, 169, 116, + 170, 116, 171, 116, 172, 116, 174, 175, + 173, 176, 173, 177, 173, 178, 173, 179, + 173, 180, 173, 181, 173, 182, 173, 183, + 173, 184, 173, 185, 173, 186, 173, 187, + 173, 188, 173, 189, 173, 190, 173, 191, + 173, 192, 173, 193, 173, 194, 173, 195, + 196, 173, 197, 173, 198, 173, 199, 173, + 200, 173, 201, 173, 202, 173, 204, 203, + 205, 203, 206, 203, 207, 203, 208, 203, + 209, 203, 210, 173, 211, 173, 212, 173, + 213, 173, 214, 173, 215, 173, 216, 173, + 217, 218, 173, 219, 173, 220, 173, 221, + 173, 222, 173, 223, 173, 224, 173, 225, + 173, 226, 173, 227, 173, 228, 229, 116, + 230, 116, 231, 116, 232, 116, 233, 116, + 234, 116, 235, 116, 236, 116, 237, 116, + 238, 116, 239, 116, 240, 116, 241, 116, + 242, 116, 243, 116, 244, 116, 245, 116, + 246, 116, 247, 116, 248, 116, 249, 116, + 250, 116, 251, 116, 252, 116, 253, 116, + 254, 116, 255, 116, 256, 116, 257, 116, + 258, 116, 259, 116, 260, 116, 261, 116, + 262, 116, 263, 116, 264, 116, 265, 116, + 266, 116, 267, 116, 268, 116, 269, 116, + 270, 116, 271, 116, 272, 116, 273, 116, + 274, 116, 275, 116, 276, 116, 277, 116, + 278, 116, 279, 116, 280, 116, 281, 116, + 282, 116, 283, 116, 284, 116, 285, 116, + 286, 287, 116, 288, 116, 289, 116, 290, + 116, 291, 116, 292, 116, 293, 116, 294, + 116, 295, 116, 296, 116, 297, 116, 298, + 116, 300, 299, 301, 299, 302, 299, 303, + 299, 304, 299, 305, 116, 306, 116, 307, + 116, 308, 116, 309, 116, 310, 116, 311, + 116, 312, 116, 313, 116, 314, 116, 315, + 116, 316, 116, 317, 116, 318, 116, 319, + 116, 320, 116, 321, 116, 322, 116, 323, + 116, 324, 116, 325, 116, 326, 116, 327, + 116, 328, 116, 329, 330, 116, 331, 116, + 332, 116, 333, 116, 334, 116, 335, 116, + 336, 116, 337, 116, 338, 116, 339, 116, + 340, 116, 341, 116, 342, 116, 343, 116, + 344, 116, 345, 116, 346, 116, 347, 116, + 348, 116, 349, 116, 350, 351, 116, 352, + 116, 353, 116, 354, 116, 355, 116, 356, + 116, 357, 116, 358, 116, 359, 116, 360, + 116, 361, 116, 362, 116, 363, 116, 364, + 116, 365, 116, 366, 116, 367, 368, 369, + 370, 116, 371, 116, 372, 116, 373, 116, + 374, 116, 375, 116, 376, 116, 377, 116, + 378, 116, 379, 116, 380, 116, 381, 116, + 382, 116, 383, 116, 384, 116, 385, 116, + 386, 116, 387, 116, 388, 389, 116, 390, + 116, 391, 116, 392, 116, 393, 116, 394, + 116, 395, 116, 396, 116, 397, 116, 398, + 116, 400, 401, 399, 402, 399, 403, 399, + 404, 399, 405, 399, 406, 399, 407, 399, + 408, 409, 410, 399, 411, 399, 412, 399, + 413, 399, 414, 399, 415, 399, 416, 399, + 417, 399, 418, 399, 419, 420, 399, 421, + 399, 422, 399, 423, 399, 424, 399, 425, + 399, 426, 399, 428, 429, 427, 430, 427, + 431, 427, 432, 427, 433, 427, 434, 427, + 435, 427, 436, 427, 437, 427, 438, 427, + 439, 427, 441, 440, 442, 440, 443, 440, + 444, 440, 445, 440, 446, 440, 447, 440, + 448, 440, 449, 440, 450, 427, 451, 427, + 452, 427, 453, 427, 454, 427, 455, 427, + 456, 427, 457, 427, 458, 427, 459, 427, + 460, 427, 461, 427, 463, 462, 464, 462, + 465, 462, 466, 462, 467, 462, 468, 462, + 469, 462, 470, 462, 471, 462, 472, 462, + 473, 116, 474, 116, 475, 116, 476, 477, + 116, 478, 116, 479, 116, 480, 116, 481, + 116, 482, 116, 483, 116, 484, 485, 486, + 487, 116, 488, 116, 489, 116, 490, 116, + 491, 116, 492, 116, 493, 116, 494, 116, + 495, 116, 496, 116, 497, 116, 498, 116, + 499, 116, 500, 116, 501, 116, 502, 116, + 503, 116, 504, 116, 505, 116, 506, 116, + 507, 116, 508, 116, 509, 116, 510, 116, + 511, 116, 512, 116, 513, 116, 514, 116, + 515, 116, 516, 116, 517, 116, 518, 116, + 519, 116, 520, 116, 521, 116, 522, 116, + 523, 116, 525, 526, 524, 527, 524, 528, + 524, 529, 524, 530, 524, 531, 524, 532, + 524, 533, 524, 534, 524, 535, 524, 536, + 524, 537, 524, 538, 524, 539, 116, 540, + 116, 541, 116, 542, 116, 543, 116, 544, + 116, 545, 116, 547, 548, 546, 549, 546, + 550, 546, 551, 546, 552, 546, 553, 546, + 554, 546, 555, 546, 556, 546, 557, 546, + 558, 546, 559, 546, 560, 546, 561, 546, + 562, 546, 563, 546, 564, 546, 565, 546, + 566, 546, 567, 546, 568, 546, 569, 546, + 570, 546, 571, 546, 572, 546, 573, 546, + 574, 546, 575, 546, 576, 546, 577, 546, + 578, 546, 579, 580, 546, 581, 546, 582, + 546, 583, 546, 584, 546, 585, 546, 586, + 546, 587, 546, 588, 546, 589, 546, 590, + 546, 591, 546, 592, 546, 593, 594, 595, + 116, 596, 597, 116, 598, 116, 599, 116, + 600, 116, 601, 116, 602, 116, 603, 116, + 604, 116, 605, 116, 606, 607, 608, 116, + 609, 116, 610, 116, 611, 116, 612, 116, + 613, 116, 614, 116, 615, 116, 616, 116, + 617, 116, 618, 116, 619, 116, 620, 116, + 621, 116, 622, 116, 623, 624, 116, 625, + 116, 626, 116, 627, 628, 116, 629, 116, + 630, 116, 631, 116, 632, 116, 633, 116, + 634, 116, 635, 116, 636, 116, 637, 116, + 638, 116, 639, 116, 640, 116, 641, 116, + 642, 116, 643, 116, 644, 116, 645, 116, + 646, 116, 647, 116, 648, 116, 650, 649, + 652, 651, 653, 649, 649, 651, 656, 657, + 654, 655, 656, 657, 658, 655, 656, 657, + 655, 660, 661, 654, 662, 663, 664, 665, + 666, 667, 668, 669, 670, 671, 672, 673, + 659, 660, 661, 654, 659, 660, 661, 674, + 659, 660, 661, 659, 660, 661, 654, 675, + 676, 677, 678, 679, 680, 681, 682, 683, + 684, 685, 659, 660, 661, 654, 686, 687, + 659, 660, 661, 654, 688, 689, 659, 660, + 661, 654, 690, 659, 660, 661, 654, 691, + 659, 692, 661, 654, 659, 660, 661, 693, + 659, 660, 661, 654, 694, 659, 660, 661, + 654, 695, 659, 696, 661, 654, 659, 660, + 661, 697, 659, 660, 661, 654, 698, 659, + 660, 661, 654, 699, 659, 660, 661, 654, + 700, 659, 701, 661, 654, 659, 660, 661, + 702, 659, 660, 661, 654, 703, 659, 660, + 661, 654, 704, 659, 660, 661, 654, 705, + 659, 660, 661, 654, 706, 659, 707, 661, + 654, 659, 660, 661, 708, 659, 660, 661, + 654, 709, 659, 660, 661, 654, 710, 659, + 660, 661, 654, 711, 659, 660, 661, 654, + 712, 659, 713, 661, 654, 659, 660, 661, + 714, 659, 660, 661, 654, 715, 659, 660, + 661, 654, 716, 659, 660, 661, 654, 717, + 659, 660, 661, 654, 718, 659, 719, 661, + 654, 659, 660, 661, 720, 659, 660, 661, + 654, 721, 659, 660, 661, 654, 722, 659, + 660, 661, 654, 723, 659, 660, 661, 654, + 724, 659, 725, 661, 654, 659, 660, 661, + 726, 659, 660, 661, 654, 727, 659, 660, + 661, 654, 728, 659, 660, 661, 654, 729, + 659, 660, 661, 654, 730, 659, 731, 661, + 654, 659, 660, 661, 732, 659, 660, 661, + 654, 733, 734, 659, 660, 661, 654, 735, + 659, 660, 661, 654, 736, 659, 660, 661, + 654, 737, 659, 738, 661, 654, 659, 660, + 661, 739, 659, 660, 661, 654, 740, 659, + 660, 661, 654, 741, 659, 660, 661, 654, + 742, 659, 743, 661, 654, 659, 660, 661, + 744, 659, 660, 661, 654, 745, 659, 660, + 661, 654, 746, 659, 660, 661, 654, 747, + 659, 660, 661, 654, 748, 659, 749, 661, + 654, 659, 660, 661, 750, 659, 660, 661, + 654, 751, 659, 660, 661, 654, 752, 659, + 660, 661, 654, 753, 659, 660, 661, 654, + 754, 659, 755, 661, 654, 659, 660, 661, + 756, 659, 660, 661, 654, 757, 659, 660, + 661, 654, 758, 659, 660, 661, 654, 759, + 659, 760, 661, 654, 659, 660, 661, 761, + 659, 660, 661, 654, 762, 659, 660, 661, + 654, 763, 659, 660, 661, 654, 764, 659, + 660, 661, 654, 765, 659, 660, 661, 654, + 766, 659, 767, 661, 654, 659, 660, 661, + 768, 659, 660, 661, 654, 769, 770, 659, + 660, 661, 654, 771, 772, 659, 660, 661, + 654, 773, 659, 660, 661, 654, 774, 659, + 775, 661, 654, 659, 660, 661, 776, 659, + 660, 661, 654, 777, 659, 660, 661, 654, + 778, 659, 779, 661, 654, 659, 660, 661, + 780, 659, 660, 661, 654, 781, 659, 660, + 661, 654, 782, 659, 660, 661, 654, 783, + 659, 784, 661, 654, 659, 660, 661, 785, + 659, 660, 661, 654, 786, 659, 660, 661, + 654, 787, 659, 660, 661, 654, 788, 659, + 660, 661, 654, 789, 659, 790, 661, 654, + 659, 660, 661, 791, 659, 660, 661, 654, + 792, 659, 660, 661, 654, 793, 659, 660, + 661, 654, 794, 659, 660, 661, 654, 795, + 659, 796, 661, 654, 659, 660, 661, 797, + 659, 660, 661, 654, 798, 659, 660, 661, + 654, 799, 659, 660, 661, 654, 800, 659, + 660, 661, 654, 801, 659, 802, 661, 654, + 659, 660, 661, 803, 659, 660, 661, 654, + 804, 659, 660, 661, 654, 805, 659, 660, + 661, 654, 806, 659, 660, 661, 654, 807, + 659, 808, 661, 654, 659, 660, 661, 809, + 659, 660, 661, 654, 810, 659, 660, 661, + 654, 811, 659, 660, 661, 654, 812, 659, + 660, 661, 654, 813, 659, 814, 661, 654, + 659, 660, 661, 815, 659, 660, 661, 654, + 816, 817, 659, 660, 661, 654, 818, 659, + 660, 661, 654, 819, 659, 660, 661, 654, + 820, 659, 821, 661, 654, 659, 660, 661, + 822, 659, 660, 661, 654, 823, 659, 660, + 661, 654, 824, 659, 660, 661, 654, 825, + 659, 826, 661, 654, 659, 660, 661, 827, + 659, 660, 661, 654, 828, 659, 660, 661, + 654, 829, 659, 660, 661, 654, 830, 659, + 660, 661, 654, 831, 659, 832, 661, 654, + 659, 660, 661, 833, 659, 660, 661, 654, + 834, 659, 660, 661, 654, 835, 659, 660, + 661, 654, 836, 659, 660, 661, 654, 837, + 659, 838, 661, 654, 659, 660, 661, 839, + 659, 660, 661, 654, 840, 659, 660, 661, + 654, 841, 659, 660, 661, 654, 842, 659, + 843, 661, 654, 659, 660, 661, 844, 659, + 660, 661, 654, 845, 659, 660, 661, 654, + 846, 659, 660, 661, 654, 847, 659, 660, + 661, 654, 848, 659, 660, 661, 654, 849, + 659, 850, 661, 654, 659, 660, 661, 851, + 659, 853, 854, 654, 852, 853, 854, 658, + 852, 853, 854, 852, 856, 855, 857, 856, + 855, 860, 859, 859, 859, 858, 862, 861, + 863, 861, 864, 861, 866, 865, 867, 865, + 868, 865, 870, 869, 871, 869, 872, 869, + 873, 876, 877, 878, 879, 880, 881, 882, + 883, 884, 885, 886, 887, 888, 875, 893, + 875, 876, 875, 875, 889, 890, 891, 892, + 889, 874, 895, 896, 894, 23, 23, 23, + 23, 897, 25, 25, 25, 25, 897, 899, + 30, 902, 903, 904, 30, 900, 901, 30, + 30, 898, 44, 897, 47, 47, 47, 47, + 44, 897, 43, 44, 897, 905, 906, 48, + 48, 48, 48, 897, 907, 897, 49, 50, + 897, 908, 909, 910, 897, 45, 53, 46, + 54, 54, 54, 54, 897, 912, 913, 911, + 915, 916, 914, 918, 919, 917, 56, 60, + 64, 920, 923, 926, 927, 928, 929, 930, + 931, 932, 933, 934, 934, 935, 936, 937, + 938, 934, 939, 940, 941, 942, 943, 944, + 945, 946, 947, 948, 949, 950, 951, 934, + 952, 953, 954, 955, 956, 957, 934, 958, + 959, 960, 961, 924, 925, 922, 963, 962, + 964, 962, 966, 967, 965, 969, 967, 968, + 967, 970, 973, 972, 975, 68, 977, 71, + 979, 978, 976, 981, 980, 982, 980, 984, + 983, 985, 983, 987, 988, 989, 986, 991, + 990, 994, 993, 999, 996, 997, 998, 995, + 1000, 1001, 1002, 995, 94, 94, 94, 1003, + 98, 1004, 1006, 1007, 1005, 1009, 1008, 1010, + 1008, 1011, 1008, 1013, 1014, 1012, 109, 108, + 109, 1016, 1017, 108, 1019, 1020, 1021, 1022, + 1023, 1024, 1025, 1026, 1027, 1028, 1029, 1030, + 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, + 1039, 1040, 1041, 1018, 1043, 1044, 1045, 1042, + 1046, 1047, 1048, 1049, 1050, 1042, 1052, 1053, + 1054, 1055, 1056, 1057, 1058, 1059, 1060, 1051, + 1062, 1063, 1061, 1064, 1042, 1065, 1066, 1042, + 1067, 1068, 1069, 1070, 1071, 1042, 1072, 1073, + 1074, 1042, 1076, 1077, 1075, 1078, 1079, 1042, + 1080, 1042, 1081, 1082, 1042, 1084, 1085, 1086, + 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1083, + 1095, 1096, 1097, 1098, 1099, 1100, 1094, 1102, + 1101, 1104, 1105, 1106, 1107, 1108, 1103, 1109, + 1110, 1111, 1112, 1042, 1114, 1115, 1116, 1117, + 1118, 1119, 1120, 1121, 1113, 1122, 1123, 1042, + 1125, 1126, 1127, 1128, 1129, 1130, 1131, 1132, + 1133, 1124, 1134, 1135, 1136, 1137, 1042, 1138, + 1042, 1139, 1042, 1140, 1141, 1142, 1143, 1042, + 1144, 1042, 1146, 1147, 1148, 1145, 649, 1150, + 1151, 1152, 1153, 1154, 1155, 1156, 1149, 1158, + 1159, 1160, 1161, 1157, 1162, 1163, 1164, 1165, + 1162, 874, 655, 1167, 852, 1166, 1169, 1173, + 1174, 1175, 1176, 1176, 1177, 1178, 1179, 1176, + 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, + 1188, 1189, 1176, 1190, 1191, 1192, 1193, 1194, + 1195, 1176, 1196, 1197, 1198, 1170, 1171, 1172, + 1172, 1168, 1200, 1199, 1201, 1199, 1203, 1202, + 1204, 1202, 1207, 1206, 1209, 1211, 1210, 1214, + 1213, 1219, 1216, 1217, 1218, 1215, 1220, 1221, + 1222, 1215, 859, 859, 859, 1223, 1225, 1224, + 1226, 1224, 1227, 1224, 1229, 1230, 1231, 1228, + 1228, 1228, 874, 1233, 1234, 1232, 1236, 1235, + 1237, 1238, 1239, 1240, 1237, 874, 1242, 1241, + 1244, 1243, 1245, 1243, 1246, 1243, 1248, 1247, + 1249, 1250, 1251, 1252, 1249, 874, 1254, 1253, + 1256, 1255, 1257, 1255, 1258, 1255, 1260, 1259, + 1262, 1261, 0 +}; + +static const short _regex_trans_targs[] = { + 746, 746, 746, 746, 746, 748, 749, 750, + 746, 751, 752, 753, 746, 754, 746, 746, + 755, 756, 757, 758, 746, 746, 746, 3, + 746, 4, 746, 6, 7, 746, 8, 746, + 9, 12, 746, 14, 746, 746, 746, 16, + 746, 18, 17, 746, 19, 746, 746, 20, + 21, 746, 22, 25, 746, 27, 28, 746, + 29, 30, 31, 746, 32, 33, 34, 746, + 35, 36, 37, 746, 38, 746, 772, 40, + 42, 46, 49, 43, 44, 746, 45, 47, + 746, 48, 746, 746, 51, 746, 53, 746, + 55, 746, 746, 57, 746, 746, 58, 746, + 746, 60, 59, 783, 61, 783, 783, 746, + 746, 64, 746, 787, 65, 787, 67, 787, + 69, 787, 70, 787, 790, 790, 73, 76, + 74, 75, 790, 77, 78, 79, 80, 790, + 82, 83, 84, 85, 790, 87, 92, 94, + 88, 89, 90, 91, 790, 93, 790, 95, + 790, 97, 98, 99, 100, 790, 102, 103, + 104, 105, 106, 790, 108, 109, 111, 110, + 790, 112, 113, 790, 115, 120, 116, 117, + 118, 119, 790, 121, 790, 790, 123, 139, + 124, 125, 126, 127, 128, 129, 130, 131, + 132, 133, 134, 135, 136, 137, 138, 790, + 140, 141, 790, 143, 144, 790, 145, 146, + 147, 148, 790, 790, 150, 151, 790, 153, + 154, 790, 156, 157, 158, 159, 160, 161, + 790, 163, 167, 164, 165, 166, 790, 168, + 169, 170, 171, 790, 173, 177, 174, 175, + 176, 790, 178, 179, 180, 181, 182, 183, + 790, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, + 200, 201, 790, 203, 204, 205, 206, 207, + 790, 209, 210, 211, 212, 213, 790, 215, + 216, 217, 218, 219, 220, 221, 790, 223, + 224, 225, 790, 227, 228, 790, 230, 235, + 231, 232, 233, 234, 790, 236, 237, 238, + 239, 790, 799, 790, 242, 790, 244, 245, + 790, 247, 248, 249, 790, 251, 252, 253, + 254, 255, 790, 257, 258, 259, 260, 261, + 262, 263, 264, 265, 266, 267, 268, 269, + 790, 271, 277, 272, 273, 274, 275, 276, + 790, 278, 279, 280, 281, 282, 283, 284, + 285, 286, 287, 288, 289, 290, 291, 295, + 292, 293, 294, 790, 296, 297, 298, 299, + 790, 301, 302, 303, 304, 305, 790, 307, + 310, 314, 319, 308, 309, 790, 311, 312, + 313, 790, 315, 316, 317, 318, 790, 320, + 321, 322, 323, 790, 325, 332, 326, 327, + 328, 329, 330, 331, 790, 333, 790, 790, + 790, 335, 336, 790, 338, 339, 340, 790, + 342, 344, 349, 343, 790, 345, 346, 347, + 348, 790, 790, 351, 354, 352, 353, 790, + 355, 356, 790, 790, 358, 364, 359, 360, + 361, 362, 363, 790, 365, 366, 367, 790, + 790, 369, 370, 371, 372, 373, 374, 375, + 376, 790, 378, 379, 380, 381, 382, 383, + 790, 385, 386, 387, 388, 790, 790, 390, + 391, 392, 393, 394, 395, 396, 397, 790, + 790, 400, 401, 790, 403, 408, 404, 405, + 406, 407, 790, 409, 410, 415, 421, 433, + 411, 412, 413, 414, 790, 416, 417, 418, + 419, 420, 790, 422, 423, 424, 425, 426, + 427, 428, 429, 430, 431, 432, 790, 434, + 435, 436, 437, 790, 439, 440, 790, 442, + 443, 444, 445, 790, 790, 447, 452, 448, + 449, 450, 451, 790, 453, 454, 455, 456, + 457, 458, 790, 460, 461, 462, 790, 464, + 465, 790, 790, 467, 473, 468, 469, 470, + 471, 472, 790, 474, 475, 476, 477, 478, + 479, 790, 481, 482, 483, 484, 790, 486, + 487, 488, 489, 790, 491, 492, 493, 494, + 495, 496, 790, 498, 507, 499, 500, 501, + 502, 503, 504, 505, 506, 790, 508, 509, + 790, 511, 519, 528, 512, 515, 513, 514, + 790, 516, 517, 518, 790, 520, 521, 522, + 525, 790, 523, 524, 790, 526, 527, 790, + 529, 790, 531, 532, 533, 790, 535, 536, + 790, 537, 790, 539, 543, 540, 541, 542, + 790, 544, 545, 546, 547, 790, 549, 550, + 551, 552, 553, 790, 790, 790, 790, 790, + 790, 0, 560, 561, 562, 817, 819, 563, + 564, 565, 819, 567, 568, 569, 570, 651, + 666, 672, 678, 684, 690, 696, 707, 713, + 719, 724, 819, 571, 586, 592, 598, 604, + 610, 616, 627, 633, 639, 644, 572, 581, + 573, 577, 574, 575, 576, 819, 578, 579, + 580, 819, 582, 583, 584, 585, 819, 587, + 588, 589, 590, 591, 819, 593, 594, 595, + 596, 597, 819, 599, 600, 601, 602, 603, + 819, 605, 606, 607, 608, 609, 819, 611, + 612, 613, 614, 615, 819, 617, 622, 618, + 619, 620, 621, 819, 623, 624, 625, 626, + 819, 628, 629, 630, 631, 632, 819, 634, + 635, 636, 637, 638, 819, 640, 641, 642, + 643, 819, 645, 646, 647, 648, 649, 650, + 819, 652, 661, 653, 657, 654, 655, 656, + 819, 658, 659, 660, 819, 662, 663, 664, + 665, 819, 667, 668, 669, 670, 671, 819, + 673, 674, 675, 676, 677, 819, 679, 680, + 681, 682, 683, 819, 685, 686, 687, 688, + 689, 819, 691, 692, 693, 694, 695, 819, + 697, 702, 698, 699, 700, 701, 819, 703, + 704, 705, 706, 819, 708, 709, 710, 711, + 712, 819, 714, 715, 716, 717, 718, 819, + 720, 721, 722, 723, 819, 725, 726, 727, + 728, 729, 730, 819, 731, 732, 733, 819, + 735, 819, 819, 736, 819, 819, 819, 739, + 819, 838, 838, 742, 838, 843, 843, 745, + 843, 746, 0, 746, 746, 746, 747, 746, + 759, 760, 746, 761, 762, 763, 746, 782, + 746, 746, 784, 785, 786, 746, 746, 1, + 2, 746, 746, 5, 9, 10, 11, 13, + 15, 746, 746, 746, 23, 24, 26, 746, + 746, 746, 746, 746, 746, 746, 746, 746, + 746, 746, 746, 764, 766, 768, 746, 746, + 746, 746, 746, 746, 746, 746, 746, 769, + 746, 746, 746, 746, 746, 746, 746, 746, + 746, 770, 746, 746, 746, 771, 746, 776, + 746, 777, 778, 746, 746, 746, 746, 746, + 779, 746, 746, 765, 746, 746, 767, 768, + 746, 768, 746, 746, 746, 746, 746, 746, + 746, 39, 774, 41, 746, 773, 746, 746, + 775, 746, 746, 50, 52, 54, 746, 56, + 746, 746, 746, 746, 780, 780, 780, 781, + 746, 746, 746, 746, 746, 746, 746, 746, + 746, 746, 62, 63, 788, 787, 789, 787, + 66, 68, 790, 791, 792, 793, 795, 796, + 797, 798, 800, 801, 802, 803, 804, 806, + 807, 808, 809, 810, 811, 812, 813, 814, + 815, 816, 790, 71, 72, 81, 86, 96, + 101, 107, 114, 790, 122, 790, 790, 142, + 790, 794, 790, 155, 162, 790, 149, 152, + 172, 184, 202, 208, 214, 222, 226, 229, + 240, 246, 250, 790, 241, 243, 256, 270, + 300, 306, 324, 790, 790, 334, 337, 341, + 790, 790, 790, 790, 790, 350, 790, 357, + 790, 805, 790, 377, 384, 790, 368, 790, + 790, 389, 398, 790, 790, 399, 402, 438, + 441, 790, 790, 790, 790, 790, 446, 790, + 790, 790, 459, 463, 790, 466, 790, 480, + 485, 790, 790, 790, 490, 497, 510, 530, + 534, 538, 548, 554, 555, 556, 557, 558, + 790, 790, 790, 790, 790, 818, 818, 818, + 818, 818, 818, 818, 818, 819, 819, 820, + 821, 819, 819, 833, 834, 835, 819, 566, + 819, 822, 824, 819, 819, 819, 819, 819, + 819, 826, 819, 819, 819, 819, 819, 819, + 827, 819, 819, 819, 819, 819, 819, 828, + 829, 819, 819, 819, 819, 819, 830, 819, + 823, 819, 819, 825, 819, 819, 819, 819, + 819, 819, 819, 734, 819, 819, 819, 819, + 831, 831, 831, 832, 819, 819, 819, 819, + 819, 819, 737, 738, 836, 837, 836, 836, + 836, 836, 836, 838, 839, 838, 840, 841, + 842, 838, 838, 838, 838, 740, 741, 843, + 844, 843, 845, 846, 847, 843, 843, 843, + 843, 743, 744, 848, 848, 849, 849 +}; + +static const short _regex_trans_actions[] = { + 827, 631, 765, 731, 723, 45, 903, 903, + 897, 45, 912, 45, 900, 903, 729, 741, + 0, 45, 45, 923, 737, 841, 747, 0, + 743, 3, 839, 3, 0, 761, 3, 759, + 870, 3, 751, 0, 749, 755, 753, 0, + 757, 3, 0, 745, 0, 725, 727, 27, + 3, 763, 0, 3, 673, 0, 25, 829, + 0, 0, 0, 603, 0, 0, 0, 601, + 0, 0, 0, 831, 0, 675, 17, 0, + 7, 870, 3, 17, 17, 663, 17, 870, + 661, 870, 665, 837, 3, 671, 3, 669, + 3, 667, 833, 0, 677, 835, 0, 679, + 845, 0, 11, 29, 13, 31, 0, 843, + 769, 0, 771, 59, 0, 53, 0, 51, + 0, 49, 0, 47, 359, 315, 0, 0, + 0, 0, 127, 0, 0, 0, 0, 129, + 0, 0, 0, 0, 131, 0, 0, 0, + 0, 0, 0, 0, 133, 0, 135, 0, + 137, 0, 0, 0, 0, 139, 0, 0, + 0, 0, 0, 141, 0, 0, 0, 0, + 143, 0, 0, 145, 0, 0, 0, 0, + 0, 0, 147, 0, 149, 341, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 151, + 0, 0, 153, 0, 0, 155, 0, 0, + 0, 0, 157, 343, 0, 0, 159, 0, + 0, 161, 0, 0, 0, 0, 0, 0, + 163, 0, 0, 0, 0, 0, 165, 0, + 0, 0, 0, 167, 0, 0, 0, 0, + 0, 169, 0, 0, 0, 0, 0, 0, + 171, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 173, 0, 0, 0, 0, 0, + 175, 0, 0, 0, 0, 0, 177, 0, + 0, 0, 0, 0, 0, 0, 179, 0, + 0, 0, 181, 0, 0, 183, 0, 0, + 0, 0, 0, 0, 185, 0, 0, 0, + 0, 187, 45, 357, 0, 189, 0, 0, + 191, 0, 0, 0, 193, 0, 0, 0, + 0, 0, 195, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 197, 0, 0, 0, 0, 0, 0, 0, + 199, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 201, 0, 0, 0, 0, + 203, 0, 0, 0, 0, 0, 205, 0, + 0, 0, 0, 0, 0, 207, 0, 0, + 0, 209, 0, 0, 0, 0, 211, 0, + 0, 0, 0, 213, 0, 0, 0, 0, + 0, 0, 0, 0, 215, 0, 217, 345, + 219, 0, 0, 221, 0, 0, 0, 223, + 0, 0, 0, 0, 225, 0, 0, 0, + 0, 227, 229, 0, 0, 0, 0, 231, + 0, 0, 233, 347, 0, 0, 0, 0, + 0, 0, 0, 235, 0, 0, 0, 237, + 349, 0, 0, 0, 0, 0, 0, 0, + 0, 239, 0, 0, 0, 0, 0, 0, + 241, 0, 0, 0, 0, 243, 351, 0, + 0, 0, 0, 0, 0, 0, 0, 245, + 247, 0, 0, 249, 0, 0, 0, 0, + 0, 0, 251, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 253, 0, 0, 0, + 0, 0, 255, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 257, 0, + 0, 0, 0, 259, 0, 0, 261, 0, + 0, 0, 0, 263, 353, 0, 0, 0, + 0, 0, 0, 265, 0, 0, 0, 0, + 0, 0, 267, 0, 0, 0, 269, 0, + 0, 271, 355, 0, 0, 0, 0, 0, + 0, 0, 273, 0, 0, 0, 0, 0, + 0, 275, 0, 0, 0, 0, 277, 0, + 0, 0, 0, 279, 0, 0, 0, 0, + 0, 0, 281, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 283, 0, 0, + 285, 0, 0, 0, 0, 0, 0, 0, + 287, 0, 0, 0, 289, 0, 0, 0, + 0, 291, 0, 0, 293, 0, 0, 295, + 0, 297, 0, 0, 0, 299, 0, 0, + 303, 0, 301, 0, 0, 0, 0, 0, + 305, 0, 0, 0, 0, 307, 0, 0, + 0, 0, 0, 309, 311, 119, 121, 123, + 125, 39, 0, 35, 33, 37, 539, 0, + 0, 0, 377, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 435, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 381, 0, 0, + 0, 385, 0, 0, 0, 0, 389, 0, + 0, 0, 0, 0, 393, 0, 0, 0, + 0, 0, 397, 0, 0, 0, 0, 0, + 401, 0, 0, 0, 0, 0, 405, 0, + 0, 0, 0, 0, 409, 0, 0, 0, + 0, 0, 0, 413, 0, 0, 0, 0, + 417, 0, 0, 0, 0, 0, 421, 0, + 0, 0, 0, 0, 425, 0, 0, 0, + 0, 429, 0, 0, 0, 0, 0, 0, + 433, 0, 0, 0, 0, 0, 0, 0, + 379, 0, 0, 0, 383, 0, 0, 0, + 0, 387, 0, 0, 0, 0, 0, 391, + 0, 0, 0, 0, 0, 395, 0, 0, + 0, 0, 0, 399, 0, 0, 0, 0, + 0, 403, 0, 0, 0, 0, 0, 407, + 0, 0, 0, 0, 0, 0, 411, 0, + 0, 0, 0, 415, 0, 0, 0, 0, + 0, 419, 0, 0, 0, 0, 0, 423, + 0, 0, 0, 0, 427, 0, 0, 0, + 0, 0, 0, 431, 0, 0, 0, 533, + 0, 471, 535, 0, 475, 537, 503, 0, + 505, 569, 557, 0, 559, 587, 575, 0, + 577, 633, 0, 777, 775, 637, 45, 597, + 0, 0, 609, 0, 45, 0, 635, 909, + 599, 773, 0, 45, 45, 629, 779, 0, + 0, 821, 819, 1, 855, 855, 1, 0, + 3, 735, 733, 739, 1, 1, 0, 783, + 615, 613, 785, 619, 617, 787, 623, 621, + 781, 817, 721, 5, 852, 915, 639, 647, + 611, 695, 607, 717, 699, 715, 683, 0, + 605, 713, 691, 703, 687, 719, 641, 657, + 645, 0, 693, 659, 655, 906, 697, 45, + 651, 45, 0, 653, 689, 649, 701, 685, + 7, 643, 791, 15, 867, 795, 858, 919, + 793, 927, 847, 811, 711, 709, 809, 681, + 801, 7, 17, 849, 799, 17, 876, 797, + 17, 873, 815, 1, 1, 1, 803, 0, + 813, 707, 705, 805, 19, 23, 21, 45, + 882, 894, 888, 807, 825, 789, 627, 625, + 823, 767, 0, 0, 45, 55, 45, 57, + 0, 0, 317, 45, 45, 45, 45, 45, + 45, 45, 45, 45, 45, 45, 45, 45, + 45, 45, 45, 45, 45, 45, 45, 45, + 0, 0, 339, 0, 0, 0, 0, 0, + 0, 0, 0, 319, 0, 61, 63, 0, + 65, 45, 67, 0, 0, 321, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 337, 0, 0, 0, 0, + 0, 0, 0, 323, 79, 0, 0, 0, + 69, 71, 73, 75, 77, 0, 325, 0, + 81, 45, 83, 0, 0, 327, 0, 329, + 85, 0, 0, 87, 89, 0, 0, 0, + 0, 331, 91, 93, 95, 97, 0, 99, + 101, 103, 0, 0, 333, 0, 105, 0, + 0, 107, 109, 111, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 313, 335, 113, 115, 117, 375, 361, 363, + 365, 367, 369, 371, 373, 509, 491, 45, + 0, 511, 507, 0, 45, 45, 531, 0, + 499, 5, 9, 473, 497, 489, 439, 457, + 493, 0, 437, 485, 461, 481, 451, 441, + 0, 487, 453, 449, 495, 455, 445, 45, + 0, 447, 483, 443, 459, 479, 7, 517, + 15, 861, 519, 15, 864, 513, 469, 467, + 527, 477, 521, 0, 515, 465, 463, 523, + 19, 23, 21, 45, 879, 891, 885, 525, + 529, 501, 0, 0, 549, 0, 543, 541, + 551, 547, 545, 563, 0, 561, 0, 45, + 45, 567, 553, 565, 555, 0, 0, 581, + 0, 579, 0, 45, 45, 585, 571, 583, + 573, 0, 0, 591, 589, 595, 593 +}; + +static const short _regex_to_state_actions[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 41, + 0, 41, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 41, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 41, 0, 0, 41, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 41, 41, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 41, 0, 41, 0, + 0, 0, 0, 41, 0, 0, 0, 0, + 41, 41 +}; + +static const short _regex_from_state_actions[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 43, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 43, 0, 0, 43, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 43, 43, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 43, 0, 43, 0, + 0, 0, 0, 43, 0, 0, 0, 0, + 43, 43 +}; + +static const short _regex_eof_actions[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 39, + 39, 39, 39, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0 +}; + +static const short _regex_eof_trans[] = { + 0, 1, 1, 22, 22, 27, 27, 27, + 27, 27, 27, 27, 27, 27, 27, 27, + 27, 27, 27, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 68, 68, + 68, 68, 68, 68, 68, 68, 68, 68, + 68, 68, 84, 84, 84, 84, 84, 84, + 91, 91, 94, 97, 97, 97, 104, 104, + 104, 108, 108, 108, 108, 108, 108, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 174, 174, 174, 174, 174, 174, + 174, 174, 174, 174, 174, 174, 174, 174, + 174, 174, 174, 174, 174, 174, 174, 174, + 174, 174, 174, 174, 174, 204, 204, 204, + 204, 204, 204, 174, 174, 174, 174, 174, + 174, 174, 174, 174, 174, 174, 174, 174, + 174, 174, 174, 174, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 300, 300, 300, 300, 300, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 400, 400, + 400, 400, 400, 400, 400, 400, 400, 400, + 400, 400, 400, 400, 400, 400, 400, 400, + 400, 400, 400, 400, 400, 428, 428, 428, + 428, 428, 428, 428, 428, 428, 428, 428, + 441, 441, 441, 441, 441, 441, 441, 441, + 441, 428, 428, 428, 428, 428, 428, 428, + 428, 428, 428, 428, 428, 463, 463, 463, + 463, 463, 463, 463, 463, 463, 463, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 525, 525, + 525, 525, 525, 525, 525, 525, 525, 525, + 525, 525, 525, 117, 117, 117, 117, 117, + 117, 117, 547, 547, 547, 547, 547, 547, + 547, 547, 547, 547, 547, 547, 547, 547, + 547, 547, 547, 547, 547, 547, 547, 547, + 547, 547, 547, 547, 547, 547, 547, 547, + 547, 547, 547, 547, 547, 547, 547, 547, + 547, 547, 547, 547, 547, 547, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 0, + 0, 0, 0, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 655, 655, + 655, 655, 655, 655, 655, 655, 856, 856, + 859, 862, 862, 862, 866, 866, 866, 870, + 870, 870, 0, 895, 898, 898, 899, 898, + 898, 898, 898, 898, 898, 898, 898, 912, + 915, 918, 921, 922, 963, 963, 966, 969, + 971, 972, 975, 977, 981, 981, 984, 984, + 987, 991, 993, 996, 996, 1004, 1005, 1006, + 1009, 1009, 1009, 0, 1016, 1016, 0, 1043, + 1043, 1052, 1062, 1043, 1043, 1043, 1043, 1076, + 1043, 1043, 1043, 1084, 1095, 1102, 1104, 1043, + 1114, 1043, 1125, 1043, 1043, 1043, 1043, 1043, + 1146, 0, 0, 0, 1167, 1167, 1200, 1200, + 1203, 1203, 1206, 1209, 1211, 1213, 1216, 1216, + 1224, 1225, 1225, 1225, 0, 1233, 0, 1242, + 1244, 1244, 1244, 0, 1254, 1256, 1256, 1256, + 0, 0 +}; + +static const int regex_start = 746; +static const int regex_error = 0; + +static const int regex_en_readVerb = 787; +static const int regex_en_readUCP = 790; +static const int regex_en_readBracedUCP = 559; +static const int regex_en_readUCPSingle = 818; +static const int regex_en_charClassGuts = 819; +static const int regex_en_readClass = 836; +static const int regex_en_readQuotedLiteral = 838; +static const int regex_en_readQuotedClass = 843; +static const int regex_en_readComment = 848; +static const int regex_en_readNewlineTerminatedComment = 849; +static const int regex_en_main = 746; + + +#line 1912 "Parser.rl" + +/** \brief Main parser call, returns root Component or nullptr. */ +unique_ptr parse(const char *ptr, ParseMode &globalMode) { + assert(ptr); + + const char *p = ptr; + const char *pe = ptr + strlen(ptr); + + // First, read the control verbs, set any global mode flags and move the + // ptr forward. + p = read_control_verbs(p, pe, 0, globalMode); + + const char *eof = pe; + int cs; + UNUSED int act; + int top; + vector stack; + const char *ts, *te; + unichar accumulator = 0; + unichar octAccumulator = 0; /* required as we are also accumulating for + * back ref when looking for octals */ + unsigned repeatN = 0; + unsigned repeatM = 0; + string label; + + ParseMode mode = globalMode; + ParseMode newMode; + + bool negated = false; + bool inComment = false; + + // Stack of sequences and flags used to store state when we enter + // sub-sequences. + vector sequences; + + // Index of the next capturing group. Note that zero is reserved for the + // root sequence. + unsigned groupIndex = 1; + + // Set storing group names that are currently in use. + flat_set groupNames; + + // Root sequence. + unique_ptr rootSeq = std::make_unique(); + rootSeq->setCaptureIndex(0); + + // Current sequence being appended to + ComponentSequence *currentSeq = rootSeq.get(); + + // The current character class being appended to. This is used as the + // accumulator for both character class and UCP properties. + unique_ptr currentCls; + + // True if the machine is currently inside a character class, i.e. square + // brackets [..]. + bool inCharClass = false; + + // True if the machine is inside a character class but it has not processed + // any "real" elements yet, i.e. it's still processing meta-characters like + // '^'. + bool inCharClassEarly = false; + + // Location at which the current character class began. + const char *currentClsBegin = p; + + // We throw exceptions on various parsing failures beyond this point: we + // use a try/catch block here to clean up our allocated memory before we + // re-throw the exception to the caller. + try { + // Embed the Ragel machine here + +#line 2533 "Parser.cpp" + { + cs = regex_start; + top = 0; + ts = 0; + te = 0; + act = 0; + } + +#line 1983 "Parser.rl" + +#line 2544 "Parser.cpp" + { + int _klen; + unsigned int _trans; + short _widec; + const short *_acts; + unsigned int _nacts; + const short *_keys; + + if ( p == pe ) + goto _test_eof; + if ( cs == 0 ) + goto _out; +_resume: + _acts = _regex_actions + _regex_from_state_actions[cs]; + _nacts = (unsigned int) *_acts++; + while ( _nacts-- > 0 ) { + switch ( *_acts++ ) { + case 24: +#line 1 "NONE" + {ts = p;} + break; +#line 2566 "Parser.cpp" + } + } + + _widec = (*p); + _klen = _regex_cond_lengths[cs]; + _keys = _regex_cond_keys + (_regex_cond_offsets[cs]*2); + if ( _klen > 0 ) { + const short *_lower = _keys; + const short *_mid; + const short *_upper = _keys + (_klen<<1) - 2; + while (1) { + if ( _upper < _lower ) + break; + + _mid = _lower + (((_upper-_lower) >> 1) & ~1); + if ( _widec < _mid[0] ) + _upper = _mid - 2; + else if ( _widec > _mid[1] ) + _lower = _mid + 2; + else { + switch ( _regex_cond_spaces[_regex_cond_offsets[cs] + ((_mid - _keys)>>1)] ) { + case 0: { + _widec = (short)(128 + ((*p) - -128)); + if ( +#line 475 "Parser.rl" + mode.utf8 ) _widec += 256; + break; + } + case 1: { + _widec = (short)(1152 + ((*p) - -128)); + if ( +#line 476 "Parser.rl" + mode.ignore_space ) _widec += 256; + break; + } + case 2: { + _widec = (short)(640 + ((*p) - -128)); + if ( +#line 477 "Parser.rl" + inCharClassEarly ) _widec += 256; + break; + } + } + break; + } + } + } + + _keys = _regex_trans_keys + _regex_key_offsets[cs]; + _trans = _regex_index_offsets[cs]; + + _klen = _regex_single_lengths[cs]; + if ( _klen > 0 ) { + const short *_lower = _keys; + const short *_mid; + const short *_upper = _keys + _klen - 1; + while (1) { + if ( _upper < _lower ) + break; + + _mid = _lower + ((_upper-_lower) >> 1); + if ( _widec < *_mid ) + _upper = _mid - 1; + else if ( _widec > *_mid ) + _lower = _mid + 1; + else { + _trans += (unsigned int)(_mid - _keys); + goto _match; + } + } + _keys += _klen; + _trans += _klen; + } + + _klen = _regex_range_lengths[cs]; + if ( _klen > 0 ) { + const short *_lower = _keys; + const short *_mid; + const short *_upper = _keys + (_klen<<1) - 2; + while (1) { + if ( _upper < _lower ) + break; + + _mid = _lower + (((_upper-_lower) >> 1) & ~1); + if ( _widec < _mid[0] ) + _upper = _mid - 2; + else if ( _widec > _mid[1] ) + _lower = _mid + 2; + else { + _trans += (unsigned int)((_mid - _keys)>>1); + goto _match; + } + } + _trans += _klen; + } + +_match: + _trans = _regex_indicies[_trans]; +_eof_trans: + cs = _regex_trans_targs[_trans]; + + if ( _regex_trans_actions[_trans] == 0 ) + goto _again; + + _acts = _regex_actions + _regex_trans_actions[_trans]; + _nacts = (unsigned int) *_acts++; + while ( _nacts-- > 0 ) + { + switch ( *_acts++ ) + { + case 0: +#line 285 "Parser.rl" + { label.clear();} + break; + case 1: +#line 286 "Parser.rl" + { label.push_back((*p));} + break; + case 2: +#line 287 "Parser.rl" + { octAccumulator = 0;} + break; + case 3: +#line 288 "Parser.rl" + { accumulator = 0;} + break; + case 4: +#line 289 "Parser.rl" + { + octAccumulator = 0; + pushOct(&octAccumulator, (*p)); + } + break; + case 5: +#line 293 "Parser.rl" + { + accumulator = 0; + pushDec(&accumulator, (*p)); + } + break; + case 6: +#line 297 "Parser.rl" + { repeatN = 0; repeatM = 0; } + break; + case 7: +#line 298 "Parser.rl" + { pushDec(&repeatN, (*p)); } + break; + case 8: +#line 299 "Parser.rl" + { pushDec(&repeatM, (*p)); } + break; + case 9: +#line 300 "Parser.rl" + { pushOct(&octAccumulator, (*p)); } + break; + case 10: +#line 301 "Parser.rl" + { pushDec(&accumulator, (*p)); } + break; + case 11: +#line 302 "Parser.rl" + { + accumulator *= 16; + accumulator += (*p) - '0'; + } + break; + case 12: +#line 306 "Parser.rl" + { + accumulator *= 16; + accumulator += 10 + (*p) - 'a'; + } + break; + case 13: +#line 310 "Parser.rl" + { + accumulator *= 16; + accumulator += 10 + (*p) - 'A'; + } + break; + case 14: +#line 430 "Parser.rl" + { + newMode = mode; + } + break; + case 15: +#line 437 "Parser.rl" + { + switch ((*p)) { + case 'i': + newMode.caseless = true; + break; + case 'm': + newMode.multiline = true; + break; + case 's': + newMode.dotall = true; + break; + case 'x': + newMode.ignore_space = true; + break; + default: + assert(0); // this action only called for [imsx] + break; + } + } + break; + case 16: +#line 456 "Parser.rl" + { + switch ((*p)) { + case 'i': + newMode.caseless = false; + break; + case 'm': + newMode.multiline = false; + break; + case 's': + newMode.dotall = false; + break; + case 'x': + newMode.ignore_space = false; + break; + default: + assert(0); // this action only called for [imsx] + break; + } + } + break; + case 17: +#line 510 "Parser.rl" + {repeatM = repeatN;} + break; + case 18: +#line 510 "Parser.rl" + {repeatM = ComponentRepeat::NoLimit;} + break; + case 19: +#line 722 "Parser.rl" + { negated = !negated; } + break; + case 20: +#line 723 "Parser.rl" + { p--; { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 790;goto _again;}} } + break; + case 21: +#line 724 "Parser.rl" + { if (!inCharClass) { // not inside [..] + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + } + break; + case 22: +#line 730 "Parser.rl" + { throw LocatedParseError("Malformed property"); } + break; + case 25: +#line 1 "NONE" + {te = p+1;} + break; + case 26: +#line 550 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("(*UTF8) must be at start of " + "expression, encountered"); + }} + break; + case 27: +#line 554 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("(*UTF) must be at start of " + "expression, encountered"); + }} + break; + case 28: +#line 558 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("(*UCP) must be at start of " + "expression, encountered"); + }} + break; + case 29: +#line 564 "Parser.rl" + {te = p+1;{ + ParseMode temp_mode; + assert(ts - 2 >= ptr); // parser needs the '(*' at the start too. + read_control_verbs(ts - 2, te, (ts - 2 - ptr), temp_mode); + assert(0); // Should have thrown a parse error. + throw LocatedParseError("Unknown control verb"); + }} + break; + case 30: +#line 571 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Unknown control verb"); + }} + break; + case 31: +#line 571 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Unknown control verb"); + }} + break; + case 32: +#line 571 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Unknown control verb"); + }} + break; + case 33: +#line 581 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_CC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 34: +#line 582 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_CF, negated); {cs = stack[--top]; goto _again;} }} + break; + case 35: +#line 583 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_CN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 36: +#line 585 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_CS, negated); {cs = stack[--top]; goto _again;} }} + break; + case 37: +#line 587 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_LL, negated); {cs = stack[--top]; goto _again;} }} + break; + case 38: +#line 588 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_LM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 39: +#line 589 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_LO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 40: +#line 590 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_LT, negated); {cs = stack[--top]; goto _again;} }} + break; + case 41: +#line 591 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_LU, negated); {cs = stack[--top]; goto _again;} }} + break; + case 42: +#line 592 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_L_AND, negated); {cs = stack[--top]; goto _again;} }} + break; + case 43: +#line 594 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_MC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 44: +#line 596 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_MN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 45: +#line 598 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_ND, negated); {cs = stack[--top]; goto _again;} }} + break; + case 46: +#line 599 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_NL, negated); {cs = stack[--top]; goto _again;} }} + break; + case 47: +#line 600 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_NO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 48: +#line 602 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_PC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 49: +#line 603 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_PD, negated); {cs = stack[--top]; goto _again;} }} + break; + case 50: +#line 604 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_PE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 51: +#line 605 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_PF, negated); {cs = stack[--top]; goto _again;} }} + break; + case 52: +#line 606 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_PI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 53: +#line 607 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_PO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 54: +#line 608 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_PS, negated); {cs = stack[--top]; goto _again;} }} + break; + case 55: +#line 610 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_SC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 56: +#line 611 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_SK, negated); {cs = stack[--top]; goto _again;} }} + break; + case 57: +#line 612 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_SM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 58: +#line 613 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_SO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 59: +#line 615 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_ZL, negated); {cs = stack[--top]; goto _again;} }} + break; + case 60: +#line 616 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_ZP, negated); {cs = stack[--top]; goto _again;} }} + break; + case 61: +#line 617 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_ZS, negated); {cs = stack[--top]; goto _again;} }} + break; + case 62: +#line 618 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_XAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 63: +#line 619 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_XPS, negated); {cs = stack[--top]; goto _again;} }} + break; + case 64: +#line 620 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_XSP, negated); {cs = stack[--top]; goto _again;} }} + break; + case 65: +#line 621 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_XWD, negated); {cs = stack[--top]; goto _again;} }} + break; + case 66: +#line 622 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_ARABIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 67: +#line 623 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_ARMENIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 68: +#line 624 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_AVESTAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 69: +#line 625 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BALINESE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 70: +#line 626 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BAMUM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 71: +#line 627 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BATAK, negated); {cs = stack[--top]; goto _again;} }} + break; + case 72: +#line 628 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BENGALI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 73: +#line 629 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BOPOMOFO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 74: +#line 630 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BRAHMI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 75: +#line 631 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BRAILLE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 76: +#line 632 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BUGINESE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 77: +#line 633 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_BUHID, negated); {cs = stack[--top]; goto _again;} }} + break; + case 78: +#line 634 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_CANADIAN_ABORIGINAL, negated); {cs = stack[--top]; goto _again;} }} + break; + case 79: +#line 635 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_CARIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 80: +#line 636 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_CHAM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 81: +#line 637 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_CHEROKEE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 82: +#line 638 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_COMMON, negated); {cs = stack[--top]; goto _again;} }} + break; + case 83: +#line 639 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_COPTIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 84: +#line 640 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_CUNEIFORM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 85: +#line 641 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_CYPRIOT, negated); {cs = stack[--top]; goto _again;} }} + break; + case 86: +#line 642 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_CYRILLIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 87: +#line 643 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_DESERET, negated); {cs = stack[--top]; goto _again;} }} + break; + case 88: +#line 644 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_DEVANAGARI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 89: +#line 645 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_EGYPTIAN_HIEROGLYPHS, negated); {cs = stack[--top]; goto _again;} }} + break; + case 90: +#line 646 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_ETHIOPIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 91: +#line 647 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_GEORGIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 92: +#line 648 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_GLAGOLITIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 93: +#line 649 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_GOTHIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 94: +#line 650 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_GREEK, negated); {cs = stack[--top]; goto _again;} }} + break; + case 95: +#line 651 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_GUJARATI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 96: +#line 652 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_GURMUKHI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 97: +#line 654 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_HANGUL, negated); {cs = stack[--top]; goto _again;} }} + break; + case 98: +#line 655 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_HANUNOO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 99: +#line 656 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_HEBREW, negated); {cs = stack[--top]; goto _again;} }} + break; + case 100: +#line 657 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_HIRAGANA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 101: +#line 658 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_IMPERIAL_ARAMAIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 102: +#line 659 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_INHERITED, negated); {cs = stack[--top]; goto _again;} }} + break; + case 103: +#line 660 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PAHLAVI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 104: +#line 661 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PARTHIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 105: +#line 662 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_JAVANESE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 106: +#line 663 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_KAITHI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 107: +#line 664 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_KANNADA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 108: +#line 665 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_KATAKANA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 109: +#line 666 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_KAYAH_LI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 110: +#line 667 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_KHAROSHTHI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 111: +#line 668 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_KHMER, negated); {cs = stack[--top]; goto _again;} }} + break; + case 112: +#line 669 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LAO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 113: +#line 670 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LATIN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 114: +#line 671 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LEPCHA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 115: +#line 672 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LIMBU, negated); {cs = stack[--top]; goto _again;} }} + break; + case 116: +#line 673 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LINEAR_B, negated); {cs = stack[--top]; goto _again;} }} + break; + case 117: +#line 674 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LISU, negated); {cs = stack[--top]; goto _again;} }} + break; + case 118: +#line 675 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LYCIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 119: +#line 676 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_LYDIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 120: +#line 677 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_MALAYALAM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 121: +#line 678 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_MANDAIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 122: +#line 679 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_MEETEI_MAYEK, negated); {cs = stack[--top]; goto _again;} }} + break; + case 123: +#line 680 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_MONGOLIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 124: +#line 681 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_MYANMAR, negated); {cs = stack[--top]; goto _again;} }} + break; + case 125: +#line 682 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_NEW_TAI_LUE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 126: +#line 683 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_NKO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 127: +#line 684 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_OGHAM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 128: +#line 685 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_OL_CHIKI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 129: +#line 686 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_OLD_ITALIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 130: +#line 687 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_OLD_PERSIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 131: +#line 688 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_OLD_SOUTH_ARABIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 132: +#line 689 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_OLD_TURKIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 133: +#line 690 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_ORIYA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 134: +#line 691 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_OSMANYA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 135: +#line 692 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_PHAGS_PA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 136: +#line 693 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_PHOENICIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 137: +#line 694 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_REJANG, negated); {cs = stack[--top]; goto _again;} }} + break; + case 138: +#line 695 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_RUNIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 139: +#line 696 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_SAMARITAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 140: +#line 697 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_SAURASHTRA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 141: +#line 698 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_SHAVIAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 142: +#line 699 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_SINHALA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 143: +#line 700 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_SUNDANESE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 144: +#line 701 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_SYLOTI_NAGRI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 145: +#line 702 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_SYRIAC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 146: +#line 703 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TAGALOG, negated); {cs = stack[--top]; goto _again;} }} + break; + case 147: +#line 704 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TAGBANWA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 148: +#line 705 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TAI_LE, negated); {cs = stack[--top]; goto _again;} }} + break; + case 149: +#line 706 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TAI_THAM, negated); {cs = stack[--top]; goto _again;} }} + break; + case 150: +#line 707 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TAI_VIET, negated); {cs = stack[--top]; goto _again;} }} + break; + case 151: +#line 708 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TAMIL, negated); {cs = stack[--top]; goto _again;} }} + break; + case 152: +#line 709 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TELUGU, negated); {cs = stack[--top]; goto _again;} }} + break; + case 153: +#line 710 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_THAANA, negated); {cs = stack[--top]; goto _again;} }} + break; + case 154: +#line 711 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_THAI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 155: +#line 712 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TIBETAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 156: +#line 713 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_TIFINAGH, negated); {cs = stack[--top]; goto _again;} }} + break; + case 157: +#line 714 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_UGARITIC, negated); {cs = stack[--top]; goto _again;} }} + break; + case 158: +#line 715 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_VAI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 159: +#line 716 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_SCRIPT_YI, negated); {cs = stack[--top]; goto _again;} }} + break; + case 160: +#line 717 "Parser.rl" + {te = p+1;{ currentCls->add(CLASS_UCP_ANY, negated); {cs = stack[--top]; goto _again;} }} + break; + case 161: +#line 718 "Parser.rl" + {te = p+1;{ throw LocatedParseError("Unknown property"); }} + break; + case 162: +#line 580 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_C, negated); {cs = stack[--top]; goto _again;} }} + break; + case 163: +#line 584 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_CO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 164: +#line 586 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_L, negated); {cs = stack[--top]; goto _again;} }} + break; + case 165: +#line 593 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_M, negated); {cs = stack[--top]; goto _again;} }} + break; + case 166: +#line 595 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_ME, negated); {cs = stack[--top]; goto _again;} }} + break; + case 167: +#line 597 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_N, negated); {cs = stack[--top]; goto _again;} }} + break; + case 168: +#line 601 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_P, negated); {cs = stack[--top]; goto _again;} }} + break; + case 169: +#line 609 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_S, negated); {cs = stack[--top]; goto _again;} }} + break; + case 170: +#line 614 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_UCP_Z, negated); {cs = stack[--top]; goto _again;} }} + break; + case 171: +#line 653 "Parser.rl" + {te = p;p--;{ currentCls->add(CLASS_SCRIPT_HAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 172: +#line 718 "Parser.rl" + {te = p;p--;{ throw LocatedParseError("Unknown property"); }} + break; + case 173: +#line 580 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_C, negated); {cs = stack[--top]; goto _again;} }} + break; + case 174: +#line 584 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_CO, negated); {cs = stack[--top]; goto _again;} }} + break; + case 175: +#line 586 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_L, negated); {cs = stack[--top]; goto _again;} }} + break; + case 176: +#line 593 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_M, negated); {cs = stack[--top]; goto _again;} }} + break; + case 177: +#line 595 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_ME, negated); {cs = stack[--top]; goto _again;} }} + break; + case 178: +#line 597 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_N, negated); {cs = stack[--top]; goto _again;} }} + break; + case 179: +#line 601 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_P, negated); {cs = stack[--top]; goto _again;} }} + break; + case 180: +#line 609 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_UCP_S, negated); {cs = stack[--top]; goto _again;} }} + break; + case 181: +#line 653 "Parser.rl" + {{p = ((te))-1;}{ currentCls->add(CLASS_SCRIPT_HAN, negated); {cs = stack[--top]; goto _again;} }} + break; + case 182: +#line 718 "Parser.rl" + {{p = ((te))-1;}{ throw LocatedParseError("Unknown property"); }} + break; + case 183: +#line 733 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UCP_C, negated); + if (!inCharClass) { + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + }} + break; + case 184: +#line 741 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UCP_L, negated); + if (!inCharClass) { + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + }} + break; + case 185: +#line 749 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UCP_M, negated); + if (!inCharClass) { + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + }} + break; + case 186: +#line 757 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UCP_N, negated); + if (!inCharClass) { + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + }} + break; + case 187: +#line 765 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UCP_P, negated); + if (!inCharClass) { + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + }} + break; + case 188: +#line 773 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UCP_S, negated); + if (!inCharClass) { + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + }} + break; + case 189: +#line 781 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UCP_Z, negated); + if (!inCharClass) { + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + } + {cs = stack[--top]; goto _again;} + }} + break; + case 190: +#line 790 "Parser.rl" + {te = p+1;{ throw LocatedParseError("Unknown property"); }} + break; + case 191: +#line 796 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Unsupported POSIX collating " + "element"); + }} + break; + case 192: +#line 803 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_ALNUM, false); + }} + break; + case 193: +#line 806 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_ALNUM, true); + }} + break; + case 194: +#line 809 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_ALPHA, false); + }} + break; + case 195: +#line 812 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_ALPHA, true); + }} + break; + case 196: +#line 815 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_ASCII, false); + }} + break; + case 197: +#line 818 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_ASCII, true); + }} + break; + case 198: +#line 821 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_BLANK, false); + }} + break; + case 199: +#line 824 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_BLANK, true); + }} + break; + case 200: +#line 827 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_CNTRL, false); + }} + break; + case 201: +#line 830 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_CNTRL, true); + }} + break; + case 202: +#line 833 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_DIGIT, false); + }} + break; + case 203: +#line 836 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_DIGIT, true); + }} + break; + case 204: +#line 839 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_GRAPH, false); + }} + break; + case 205: +#line 842 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_GRAPH, true); + }} + break; + case 206: +#line 845 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_LOWER, false); + }} + break; + case 207: +#line 848 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_LOWER, true); + }} + break; + case 208: +#line 851 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_PRINT, false); + }} + break; + case 209: +#line 854 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_PRINT, true); + }} + break; + case 210: +#line 857 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_PUNCT, false); + }} + break; + case 211: +#line 860 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_PUNCT, true); + }} + break; + case 212: +#line 864 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_SPACE, false); + }} + break; + case 213: +#line 867 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_SPACE, true); + }} + break; + case 214: +#line 870 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UPPER, false); + }} + break; + case 215: +#line 873 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_UPPER, true); + }} + break; + case 216: +#line 876 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_WORD, false); + }} + break; + case 217: +#line 879 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_WORD, true); + }} + break; + case 218: +#line 882 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_XDIGIT, false); + }} + break; + case 219: +#line 885 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_XDIGIT, true); + }} + break; + case 220: +#line 890 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Invalid POSIX named class"); + }} + break; + case 221: +#line 893 "Parser.rl" + {te = p+1;{ + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 843;goto _again;}} + }} + break; + case 222: +#line 896 "Parser.rl" + {te = p+1;{ /*noop*/}} + break; + case 223: +#line 898 "Parser.rl" + {te = p+1;{ + currentCls->add('\x08'); + }} + break; + case 224: +#line 902 "Parser.rl" + {te = p+1;{ + currentCls->add('\x09'); + }} + break; + case 225: +#line 906 "Parser.rl" + {te = p+1;{ + currentCls->add('\x0a'); + }} + break; + case 226: +#line 910 "Parser.rl" + {te = p+1;{ + currentCls->add('\x0d'); + }} + break; + case 227: +#line 914 "Parser.rl" + {te = p+1;{ + currentCls->add('\x0c'); + }} + break; + case 228: +#line 918 "Parser.rl" + {te = p+1;{ + currentCls->add('\x07'); + }} + break; + case 229: +#line 922 "Parser.rl" + {te = p+1;{ + currentCls->add('\x1b'); + }} + break; + case 230: +#line 926 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_HORZ, false); + }} + break; + case 231: +#line 930 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_HORZ, true); + }} + break; + case 232: +#line 934 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_VERT, false); + }} + break; + case 233: +#line 938 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_VERT, true); + }} + break; + case 234: +#line 942 "Parser.rl" + {te = p+1;{ + negated = false; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 559;goto _again;}} + }} + break; + case 235: +#line 948 "Parser.rl" + {te = p+1;{ + negated = false; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 818;goto _again;}} + }} + break; + case 236: +#line 954 "Parser.rl" + {te = p+1;{ + negated = true; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 559;goto _again;}} + }} + break; + case 237: +#line 960 "Parser.rl" + {te = p+1;{ + negated = true; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 818;goto _again;}} + }} + break; + case 238: +#line 970 "Parser.rl" + {te = p+1;{ + currentCls->add(octAccumulator); + }} + break; + case 239: +#line 973 "Parser.rl" + {te = p+1;{ + currentCls->add(octAccumulator); + }} + break; + case 240: +#line 977 "Parser.rl" + {te = p+1;{ + string oct(ts + 3, te - ts - 4); + unsigned long val; + try { + val = stoul(oct, nullptr, 8); + } catch (const std::out_of_range &) { + val = MAX_UNICODE + 1; + } + if ((!mode.utf8 && val > 255) || val > MAX_UNICODE) { + throw LocatedParseError("Value in \\o{...} sequence is too large"); + } + currentCls->add((unichar)val); + }} + break; + case 241: +#line 997 "Parser.rl" + {te = p+1;{ + currentCls->add(accumulator); + }} + break; + case 242: +#line 1001 "Parser.rl" + {te = p+1;{ + // whatever we found here + currentCls->add(*(ts + 1)); + + }} + break; + case 243: +#line 1007 "Parser.rl" + {te = p+1;{ + string hex(ts + 3, te - ts - 4); + unsigned long val; + try { + val = stoul(hex, nullptr, 16); + } catch (const std::out_of_range &) { + val = MAX_UNICODE + 1; + } + if (val > MAX_UNICODE) { + throw LocatedParseError("Value in \\x{...} sequence is too large"); + } + currentCls->add((unichar)val); + }} + break; + case 244: +#line 1025 "Parser.rl" + {te = p+1;{ + if (te - ts < 3) { + assert(te - ts == 2); + throw LocatedParseError(SLASH_C_ERROR); + } else { + assert(te - ts == 3); + currentCls->add(decodeCtrl(ts[2])); + } + }} + break; + case 245: +#line 1035 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_WORD, false); + }} + break; + case 246: +#line 1039 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_WORD, true); + }} + break; + case 247: +#line 1043 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_SPACE, false); + }} + break; + case 248: +#line 1047 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_SPACE, true); + }} + break; + case 249: +#line 1051 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_DIGIT, false); + }} + break; + case 250: +#line 1055 "Parser.rl" + {te = p+1;{ + currentCls->add(CLASS_DIGIT, true); + }} + break; + case 251: +#line 1058 "Parser.rl" + {te = p+1;{ + currentCls->addDash(); + }} + break; + case 252: +#line 276 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "'\\" << *(ts + 1) << "' at index " << ts - ptr + << " not supported in a character class."; + throw ParseError(str.str()); + }} + break; + case 253: +#line 276 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "'\\" << *(ts + 1) << "' at index " << ts - ptr + << " not supported in a character class."; + throw ParseError(str.str()); + }} + break; + case 254: +#line 276 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "'\\" << *(ts + 1) << "' at index " << ts - ptr + << " not supported in a character class."; + throw ParseError(str.str()); + }} + break; + case 255: +#line 1075 "Parser.rl" + {te = p+1;{ + // add the literal char + currentCls->add(*(ts + 1)); + }} + break; + case 256: +#line 1081 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + currentCls->add(readUtf8CodePoint2c(ts)); + }} + break; + case 257: +#line 1086 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + currentCls->add(readUtf8CodePoint3c(ts)); + }} + break; + case 258: +#line 1091 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + currentCls->add(readUtf8CodePoint4c(ts)); + }} + break; + case 259: +#line 1096 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 260: +#line 1102 "Parser.rl" + {te = p+1;{ + currentCls->add((u8)*ts); + }} + break; + case 261: +#line 1106 "Parser.rl" + {te = p+1;{ + currentCls->finalize(); + currentSeq->addComponent(move(currentCls)); + inCharClass = false; + {cs = 746;goto _again;} + }} + break; + case 262: +#line 966 "Parser.rl" + {te = p;p--;{ throw LocatedParseError("Malformed property"); }} + break; + case 263: +#line 967 "Parser.rl" + {te = p;p--;{ throw LocatedParseError("Malformed property"); }} + break; + case 264: +#line 970 "Parser.rl" + {te = p;p--;{ + currentCls->add(octAccumulator); + }} + break; + case 265: +#line 973 "Parser.rl" + {te = p;p--;{ + currentCls->add(octAccumulator); + }} + break; + case 266: +#line 992 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); + }} + break; + case 267: +#line 997 "Parser.rl" + {te = p;p--;{ + currentCls->add(accumulator); + }} + break; + case 268: +#line 1021 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); + }} + break; + case 269: +#line 1025 "Parser.rl" + {te = p;p--;{ + if (te - ts < 3) { + assert(te - ts == 2); + throw LocatedParseError(SLASH_C_ERROR); + } else { + assert(te - ts == 3); + currentCls->add(decodeCtrl(ts[2])); + } + }} + break; + case 270: +#line 1096 "Parser.rl" + {te = p;p--;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 271: +#line 1102 "Parser.rl" + {te = p;p--;{ + currentCls->add((u8)*ts); + }} + break; + case 272: +#line 992 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); + }} + break; + case 273: +#line 1021 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); + }} + break; + case 274: +#line 1096 "Parser.rl" + {{p = ((te))-1;}{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 275: +#line 1102 "Parser.rl" + {{p = ((te))-1;}{ + currentCls->add((u8)*ts); + }} + break; + case 276: +#line 1120 "Parser.rl" + {te = p+1;{ + if (currentCls->isNegated()) { + // Already seen a caret; the second one is not a meta-character. + inCharClassEarly = false; + p--; {cs = 819;goto _again;} + } else { + currentCls->negate(); + // Note: we cannot switch off inCharClassEarly here, as /[^]]/ + // needs to use the right square bracket path below. + } + }} + break; + case 277: +#line 1133 "Parser.rl" + {te = p+1;{ + currentCls->add(']'); + inCharClassEarly = false; + }} + break; + case 278: +#line 1138 "Parser.rl" + {te = p+1;{ { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 843;goto _again;}} }} + break; + case 279: +#line 1139 "Parser.rl" + {te = p+1;{ /*noop*/}} + break; + case 280: +#line 1142 "Parser.rl" + {te = p+1;{ + inCharClassEarly = false; + p--; + {cs = 819;goto _again;} + }} + break; + case 281: +#line 1142 "Parser.rl" + {te = p;p--;{ + inCharClassEarly = false; + p--; + {cs = 819;goto _again;} + }} + break; + case 282: +#line 1154 "Parser.rl" + {te = p+1;{ + {cs = 746;goto _again;} + }} + break; + case 283: +#line 1159 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + cc->add(readUtf8CodePoint2c(ts)); + cc->finalize(); + currentSeq->addComponent(move(cc)); + }} + break; + case 284: +#line 1168 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + cc->add(readUtf8CodePoint3c(ts)); + cc->finalize(); + currentSeq->addComponent(move(cc)); + }} + break; + case 285: +#line 1177 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + cc->add(readUtf8CodePoint4c(ts)); + cc->finalize(); + currentSeq->addComponent(move(cc)); + }} + break; + case 286: +#line 1186 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 287: +#line 1192 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, *ts, mode); + }} + break; + case 288: +#line 1186 "Parser.rl" + {te = p;p--;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 289: +#line 1192 "Parser.rl" + {te = p;p--;{ + addLiteral(currentSeq, *ts, mode); + }} + break; + case 290: +#line 1186 "Parser.rl" + {{p = ((te))-1;}{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 291: +#line 1202 "Parser.rl" + {te = p+1;{ + {cs = stack[--top]; goto _again;} + }} + break; + case 292: +#line 1207 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + currentCls->add(readUtf8CodePoint2c(ts)); + inCharClassEarly = false; + }} + break; + case 293: +#line 1213 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + currentCls->add(readUtf8CodePoint3c(ts)); + inCharClassEarly = false; + }} + break; + case 294: +#line 1219 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + currentCls->add(readUtf8CodePoint4c(ts)); + inCharClassEarly = false; + }} + break; + case 295: +#line 1225 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 296: +#line 1231 "Parser.rl" + {te = p+1;{ + currentCls->add(*ts); + inCharClassEarly = false; + }} + break; + case 297: +#line 1225 "Parser.rl" + {te = p;p--;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 298: +#line 1231 "Parser.rl" + {te = p;p--;{ + currentCls->add(*ts); + inCharClassEarly = false; + }} + break; + case 299: +#line 1225 "Parser.rl" + {{p = ((te))-1;}{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 300: +#line 1243 "Parser.rl" + {te = p+1;{ inComment = false; {cs = 746;goto _again;} }} + break; + case 301: +#line 1247 "Parser.rl" + {te = p+1;} + break; + case 302: +#line 1255 "Parser.rl" + {te = p+1;{ inComment = false; {cs = 746;goto _again;} }} + break; + case 303: +#line 1259 "Parser.rl" + {te = p+1;} + break; + case 304: +#line 1491 "Parser.rl" + {act = 288;} + break; + case 305: +#line 1508 "Parser.rl" + {act = 290;} + break; + case 306: +#line 1737 "Parser.rl" + {act = 330;} + break; + case 307: +#line 362 "Parser.rl" + {te = p+1;{ + if (sequences.empty()) { + throw LocatedParseError("Unmatched parentheses"); + } + currentSeq->finalize(); + POP_SEQUENCE; + }} + break; + case 308: +#line 1274 "Parser.rl" + {te = p+1;{ + currentSeq->addAlternation(); + }} + break; + case 309: +#line 1279 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("POSIX named classes are only " + "supported inside a class"); + }} + break; + case 310: +#line 1286 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Unsupported POSIX collating " + "element"); + }} + break; + case 311: +#line 1293 "Parser.rl" + {te = p+1;{ + {cs = 838;goto _again;} + }} + break; + case 312: +#line 1297 "Parser.rl" + {te = p+1;{ /* noop */ }} + break; + case 313: +#line 1299 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent(generateComponent(CLASS_ANY, false, mode)); + }} + break; + case 314: +#line 1303 "Parser.rl" + {te = p+1;{ + if (mode.utf8) { + throw LocatedParseError("\\C is unsupported in UTF8"); + } + currentSeq->addComponent(std::make_unique()); + }} + break; + case 315: +#line 1317 "Parser.rl" + {te = p+1;{ + if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, + ComponentRepeat::REPEAT_NONGREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 316: +#line 1324 "Parser.rl" + {te = p+1;{ + if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, + ComponentRepeat::REPEAT_POSSESSIVE)) { + throwInvalidRepeat(); + } + }} + break; + case 317: +#line 1338 "Parser.rl" + {te = p+1;{ + if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, + ComponentRepeat::REPEAT_NONGREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 318: +#line 1345 "Parser.rl" + {te = p+1;{ + if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, + ComponentRepeat::REPEAT_POSSESSIVE)) { + throwInvalidRepeat(); + } + }} + break; + case 319: +#line 1359 "Parser.rl" + {te = p+1;{ + if (!currentSeq->addRepeat( + 0, 1, ComponentRepeat::REPEAT_NONGREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 320: +#line 1366 "Parser.rl" + {te = p+1;{ + if (!currentSeq->addRepeat( + 0, 1, ComponentRepeat::REPEAT_POSSESSIVE)) { + throwInvalidRepeat(); + } + }} + break; + case 321: +#line 1383 "Parser.rl" + {te = p+1;{ + if (repeatN > repeatM || repeatM == 0) { + throwInvalidRepeat(); + } else if (!currentSeq->addRepeat( + repeatN, repeatM, + ComponentRepeat::REPEAT_NONGREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 322: +#line 1393 "Parser.rl" + {te = p+1;{ + if (repeatN > repeatM || repeatM == 0) { + throwInvalidRepeat(); + } else if (!currentSeq->addRepeat( + repeatN, repeatM, + ComponentRepeat::REPEAT_POSSESSIVE)) { + throwInvalidRepeat(); + } + }} + break; + case 323: +#line 322 "Parser.rl" + {te = p+1;{ + inComment = true; + {cs = 849;goto _again;} + }} + break; + case 324: +#line 1410 "Parser.rl" + {te = p+1;{ p--; { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 787;goto _again;}} }} + break; + case 325: +#line 1414 "Parser.rl" + {te = p+1;{ assert(0); {p++; goto _out; } }} + break; + case 326: +#line 1421 "Parser.rl" + {te = p+1;{ + auto bound = mode.multiline ? ComponentBoundary::BEGIN_LINE + : ComponentBoundary::BEGIN_STRING; + currentSeq->addComponent(std::make_unique(bound)); + }} + break; + case 327: +#line 1428 "Parser.rl" + {te = p+1;{ + auto bound = mode.multiline ? ComponentBoundary::END_LINE + : ComponentBoundary::END_STRING_OPTIONAL_LF; + currentSeq->addComponent(std::make_unique(bound)); + }} + break; + case 328: +#line 1434 "Parser.rl" + {te = p+1;{ + auto bound = ComponentBoundary::BEGIN_STRING; + currentSeq->addComponent(std::make_unique(bound)); + }} + break; + case 329: +#line 1439 "Parser.rl" + {te = p+1;{ + auto bound = ComponentBoundary::END_STRING_OPTIONAL_LF; + currentSeq->addComponent(std::make_unique(bound)); + }} + break; + case 330: +#line 1444 "Parser.rl" + {te = p+1;{ + auto bound = ComponentBoundary::END_STRING; + currentSeq->addComponent(std::make_unique(bound)); + }} + break; + case 331: +#line 1449 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent( + std::make_unique(ts - ptr, false, mode)); + }} + break; + case 332: +#line 1454 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent( + std::make_unique(ts - ptr, true, mode)); + }} + break; + case 333: +#line 1464 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, '\x09', mode); + }} + break; + case 334: +#line 1468 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, '\x0a', mode); + }} + break; + case 335: +#line 1472 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, '\x0d', mode); + }} + break; + case 336: +#line 1476 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, '\x0c', mode); + }} + break; + case 337: +#line 1480 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, '\x07', mode); + }} + break; + case 338: +#line 1484 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, '\x1b', mode); + }} + break; + case 339: +#line 1488 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, octAccumulator, mode); + }} + break; + case 340: +#line 479 "Parser.rl" + {te = p+1;{ + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + currentSeq->addComponent(std::make_unique(accumulator)); + }} + break; + case 341: +#line 486 "Parser.rl" + {te = p+1;{ + // Accumulator is a negative offset. + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + if (accumulator >= groupIndex) { + throw LocatedParseError("Invalid reference"); + } + unsigned idx = groupIndex - accumulator; + currentSeq->addComponent(std::make_unique(idx)); + }} + break; + case 342: +#line 479 "Parser.rl" + {te = p+1;{ + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + currentSeq->addComponent(std::make_unique(accumulator)); + }} + break; + case 343: +#line 486 "Parser.rl" + {te = p+1;{ + // Accumulator is a negative offset. + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + if (accumulator >= groupIndex) { + throw LocatedParseError("Invalid reference"); + } + unsigned idx = groupIndex - accumulator; + currentSeq->addComponent(std::make_unique(idx)); + }} + break; + case 344: +#line 498 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent(std::make_unique(label)); + }} + break; + case 345: +#line 498 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent(std::make_unique(label)); + }} + break; + case 346: +#line 498 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent(std::make_unique(label)); + }} + break; + case 347: +#line 498 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent(std::make_unique(label)); + }} + break; + case 348: +#line 498 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent(std::make_unique(label)); + }} + break; + case 349: +#line 1549 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "Onigiruma subroutine call at index " << ts - ptr << + " not supported."; + throw ParseError(str.str()); + }} + break; + case 350: +#line 1560 "Parser.rl" + {te = p+1;{ + string oct(ts + 3, te - ts - 4); + unsigned long val; + try { + val = stoul(oct, nullptr, 8); + } catch (const std::out_of_range &) { + val = MAX_UNICODE + 1; + } + if ((!mode.utf8 && val > 255) || val > MAX_UNICODE) { + throw LocatedParseError("Value in \\o{...} sequence is too large"); + } + addEscapedOctal(currentSeq, (unichar)val, mode); + }} + break; + case 351: +#line 1578 "Parser.rl" + {te = p+1;{ + addEscapedHex(currentSeq, accumulator, mode); + }} + break; + case 352: +#line 1582 "Parser.rl" + {te = p+1;{ + string hex(ts + 3, te - ts - 4); + unsigned long val; + try { + val = stoul(hex, nullptr, 16); + } catch (const std::out_of_range &) { + val = MAX_UNICODE + 1; + } + if (val > MAX_UNICODE) { + throw LocatedParseError("Value in \\x{...} sequence is too large"); + } + addEscapedHex(currentSeq, (unichar)val, mode); + }} + break; + case 353: +#line 1600 "Parser.rl" + {te = p+1;{ + if (te - ts < 3) { + assert(te - ts == 2); + throw LocatedParseError(SLASH_C_ERROR); + } else { + assert(te - ts == 3); + addLiteral(currentSeq, decodeCtrl(ts[2]), mode); + } + }} + break; + case 354: +#line 1610 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "'\\" << *(ts + 1) << "' at index " << ts - ptr + << " not supported."; + throw ParseError(str.str()); + }} + break; + case 355: +#line 1618 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_WORD, false, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 356: +#line 1623 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_WORD, true, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 357: +#line 1628 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_SPACE, false, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 358: +#line 1633 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_SPACE, true, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 359: +#line 1638 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_DIGIT, false, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 360: +#line 1643 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_DIGIT, true, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 361: +#line 1648 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_HORZ, false, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 362: +#line 1653 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_HORZ, true, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 363: +#line 1658 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_VERT, false, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 364: +#line 1663 "Parser.rl" + {te = p+1;{ + auto cc = generateComponent(CLASS_VERT, true, mode); + currentSeq->addComponent(move(cc)); + }} + break; + case 365: +#line 1668 "Parser.rl" + {te = p+1;{ + assert(!currentCls && !inCharClass); + currentCls = getComponentClass(mode); + negated = false; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 559;goto _again;}} + }} + break; + case 366: +#line 1676 "Parser.rl" + {te = p+1;{ + assert(!currentCls && !inCharClass); + currentCls = getComponentClass(mode); + negated = false; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 818;goto _again;}} + }} + break; + case 367: +#line 1684 "Parser.rl" + {te = p+1;{ + assert(!currentCls && !inCharClass); + currentCls = getComponentClass(mode); + negated = true; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 559;goto _again;}} + }} + break; + case 368: +#line 1692 "Parser.rl" + {te = p+1;{ + assert(!currentCls && !inCharClass); + currentCls = getComponentClass(mode); + negated = true; + p--; + { + DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); + if ((int)stack.size() == top) { + stack.resize(2 * (top + 1)); + } + {stack[top++] = cs; cs = 818;goto _again;}} + }} + break; + case 369: +#line 1704 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "\\R at index " << ts - ptr << " not supported."; + throw ParseError(str.str()); + }} + break; + case 370: +#line 1711 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "\\K at index " << ts - ptr << " not supported."; + throw ParseError(str.str()); + }} + break; + case 371: +#line 1726 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "\\G at index " << ts - ptr << " not supported."; + throw ParseError(str.str()); + }} + break; + case 372: +#line 1732 "Parser.rl" + {te = p+1;{ + currentSeq->addComponent(std::make_unique(ts - ptr, mode)); + }} + break; + case 373: +#line 1737 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, *(ts + 1), mode); + }} + break; + case 374: +#line 316 "Parser.rl" + {te = p+1;{ + inComment = true; + {cs = 848;goto _again;} + }} + break; + case 375: +#line 433 "Parser.rl" + {te = p+1;{ + mode = newMode; + currentSeq->addComponent(std::make_unique()); + }} + break; + case 376: +#line 355 "Parser.rl" + {te = p+1;{ + PUSH_SEQUENCE; + mode = newMode; + currentSeq = + enterSequence(currentSeq, std::make_unique()); + }} + break; + case 377: +#line 369 "Parser.rl" + {te = p+1;{ + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(ComponentAssertion::LOOKAHEAD, + ComponentAssertion::POS)); + }} + break; + case 378: +#line 375 "Parser.rl" + {te = p+1;{ + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(ComponentAssertion::LOOKAHEAD, + ComponentAssertion::NEG)); + }} + break; + case 379: +#line 381 "Parser.rl" + {te = p+1;{ + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(ComponentAssertion::LOOKBEHIND, + ComponentAssertion::POS)); + }} + break; + case 380: +#line 387 "Parser.rl" + {te = p+1;{ + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(ComponentAssertion::LOOKBEHIND, + ComponentAssertion::NEG)); + }} + break; + case 381: +#line 393 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Embedded code is not supported"); + }} + break; + case 382: +#line 393 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Embedded code is not supported"); + }} + break; + case 383: +#line 416 "Parser.rl" + {te = p+1;{ + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique()); + }} + break; + case 384: +#line 336 "Parser.rl" + {te = p+1;{ + assert(!label.empty()); // should be guaranteed by machine + char c = *label.begin(); + if (c >= '0' && c <= '9') { + throw LocatedParseError("Group name cannot begin with a digit"); + } + if (!groupNames.insert(label).second) { + throw LocatedParseError("Two named subpatterns use the name '" + label + "'"); + } + PUSH_SEQUENCE; + auto seq = std::make_unique(); + seq->setCaptureIndex(groupIndex++); + seq->setCaptureName(label); + currentSeq = enterSequence(currentSeq, move(seq)); + }} + break; + case 385: +#line 399 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Subpattern reference unsupported"); + }} + break; + case 386: +#line 399 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Subpattern reference unsupported"); + }} + break; + case 387: +#line 1783 "Parser.rl" + {te = p+1;{ + auto a = std::make_unique( + ComponentAssertion::LOOKAHEAD, ComponentAssertion::POS); + ComponentAssertion *a_seq = a.get(); + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(move(a))); + PUSH_SEQUENCE; + currentSeq = a_seq; + }} + break; + case 388: +#line 1794 "Parser.rl" + {te = p+1;{ + auto a = std::make_unique( + ComponentAssertion::LOOKAHEAD, ComponentAssertion::NEG); + ComponentAssertion *a_seq = a.get(); + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(move(a))); + PUSH_SEQUENCE; + currentSeq = a_seq; + }} + break; + case 389: +#line 1805 "Parser.rl" + {te = p+1;{ + auto a = std::make_unique( + ComponentAssertion::LOOKBEHIND, ComponentAssertion::POS); + ComponentAssertion *a_seq = a.get(); + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(move(a))); + PUSH_SEQUENCE; + currentSeq = a_seq; + }} + break; + case 390: +#line 1816 "Parser.rl" + {te = p+1;{ + auto a = std::make_unique( + ComponentAssertion::LOOKBEHIND, ComponentAssertion::NEG); + ComponentAssertion *a_seq = a.get(); + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(move(a))); + PUSH_SEQUENCE; + currentSeq = a_seq; + }} + break; + case 391: +#line 1828 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Pattern recursion not supported"); + }} + break; + case 392: +#line 402 "Parser.rl" + {te = p+1;{ + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + PUSH_SEQUENCE; + currentSeq = enterSequence(currentSeq, + std::make_unique(accumulator)); + }} + break; + case 393: +#line 410 "Parser.rl" + {te = p+1;{ + PUSH_SEQUENCE; + assert(!label.empty()); + currentSeq = enterSequence(currentSeq, + std::make_unique(label)); + }} + break; + case 394: +#line 1844 "Parser.rl" + {te = p+1;{ + ostringstream str; + str << "Callout at index " << ts - ptr << " not supported."; + throw ParseError(str.str()); + }} + break; + case 395: +#line 1852 "Parser.rl" + {te = p+1;{ + throw LocatedParseError("Unrecognised character after (?"); + }} + break; + case 396: +#line 1857 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + cc->add(readUtf8CodePoint2c(ts)); + cc->finalize(); + currentSeq->addComponent(move(cc)); + }} + break; + case 397: +#line 1866 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + cc->add(readUtf8CodePoint3c(ts)); + cc->finalize(); + currentSeq->addComponent(move(cc)); + }} + break; + case 398: +#line 1875 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + /* leverage ComponentClass to generate the vertices */ + auto cc = getComponentClass(mode); + cc->add(readUtf8CodePoint4c(ts)); + cc->finalize(); + currentSeq->addComponent(move(cc)); + }} + break; + case 399: +#line 1884 "Parser.rl" + {te = p+1;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 400: +#line 1893 "Parser.rl" + {te = p+1;{ + if (mode.ignore_space == false) { + addLiteral(currentSeq, *ts, mode); + } + }} + break; + case 401: +#line 1898 "Parser.rl" + {te = p+1;{ + addLiteral(currentSeq, *ts, mode); + }} + break; + case 402: +#line 328 "Parser.rl" + {te = p;p--;{ + PUSH_SEQUENCE; + auto seq = std::make_unique(); + seq->setCaptureIndex(groupIndex++); + currentSeq = enterSequence(currentSeq, move(seq)); + }} + break; + case 403: +#line 421 "Parser.rl" + {te = p;p--;{ + assert(!currentCls); + assert(!inCharClass); // not reentrant + currentCls = getComponentClass(mode); + inCharClass = true; + inCharClassEarly = true; + currentClsBegin = ts; + {cs = 836;goto _again;} + }} + break; + case 404: +#line 1310 "Parser.rl" + {te = p;p--;{ + if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, + ComponentRepeat::REPEAT_GREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 405: +#line 1331 "Parser.rl" + {te = p;p--;{ + if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, + ComponentRepeat::REPEAT_GREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 406: +#line 1352 "Parser.rl" + {te = p;p--;{ + if (!currentSeq->addRepeat( + 0, 1, ComponentRepeat::REPEAT_GREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 407: +#line 1373 "Parser.rl" + {te = p;p--;{ + if (repeatN > repeatM || repeatM == 0) { + throwInvalidRepeat(); + } else if (!currentSeq->addRepeat( + repeatN, repeatM, + ComponentRepeat::REPEAT_GREEDY)) { + throwInvalidRepeat(); + } + }} + break; + case 408: +#line 1488 "Parser.rl" + {te = p;p--;{ + addLiteral(currentSeq, octAccumulator, mode); + }} + break; + case 409: +#line 1491 "Parser.rl" + {te = p;p--;{ + // If there are enough capturing sub expressions, this may be + // a back reference + accumulator = parseAsDecimal(octAccumulator); + if (accumulator < groupIndex) { + currentSeq->addComponent(std::make_unique(accumulator)); + } else { + addEscapedOctal(currentSeq, octAccumulator, mode); + } + }} + break; + case 410: +#line 479 "Parser.rl" + {te = p;p--;{ + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + currentSeq->addComponent(std::make_unique(accumulator)); + }} + break; + case 411: +#line 479 "Parser.rl" + {te = p;p--;{ + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + currentSeq->addComponent(std::make_unique(accumulator)); + }} + break; + case 412: +#line 486 "Parser.rl" + {te = p;p--;{ + // Accumulator is a negative offset. + if (accumulator == 0) { + throw LocatedParseError("Numbered reference cannot be zero"); + } + if (accumulator >= groupIndex) { + throw LocatedParseError("Invalid reference"); + } + unsigned idx = groupIndex - accumulator; + currentSeq->addComponent(std::make_unique(idx)); + }} + break; + case 413: +#line 1557 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Invalid reference after \\g"); + }} + break; + case 414: +#line 1574 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); + }} + break; + case 415: +#line 1578 "Parser.rl" + {te = p;p--;{ + addEscapedHex(currentSeq, accumulator, mode); + }} + break; + case 416: +#line 1596 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); + }} + break; + case 417: +#line 1600 "Parser.rl" + {te = p;p--;{ + if (te - ts < 3) { + assert(te - ts == 2); + throw LocatedParseError(SLASH_C_ERROR); + } else { + assert(te - ts == 3); + addLiteral(currentSeq, decodeCtrl(ts[2]), mode); + } + }} + break; + case 418: +#line 1700 "Parser.rl" + {te = p;p--;{ throw LocatedParseError("Malformed property"); }} + break; + case 419: +#line 1701 "Parser.rl" + {te = p;p--;{ throw LocatedParseError("Malformed property"); }} + break; + case 420: +#line 1719 "Parser.rl" + {te = p;p--;{ + ostringstream str; + str << "\\k at index " << ts - ptr << " not supported."; + throw ParseError(str.str()); + }} + break; + case 421: +#line 1742 "Parser.rl" + {te = p;p--;{ + assert(ts + 1 == pe); + ostringstream str; + str << "Unescaped \\ at end of input, index " << ts - ptr << "."; + throw ParseError(str.str()); + }} + break; + case 422: +#line 396 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Conditional subpattern unsupported"); + }} + break; + case 423: +#line 1852 "Parser.rl" + {te = p;p--;{ + throw LocatedParseError("Unrecognised character after (?"); + }} + break; + case 424: +#line 1884 "Parser.rl" + {te = p;p--;{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 425: +#line 1898 "Parser.rl" + {te = p;p--;{ + addLiteral(currentSeq, *ts, mode); + }} + break; + case 426: +#line 328 "Parser.rl" + {{p = ((te))-1;}{ + PUSH_SEQUENCE; + auto seq = std::make_unique(); + seq->setCaptureIndex(groupIndex++); + currentSeq = enterSequence(currentSeq, move(seq)); + }} + break; + case 427: +#line 421 "Parser.rl" + {{p = ((te))-1;}{ + assert(!currentCls); + assert(!inCharClass); // not reentrant + currentCls = getComponentClass(mode); + inCharClass = true; + inCharClassEarly = true; + currentClsBegin = ts; + {cs = 836;goto _again;} + }} + break; + case 428: +#line 1557 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Invalid reference after \\g"); + }} + break; + case 429: +#line 1574 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); + }} + break; + case 430: +#line 1596 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); + }} + break; + case 431: +#line 1719 "Parser.rl" + {{p = ((te))-1;}{ + ostringstream str; + str << "\\k at index " << ts - ptr << " not supported."; + throw ParseError(str.str()); + }} + break; + case 432: +#line 396 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Conditional subpattern unsupported"); + }} + break; + case 433: +#line 1852 "Parser.rl" + {{p = ((te))-1;}{ + throw LocatedParseError("Unrecognised character after (?"); + }} + break; + case 434: +#line 1884 "Parser.rl" + {{p = ((te))-1;}{ + assert(mode.utf8); + throwInvalidUtf8(); + }} + break; + case 435: +#line 1898 "Parser.rl" + {{p = ((te))-1;}{ + addLiteral(currentSeq, *ts, mode); + }} + break; + case 436: +#line 1 "NONE" + { switch( act ) { + case 288: + {{p = ((te))-1;} + // If there are enough capturing sub expressions, this may be + // a back reference + accumulator = parseAsDecimal(octAccumulator); + if (accumulator < groupIndex) { + currentSeq->addComponent(std::make_unique(accumulator)); + } else { + addEscapedOctal(currentSeq, octAccumulator, mode); + } + } + break; + case 290: + {{p = ((te))-1;} + // if there are enough left parens to this point, back ref + if (accumulator < groupIndex) { + currentSeq->addComponent(std::make_unique(accumulator)); + } else { + // Otherwise, we interpret the first three digits as an + // octal escape, and the remaining characters stand for + // themselves as literals. + const char *s = ts; + unsigned int accum = 0; + unsigned int oct_digits = 0; + assert(*s == '\\'); // token starts at backslash + for (++s; s < te && oct_digits < 3; ++oct_digits, ++s) { + u8 digit = *s - '0'; + if (digit < 8) { + accum = digit + accum * 8; + } else { + break; + } + } + + if (oct_digits > 0) { + addEscapedOctal(currentSeq, accum, mode); + } + + // And then the rest of the digits, if any, are literal. + for (; s < te; ++s) { + addLiteral(currentSeq, *s, mode); + } + } + } + break; + case 330: + {{p = ((te))-1;} + addLiteral(currentSeq, *(ts + 1), mode); + } + break; + } + } + break; +#line 5508 "Parser.cpp" + } + } + +_again: + _acts = _regex_actions + _regex_to_state_actions[cs]; + _nacts = (unsigned int) *_acts++; + while ( _nacts-- > 0 ) { + switch ( *_acts++ ) { + case 23: +#line 1 "NONE" + {ts = 0;} + break; +#line 5521 "Parser.cpp" + } + } + + if ( cs == 0 ) + goto _out; + if ( ++p != pe ) + goto _resume; + _test_eof: {} + if ( p == eof ) + { + if ( _regex_eof_trans[cs] > 0 ) { + _trans = _regex_eof_trans[cs] - 1; + goto _eof_trans; + } + const short *__acts = _regex_actions + _regex_eof_actions[cs]; + unsigned int __nacts = (unsigned int) *__acts++; + while ( __nacts-- > 0 ) { + switch ( *__acts++ ) { + case 22: +#line 730 "Parser.rl" + { throw LocatedParseError("Malformed property"); } + break; +#line 5544 "Parser.cpp" + } + } + } + + _out: {} + } + +#line 1984 "Parser.rl" + + if (p != pe && *p != '\0') { + // didn't make it to the end of our input, but we didn't throw a ParseError? + assert(0); + ostringstream str; + str << "Parse error at index " << (p - ptr) << "."; + throw ParseError(str.str()); + } + + if (currentCls) { + assert(inCharClass); + assert(currentClsBegin); + ostringstream oss; + oss << "Unterminated character class starting at index " + << currentClsBegin - ptr << "."; + throw ParseError(oss.str()); + } + + if (inComment) { + throw ParseError("Unterminated comment."); + } + + if (!sequences.empty()) { + ostringstream str; + str << "Missing close parenthesis for group started at index " + << sequences.back().seqOffset << "."; + throw ParseError(str.str()); + } + + // Unlikely, but possible + if (groupIndex > 65535) { + throw ParseError("The maximum number of capturing subexpressions is 65535."); + } + + // Finalize the top-level sequence, which will take care of any + // top-level alternation. + currentSeq->finalize(); + assert(currentSeq == rootSeq.get()); + + // Ensure that all references are valid. + checkReferences(*rootSeq, groupIndex, groupNames); + + return move(rootSeq); + } catch (LocatedParseError &error) { + if (ts >= ptr && ts <= pe) { + error.locate(ts - ptr); + } else { + error.locate(0); + } + throw; + } +} + +} // namespace ue2 diff --git a/contrib/vectorscan-cmake/rageled_files/control_verbs.cpp b/contrib/vectorscan-cmake/rageled_files/control_verbs.cpp new file mode 100644 index 00000000000..19b5c6955e1 --- /dev/null +++ b/contrib/vectorscan-cmake/rageled_files/control_verbs.cpp @@ -0,0 +1,443 @@ + +#line 1 "control_verbs.rl" +/* + * Copyright (c) 2017, Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Intel Corporation nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * \file + * \brief Parser for control verbs that can occur at the beginning of a pattern. + */ + +#include "parser/control_verbs.h" + +#include "parser/Parser.h" +#include "parser/parse_error.h" + +#include +#include + +using namespace std; + +namespace ue2 { + +const char *read_control_verbs(const char *ptr, const char *end, size_t start, + ParseMode &mode) { + const char *p = ptr; + const char *pe = end; + const char *eof = pe; + const char *ts, *te; + int cs; + UNUSED int act; + + +#line 59 "control_verbs.cpp" +static const char _ControlVerbs_actions[] = { + 0, 1, 0, 1, 1, 1, 2, 1, + 3, 1, 4, 1, 5, 1, 6, 1, + 7, 1, 8, 1, 9 +}; + +static const unsigned char _ControlVerbs_key_offsets[] = { + 0, 7, 8, 10, 12, 14, 16, 18, + 20, 21, 23, 25, 27, 30, 32, 34, + 36, 38, 40, 42, 44, 46, 48, 50, + 52, 55, 57, 59, 61, 63, 66, 68, + 70, 72, 74, 76, 79, 82, 84, 86, + 88, 90, 92, 94, 96, 98, 100, 102, + 105, 107, 109, 111, 113, 115, 117, 119, + 121, 123, 125, 127, 129, 131, 133, 135, + 137, 139, 141, 143, 146, 148, 149, 151, + 155, 157, 159, 160, 161 +}; + +static const char _ControlVerbs_trans_keys[] = { + 41, 65, 66, 67, 76, 78, 85, 41, + 41, 78, 41, 89, 41, 67, 41, 82, + 41, 76, 41, 70, 41, 41, 83, 41, + 82, 41, 95, 41, 65, 85, 41, 78, + 41, 89, 41, 67, 41, 78, 41, 73, + 41, 67, 41, 79, 41, 68, 41, 69, + 41, 82, 41, 76, 41, 70, 73, 41, + 77, 41, 73, 41, 84, 41, 95, 41, + 77, 82, 41, 65, 41, 84, 41, 67, + 41, 72, 41, 61, 41, 48, 57, 41, + 48, 57, 41, 69, 41, 67, 41, 85, + 41, 82, 41, 83, 41, 73, 41, 79, + 41, 78, 41, 79, 41, 95, 41, 65, + 83, 41, 85, 41, 84, 41, 79, 41, + 95, 41, 80, 41, 79, 41, 83, 41, + 83, 41, 69, 41, 83, 41, 83, 41, + 84, 41, 65, 41, 82, 41, 84, 41, + 95, 41, 79, 41, 80, 41, 84, 41, + 67, 84, 41, 80, 41, 41, 70, 41, + 49, 51, 56, 41, 54, 41, 50, 41, + 40, 42, 0 +}; + +static const char _ControlVerbs_single_lengths[] = { + 7, 1, 2, 2, 2, 2, 2, 2, + 1, 2, 2, 2, 3, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, + 3, 2, 2, 2, 2, 3, 2, 2, + 2, 2, 2, 1, 1, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 3, + 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 3, 2, 1, 2, 4, + 2, 2, 1, 1, 1 +}; + +static const char _ControlVerbs_range_lengths[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0 +}; + +static const short _ControlVerbs_index_offsets[] = { + 0, 8, 10, 13, 16, 19, 22, 25, + 28, 30, 33, 36, 39, 43, 46, 49, + 52, 55, 58, 61, 64, 67, 70, 73, + 76, 80, 83, 86, 89, 92, 96, 99, + 102, 105, 108, 111, 114, 117, 120, 123, + 126, 129, 132, 135, 138, 141, 144, 147, + 151, 154, 157, 160, 163, 166, 169, 172, + 175, 178, 181, 184, 187, 190, 193, 196, + 199, 202, 205, 208, 212, 215, 217, 220, + 225, 228, 231, 233, 235 +}; + +static const char _ControlVerbs_indicies[] = { + 0, 2, 3, 4, 5, 6, 7, 1, + 8, 1, 8, 9, 1, 8, 10, 1, + 11, 12, 1, 8, 13, 1, 8, 14, + 1, 8, 15, 1, 11, 1, 8, 16, + 1, 8, 17, 1, 8, 18, 1, 8, + 19, 20, 1, 8, 21, 1, 8, 22, + 1, 8, 12, 1, 8, 23, 1, 8, + 24, 1, 8, 25, 1, 8, 26, 1, + 8, 27, 1, 8, 15, 1, 8, 28, + 1, 11, 14, 1, 8, 15, 29, 1, + 8, 30, 1, 8, 31, 1, 8, 32, + 1, 8, 33, 1, 8, 34, 35, 1, + 8, 36, 1, 8, 37, 1, 8, 38, + 1, 8, 39, 1, 8, 40, 1, 8, + 41, 1, 11, 41, 1, 8, 42, 1, + 8, 43, 1, 8, 44, 1, 8, 45, + 1, 8, 46, 1, 8, 47, 1, 8, + 48, 1, 8, 39, 1, 8, 49, 1, + 8, 50, 1, 8, 51, 52, 1, 8, + 53, 1, 8, 54, 1, 8, 55, 1, + 8, 56, 1, 8, 57, 1, 8, 58, + 1, 8, 59, 1, 8, 60, 1, 8, + 61, 1, 8, 62, 1, 8, 15, 1, + 8, 63, 1, 8, 64, 1, 8, 65, + 1, 8, 66, 1, 8, 67, 1, 8, + 68, 1, 8, 69, 1, 8, 15, 1, + 8, 70, 71, 1, 8, 72, 1, 73, + 1, 8, 74, 1, 75, 76, 77, 78, + 1, 8, 15, 1, 8, 15, 1, 75, + 1, 80, 79, 82, 81, 0 +}; + +static const char _ControlVerbs_trans_targs[] = { + 75, 1, 2, 9, 22, 24, 45, 67, + 75, 3, 4, 75, 5, 6, 7, 8, + 10, 11, 12, 13, 16, 14, 15, 17, + 18, 19, 20, 21, 23, 25, 26, 27, + 28, 29, 30, 37, 31, 32, 33, 34, + 35, 36, 38, 39, 40, 41, 42, 43, + 44, 46, 47, 48, 59, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 60, + 61, 62, 63, 64, 65, 66, 68, 70, + 69, 75, 71, 75, 72, 73, 74, 75, + 76, 75, 0 +}; + +static const char _ControlVerbs_trans_actions[] = { + 19, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 11, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 9, 0, 7, 0, 0, 0, 15, + 5, 17, 0 +}; + +static const char _ControlVerbs_to_state_actions[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0 +}; + +static const char _ControlVerbs_from_state_actions[] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 3, 0 +}; + +static const short _ControlVerbs_eof_trans[] = { + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 0, 82 +}; + +static const int ControlVerbs_start = 75; +static const int ControlVerbs_first_final = 75; +static const int ControlVerbs_error = -1; + +static const int ControlVerbs_en_main = 75; + + +#line 249 "control_verbs.cpp" + { + cs = ControlVerbs_start; + ts = 0; + te = 0; + act = 0; + } + +#line 105 "control_verbs.rl" + + + try { + +#line 262 "control_verbs.cpp" + { + int _klen; + unsigned int _trans; + const char *_acts; + unsigned int _nacts; + const char *_keys; + + if ( p == pe ) + goto _test_eof; +_resume: + _acts = _ControlVerbs_actions + _ControlVerbs_from_state_actions[cs]; + _nacts = (unsigned int) *_acts++; + while ( _nacts-- > 0 ) { + switch ( *_acts++ ) { + case 1: +#line 1 "NONE" + {ts = p;} + break; +#line 281 "control_verbs.cpp" + } + } + + _keys = _ControlVerbs_trans_keys + _ControlVerbs_key_offsets[cs]; + _trans = _ControlVerbs_index_offsets[cs]; + + _klen = _ControlVerbs_single_lengths[cs]; + if ( _klen > 0 ) { + const char *_lower = _keys; + const char *_mid; + const char *_upper = _keys + _klen - 1; + while (1) { + if ( _upper < _lower ) + break; + + _mid = _lower + ((_upper-_lower) >> 1); + if ( (*p) < *_mid ) + _upper = _mid - 1; + else if ( (*p) > *_mid ) + _lower = _mid + 1; + else { + _trans += (unsigned int)(_mid - _keys); + goto _match; + } + } + _keys += _klen; + _trans += _klen; + } + + _klen = _ControlVerbs_range_lengths[cs]; + if ( _klen > 0 ) { + const char *_lower = _keys; + const char *_mid; + const char *_upper = _keys + (_klen<<1) - 2; + while (1) { + if ( _upper < _lower ) + break; + + _mid = _lower + (((_upper-_lower) >> 1) & ~1); + if ( (*p) < _mid[0] ) + _upper = _mid - 2; + else if ( (*p) > _mid[1] ) + _lower = _mid + 2; + else { + _trans += (unsigned int)((_mid - _keys)>>1); + goto _match; + } + } + _trans += _klen; + } + +_match: + _trans = _ControlVerbs_indicies[_trans]; +_eof_trans: + cs = _ControlVerbs_trans_targs[_trans]; + + if ( _ControlVerbs_trans_actions[_trans] == 0 ) + goto _again; + + _acts = _ControlVerbs_actions + _ControlVerbs_trans_actions[_trans]; + _nacts = (unsigned int) *_acts++; + while ( _nacts-- > 0 ) + { + switch ( *_acts++ ) + { + case 2: +#line 1 "NONE" + {te = p+1;} + break; + case 3: +#line 76 "control_verbs.rl" + {te = p+1;{ + mode.utf8 = true; + }} + break; + case 4: +#line 80 "control_verbs.rl" + {te = p+1;{ + mode.ucp = true; + }} + break; + case 5: +#line 84 "control_verbs.rl" + {te = p+1;{ + ostringstream str; + str << "Unsupported control verb " << string(ts, te - ts); + throw LocatedParseError(str.str()); + }} + break; + case 6: +#line 90 "control_verbs.rl" + {te = p+1;{ + ostringstream str; + str << "Unknown control verb " << string(ts, te - ts); + throw LocatedParseError(str.str()); + }} + break; + case 7: +#line 97 "control_verbs.rl" + {te = p+1;{ + p--; + {p++; goto _out; } + }} + break; + case 8: +#line 97 "control_verbs.rl" + {te = p;p--;{ + p--; + {p++; goto _out; } + }} + break; + case 9: +#line 97 "control_verbs.rl" + {{p = ((te))-1;}{ + p--; + {p++; goto _out; } + }} + break; +#line 400 "control_verbs.cpp" + } + } + +_again: + _acts = _ControlVerbs_actions + _ControlVerbs_to_state_actions[cs]; + _nacts = (unsigned int) *_acts++; + while ( _nacts-- > 0 ) { + switch ( *_acts++ ) { + case 0: +#line 1 "NONE" + {ts = 0;} + break; +#line 413 "control_verbs.cpp" + } + } + + if ( ++p != pe ) + goto _resume; + _test_eof: {} + if ( p == eof ) + { + if ( _ControlVerbs_eof_trans[cs] > 0 ) { + _trans = _ControlVerbs_eof_trans[cs] - 1; + goto _eof_trans; + } + } + + _out: {} + } + +#line 109 "control_verbs.rl" + } catch (LocatedParseError &error) { + if (ts >= ptr && ts <= pe) { + error.locate(ts - ptr + start); + } else { + error.locate(0); + } + throw; + } + + return p; +} + +} // namespace ue2 diff --git a/contrib/hyperscan-cmake/x86_64/config.h b/contrib/vectorscan-cmake/x86_64/config.h similarity index 73% rename from contrib/hyperscan-cmake/x86_64/config.h rename to contrib/vectorscan-cmake/x86_64/config.h index 4786e3f4e21..eab2f3eb079 100644 --- a/contrib/hyperscan-cmake/x86_64/config.h +++ b/contrib/vectorscan-cmake/x86_64/config.h @@ -15,15 +15,42 @@ /* "Define if building for EM64T" */ #define ARCH_X86_64 +/* "Define if building for ARM32" */ +/* #undef ARCH_ARM32 */ + +/* "Define if building for AARCH64" */ +/* #undef ARCH_AARCH64 */ + +/* "Define if building for PPC64EL" */ +/* #undef ARCH_PPC64EL */ + +/* "Define if cross compiling for AARCH64" */ +/* #undef CROSS_COMPILE_AARCH64 */ + +/* Define if building SVE for AARCH64. */ +/* #undef BUILD_SVE */ + +/* Define if building SVE2 for AARCH64. */ +/* #undef BUILD_SVE2 */ + +/* Define if building SVE2+BITPERM for AARCH64. */ +/* #undef BUILD_SVE2_BITPERM */ + /* internal build, switch on dump support. */ /* #undef DUMP_SUPPORT */ /* Define if building "fat" runtime. */ /* #undef FAT_RUNTIME */ +/* Define if building AVX2 in the fat runtime. */ +/* #undef BUILD_AVX2 */ + /* Define if building AVX-512 in the fat runtime. */ /* #undef BUILD_AVX512 */ +/* Define if building AVX512VBMI in the fat runtime. */ +/* #undef BUILD_AVX512VBMI */ + /* Define to 1 if `backtrace' works. */ #define HAVE_BACKTRACE @@ -45,6 +72,15 @@ /* C compiler has intrin.h */ /* #undef HAVE_C_INTRIN_H */ +/* C compiler has arm_neon.h */ +/* #undef HAVE_C_ARM_NEON_H */ + +/* C compiler has arm_sve.h */ +/* #undef HAVE_C_ARM_SVE_H */ + +/* C compiler has arm_neon.h */ +/* #undef HAVE_C_PPC64EL_ALTIVEC_H */ + /* Define to 1 if you have the declaration of `pthread_setaffinity_np', and to 0 if you don't. */ /* #undef HAVE_DECL_PTHREAD_SETAFFINITY_NP */ @@ -85,7 +121,7 @@ /* #undef HAVE__ALIGNED_MALLOC */ /* Define if compiler has __builtin_constant_p */ -#define HAVE__BUILTIN_CONSTANT_P +/* #undef HAVE__BUILTIN_CONSTANT_P */ /* Optimize, inline critical functions */ #define HS_OPTIMIZE diff --git a/docs/en/development/contrib.md b/docs/en/development/contrib.md index 3936b613bcb..1fbbff7dcca 100644 --- a/docs/en/development/contrib.md +++ b/docs/en/development/contrib.md @@ -40,7 +40,7 @@ The list of third-party libraries: | googletest | [BSD 3-clause](https://github.com/google/googletest/blob/e7e591764baba0a0c3c9ad0014430e7a27331d16/LICENSE) | | grpc | [Apache](https://github.com/ClickHouse-Extras/grpc/blob/60c986e15cae70aade721d26badabab1f822fdd6/LICENSE) | | h3 | [Apache](https://github.com/ClickHouse-Extras/h3/blob/c7f46cfd71fb60e2fefc90e28abe81657deff735/LICENSE) | -| hyperscan | [Boost](https://github.com/ClickHouse-Extras/hyperscan/blob/e9f08df0213fc637aac0a5bbde9beeaeba2fe9fa/LICENSE) | +| vectorscan | [Boost](https://github.com/ClickHouse-Extras/hyperscan/blob/73695e419c27af7fe2a099c7aa57931cc02aea5d/LICENSE) | | icu | [Public Domain](https://github.com/unicode-org/icu/blob/a56dde820dc35665a66f2e9ee8ba58e75049b668/icu4c/LICENSE) | | icudata | [Public Domain](https://github.com/ClickHouse-Extras/icudata/blob/72d9a4a7febc904e2b0a534ccb25ae40fac5f1e5/LICENSE) | | jemalloc | [BSD 2-clause](https://github.com/ClickHouse-Extras/jemalloc/blob/e6891d9746143bf2cf617493d880ba5a0b9a3efd/COPYING) | diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 60386908f01..a37891377f4 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -86,8 +86,8 @@ if (TARGET ch_contrib::h3) target_link_libraries (clickhouse_functions PRIVATE ch_contrib::h3) endif() -if (TARGET ch_contrib::hyperscan) - target_link_libraries(clickhouse_functions PRIVATE ch_contrib::hyperscan) +if (TARGET ch_contrib::vectorscan) + target_link_libraries(clickhouse_functions PRIVATE ch_contrib::vectorscan) endif() if (TARGET ch_contrib::simdjson) diff --git a/src/Functions/MultiMatchAllIndicesImpl.h b/src/Functions/MultiMatchAllIndicesImpl.h index adf9e9b585f..80a71548deb 100644 --- a/src/Functions/MultiMatchAllIndicesImpl.h +++ b/src/Functions/MultiMatchAllIndicesImpl.h @@ -9,7 +9,7 @@ #include "config_functions.h" #include -#if USE_HYPERSCAN +#if USE_VECTORSCAN # include #endif @@ -60,7 +60,7 @@ struct MultiMatchAllIndicesImpl [[maybe_unused]] std::optional edit_distance) { offsets.resize(haystack_offsets.size()); -#if USE_HYPERSCAN +#if USE_VECTORSCAN const auto & hyperscan_regex = MultiRegexps::get(needles, edit_distance); hs_scratch_t * scratch = nullptr; hs_error_t err = hs_clone_scratch(hyperscan_regex->getScratch(), &scratch); @@ -97,7 +97,7 @@ struct MultiMatchAllIndicesImpl on_match, &res); if (err != HS_SUCCESS) - throw Exception("Failed to scan with hyperscan", ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT); + throw Exception("Failed to scan with vectorscan", ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT); offsets[i] = res.size(); offset = haystack_offsets[i]; } @@ -108,9 +108,9 @@ struct MultiMatchAllIndicesImpl (void)res; (void)offsets; throw Exception( - "multi-search all indices is not implemented when hyperscan is off (is it x86 processor?)", + "multi-search all indices is not implemented when vectorscan is off", ErrorCodes::NOT_IMPLEMENTED); -#endif // USE_HYPERSCAN +#endif // USE_VECTORSCAN } }; diff --git a/src/Functions/MultiMatchAnyImpl.h b/src/Functions/MultiMatchAnyImpl.h index 8a65c8cb2b4..fbbefe7be1d 100644 --- a/src/Functions/MultiMatchAnyImpl.h +++ b/src/Functions/MultiMatchAnyImpl.h @@ -8,7 +8,7 @@ #include "config_functions.h" #include -#if USE_HYPERSCAN +#if USE_VECTORSCAN # include #else # include "MatchImpl.h" @@ -64,13 +64,13 @@ struct MultiMatchAnyImpl (void)FindAny; (void)FindAnyIndex; res.resize(haystack_offsets.size()); -#if USE_HYPERSCAN +#if USE_VECTORSCAN const auto & hyperscan_regex = MultiRegexps::get(needles, edit_distance); hs_scratch_t * scratch = nullptr; hs_error_t err = hs_clone_scratch(hyperscan_regex->getScratch(), &scratch); if (err != HS_SUCCESS) - throw Exception("Could not clone scratch space for hyperscan", ErrorCodes::CANNOT_ALLOCATE_MEMORY); + throw Exception("Could not clone scratch space for vectorscan", ErrorCodes::CANNOT_ALLOCATE_MEMORY); MultiRegexps::ScratchPtr smart_scratch(scratch); @@ -92,7 +92,7 @@ struct MultiMatchAnyImpl for (size_t i = 0; i < haystack_offsets_size; ++i) { UInt64 length = haystack_offsets[i] - offset - 1; - /// Hyperscan restriction. + /// Vectorscan restriction. if (length > std::numeric_limits::max()) throw Exception("Too long string to search", ErrorCodes::TOO_MANY_BYTES); /// Zero the result, scan, check, update the offset. @@ -106,14 +106,14 @@ struct MultiMatchAnyImpl on_match, &res[i]); if (err != HS_SUCCESS && err != HS_SCAN_TERMINATED) - throw Exception("Failed to scan with hyperscan", ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT); + throw Exception("Failed to scan with vectorscan", ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT); offset = haystack_offsets[i]; } #else - /// Fallback if do not use hyperscan + /// Fallback if do not use vectorscan if constexpr (MultiSearchDistance) throw Exception( - "Edit distance multi-search is not implemented when hyperscan is off (is it x86 processor?)", + "Edit distance multi-search is not implemented when vectorscan is off", ErrorCodes::NOT_IMPLEMENTED); PaddedPODArray accum(res.size()); memset(res.data(), 0, res.size() * sizeof(res.front())); @@ -129,7 +129,7 @@ struct MultiMatchAnyImpl res[i] = j + 1; } } -#endif // USE_HYPERSCAN +#endif // USE_VECTORSCAN } }; diff --git a/src/Functions/Regexps.h b/src/Functions/Regexps.h index 952e27b29bc..ac37875f91e 100644 --- a/src/Functions/Regexps.h +++ b/src/Functions/Regexps.h @@ -16,7 +16,7 @@ #include "config_functions.h" -#if USE_HYPERSCAN +#if USE_VECTORSCAN # include #endif @@ -103,7 +103,7 @@ private: } -#if USE_HYPERSCAN +#if USE_VECTORSCAN namespace MultiRegexps { @@ -312,6 +312,6 @@ inline Regexps * get(const std::vector & patterns, std::optional Date: Fri, 24 Jun 2022 17:23:01 +0500 Subject: [PATCH 065/123] fixed tests --- tests/queries/0_stateless/02337_base58.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02337_base58.sql b/tests/queries/0_stateless/02337_base58.sql index 68dac97a20b..34da1da4c86 100644 --- a/tests/queries/0_stateless/02337_base58.sql +++ b/tests/queries/0_stateless/02337_base58.sql @@ -14,4 +14,4 @@ SELECT base58Decode(encoded, 'bitcoin') FROM (SELECT base58Encode(val, 'bitcoin' SELECT base58Encode(val) FROM (select arrayJoin(['', 'f', 'fo', 'foo', 'foob', 'fooba', 'foobar']) val); SELECT base58Decode(val) FROM (select arrayJoin(['', '2m', '8o8', 'bQbp', '3csAg9', 'CZJRhmz', 't1Zv2yaZ']) val); -SELECT base58Decode('Why_not?'); -- { serverError 1001 } +SELECT base58Decode('Why_not?'); -- { serverError 36 } From 571410a1723aa31cc9ff420d2c71860ada8c5109 Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Fri, 24 Jun 2022 08:48:18 -0400 Subject: [PATCH 066/123] move settings to H3 level --- docs/en/engines/database-engines/index.md | 2 + .../database-engines/materialized-mysql.md | 50 ++++++++++++++----- .../materialized-postgresql.md | 14 +++--- 3 files changed, 47 insertions(+), 19 deletions(-) diff --git a/docs/en/engines/database-engines/index.md b/docs/en/engines/database-engines/index.md index 8e36aca695c..72689d29780 100644 --- a/docs/en/engines/database-engines/index.md +++ b/docs/en/engines/database-engines/index.md @@ -20,6 +20,8 @@ Here is a complete list of available database engines. Follow the links for more - [PostgreSQL](../../engines/database-engines/postgresql.md) +- [MaterializedMySQL](../../engines/database-engines/materialized-postgresql.md) + - [Replicated](../../engines/database-engines/replicated.md) - [SQLite](../../engines/database-engines/sqlite.md) diff --git a/docs/en/engines/database-engines/materialized-mysql.md b/docs/en/engines/database-engines/materialized-mysql.md index cbc40993da8..4b16d877210 100644 --- a/docs/en/engines/database-engines/materialized-mysql.md +++ b/docs/en/engines/database-engines/materialized-mysql.md @@ -26,14 +26,30 @@ ENGINE = MaterializedMySQL('host:port', ['database' | database], 'user', 'passwo - `user` — MySQL user. - `password` — User password. -**Engine Settings** +## Engine Settings -- `max_rows_in_buffer` — Maximum number of rows that data is allowed to cache in memory (for single table and the cache data unable to query). When this number is exceeded, the data will be materialized. Default: `65 505`. -- `max_bytes_in_buffer` — Maximum number of bytes that data is allowed to cache in memory (for single table and the cache data unable to query). When this number is exceeded, the data will be materialized. Default: `1 048 576`. -- `max_flush_data_time` — Maximum number of milliseconds that data is allowed to cache in memory (for database and the cache data unable to query). When this time is exceeded, the data will be materialized. Default: `1000`. -- `max_wait_time_when_mysql_unavailable` — Retry interval when MySQL is not available (milliseconds). Negative value disables retry. Default: `1000`. -- `allows_query_when_mysql_lost` — Allows to query a materialized table when MySQL is lost. Default: `0` (`false`). -- `materialized_mysql_tables_list` — a comma-separated list of mysql database tables, which will be replicated by MaterializedMySQL database engine. Default value: empty list — means whole tables will be replicated. +### max_rows_in_buffer + +`max_rows_in_buffer` — Maximum number of rows that data is allowed to cache in memory (for single table and the cache data unable to query). When this number is exceeded, the data will be materialized. Default: `65 505`. + +### max_bytes_in_buffer + +`max_bytes_in_buffer` — Maximum number of bytes that data is allowed to cache in memory (for single table and the cache data unable to query). When this number is exceeded, the data will be materialized. Default: `1 048 576`. + +### max_flush_data_time + +`max_flush_data_time` — Maximum number of milliseconds that data is allowed to cache in memory (for database and the cache data unable to query). When this time is exceeded, the data will be materialized. Default: `1000`. + +### max_wait_time_when_mysql_unavailable + +`max_wait_time_when_mysql_unavailable` — Retry interval when MySQL is not available (milliseconds). Negative value disables retry. Default: `1000`. + +### allows_query_when_mysql_lost +`allows_query_when_mysql_lost` — Allows to query a materialized table when MySQL is lost. Default: `0` (`false`). + +### materialized_mysql_tables_list + +`materialized_mysql_tables_list` — a comma-separated list of mysql database tables, which will be replicated by MaterializedMySQL database engine. Default value: empty list — means whole tables will be replicated. ```sql CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***') @@ -42,12 +58,17 @@ CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', max_wait_time_when_mysql_unavailable=10000; ``` -**Settings on MySQL-server Side** +## Settings on MySQL-server Side For the correct work of `MaterializedMySQL`, there are few mandatory `MySQL`-side configuration settings that must be set: -- `default_authentication_plugin = mysql_native_password` since `MaterializedMySQL` can only authorize with this method. -- `gtid_mode = on` since GTID based logging is a mandatory for providing correct `MaterializedMySQL` replication. +### default_authentication_plugin + +`default_authentication_plugin = mysql_native_password` since `MaterializedMySQL` can only authorize with this method. + +### gtid_mode + +`gtid_mode = on` since GTID based logging is a mandatory for providing correct `MaterializedMySQL` replication. :::note While turning on `gtid_mode` you should also specify `enforce_gtid_consistency = on`. @@ -57,8 +78,13 @@ While turning on `gtid_mode` you should also specify `enforce_gtid_consistency = When working with the `MaterializedMySQL` database engine, [ReplacingMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md) tables are used with virtual `_sign` and `_version` columns. -- `_version` — Transaction counter. Type [UInt64](../../sql-reference/data-types/int-uint.md). -- `_sign` — Deletion mark. Type [Int8](../../sql-reference/data-types/int-uint.md). Possible values: +### \_version + +`_version` — Transaction counter. Type [UInt64](../../sql-reference/data-types/int-uint.md). + +### \_sign + +`_sign` — Deletion mark. Type [Int8](../../sql-reference/data-types/int-uint.md). Possible values: - `1` — Row is not deleted, - `-1` — Row is deleted. diff --git a/docs/en/engines/database-engines/materialized-postgresql.md b/docs/en/engines/database-engines/materialized-postgresql.md index 66f918f01d6..dc05c58f092 100644 --- a/docs/en/engines/database-engines/materialized-postgresql.md +++ b/docs/en/engines/database-engines/materialized-postgresql.md @@ -150,21 +150,21 @@ Replication of [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.htm ## Settings {#settings} -1. `materialized_postgresql_tables_list` {#materialized-postgresql-tables-list} +### `materialized_postgresql_tables_list` {#materialized-postgresql-tables-list} Sets a comma-separated list of PostgreSQL database tables, which will be replicated via [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) database engine. Default value: empty list — means whole PostgreSQL database will be replicated. -2. `materialized_postgresql_schema` {#materialized-postgresql-schema} +### `materialized_postgresql_schema` {#materialized-postgresql-schema} Default value: empty string. (Default schema is used) -3. `materialized_postgresql_schema_list` {#materialized-postgresql-schema-list} +### `materialized_postgresql_schema_list` {#materialized-postgresql-schema-list} Default value: empty list. (Default schema is used) -4. `materialized_postgresql_allow_automatic_update` {#materialized-postgresql-allow-automatic-update} +### `materialized_postgresql_allow_automatic_update` {#materialized-postgresql-allow-automatic-update} Do not use this setting before 22.1 version. @@ -177,7 +177,7 @@ Replication of [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.htm Default value: `0`. -5. `materialized_postgresql_max_block_size` {#materialized-postgresql-max-block-size} +### `materialized_postgresql_max_block_size` {#materialized-postgresql-max-block-size} Sets the number of rows collected in memory before flushing data into PostgreSQL database table. @@ -187,11 +187,11 @@ Replication of [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.htm Default value: `65536`. -6. `materialized_postgresql_replication_slot` {#materialized-postgresql-replication-slot} +### `materialized_postgresql_replication_slot` {#materialized-postgresql-replication-slot} A user-created replication slot. Must be used together with `materialized_postgresql_snapshot`. -7. `materialized_postgresql_snapshot` {#materialized-postgresql-snapshot} +### `materialized_postgresql_snapshot` {#materialized-postgresql-snapshot} A text string identifying a snapshot, from which [initial dump of PostgreSQL tables](../../engines/database-engines/materialized-postgresql.md) will be performed. Must be used together with `materialized_postgresql_replication_slot`. From bd2f5eb96d6027b43f2479aa2f177f89eee81de8 Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Fri, 24 Jun 2022 08:57:51 -0400 Subject: [PATCH 067/123] move settings to H3 level --- docs/en/engines/database-engines/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/database-engines/index.md b/docs/en/engines/database-engines/index.md index 72689d29780..237112a5bee 100644 --- a/docs/en/engines/database-engines/index.md +++ b/docs/en/engines/database-engines/index.md @@ -20,7 +20,7 @@ Here is a complete list of available database engines. Follow the links for more - [PostgreSQL](../../engines/database-engines/postgresql.md) -- [MaterializedMySQL](../../engines/database-engines/materialized-postgresql.md) +- [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) - [Replicated](../../engines/database-engines/replicated.md) From 3bb765ae1fcfcb0687342ad0c93c4ab868e62cbb Mon Sep 17 00:00:00 2001 From: xinhuitian Date: Fri, 24 Jun 2022 22:24:48 +0800 Subject: [PATCH 068/123] fix some wrong titles and links in alter docs --- .../sql-reference/statements/alter/column.md | 12 +++++----- .../statements/alter/partition.md | 24 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index 07266eb09a3..2a5e36eaa00 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -18,12 +18,12 @@ Each action is an operation on a column. The following actions are supported: -- [ADD COLUMN](#alter_add-column) — Adds a new column to the table. -- [DROP COLUMN](#alter_drop-column) — Deletes the column. -- [RENAME COLUMN](#alter_rename-column) — Renames an existing column. -- [CLEAR COLUMN](#alter_clear-column) — Resets column values. -- [COMMENT COLUMN](#alter_comment-column) — Adds a text comment to the column. -- [MODIFY COLUMN](#alter_modify-column) — Changes column’s type, default expression and TTL. +- [ADD COLUMN](#add-column) — Adds a new column to the table. +- [DROP COLUMN](#drop-column) — Deletes the column. +- [RENAME COLUMN](#rename-column) — Renames an existing column. +- [CLEAR COLUMN](#clear-column) — Resets column values. +- [COMMENT COLUMN](#comment-column) — Adds a text comment to the column. +- [MODIFY COLUMN](#modify-column) — Changes column’s type, default expression and TTL. - [MODIFY COLUMN REMOVE](#modify-remove) — Removes one of the column properties. - [MATERIALIZE COLUMN](#materialize-column) — Materializes the column in the parts where the column is missing. diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md index 75c80add9b7..27178c91de8 100644 --- a/docs/en/sql-reference/statements/alter/partition.md +++ b/docs/en/sql-reference/statements/alter/partition.md @@ -7,18 +7,18 @@ sidebar_label: PARTITION The following operations with [partitions](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md) are available: -- [DETACH PARTITION](#alter_detach-partition) — Moves a partition to the `detached` directory and forget it. -- [DROP PARTITION](#alter_drop-partition) — Deletes a partition. -- [ATTACH PART\|PARTITION](#alter_attach-partition) — Adds a part or partition from the `detached` directory to the table. -- [ATTACH PARTITION FROM](#alter_attach-partition-from) — Copies the data partition from one table to another and adds. -- [REPLACE PARTITION](#alter_replace-partition) — Copies the data partition from one table to another and replaces. -- [MOVE PARTITION TO TABLE](#alter_move_to_table-partition) — Moves the data partition from one table to another. -- [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) — Resets the value of a specified column in a partition. -- [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) — Resets the specified secondary index in a partition. -- [FREEZE PARTITION](#alter_freeze-partition) — Creates a backup of a partition. -- [UNFREEZE PARTITION](#alter_unfreeze-partition) — Removes a backup of a partition. -- [FETCH PARTITION\|PART](#alter_fetch-partition) — Downloads a part or partition from another server. -- [MOVE PARTITION\|PART](#alter_move-partition) — Move partition/data part to another disk or volume. +- [DETACH PARTITION\|Part](#detach-partitionpart) — Moves a partition or part to the `detached` directory and forget it. +- [DROP PARTITION\|Part](#drop-partitionpart) — Deletes a partition or part. +- [ATTACH PARTITION\|Part](#attach-partitionpart) — Adds a partition or part from the `detached` directory to the table. +- [ATTACH PARTITION FROM](#attach-partition-from) — Copies the data partition from one table to another and adds. +- [REPLACE PARTITION](#replace-partition) — Copies the data partition from one table to another and replaces. +- [MOVE PARTITION TO TABLE](#move_to_table-partition) — Moves the data partition from one table to another. +- [CLEAR COLUMN IN PARTITION](#clear-column-partition) — Resets the value of a specified column in a partition. +- [CLEAR INDEX IN PARTITION](#clear-index-partition) — Resets the specified secondary index in a partition. +- [FREEZE PARTITION](#freeze-partition) — Creates a backup of a partition. +- [UNFREEZE PARTITION](#unfreeze-partition) — Removes a backup of a partition. +- [FETCH PARTITION\|PART](#fetch-partition) — Downloads a part or partition from another server. +- [MOVE PARTITION\|PART](#move-partition) — Move partition/data part to another disk or volume. - [UPDATE IN PARTITION](#update-in-partition) — Update data inside the partition by condition. - [DELETE IN PARTITION](#delete-in-partition) — Delete data inside the partition by condition. From 848ae7b1304f5b1389ed7c812f47edb130d1112b Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 24 Jun 2022 16:49:48 +0200 Subject: [PATCH 069/123] Update docker-compose to try get rid of v1 errors --- docker/test/integration/runner/Dockerfile | 2 +- docker/test/testflows/runner/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index 57e4dfdeda1..80a2158b17d 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -67,7 +67,7 @@ RUN python3 -m pip install \ dict2xml \ dicttoxml \ docker \ - docker-compose==1.28.2 \ + docker-compose==1.29.2 \ grpcio \ grpcio-tools \ kafka-python \ diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index fbff6fd5e97..bfc3ed5e39f 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -38,7 +38,7 @@ RUN apt-get update \ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN pip3 install urllib3 testflows==1.7.20 docker-compose==1.29.1 docker==5.0.0 dicttoxml kazoo tzlocal==2.1 pytz python-dateutil numpy +RUN pip3 install urllib3 testflows==1.7.20 docker-compose==1.29.2 docker==5.0.0 dicttoxml kazoo tzlocal==2.1 pytz python-dateutil numpy ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 20.10.6 From 072f64c80088750c996b8056b0d7c459cdaa5669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 24 Jun 2022 16:54:47 +0200 Subject: [PATCH 070/123] Improvements based on review --- src/Interpreters/executeQuery.cpp | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index ae622e5e1f0..4b328f0466e 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -626,8 +626,10 @@ static std::tuple executeQueryImpl( if (!table_id.empty()) context->setInsertionTable(table_id); - if (context->getCurrentTransaction() && context->getSettingsRef().throw_on_unsupported_query_inside_transaction) + if (context->getCurrentTransaction() && settings.throw_on_unsupported_query_inside_transaction) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Async inserts inside transactions are not supported"); + if (settings.implicit_transaction && settings.throw_on_unsupported_query_inside_transaction) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Async inserts with 'implicit_transaction' are not supported"); } else { @@ -636,6 +638,9 @@ static std::tuple executeQueryImpl( { try { + if (context->isGlobalContext()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Global context cannot create transactions"); + /// If there is no session (which is the default for the HTTP Handler), set up one just for this as it is necessary /// to control the transaction lifetime if (!context->hasSessionContext()) @@ -972,8 +977,18 @@ static std::tuple executeQueryImpl( if (implicit_txn_control) { - implicit_txn_control->executeCommit(context->getSessionContext()); - implicit_txn_control.reset(); + try + { + implicit_txn_control->executeCommit(context->getSessionContext()); + implicit_txn_control.reset(); + } + catch (const Exception &) + { + /// An exception might happen when trying to commit the transaction. For example we might get an immediate exception + /// because ZK is down and wait_changes_become_visible_after_commit_mode == WAIT_UNKNOWN + implicit_txn_control.reset(); + throw; + } } }; From 6ef534c86460b4e658be8fe0d3fadb499e3c386e Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Fri, 24 Jun 2022 11:13:15 -0400 Subject: [PATCH 071/123] move settings to H3 level --- .../mergetree-family/aggregatingmergetree.md | 6 +- .../mergetree-family/collapsingmergetree.md | 10 +- .../mergetree-family/graphitemergetree.md | 17 ++- .../mergetree-family/mergetree.md | 123 +++++++++++++----- .../mergetree-family/replacingmergetree.md | 8 +- .../mergetree-family/replication.md | 16 ++- .../mergetree-family/summingmergetree.md | 8 +- .../versionedcollapsingmergetree.md | 12 +- 8 files changed, 142 insertions(+), 58 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md b/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md index 5c3143c6c18..b2eea820139 100644 --- a/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md @@ -11,8 +11,8 @@ You can use `AggregatingMergeTree` tables for incremental data aggregation, incl The engine processes all columns with the following types: -- [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) -- [SimpleAggregateFunction](../../../sql-reference/data-types/simpleaggregatefunction.md) +## [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md) +## [SimpleAggregateFunction](../../../sql-reference/data-types/simpleaggregatefunction.md) It is appropriate to use `AggregatingMergeTree` if it reduces the number of rows by orders. @@ -36,7 +36,7 @@ For a description of request parameters, see [request description](../../../sql- **Query clauses** -When creating a `AggregatingMergeTree` table the same [clauses](../../../engines/table-engines/mergetree-family/mergetree.md) are required, as when creating a `MergeTree` table. +When creating an `AggregatingMergeTree` table the same [clauses](../../../engines/table-engines/mergetree-family/mergetree.md) are required, as when creating a `MergeTree` table.
diff --git a/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md b/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md index afe323441ab..1b37e20d0da 100644 --- a/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/collapsingmergetree.md @@ -7,7 +7,7 @@ sidebar_label: CollapsingMergeTree The engine inherits from [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) and adds the logic of rows collapsing to data parts merge algorithm. -`CollapsingMergeTree` asynchronously deletes (collapses) pairs of rows if all of the fields in a sorting key (`ORDER BY`) are equivalent excepting the particular field `Sign` which can have `1` and `-1` values. Rows without a pair are kept. For more details see the [Collapsing](#table_engine-collapsingmergetree-collapsing) section of the document. +`CollapsingMergeTree` asynchronously deletes (collapses) pairs of rows if all of the fields in a sorting key (`ORDER BY`) are equivalent except the particular field `Sign`, which can have `1` and `-1` values. Rows without a pair are kept. For more details see the [Collapsing](#table_engine-collapsingmergetree-collapsing) section of the document. The engine may significantly reduce the volume of storage and increase the efficiency of `SELECT` query as a consequence. @@ -28,13 +28,15 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] For a description of query parameters, see [query description](../../../sql-reference/statements/create/table.md). -**CollapsingMergeTree Parameters** +## CollapsingMergeTree Parameters -- `sign` — Name of the column with the type of row: `1` is a “state” row, `-1` is a “cancel” row. +### sign + +`sign` — Name of the column with the type of row: `1` is a “state” row, `-1` is a “cancel” row. Column data type — `Int8`. -**Query clauses** +## Query clauses When creating a `CollapsingMergeTree` table, the same [query clauses](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-creating-a-table) are required, as when creating a `MergeTree` table. diff --git a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md index c1011e69ba6..9062dd3c423 100644 --- a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md @@ -87,10 +87,18 @@ Rollup configuration structure: ### Required Columns {#required-columns} -- `path_column_name` — The name of the column storing the metric name (Graphite sensor). Default value: `Path`. -- `time_column_name` — The name of the column storing the time of measuring the metric. Default value: `Time`. -- `value_column_name` — The name of the column storing the value of the metric at the time set in `time_column_name`. Default value: `Value`. -- `version_column_name` — The name of the column storing the version of the metric. Default value: `Timestamp`. +#### path_column_name + +`path_column_name` — The name of the column storing the metric name (Graphite sensor). Default value: `Path`. + +#### time_column_name +`time_column_name` — The name of the column storing the time of measuring the metric. Default value: `Time`. + +#### value_column_name +`value_column_name` — The name of the column storing the value of the metric at the time set in `time_column_name`. Default value: `Value`. + +#### version_column_name +`version_column_name` — The name of the column storing the version of the metric. Default value: `Timestamp`. ### Patterns {#patterns} @@ -254,7 +262,6 @@ Valid values: ``` - :::warning Data rollup is performed during merges. Usually, for old partitions, merges are not started, so for rollup it is necessary to trigger an unscheduled merge using [optimize](../../../sql-reference/statements/optimize.md). Or use additional tools, for example [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). ::: diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 15f66d2695f..103272fb250 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -60,7 +60,11 @@ For a description of parameters, see the [CREATE query description](../../../sql ### Query Clauses {#mergetree-query-clauses} -- `ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. The `MergeTree` engine does not have parameters. +#### ENGINE + +`ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. The `MergeTree` engine does not have parameters. + +#### - `ORDER BY` — The sorting key. @@ -70,18 +74,23 @@ For a description of parameters, see the [CREATE query description](../../../sql Use the `ORDER BY tuple()` syntax, if you do not need sorting. See [Selecting the Primary Key](#selecting-the-primary-key). +#### + - `PARTITION BY` — The [partitioning key](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md). Optional. In most cases you don't need partition key, and in most other cases you don't need partition key more granular than by months. Partitioning does not speed up queries (in contrast to the ORDER BY expression). You should never use too granular partitioning. Don't partition your data by client identifiers or names (instead make client identifier or name the first column in the ORDER BY expression). For partitioning by month, use the `toYYYYMM(date_column)` expression, where `date_column` is a column with a date of the type [Date](../../../sql-reference/data-types/date.md). The partition names here have the `"YYYYMM"` format. +#### - `PRIMARY KEY` — The primary key if it [differs from the sorting key](#choosing-a-primary-key-that-differs-from-the-sorting-key). Optional. By default the primary key is the same as the sorting key (which is specified by the `ORDER BY` clause). Thus in most cases it is unnecessary to specify a separate `PRIMARY KEY` clause. +#### - `SAMPLE BY` — An expression for sampling. Optional. If a sampling expression is used, the primary key must contain it. The result of a sampling expression must be an unsigned integer. Example: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. +#### - `TTL` — A list of rules specifying storage duration of rows and defining logic of automatic parts movement [between disks and volumes](#table_engine-mergetree-multiple-volumes). Optional. Expression must have one `Date` or `DateTime` column as a result. Example: @@ -91,26 +100,76 @@ For a description of parameters, see the [CREATE query description](../../../sql For more details, see [TTL for columns and tables](#table_engine-mergetree-ttl) -- `SETTINGS` — Additional parameters that control the behavior of the `MergeTree` (optional): +### SETTINGS +Additional parameters that control the behavior of the `MergeTree` (optional): - - `index_granularity` — Maximum number of data rows between the marks of an index. Default value: 8192. See [Data Storage](#mergetree-data-storage). - - `index_granularity_bytes` — Maximum size of data granules in bytes. Default value: 10Mb. To restrict the granule size only by number of rows, set to 0 (not recommended). See [Data Storage](#mergetree-data-storage). - - `min_index_granularity_bytes` — Min allowed size of data granules in bytes. Default value: 1024b. To provide a safeguard against accidentally creating tables with very low index_granularity_bytes. See [Data Storage](#mergetree-data-storage). - - `enable_mixed_granularity_parts` — Enables or disables transitioning to control the granule size with the `index_granularity_bytes` setting. Before version 19.11, there was only the `index_granularity` setting for restricting granule size. The `index_granularity_bytes` setting improves ClickHouse performance when selecting data from tables with big rows (tens and hundreds of megabytes). If you have tables with big rows, you can enable this setting for the tables to improve the efficiency of `SELECT` queries. - - `use_minimalistic_part_header_in_zookeeper` — Storage method of the data parts headers in ZooKeeper. If `use_minimalistic_part_header_in_zookeeper=1`, then ZooKeeper stores less data. For more information, see the [setting description](../../../operations/server-configuration-parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) in “Server configuration parameters”. - - `min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation that is required for using direct I/O access to the storage disk. When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, ClickHouse reads and writes the data to the storage disk using the direct I/O interface (`O_DIRECT` option). If `min_merge_bytes_to_use_direct_io = 0`, then direct I/O is disabled. Default value: `10 * 1024 * 1024 * 1024` bytes. +#### index_granularity + +`index_granularity` — Maximum number of data rows between the marks of an index. Default value: 8192. See [Data Storage](#mergetree-data-storage). + +#### index_granularity_bytes + +`index_granularity_bytes` — Maximum size of data granules in bytes. Default value: 10Mb. To restrict the granule size only by number of rows, set to 0 (not recommended). See [Data Storage](#mergetree-data-storage). + +#### min_index_granularity_bytes + +`min_index_granularity_bytes` — Min allowed size of data granules in bytes. Default value: 1024b. To provide a safeguard against accidentally creating tables with very low index_granularity_bytes. See [Data Storage](#mergetree-data-storage). + +#### enable_mixed_granularity_parts + +`enable_mixed_granularity_parts` — Enables or disables transitioning to control the granule size with the `index_granularity_bytes` setting. Before version 19.11, there was only the `index_granularity` setting for restricting granule size. The `index_granularity_bytes` setting improves ClickHouse performance when selecting data from tables with big rows (tens and hundreds of megabytes). If you have tables with big rows, you can enable this setting for the tables to improve the efficiency of `SELECT` queries. + +#### use_minimalistic_part_header_in_zookeeper + +`use_minimalistic_part_header_in_zookeeper` — Storage method of the data parts headers in ZooKeeper. If `use_minimalistic_part_header_in_zookeeper=1`, then ZooKeeper stores less data. For more information, see the [setting description](../../../operations/server-configuration-parameters/settings.md#server-settings-use_minimalistic_part_header_in_zookeeper) in “Server configuration parameters”. + +#### min_merge_bytes_to_use_direct_io + +`min_merge_bytes_to_use_direct_io` — The minimum data volume for merge operation that is required for using direct I/O access to the storage disk. When merging data parts, ClickHouse calculates the total storage volume of all the data to be merged. If the volume exceeds `min_merge_bytes_to_use_direct_io` bytes, ClickHouse reads and writes the data to the storage disk using the direct I/O interface (`O_DIRECT` option). If `min_merge_bytes_to_use_direct_io = 0`, then direct I/O is disabled. Default value: `10 * 1024 * 1024 * 1024` bytes. - - `merge_with_ttl_timeout` — Minimum delay in seconds before repeating a merge with delete TTL. Default value: `14400` seconds (4 hours). - - `merge_with_recompression_ttl_timeout` — Minimum delay in seconds before repeating a merge with recompression TTL. Default value: `14400` seconds (4 hours). - - `try_fetch_recompressed_part_timeout` — Timeout (in seconds) before starting merge with recompression. During this time ClickHouse tries to fetch recompressed part from replica which assigned this merge with recompression. Default value: `7200` seconds (2 hours). - - `write_final_mark` — Enables or disables writing the final index mark at the end of data part (after the last byte). Default value: 1. Don’t turn it off. - - `merge_max_block_size` — Maximum number of rows in block for merge operations. Default value: 8192. - - `storage_policy` — Storage policy. See [Using Multiple Block Devices for Data Storage](#table_engine-mergetree-multiple-volumes). - - `min_bytes_for_wide_part`, `min_rows_for_wide_part` — Minimum number of bytes/rows in a data part that can be stored in `Wide` format. You can set one, both or none of these settings. See [Data Storage](#mergetree-data-storage). - - `max_parts_in_total` — Maximum number of parts in all partitions. - - `max_compress_block_size` — Maximum size of blocks of uncompressed data before compressing for writing to a table. You can also specify this setting in the global settings (see [max_compress_block_size](../../../operations/settings/settings.md#max-compress-block-size) setting). The value specified when table is created overrides the global value for this setting. - - `min_compress_block_size` — Minimum size of blocks of uncompressed data required for compression when writing the next mark. You can also specify this setting in the global settings (see [min_compress_block_size](../../../operations/settings/settings.md#min-compress-block-size) setting). The value specified when table is created overrides the global value for this setting. - - `max_partitions_to_read` — Limits the maximum number of partitions that can be accessed in one query. You can also specify setting [max_partitions_to_read](../../../operations/settings/merge-tree-settings.md#max-partitions-to-read) in the global setting. + +#### merge_with_ttl_timeout + +`merge_with_ttl_timeout` — Minimum delay in seconds before repeating a merge with delete TTL. Default value: `14400` seconds (4 hours). +#### merge_with_recompression_ttl_timeout + +`merge_with_recompression_ttl_timeout` — Minimum delay in seconds before repeating a merge with recompression TTL. Default value: `14400` seconds (4 hours). + +#### try_fetch_recompressed_part_timeout + +`try_fetch_recompressed_part_timeout` — Timeout (in seconds) before starting merge with recompression. During this time ClickHouse tries to fetch recompressed part from replica which assigned this merge with recompression. Default value: `7200` seconds (2 hours). + +#### write_final_mark + +`write_final_mark` — Enables or disables writing the final index mark at the end of data part (after the last byte). Default value: 1. Don’t turn it off. + +#### merge_max_block_size + +`merge_max_block_size` — Maximum number of rows in block for merge operations. Default value: 8192. + +#### storage_policy + +`storage_policy` — Storage policy. See [Using Multiple Block Devices for Data Storage](#table_engine-mergetree-multiple-volumes). + +#### min_bytes_for_wide_part + +`min_bytes_for_wide_part`, `min_rows_for_wide_part` — Minimum number of bytes/rows in a data part that can be stored in `Wide` format. You can set one, both or none of these settings. See [Data Storage](#mergetree-data-storage). + +#### max_parts_in_total + +`max_parts_in_total` — Maximum number of parts in all partitions. + +#### max_compress_block_size + +`max_compress_block_size` — Maximum size of blocks of uncompressed data before compressing for writing to a table. You can also specify this setting in the global settings (see [max_compress_block_size](../../../operations/settings/settings.md#max-compress-block-size) setting). The value specified when table is created overrides the global value for this setting. + +#### min_compress_block_size + +`min_compress_block_size` — Minimum size of blocks of uncompressed data required for compression when writing the next mark. You can also specify this setting in the global settings (see [min_compress_block_size](../../../operations/settings/settings.md#min-compress-block-size) setting). The value specified when table is created overrides the global value for this setting. + +#### max_partitions_to_read + +`max_partitions_to_read` — Limits the maximum number of partitions that can be accessed in one query. You can also specify setting [max_partitions_to_read](../../../operations/settings/merge-tree-settings.md#max-partitions-to-read) in the global setting. **Example of Sections Setting** @@ -310,17 +369,17 @@ SELECT count() FROM table WHERE s < 'z' SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 ``` -#### Available Types of Indices {#available-types-of-indices} +### Available Types of Indices {#available-types-of-indices} -- `minmax` +#### `minmax` Stores extremes of the specified expression (if the expression is `tuple`, then it stores extremes for each element of `tuple`), uses stored info for skipping blocks of data like the primary key. -- `set(max_rows)` +#### `set(max_rows)` Stores unique values of the specified expression (no more than `max_rows` rows, `max_rows=0` means “no limits”). Uses the values to check if the `WHERE` expression is not satisfiable on a block of data. -- `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` +#### `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` Stores a [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) that contains all ngrams from a block of data. Works only with datatypes: [String](../../../sql-reference/data-types/string.md), [FixedString](../../../sql-reference/data-types/fixedstring.md) and [Map](../../../sql-reference/data-types/map.md). Can be used for optimization of `EQUALS`, `LIKE` and `IN` expressions. @@ -329,11 +388,11 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 - `number_of_hash_functions` — The number of hash functions used in the Bloom filter. - `random_seed` — The seed for Bloom filter hash functions. -- `tokenbf_v1(size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` +#### `tokenbf_v1(size_of_bloom_filter_in_bytes, number_of_hash_functions, random_seed)` The same as `ngrambf_v1`, but stores tokens instead of ngrams. Tokens are sequences separated by non-alphanumeric characters. -- `bloom_filter([false_positive])` — Stores a [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) for the specified columns. +#### `bloom_filter([false_positive])` — Stores a [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) for the specified columns. The optional `false_positive` parameter is the probability of receiving a false positive response from the filter. Possible values: (0, 1). Default value: 0.025. @@ -357,7 +416,7 @@ INDEX sample_index2 (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100 INDEX sample_index3 (lower(str), str) TYPE ngrambf_v1(3, 256, 2, 0) GRANULARITY 4 ``` -#### Functions Support {#functions-support} +### Functions Support {#functions-support} Conditions in the `WHERE` clause contains calls of the functions that operate with columns. If the column is a part of an index, ClickHouse tries to use this index when performing the functions. ClickHouse supports different subsets of functions for using indexes. @@ -466,7 +525,7 @@ The `TTL` clause can’t be used for key columns. **Examples** -Creating a table with `TTL`: +#### Creating a table with `TTL`: ``` sql CREATE TABLE example_table @@ -481,7 +540,7 @@ PARTITION BY toYYYYMM(d) ORDER BY d; ``` -Adding TTL to a column of an existing table +#### Adding TTL to a column of an existing table ``` sql ALTER TABLE example_table @@ -489,7 +548,7 @@ ALTER TABLE example_table c String TTL d + INTERVAL 1 DAY; ``` -Altering TTL of the column +#### Altering TTL of the column ``` sql ALTER TABLE example_table @@ -524,7 +583,7 @@ If a column is not part of the `GROUP BY` expression and is not set explicitly i **Examples** -Creating a table with `TTL`: +#### Creating a table with `TTL`: ``` sql CREATE TABLE example_table @@ -540,7 +599,7 @@ TTL d + INTERVAL 1 MONTH [DELETE], d + INTERVAL 2 WEEK TO DISK 'bbb'; ``` -Altering `TTL` of the table: +#### Altering `TTL` of the table: ``` sql ALTER TABLE example_table @@ -561,7 +620,7 @@ ORDER BY d TTL d + INTERVAL 1 MONTH DELETE WHERE toDayOfWeek(d) = 1; ``` -Creating a table, where expired rows are recompressed: +#### Creating a table, where expired rows are recompressed: ```sql CREATE TABLE table_for_recompression diff --git a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md index 5586d108ead..daa507cab66 100644 --- a/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md @@ -33,16 +33,18 @@ For a description of request parameters, see [statement description](../../../sq Uniqueness of rows is determined by the `ORDER BY` table section, not `PRIMARY KEY`. ::: -**ReplacingMergeTree Parameters** +## ReplacingMergeTree Parameters -- `ver` — column with the version number. Type `UInt*`, `Date`, `DateTime` or `DateTime64`. Optional parameter. +### ver + +`ver` — column with the version number. Type `UInt*`, `Date`, `DateTime` or `DateTime64`. Optional parameter. When merging, `ReplacingMergeTree` from all the rows with the same sorting key leaves only one: - The last in the selection, if `ver` not set. A selection is a set of rows in a set of parts participating in the merge. The most recently created part (the last insert) will be the last one in the selection. Thus, after deduplication, the very last row from the most recent insert will remain for each unique sorting key. - With the maximum version, if `ver` specified. If `ver` is the same for several rows, then it will use "if `ver` is not specified" rule for them, i.e. the most recent inserted row will remain. -**Query clauses** +## Query clauses When creating a `ReplacingMergeTree` table the same [clauses](../../../engines/table-engines/mergetree-family/mergetree.md) are required, as when creating a `MergeTree` table. diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 3562bdf6d3a..0dfcdccb029 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -120,11 +120,19 @@ The system monitors data synchronicity on replicas and is able to recover after The `Replicated` prefix is added to the table engine name. For example:`ReplicatedMergeTree`. -**Replicated\*MergeTree parameters** +### Replicated\*MergeTree parameters -- `zoo_path` — The path to the table in ClickHouse Keeper. -- `replica_name` — The replica name in ClickHouse Keeper. -- `other_parameters` — Parameters of an engine which is used for creating the replicated version, for example, version in `ReplacingMergeTree`. +#### zoo_path + +`zoo_path` — The path to the table in ClickHouse Keeper. + +#### replica_name + +`replica_name` — The replica name in ClickHouse Keeper. + +#### other_parameters + +`other_parameters` — Parameters of an engine which is used for creating the replicated version, for example, version in `ReplacingMergeTree`. Example: diff --git a/docs/en/engines/table-engines/mergetree-family/summingmergetree.md b/docs/en/engines/table-engines/mergetree-family/summingmergetree.md index b532aef1980..7afa7cf028e 100644 --- a/docs/en/engines/table-engines/mergetree-family/summingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/summingmergetree.md @@ -26,14 +26,16 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] For a description of request parameters, see [request description](../../../sql-reference/statements/create/table.md). -**Parameters of SummingMergeTree** +### Parameters of SummingMergeTree -- `columns` - a tuple with the names of columns where values will be summarized. Optional parameter. +#### columns + +`columns` - a tuple with the names of columns where values will be summarized. Optional parameter. The columns must be of a numeric type and must not be in the primary key. If `columns` not specified, ClickHouse summarizes the values in all columns with a numeric data type that are not in the primary key. -**Query clauses** +### Query clauses When creating a `SummingMergeTree` table the same [clauses](../../../engines/table-engines/mergetree-family/mergetree.md) are required, as when creating a `MergeTree` table. diff --git a/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md index c1fe5dfffdf..5642602f4a1 100644 --- a/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md @@ -31,21 +31,25 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] For a description of query parameters, see the [query description](../../../sql-reference/statements/create/table.md). -**Engine Parameters** +### Engine Parameters ``` sql VersionedCollapsingMergeTree(sign, version) ``` -- `sign` — Name of the column with the type of row: `1` is a “state” row, `-1` is a “cancel” row. +#### sign + +`sign` — Name of the column with the type of row: `1` is a “state” row, `-1` is a “cancel” row. The column data type should be `Int8`. -- `version` — Name of the column with the version of the object state. +#### version + +`version` — Name of the column with the version of the object state. The column data type should be `UInt*`. -**Query Clauses** +### Query Clauses When creating a `VersionedCollapsingMergeTree` table, the same [clauses](../../../engines/table-engines/mergetree-family/mergetree.md) are required as when creating a `MergeTree` table. From 379fecf7f9069fe8e826c40a55b6d82148511589 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 24 Jun 2022 17:15:15 +0200 Subject: [PATCH 072/123] try suppress failures with MaterializedPostgreSQL --- src/Databases/DatabaseOrdinary.cpp | 4 +++- src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp | 1 + src/Interpreters/DatabaseCatalog.cpp | 1 - src/Interpreters/InterpreterCreateQuery.cpp | 6 +++++- tests/clickhouse-test | 3 +++ 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 0bdabf7ef79..2b88fbbfcf7 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -188,7 +188,9 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables else if (!DatabaseCatalog::instance().hasUUIDMapping(create_query->uuid)) { /// It's ATTACH DATABASE. UUID for permanently detached table must be already locked. - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create_query->uuid); + /// FIXME MaterializedPostgreSQL works with UUIDs incorrectly and breaks invariants + if (getEngineName() != "MaterializedPostgreSQL") + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create_query->uuid); } } diff --git a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp index 60d2fa0d2c8..db184342a97 100644 --- a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp @@ -259,6 +259,7 @@ void DatabaseMaterializedPostgreSQL::createTable(ContextPtr local_context, const auto * create_query = assert_cast(query_copy.get()); create_query->attach = false; create_query->attach_short_syntax = false; + DatabaseCatalog::instance().addUUIDMapping(create->uuid); DatabaseAtomic::createTable(StorageMaterializedPostgreSQL::makeNestedTableContext(local_context), table_name, table, query_copy); /// Attach MaterializedPostgreSQL table. diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 159003138c2..1df3bf30396 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -31,7 +31,6 @@ #endif #if USE_LIBPQXX -# include # include # include #endif diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 54a11e9100f..75d00fcb8a7 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1273,7 +1273,11 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, if (need_lock_uuid) uuid_lock = TemporaryLockForUUIDDirectory{create.uuid}; else if (create.uuid != UUIDHelpers::Nil && !DatabaseCatalog::instance().hasUUIDMapping(create.uuid)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create.uuid); + { + /// FIXME MaterializedPostgreSQL works with UUIDs incorrectly and breaks invariants + if (database->getEngineName() != "MaterializedPostgreSQL") + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create.uuid); + } StoragePtr res; /// NOTE: CREATE query may be rewritten by Storage creator or table function diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 3e0d4e822b4..75159053f26 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -2172,4 +2172,7 @@ if __name__ == "__main__": if args.jobs is None: args.jobs = multiprocessing.cpu_count() + if args.db_engine and args.db_engine == "Ordinary": + MESSAGES_TO_RETRY.append(" locking attempt on ") + main(args) From 59a14d2a509a45de46066bfc8e0a9ad3327d35a7 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 24 Jun 2022 17:34:41 +0200 Subject: [PATCH 073/123] add docs --- .../settings.md | 29 +++++++++++++++++++ src/Interpreters/DatabaseCatalog.cpp | 5 +++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index ad879679a3d..d3a50969a39 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -193,6 +193,35 @@ Sets the delay before remove table data in seconds. If the query has `SYNC` modi Default value: `480` (8 minute). +## database_catalog_unused_dir_hide_timeout_sec {#database_catalog_unused_dir_hide_timeout_sec} + +Parameter of a task that cleans up garbage from `store/` directory. +If some subdirectory is not used by clickhouse-server and this directory was not modified for last +`database_catalog_unused_dir_hide_timeout_sec` seconds, the task will "hide" this directory by +removing all access rights. It also works for directories that clickhouse-server does not +expect to see inside `store/`. Zero means "immediately". + +Default value: `3600` (1 hour). + +## database_catalog_unused_dir_rm_timeout_sec {#database_catalog_unused_dir_rm_timeout_sec} + +Parameter of a task that cleans up garbage from `store/` directory. +If some subdirectory is not used by clickhouse-server and it was previousely "hidden" +(see [database_catalog_unused_dir_hide_timeout_sec](../../operations/server-configuration-parameters/settings.md#database_catalog_unused_dir_hide_timeout_sec)) +and this directory was not modified for last +`database_catalog_unused_dir_rm_timeout_sec` seconds, the task will remove this directory. +It also works for directories that clickhouse-server does not +expect to see inside `store/`. Zero means "never". + +Default value: `2592000` (30 days). + +## database_catalog_unused_dir_cleanup_period_sec {#database_catalog_unused_dir_cleanup_period_sec} + +Parameter of a task that cleans up garbage from `store/` directory. +Sets scheduling period of the task. Zero means "never". + +Default value: `86400` (1 day). + ## default_database {#default-database} The default database. diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 1df3bf30396..13340221f38 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -156,7 +156,7 @@ void DatabaseCatalog::initializeAndLoadTemporaryDatabase() void DatabaseCatalog::loadDatabases() { - if (Context::getGlobalContextInstance()->getApplicationType() == Context::ApplicationType::SERVER) + if (Context::getGlobalContextInstance()->getApplicationType() == Context::ApplicationType::SERVER && unused_dir_cleanup_period_sec) { auto cleanup_task_holder = getContext()->getSchedulePool().createTask("DatabaseCatalog", [this]() { this->cleanupStoreDirectoryTask(); }); @@ -1219,6 +1219,9 @@ bool DatabaseCatalog::maybeRemoveDirectory(const fs::path & unused_dir) } else { + if (!unused_dir_rm_timeout_sec) + return false; + if (current_time <= max_modification_time + unused_dir_rm_timeout_sec) return false; From 75cd941956b322ebda0156035ddaa063e59e588c Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Fri, 24 Jun 2022 11:23:02 -0400 Subject: [PATCH 074/123] move settings to H3 level --- .../mergetree-family/mergetree.md | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 103272fb250..20d9a14b194 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -64,9 +64,9 @@ For a description of parameters, see the [CREATE query description](../../../sql `ENGINE` — Name and parameters of the engine. `ENGINE = MergeTree()`. The `MergeTree` engine does not have parameters. -#### +#### ORDER_BY -- `ORDER BY` — The sorting key. +`ORDER BY` — The sorting key. A tuple of column names or arbitrary expressions. Example: `ORDER BY (CounterID, EventDate)`. @@ -74,24 +74,27 @@ For a description of parameters, see the [CREATE query description](../../../sql Use the `ORDER BY tuple()` syntax, if you do not need sorting. See [Selecting the Primary Key](#selecting-the-primary-key). -#### +#### PARTITION BY -- `PARTITION BY` — The [partitioning key](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md). Optional. In most cases you don't need partition key, and in most other cases you don't need partition key more granular than by months. Partitioning does not speed up queries (in contrast to the ORDER BY expression). You should never use too granular partitioning. Don't partition your data by client identifiers or names (instead make client identifier or name the first column in the ORDER BY expression). +`PARTITION BY` — The [partitioning key](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md). Optional. In most cases you don't need partition key, and in most other cases you don't need partition key more granular than by months. Partitioning does not speed up queries (in contrast to the ORDER BY expression). You should never use too granular partitioning. Don't partition your data by client identifiers or names (instead make client identifier or name the first column in the ORDER BY expression). For partitioning by month, use the `toYYYYMM(date_column)` expression, where `date_column` is a column with a date of the type [Date](../../../sql-reference/data-types/date.md). The partition names here have the `"YYYYMM"` format. -#### -- `PRIMARY KEY` — The primary key if it [differs from the sorting key](#choosing-a-primary-key-that-differs-from-the-sorting-key). Optional. +#### PRIMARY KEY + +`PRIMARY KEY` — The primary key if it [differs from the sorting key](#choosing-a-primary-key-that-differs-from-the-sorting-key). Optional. By default the primary key is the same as the sorting key (which is specified by the `ORDER BY` clause). Thus in most cases it is unnecessary to specify a separate `PRIMARY KEY` clause. -#### -- `SAMPLE BY` — An expression for sampling. Optional. +#### SAMPLE BY + +`SAMPLE BY` — An expression for sampling. Optional. If a sampling expression is used, the primary key must contain it. The result of a sampling expression must be an unsigned integer. Example: `SAMPLE BY intHash32(UserID) ORDER BY (CounterID, EventDate, intHash32(UserID))`. -#### -- `TTL` — A list of rules specifying storage duration of rows and defining logic of automatic parts movement [between disks and volumes](#table_engine-mergetree-multiple-volumes). Optional. +#### TTL + +`TTL` — A list of rules specifying storage duration of rows and defining logic of automatic parts movement [between disks and volumes](#table_engine-mergetree-multiple-volumes). Optional. Expression must have one `Date` or `DateTime` column as a result. Example: `TTL date + INTERVAL 1 DAY` From 70de1afad71ab62e1511c444bf01a62b0e63f162 Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Fri, 24 Jun 2022 12:16:20 -0400 Subject: [PATCH 075/123] move settings to H3 level --- .../engines/table-engines/special/buffer.md | 41 +++++++++--- .../table-engines/special/distributed.md | 64 ++++++++++++++----- docs/en/engines/table-engines/special/join.md | 44 ++++++++++--- .../en/engines/table-engines/special/merge.md | 10 ++- 4 files changed, 119 insertions(+), 40 deletions(-) diff --git a/docs/en/engines/table-engines/special/buffer.md b/docs/en/engines/table-engines/special/buffer.md index 5f81bd76ae4..bcd7c390eb1 100644 --- a/docs/en/engines/table-engines/special/buffer.md +++ b/docs/en/engines/table-engines/special/buffer.md @@ -11,24 +11,45 @@ Buffers the data to write in RAM, periodically flushing it to another table. Dur Buffer(database, table, num_layers, min_time, max_time, min_rows, max_rows, min_bytes, max_bytes) ``` -Engine parameters: +### Engine parameters: -- `database` – Database name. You can use `currentDatabase()` or another constant expression that returns a string. -- `table` – Table to flush data to. -- `num_layers` – Parallelism layer. Physically, the table will be represented as `num_layers` of independent buffers. Recommended value: 16. -- `min_time`, `max_time`, `min_rows`, `max_rows`, `min_bytes`, and `max_bytes` – Conditions for flushing data from the buffer. +#### database -Optional engine parameters: +`database` – Database name. You can use `currentDatabase()` or another constant expression that returns a string. -- `flush_time`, `flush_rows`, `flush_bytes` – Conditions for flushing data from the buffer, that will happen only in background (omitted or zero means no `flush*` parameters). +#### table + +`table` – Table to flush data to. + +#### num_layers + +`num_layers` – Parallelism layer. Physically, the table will be represented as `num_layers` of independent buffers. Recommended value: 16. + +#### min_time, max_time, min_rows, max_rows, min_bytes, and max_bytes + +Conditions for flushing data from the buffer. + +### Optional engine parameters: + +#### flush_time, flush_rows, and flush_bytes + +Conditions for flushing data from the buffer, that will happen only in background (omitted or zero means no `flush*` parameters). Data is flushed from the buffer and written to the destination table if all the `min*` conditions or at least one `max*` condition are met. Also, if at least one `flush*` condition are met flush initiated in background, this is different from `max*`, since `flush*` allows you to configure background flushes separately to avoid adding latency for `INSERT` (into `Buffer`) queries. -- `min_time`, `max_time`, `flush_time` – Condition for the time in seconds from the moment of the first write to the buffer. -- `min_rows`, `max_rows`, `flush_rows` – Condition for the number of rows in the buffer. -- `min_bytes`, `max_bytes`, `flush_bytes` – Condition for the number of bytes in the buffer. +#### min_time, max_time, and flush_time + +Condition for the time in seconds from the moment of the first write to the buffer. + +#### min_rows, max_rows, and flush_rows + +Condition for the number of rows in the buffer. + +#### min_bytes, max_bytes, and flush_bytes + +Condition for the number of bytes in the buffer. During the write operation, data is inserted to a `num_layers` number of random buffers. Or, if the data part to insert is large enough (greater than `max_rows` or `max_bytes`), it is written directly to the destination table, omitting the buffer. diff --git a/docs/en/engines/table-engines/special/distributed.md b/docs/en/engines/table-engines/special/distributed.md index fe3348c4d78..d643d4b3c68 100644 --- a/docs/en/engines/table-engines/special/distributed.md +++ b/docs/en/engines/table-engines/special/distributed.md @@ -27,42 +27,70 @@ When the `Distributed` table is pointing to a table on the current server you ca CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] AS [db2.]name2 ENGINE = Distributed(cluster, database, table[, sharding_key[, policy_name]]) [SETTINGS name=value, ...] ``` -**Distributed Parameters** +### Distributed Parameters -- `cluster` - the cluster name in the server’s config file +#### cluster -- `database` - the name of a remote database +`cluster` - the cluster name in the server’s config file -- `table` - the name of a remote table +#### database -- `sharding_key` - (optionally) sharding key +`database` - the name of a remote database -- `policy_name` - (optionally) policy name, it will be used to store temporary files for async send +#### table + +`table` - the name of a remote table + +#### sharding_key + +`sharding_key` - (optionally) sharding key + +#### policy_name + +`policy_name` - (optionally) policy name, it will be used to store temporary files for async send **See Also** - [insert_distributed_sync](../../../operations/settings/settings.md#insert_distributed_sync) setting - [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes) for the examples -**Distributed Settings** +### Distributed Settings -- `fsync_after_insert` - do the `fsync` for the file data after asynchronous insert to Distributed. Guarantees that the OS flushed the whole inserted data to a file **on the initiator node** disk. +#### fsync_after_insert -- `fsync_directories` - do the `fsync` for directories. Guarantees that the OS refreshed directory metadata after operations related to asynchronous inserts on Distributed table (after insert, after sending the data to shard, etc). +`fsync_after_insert` - do the `fsync` for the file data after asynchronous insert to Distributed. Guarantees that the OS flushed the whole inserted data to a file **on the initiator node** disk. -- `bytes_to_throw_insert` - if more than this number of compressed bytes will be pending for async INSERT, an exception will be thrown. 0 - do not throw. Default 0. +#### fsync_directories -- `bytes_to_delay_insert` - if more than this number of compressed bytes will be pending for async INSERT, the query will be delayed. 0 - do not delay. Default 0. +`fsync_directories` - do the `fsync` for directories. Guarantees that the OS refreshed directory metadata after operations related to asynchronous inserts on Distributed table (after insert, after sending the data to shard, etc). -- `max_delay_to_insert` - max delay of inserting data into Distributed table in seconds, if there are a lot of pending bytes for async send. Default 60. +#### bytes_to_throw_insert -- `monitor_batch_inserts` - same as [distributed_directory_monitor_batch_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) +`bytes_to_throw_insert` - if more than this number of compressed bytes will be pending for async INSERT, an exception will be thrown. 0 - do not throw. Default 0. -- `monitor_split_batch_on_failure` - same as [distributed_directory_monitor_split_batch_on_failure](../../../operations/settings/settings.md#distributed_directory_monitor_split_batch_on_failure) +#### bytes_to_delay_insert -- `monitor_sleep_time_ms` - same as [distributed_directory_monitor_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) +`bytes_to_delay_insert` - if more than this number of compressed bytes will be pending for async INSERT, the query will be delayed. 0 - do not delay. Default 0. -- `monitor_max_sleep_time_ms` - same as [distributed_directory_monitor_max_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) +#### max_delay_to_insert + +`max_delay_to_insert` - max delay of inserting data into Distributed table in seconds, if there are a lot of pending bytes for async send. Default 60. + +#### monitor_batch_inserts + +`monitor_batch_inserts` - same as [distributed_directory_monitor_batch_inserts](../../../operations/settings/settings.md#distributed_directory_monitor_batch_inserts) + +#### monitor_split_batch_on_failure + +`monitor_split_batch_on_failure` - same as [distributed_directory_monitor_split_batch_on_failure](../../../operations/settings/settings.md#distributed_directory_monitor_split_batch_on_failure) + +#### monitor_sleep_time_ms + +`monitor_sleep_time_ms` - same as [distributed_directory_monitor_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_sleep_time_ms) + +#### monitor_max_sleep_time_ms + +`monitor_max_sleep_time_ms` - same as [distributed_directory_monitor_max_sleep_time_ms](../../../operations/settings/settings.md#distributed_directory_monitor_max_sleep_time_ms) :::note **Durability settings** (`fsync_...`): @@ -213,7 +241,9 @@ To learn more about how distibuted `in` and `global in` queries are processed, r ## Virtual Columns {#virtual-columns} -- `_shard_num` — Contains the `shard_num` value from the table `system.clusters`. Type: [UInt32](../../../sql-reference/data-types/int-uint.md). +#### _shard_num + +`_shard_num` — Contains the `shard_num` value from the table `system.clusters`. Type: [UInt32](../../../sql-reference/data-types/int-uint.md). :::note Since [remote](../../../sql-reference/table-functions/remote.md) and [cluster](../../../sql-reference/table-functions/cluster.md) table functions internally create temporary Distributed table, `_shard_num` is available there too. diff --git a/docs/en/engines/table-engines/special/join.md b/docs/en/engines/table-engines/special/join.md index bb9744103f7..c95ebe19c31 100644 --- a/docs/en/engines/table-engines/special/join.md +++ b/docs/en/engines/table-engines/special/join.md @@ -23,11 +23,19 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] See the detailed description of the [CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query) query. -**Engine Parameters** +## Engine Parameters -- `join_strictness` – [JOIN strictness](../../../sql-reference/statements/select/join.md#select-join-types). -- `join_type` – [JOIN type](../../../sql-reference/statements/select/join.md#select-join-types). -- `k1[, k2, ...]` – Key columns from the `USING` clause that the `JOIN` operation is made with. +### join_strictness + +`join_strictness` – [JOIN strictness](../../../sql-reference/statements/select/join.md#select-join-types). + +### join_type + +`join_type` – [JOIN type](../../../sql-reference/statements/select/join.md#select-join-types). + +### Key columns + +`k1[, k2, ...]` – Key columns from the `USING` clause that the `JOIN` operation is made with. Enter `join_strictness` and `join_type` parameters without quotes, for example, `Join(ANY, LEFT, col1)`. They must match the `JOIN` operation that the table will be used for. If the parameters do not match, ClickHouse does not throw an exception and may return incorrect data. @@ -56,12 +64,28 @@ Main use-cases for `Join`-engine tables are following: When creating a table, the following settings are applied: -- [join_use_nulls](../../../operations/settings/settings.md#join_use_nulls) -- [max_rows_in_join](../../../operations/settings/query-complexity.md#settings-max_rows_in_join) -- [max_bytes_in_join](../../../operations/settings/query-complexity.md#settings-max_bytes_in_join) -- [join_overflow_mode](../../../operations/settings/query-complexity.md#settings-join_overflow_mode) -- [join_any_take_last_row](../../../operations/settings/settings.md#settings-join_any_take_last_row) -- [persistent](../../../operations/settings/settings.md#persistent) +#### join_use_nulls + +[join_use_nulls](../../../operations/settings/settings.md#join_use_nulls) + +#### max_rows_in_join + +[max_rows_in_join](../../../operations/settings/query-complexity.md#settings-max_rows_in_join) + +#### max_bytes_in_join + +[max_bytes_in_join](../../../operations/settings/query-complexity.md#settings-max_bytes_in_join) + +#### join_overflow_mode + +[join_overflow_mode](../../../operations/settings/query-complexity.md#settings-join_overflow_mode) + +#### join_any_take_last_row + +[join_any_take_last_row](../../../operations/settings/settings.md#settings-join_any_take_last_row) +#### join_use_nulls + +[persistent](../../../operations/settings/settings.md#persistent) The `Join`-engine tables can’t be used in `GLOBAL JOIN` operations. diff --git a/docs/en/engines/table-engines/special/merge.md b/docs/en/engines/table-engines/special/merge.md index 0f97acda8b5..ab15ad8dc76 100644 --- a/docs/en/engines/table-engines/special/merge.md +++ b/docs/en/engines/table-engines/special/merge.md @@ -15,14 +15,18 @@ Reading is automatically parallelized. Writing to a table is not supported. When CREATE TABLE ... Engine=Merge(db_name, tables_regexp) ``` -**Engine Parameters** +## Engine Parameters -- `db_name` — Possible values: +### db_name + +`db_name` — Possible values: - database name, - constant expression that returns a string with a database name, for example, `currentDatabase()`, - `REGEXP(expression)`, where `expression` is a regular expression to match the DB names. -- `tables_regexp` — A regular expression to match the table names in the specified DB or DBs. +### tables_regexp + +`tables_regexp` — A regular expression to match the table names in the specified DB or DBs. Regular expressions — [re2](https://github.com/google/re2) (supports a subset of PCRE), case-sensitive. See the notes about escaping symbols in regular expressions in the "match" section. From d52dc3201070a9dd2a57858d6c3772e6652cc8fe Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Fri, 24 Jun 2022 19:31:41 +0200 Subject: [PATCH 076/123] Update fetchPostgreSQLTableStructure.cpp --- src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp index 08a7e78d0e9..10cde43e9e1 100644 --- a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp +++ b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp @@ -300,10 +300,11 @@ PostgreSQLTableStructure fetchPostgreSQLTableStructure( "and a.attnum = ANY(ix.indkey) " "and t.relkind in ('r', 'p') " /// simple tables "and t.relname = {} " /// Connection is already done to a needed database, only table name is needed. - "and t.relnamespace = {} " + "{}" "and ix.indisreplident = 't' " /// index is is replica identity index - "ORDER BY a.attname", /// column names - quoteString(postgres_table), quoteString(postgres_schema.empty() ? "public" : postgres_schema)); + "ORDER BY a.attname", /// column name + (postgres_schema.empty() ? "" : "and t.relnamespace = " + quoteString(postgres_schema)) + " ", + quoteString(postgres_table)); table.replica_identity_columns = readNamesAndTypesList(tx, postgres_table_with_schema, query, use_nulls, true); } From a88ae9ca99dae0490c5076c07b71e1b447ad4734 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 24 Jun 2022 19:48:38 +0200 Subject: [PATCH 077/123] Review fix --- src/Databases/DatabaseFactory.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 82a7dff7125..5c7c1dedf9b 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "config_core.h" @@ -334,18 +335,25 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String configuration.username = safeGetLiteralValue(engine_args[2], engine_name); configuration.password = safeGetLiteralValue(engine_args[3], engine_name); + bool is_deprecated_syntax = false; if (engine_args.size() >= 5) { auto arg_value = engine_args[4]->as()->value; if (arg_value.getType() == Field::Types::Which::String) + { configuration.schema = safeGetLiteralValue(engine_args[4], engine_name); + } else + { use_table_cache = safeGetLiteralValue(engine_args[4], engine_name); + LOG_WARNING(&Poco::Logger::get("DatabaseFactory"), "A deprecated syntax of PostgreSQL database engine is used"); + is_deprecated_syntax = true; + } } - } - if (engine_args.size() >= 6) - use_table_cache = safeGetLiteralValue(engine_args[5], engine_name); + if (!is_deprecated_syntax && engine_args.size() >= 6) + use_table_cache = safeGetLiteralValue(engine_args[5], engine_name); + } auto pool = std::make_shared(configuration, context->getSettingsRef().postgresql_connection_pool_size, From 8d3772948a5ec5977a9417e8019d94792c504fa2 Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Fri, 24 Jun 2022 15:31:42 -0400 Subject: [PATCH 078/123] move title to frontmatter --- docs/en/operations/access-rights.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/en/operations/access-rights.md b/docs/en/operations/access-rights.md index 34d79aa99d9..a431f10fbad 100644 --- a/docs/en/operations/access-rights.md +++ b/docs/en/operations/access-rights.md @@ -1,10 +1,9 @@ --- sidebar_position: 48 sidebar_label: Access Control and Account Management +title: Access Control and Account Management --- -# Access Control and Account Management - ClickHouse supports access control management based on [RBAC](https://en.wikipedia.org/wiki/Role-based_access_control) approach. ClickHouse access entities: From 13a6254e1e7f4f302bd767472c105b7b57210c93 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Jun 2022 01:45:38 +0200 Subject: [PATCH 079/123] Correct submodule --- .gitmodules | 6 +- contrib/base-x | 1 + contrib/base-x/.gitignore | 4 - contrib/base-x/.travis.yml | 36 - contrib/base-x/LICENSE | 21 - contrib/base-x/README.md | 97 - contrib/base-x/base_x.hh | 614 ------ contrib/base-x/tests/test.cc | 30 - contrib/base-x/tests/testcases/tests.cc | 359 ---- contrib/base-x/uinteger_t.hh | 2546 ----------------------- 10 files changed, 4 insertions(+), 3710 deletions(-) create mode 160000 contrib/base-x delete mode 100644 contrib/base-x/.gitignore delete mode 100755 contrib/base-x/.travis.yml delete mode 100644 contrib/base-x/LICENSE delete mode 100644 contrib/base-x/README.md delete mode 100644 contrib/base-x/base_x.hh delete mode 100644 contrib/base-x/tests/test.cc delete mode 100644 contrib/base-x/tests/testcases/tests.cc delete mode 100644 contrib/base-x/uinteger_t.hh diff --git a/.gitmodules b/.gitmodules index a8924e3aaba..f65806c1da5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -268,9 +268,9 @@ [submodule "contrib/hashidsxx"] path = contrib/hashidsxx url = https://github.com/schoentoon/hashidsxx.git -[submodule "contrib/base-x"] - path = contrib/base-x - url = https://github.com/ClickHouse/base-x.git [submodule "contrib/liburing"] path = contrib/liburing url = https://github.com/axboe/liburing.git +[submodule "contrib/base-x"] + path = contrib/base-x + url = https://github.com/ClickHouse/base-x.git diff --git a/contrib/base-x b/contrib/base-x new file mode 160000 index 00000000000..a85f98fb4ed --- /dev/null +++ b/contrib/base-x @@ -0,0 +1 @@ +Subproject commit a85f98fb4ed52c2f4029a4b6ac1ef0bafdfc56f5 diff --git a/contrib/base-x/.gitignore b/contrib/base-x/.gitignore deleted file mode 100644 index b63b40c8b71..00000000000 --- a/contrib/base-x/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.DS_Store -test -*.o -*.dSYM \ No newline at end of file diff --git a/contrib/base-x/.travis.yml b/contrib/base-x/.travis.yml deleted file mode 100755 index f55132e614f..00000000000 --- a/contrib/base-x/.travis.yml +++ /dev/null @@ -1,36 +0,0 @@ -sudo: false - -language: cpp - -compiler: - - clang - - gcc - -addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.8 - packages: - - g++-6 - - clang-3.8 - -install: - - if [ "$CXX" = "g++" ]; then export CXX="g++-6"; fi - - if [ "$CXX" == "clang++" ]; then export CXX="clang++-3.8"; fi - - sudo apt-get install -qq git cmake - -before_script: - # not much better than git submodules, but there was never a need/want for the repo in this repo - - cd .. - - git clone https://github.com/google/googletest.git - - cd googletest - - git reset --hard d62d6c6556d96dda924382547c54a4b3afedb22c - - cmake CMakeLists.txt - - make - - - cd ../base-x/tests - - make - -script: - - make run diff --git a/contrib/base-x/LICENSE b/contrib/base-x/LICENSE deleted file mode 100644 index f7b3408abac..00000000000 --- a/contrib/base-x/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2017 German Mendez Bravo (Kronuz) @ german dot mb at gmail.com - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/contrib/base-x/README.md b/contrib/base-x/README.md deleted file mode 100644 index 5dc4a068043..00000000000 --- a/contrib/base-x/README.md +++ /dev/null @@ -1,97 +0,0 @@ -# base-x [![License][license-img]][license-url] [![GitHub Stars][stars-img]][stars-url] [![GitHub Forks][forks-img]][forks-url] [![GitHub Watchers][watchers-img]][watchers-url] [![Tweet][tweet-img]][tweet-url] - -[![Build Status](https://travis-ci.org/Kronuz/base-x.svg?branch=master)](https://travis-ci.org/Kronuz/base-x) - - -### BaseX encoder / decoder for C++ - -This is a fast base encoder / decoder of any given alphabet. - - -#### Example - -``` cpp -// example.cc -// g++ -std=c++14 -o example example.cc - -#include -#include "base_x.hh" - -int main() { - auto encoded = Base58::base58().encode("Hello world!"); - - std::cout << encoded << std::endl; - // => 1LDlk6QWOejX6rPrJ - - return 0; -} -``` - - -#### Compilation - -* g++ and clang++ are supported. -* C++14 is required. - - -### Alphabets - -See below for a list of commonly recognized alphabets, and their respective base. - -Base | Factory | Alphabet ------|---------------------|------------- - 2 | base2::base2() | `01` - 2 | base8::base8() | `01234567` - 11 | bas11::bas11() | `0123456789a` - 16 | base16::base16() | `0123456789abcdef` - 32 | base32::base32() | `0123456789ABCDEFGHJKMNPQRSTVWXYZ` - 36 | base36::base36() | `0123456789abcdefghijklmnopqrstuvwxyz` - 58 | base58::base58() | `123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz` - 58 | base58::bitcoin() | `123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz` - 58 | base58::gmp() | `0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv` - 58 | base58::ripple() | `rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz` - 58 | base58::flickr() | `123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ` - 62 | base62::base62() | `0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz` - 62 | base62::inverted() | `0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ` - 64 | base64::base64() | `ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/` - 64 | base64::urlsafe() | `ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_` - 66 | base66::base66() | `ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.!~` - - -### How it works - -It encodes octet arrays by doing long divisions on all significant digits in the -array, creating a representation of that number in the new base. - -**If you need standard hex encoding, or base64 encoding, this module is NOT -appropriate.** - - -## Author -[**German Mendez Bravo (Kronuz)**](https://kronuz.io/) - -[![Follow on GitHub][github-follow-img]][github-follow-url] -[![Follow on Twitter][twitter-follow-img]][twitter-follow-url] - - -## License - -MIT License. See [LICENSE](LICENSE) for details. - -Copyright (c) 2017 German Mendez Bravo (Kronuz) @ german dot mb at gmail.com - - -[license-url]: https://github.com/Kronuz/base-x/blob/master/LICENSE -[license-img]: https://img.shields.io/github/license/Kronuz/base-x.svg -[stars-url]: https://github.com/Kronuz/base-x/stargazers -[stars-img]: https://img.shields.io/github/stars/Kronuz/base-x.svg?style=social&label=Stars -[forks-url]: https://github.com/Kronuz/base-x/network/members -[forks-img]: https://img.shields.io/github/forks/Kronuz/base-x.svg?style=social&label=Forks -[watchers-url]: https://github.com/Kronuz/base-x/watchers -[watchers-img]: https://img.shields.io/github/watchers/Kronuz/base-x.svg?style=social&label=Watchers -[tweet-img]: https://img.shields.io/twitter/url/https/github.com/Kronuz/base-x.svg?style=social -[tweet-url]: https://twitter.com/intent/tweet?text=Base-X+encoding%2Fdecoding+for+modern+C%2B%2B+by+%40germbravo:&url=https%3A%2F%2Fgithub.com%2FKronuz%2Fbase-x -[github-follow-url]: https://github.com/Kronuz -[github-follow-img]: https://img.shields.io/github/followers/Kronuz.svg?style=social&label=Follow -[twitter-follow-url]: https://twitter.com/intent/follow?screen_name=germbravo -[twitter-follow-img]: https://img.shields.io/twitter/follow/germbravo.svg?style=social&label=Follow diff --git a/contrib/base-x/base_x.hh b/contrib/base-x/base_x.hh deleted file mode 100644 index fdc06fead2f..00000000000 --- a/contrib/base-x/base_x.hh +++ /dev/null @@ -1,614 +0,0 @@ -/* -base_x.hh -BaseX encoder / decoder for C++ - -Copyright (c) 2017 German Mendez Bravo (Kronuz) @ german dot mb at gmail.com - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#ifndef __BASE_X__H_ -#define __BASE_X__H_ - -#include // for std::find_if, std::reverse -#include // for std::invalid_argument -#include // for std::string -#include // for std::enable_if_t - -#include "uinteger_t.hh" - - -class BaseX { - char _chr[256]; - int _ord[256]; - - const int size; - const int alphabet_base; - const unsigned base_size; - const unsigned alphabet_base_bits; - const unsigned block_size; - const uinteger_t::digit alphabet_base_mask; - const unsigned padding_size; - const char padding; - const int flags; - - constexpr char chr(unsigned char ord) const { - return _chr[ord]; - } - - constexpr int ord(unsigned char chr) const { - return _ord[chr]; - } - -public: - static constexpr int ignore_case = (1 << 0); - static constexpr int with_checksum = (1 << 1); - static constexpr int with_check = (1 << 2); - static constexpr int block_padding = (1 << 3); - - template - constexpr BaseX(int flgs, const char (&alphabet)[alphabet_size1], const char (&extended)[extended_size1], const char (&padding_string)[padding_size1], const char (&translate)[translate_size1]) : - _chr(), - _ord(), - size(alphabet_size1 - 1 + extended_size1 - 1), - alphabet_base(alphabet_size1 - 1), - base_size(uinteger_t::base_size(alphabet_base)), - alphabet_base_bits(uinteger_t::base_bits(alphabet_base)), - block_size((flgs & BaseX::block_padding) ? alphabet_base_bits : 0), - alphabet_base_mask(alphabet_base - 1), - padding_size(padding_size1 - 1), - padding(padding_size ? padding_string[0] : '\0'), - flags(flgs) - { - for (int c = 0; c < 256; ++c) { - _chr[c] = 0; - _ord[c] = alphabet_base; - } - for (int cp = 0; cp < alphabet_base; ++cp) { - auto ch = alphabet[cp]; - _chr[cp] = ch; - ASSERT(_ord[(unsigned char)ch] == alphabet_base); // Duplicate character in the alphabet - _ord[(unsigned char)ch] = cp; - if (flags & BaseX::ignore_case) { - if (ch >= 'A' && ch <='Z') { - _ord[(unsigned char)ch - 'A' + 'a'] = cp; - } else if (ch >= 'a' && ch <='z') { - _ord[(unsigned char)ch - 'a' + 'A'] = cp; - } - } - } - for (std::size_t i = 0; i < extended_size1 - 1; ++i) { - auto ch = extended[i]; - auto cp = alphabet_base + i; - _chr[cp] = ch; - ASSERT(_ord[(unsigned char)ch] == alphabet_base); // Duplicate character in the extended alphabet - _ord[(unsigned char)ch] = cp; - if (flags & BaseX::ignore_case) { - if (ch >= 'A' && ch <='Z') { - _ord[(unsigned char)ch - 'A' + 'a'] = cp; - } else if (ch >= 'a' && ch <='z') { - _ord[(unsigned char)ch - 'a' + 'A'] = cp; - } - } - } - int cp = -1; - for (std::size_t i = 0; i < translate_size1 - 1; ++i) { - auto ch = translate[i]; - auto ncp = _ord[(unsigned char)ch]; - if (ncp >= alphabet_base) { - ASSERT(_ord[(unsigned char)ch] == alphabet_base); // Invalid translation character - _ord[(unsigned char)ch] = cp; - if (flags & BaseX::ignore_case) { - if (ch >= 'A' && ch <='Z') { - _ord[(unsigned char)ch - 'A' + 'a'] = cp; - } else if (ch >= 'a' && ch <='z') { - _ord[(unsigned char)ch - 'a' + 'A'] = cp; - } - } - } else { - cp = ncp; - } - } - } - - // Get string representation of value - template ::value>> - void encode(Result& result, const uinteger_t& input) const { - std::size_t bp = 0; - uinteger_t quotient; - if (block_size) { - bp = ((input.bits() + 7) & 0xf8) % block_size; - bp = bp ? (block_size - bp) % block_size : 0; - if (bp) { - quotient = input << bp; - } - } - const uinteger_t& num = bp ? quotient : input; - auto num_sz = num.size(); - if (num_sz) { - int sum = 0; - result.reserve(num_sz * base_size); - if (alphabet_base_bits) { - std::size_t shift = 0; - auto ptr = reinterpret_cast(num.data()); - uinteger_t::digit v = *ptr++; - v <<= uinteger_t::half_digit_bits; - for (auto i = num_sz * 2 - 1; i; --i) { - v >>= uinteger_t::half_digit_bits; - v |= (static_cast(*ptr++) << uinteger_t::half_digit_bits); - do { - auto d = static_cast((v >> shift) & alphabet_base_mask); - result.push_back(chr(d)); - shift += alphabet_base_bits; - sum += d; - } while (shift <= uinteger_t::half_digit_bits); - shift -= uinteger_t::half_digit_bits; - } - v >>= (shift + uinteger_t::half_digit_bits); - while (v) { - auto d = static_cast(v & alphabet_base_mask); - result.push_back(chr(d)); - v >>= alphabet_base_bits; - sum += d; - } - auto s = chr(0); - auto rit_f = std::find_if(result.rbegin(), result.rend(), [s](const char& c) { return c != s; }); - result.resize(result.rend() - rit_f); // shrink - } else { - uinteger_t uint_base = alphabet_base; - if (!bp) { - quotient = num; - } - do { - auto r = quotient.divmod(uint_base); - auto d = static_cast(r.second); - result.push_back(chr(d)); - quotient = std::move(r.first); - sum += d; - } while (quotient); - } - std::reverse(result.begin(), result.end()); - if (padding_size) { - Result p; - p.resize((padding_size - (result.size() % padding_size)) % padding_size, padding); - result.append(p); - } - if (flags & BaseX::with_check) { - auto chk = static_cast(num % size); - result.push_back(chr(chk)); - sum += chk; - } - if (flags & BaseX::with_checksum) { - auto sz = result.size(); - sz = (sz + sz / size) % size; - sum += sz; - sum = (size - sum % size) % size; - result.push_back(chr(sum)); - } - } else { - result.push_back(chr(0)); - } - } - - template ::value>> - Result encode(const uinteger_t& num) const { - Result result; - encode(result, num); - return result; - } - - template ::value>> - void encode(Result& result, const unsigned char* decoded, std::size_t decoded_size) const { - encode(result, uinteger_t(decoded, decoded_size, 256)); - } - - template ::value>> - Result encode(const unsigned char* decoded, std::size_t decoded_size) const { - Result result; - encode(result, uinteger_t(decoded, decoded_size, 256)); - return result; - } - - template ::value>> - void encode(Result& result, const char* decoded, std::size_t decoded_size) const { - encode(result, uinteger_t(decoded, decoded_size, 256)); - } - - template ::value>> - Result encode(const char* decoded, std::size_t decoded_size) const { - Result result; - encode(result, uinteger_t(decoded, decoded_size, 256)); - return result; - } - - template ::value>> - void encode(Result& result, T (&s)[N]) const { - encode(result, s, N - 1); - } - - template ::value>> - Result encode(T (&s)[N]) const { - Result result; - encode(result, s, N - 1); - return result; - } - - template ::value>> - void encode(Result& result, const std::string& binary) const { - return encode(result, binary.data(), binary.size()); - } - - template ::value>> - Result encode(const std::string& binary) const { - Result result; - encode(result, binary.data(), binary.size()); - return result; - } - - void decode(uinteger_t& result, const char* encoded, std::size_t encoded_size) const { - result = 0; - int sum = 0; - int sumsz = 0; - int direction = 1; - - auto sz = encoded_size; - if (flags & BaseX::with_checksum) --sz; - if (flags & BaseX::with_check) --sz; - - int bp = 0; - - if (alphabet_base_bits) { - for (; sz; --sz, encoded += direction) { - auto c = *encoded; - if (c == padding) break; - auto d = ord(static_cast(c)); - if (d < 0) continue; // ignored character - if (d >= alphabet_base) { - throw std::invalid_argument("Error: Invalid character: '" + std::string(1, c) + "' at " + std::to_string(encoded_size - sz)); - } - sum += d; - ++sumsz; - result = (result << alphabet_base_bits) | d; - bp += block_size; - } - } else { - uinteger_t uint_base = alphabet_base; - for (; sz; --sz, encoded += direction) { - auto c = *encoded; - if (c == padding) break; - auto d = ord(static_cast(c)); - if (d < 0) continue; // ignored character - if (d >= alphabet_base) { - throw std::invalid_argument("Error: Invalid character: '" + std::string(1, c) + "' at " + std::to_string(encoded_size - sz)); - } - sum += d; - ++sumsz; - result = (result * uint_base) + d; - bp += block_size; - } - } - - for (; sz && *encoded == padding; --sz, ++encoded); - - result >>= (bp & 7); - - if (flags & BaseX::with_check) { - auto c = *encoded; - auto d = ord(static_cast(c)); - if (d < 0 || d >= size) { - throw std::invalid_argument("Error: Invalid character: '" + std::string(1, c) + "' at " + std::to_string(encoded_size - sz)); - } - auto chk = static_cast(result % size); - if (d != chk) { - throw std::invalid_argument("Error: Invalid check"); - } - sum += chk; - ++sumsz; - ++encoded; - } - - if (flags & BaseX::with_checksum) { - auto c = *encoded; - auto d = ord(static_cast(c)); - if (d < 0 || d >= size) { - throw std::invalid_argument("Error: Invalid character: '" + std::string(1, c) + "' at " + std::to_string(encoded_size - sz)); - } - sum += d; - sum += (sumsz + sumsz / size) % size; - if (sum % size) { - throw std::invalid_argument("Error: Invalid checksum"); - } - } - } - - template ::value>> - void decode(Result& result, const char* encoded, std::size_t encoded_size) const { - uinteger_t num; - decode(num, encoded, encoded_size); - result = num.template str(256); - } - - template ::value or std::is_integral::value>> - Result decode(const char* encoded, std::size_t encoded_size) const { - Result result; - decode(result, encoded, encoded_size); - return result; - } - - template ::value or std::is_integral::value>> - void decode(Result& result, T (&s)[N]) const { - decode(result, s, N - 1); - } - - template ::value or std::is_integral::value>> - Result decode(T (&s)[N]) const { - Result result; - decode(result, s, N - 1); - return result; - } - - template ::value or std::is_integral::value>> - void decode(Result& result, const std::string& encoded) const { - decode(result, encoded.data(), encoded.size()); - } - - template ::value or std::is_integral::value>> - Result decode(const std::string& encoded) const { - Result result; - decode(result, encoded.data(), encoded.size()); - return result; - } - - bool is_valid(const char* encoded, std::size_t encoded_size) const { - int sum = 0; - int sumsz = 0; - if (flags & BaseX::with_checksum) --sumsz; - for (; encoded_size; --encoded_size, ++encoded) { - auto d = ord(static_cast(*encoded)); - if (d < 0) continue; // ignored character - if (d >= alphabet_base) { - return false; - } - sum += d; - ++sumsz; - } - if (flags & BaseX::with_checksum) { - sum += (sumsz + sumsz / size) % size; - if (sum % size) { - return false; - } - } - return true; - } - - template - bool is_valid(T (&s)[N]) const { - return is_valid(s, N - 1); - } - - bool is_valid(const std::string& encoded) const { - return is_valid(encoded.data(), encoded.size()); - } -}; - -// base2 -struct Base2 { - static const BaseX& base2() { - static constexpr BaseX encoder(0, "01", "", "", ""); - return encoder; - } - static const BaseX& base2chk() { - static constexpr BaseX encoder(BaseX::with_checksum, "01", "", "", ""); - return encoder; - } -}; - -// base8 -struct Base8 { - static const BaseX& base8() { - static constexpr BaseX encoder(0, "01234567", "", "", ""); - return encoder; - } - static const BaseX& base8chk() { - static constexpr BaseX encoder(BaseX::with_checksum, "01234567", "", "", ""); - return encoder; - } -}; - -// base11 -struct Base11 { - static const BaseX& base11() { - static constexpr BaseX encoder(BaseX::ignore_case, "0123456789a", "", "", ""); - return encoder; - } - static const BaseX& base11chk() { - static constexpr BaseX encoder(BaseX::ignore_case | BaseX::with_checksum, "0123456789a", "", "", ""); - return encoder; - } -}; - -// base16 -struct Base16 { - static const BaseX& base16() { - static constexpr BaseX encoder(BaseX::ignore_case, "0123456789abcdef", "", "", ""); - return encoder; - } - static const BaseX& base16chk() { - static constexpr BaseX encoder(BaseX::ignore_case | BaseX::with_checksum, "0123456789abcdef", "", "", ""); - return encoder; - } - static const BaseX& rfc4648() { - static constexpr BaseX encoder(0, "0123456789ABCDEF", "", "", ""); - return encoder; - } -}; - -// base32 -struct Base32 { - static const BaseX& base32() { - static constexpr BaseX encoder(BaseX::ignore_case, "0123456789abcdefghijklmnopqrstuv", "", "", ""); - return encoder; - } - static const BaseX& base32chk() { - static constexpr BaseX encoder(BaseX::ignore_case | BaseX::with_checksum, "0123456789abcdefghijklmnopqrstuv", "", "", ""); - return encoder; - } - static const BaseX& crockford() { - static constexpr BaseX encoder(BaseX::ignore_case, "0123456789ABCDEFGHJKMNPQRSTVWXYZ", "", "", "-0O1IL"); - return encoder; - } - static const BaseX& crockfordchk() { - static constexpr BaseX encoder(BaseX::ignore_case | BaseX::with_check, "0123456789ABCDEFGHJKMNPQRSTVWXYZ", "*~$=U", "", "-0O1IL"); - return encoder; - } - static const BaseX& rfc4648() { - static constexpr BaseX encoder(BaseX::block_padding, "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567", "", "========", "\n\r"); - return encoder; - } - static const BaseX& rfc4648hex() { - static constexpr BaseX encoder(BaseX::block_padding, "0123456789ABCDEFGHIJKLMNOPQRSTUV", "", "========", "\n\r"); - return encoder; - } -}; - -// base36 -struct Base36 { - static const BaseX& base36() { - static constexpr BaseX encoder(BaseX::ignore_case, "0123456789abcdefghijklmnopqrstuvwxyz", "", "", ""); - return encoder; - } - static const BaseX& base36chk() { - static constexpr BaseX encoder(BaseX::ignore_case | BaseX::with_checksum, "0123456789abcdefghijklmnopqrstuvwxyz", "", "", ""); - return encoder; - } -}; - -// base58 -struct Base58 { - static const BaseX& base58() { - static constexpr BaseX encoder(0, "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv", "", "", ""); - return encoder; - } - static const BaseX& base58chk() { - static constexpr BaseX encoder(BaseX::with_checksum, "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv", "", "", ""); - return encoder; - } - static const BaseX& bitcoin() { - static constexpr BaseX encoder(0, "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", "", "", ""); - return encoder; - } - static const BaseX& bitcoinchk() { - static constexpr BaseX encoder(BaseX::with_checksum, "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", "", "", ""); - return encoder; - } - static const BaseX& ripple() { - static constexpr BaseX encoder(0, "rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz", "", "", ""); - return encoder; - } - static const BaseX& ripplechk() { - static constexpr BaseX encoder(BaseX::with_checksum, "rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz", "", "", ""); - return encoder; - } - static const BaseX& flickr() { - static constexpr BaseX encoder(0, "123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ", "", "", ""); - return encoder; - } - static const BaseX& flickrchk() { - static constexpr BaseX encoder(BaseX::with_checksum, "123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ", "", "", ""); - return encoder; - } -}; - -// base59 -struct Base59 { - static const BaseX& base59() { - static constexpr BaseX encoder(0, "23456789abcdefghijklmnopqrstuvwxyzABCDEFGHJKLMNOPQRSTUVWXYZ", "", "", "l1IO0"); - return encoder; - } - static const BaseX& base59chk() { - static constexpr BaseX encoder(BaseX::with_checksum, "23456789abcdefghijklmnopqrstuvwxyzABCDEFGHJKLMNOPQRSTUVWXYZ", "", "", "l1IO0"); - return encoder; - } - static const BaseX& dubaluchk() { - static constexpr BaseX encoder(BaseX::with_checksum, "zy9MalDxwpKLdvW2AtmscgbYUq6jhP7E53TiXenZRkVCrouBH4GSQf8FNJO", "", "", "-l1IO0"); - return encoder; - } -}; - -// base62 -struct Base62 { - static const BaseX& base62() { - static constexpr BaseX encoder(0, "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", "", "", ""); - return encoder; - } - static const BaseX& base62chk() { - static constexpr BaseX encoder(BaseX::with_checksum, "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", "", "", ""); - return encoder; - } - static const BaseX& inverted() { - static constexpr BaseX encoder(0, "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ", "", "", ""); - return encoder; - } - static const BaseX& invertedchk() { - static constexpr BaseX encoder(BaseX::with_checksum, "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ", "", "", ""); - return encoder; - } -}; - -// base64 -struct Base64 { - static const BaseX& base64() { - static constexpr BaseX encoder(0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", "", "", ""); - return encoder; - } - static const BaseX& base64chk() { - static constexpr BaseX encoder(BaseX::with_checksum, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", "", "", ""); - return encoder; - } - static const BaseX& url() { - static constexpr BaseX encoder(0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", "", "", ""); - return encoder; - } - static const BaseX& urlchk() { - static constexpr BaseX encoder(BaseX::with_checksum, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", "", "", ""); - return encoder; - } - static const BaseX& rfc4648() { - static constexpr BaseX encoder(BaseX::block_padding, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", "", "====", "\n\r"); - return encoder; - } - static const BaseX& rfc4648url() { - static constexpr BaseX encoder(BaseX::block_padding, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", "", "====", "\n\r"); - return encoder; - } -}; - -// base66 -struct Base66 { - static const BaseX& base66() { - static constexpr BaseX encoder(0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.!~", "", "", ""); - return encoder; - } - static const BaseX& base66chk() { - static constexpr BaseX encoder(BaseX::with_checksum, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.!~", "", "", ""); - return encoder; - } -}; - -#endif diff --git a/contrib/base-x/tests/test.cc b/contrib/base-x/tests/test.cc deleted file mode 100644 index d47d211173e..00000000000 --- a/contrib/base-x/tests/test.cc +++ /dev/null @@ -1,30 +0,0 @@ -/* -The MIT License (MIT) - -Copyright (c) 2017 German Mendez Bravo (Kronuz) @ german dot mb at gmail.com - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#include - -int main(int argc, char * argv[]){ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} \ No newline at end of file diff --git a/contrib/base-x/tests/testcases/tests.cc b/contrib/base-x/tests/testcases/tests.cc deleted file mode 100644 index c5bebfc8288..00000000000 --- a/contrib/base-x/tests/testcases/tests.cc +++ /dev/null @@ -1,359 +0,0 @@ -/* -The MIT License (MIT) - -Copyright (c) 2017 German Mendez Bravo (Kronuz) @ german dot mb at gmail.com - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#include - -#include "base_x.hh" - - -static constexpr BaseX test_base2(0, "01", "", "", ""); -static constexpr BaseX test_base16(0, "0123456789abcdef", "", "", ""); -static constexpr BaseX test_base58(0, "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", "", "", ""); - - -TEST(UUID, Encode) { - EXPECT_EQ(Base62::base62().encode("\330\105\140\310\23\117\21\346\241\342\64\66\73\322\155\256"), "6a630O1jrtMjCrQDyG3D3O"); - EXPECT_EQ(Base58::bitcoin().encode("\330\105\140\310\23\117\21\346\241\342\64\66\73\322\155\256"), "ThxCy1Ek2q6UhWQhj9CK1o"); - EXPECT_EQ(Base58::base58().encode("\330\105\140\310\23\117\21\346\241\342\64\66\73\322\155\256"), "QetBu0Dh1m5ReTNeg8BI0k"); -} - -TEST(BaseX, checksums) { - EXPECT_EQ(Base64::base64().encode("Hello world!"), "SGVsbG8gd29ybGQh"); - EXPECT_EQ(Base64::base64chk().encode("Hello world!"), "SGVsbG8gd29ybGQhG"); - - EXPECT_EQ(Base64::base64().decode("SGVsbG8gd29ybGQh"), "Hello world!"); - EXPECT_EQ(Base64::base64chk().decode("SGVsbG8gd29ybGQhG"), "Hello world!"); - - EXPECT_EQ(Base62::base62().encode("Hello world!"), "T8dgcjRGuYUueWht"); - EXPECT_EQ(Base62::base62chk().encode("Hello world!"), "T8dgcjRGuYUueWhtE"); - - EXPECT_EQ(Base62::base62().decode("T8dgcjRGuYUueWht"), "Hello world!"); - EXPECT_EQ(Base62::base62chk().decode("T8dgcjRGuYUueWhtE"), "Hello world!"); - - EXPECT_EQ(Base62::base62chk().is_valid("T8dgcjRGuYUueWhtE"), true); - EXPECT_EQ(Base62::base62chk().is_valid("Some random text!"), false); -} - -TEST(base16, Encoder) { - EXPECT_EQ(Base16::base16().encode("A"), "41"); - EXPECT_EQ(Base16::base16().encode("AB"), "4142"); - EXPECT_EQ(Base16::base16().encode("ABC"), "414243"); - EXPECT_EQ(Base16::base16().encode("ABCD"), "41424344"); - EXPECT_EQ(Base16::base16().encode("ABCDE"), "4142434445"); - EXPECT_EQ(Base16::base16().encode("ABCDEF"), "414243444546"); - - EXPECT_EQ(Base16::rfc4648().encode("A"), "41"); - EXPECT_EQ(Base16::rfc4648().encode("AB"), "4142"); - EXPECT_EQ(Base16::rfc4648().encode("ABC"), "414243"); - EXPECT_EQ(Base16::rfc4648().encode("ABCD"), "41424344"); - EXPECT_EQ(Base16::rfc4648().encode("ABCDE"), "4142434445"); - EXPECT_EQ(Base16::rfc4648().encode("ABCDEF"), "414243444546"); -} - -TEST(base16, Decoder) { - EXPECT_EQ(Base16::base16().decode("41"), "A"); - EXPECT_EQ(Base16::base16().decode("4142"), "AB"); - EXPECT_EQ(Base16::base16().decode("414243"), "ABC"); - EXPECT_EQ(Base16::base16().decode("41424344"), "ABCD"); - EXPECT_EQ(Base16::base16().decode("4142434445"), "ABCDE"); - EXPECT_EQ(Base16::base16().decode("414243444546"), "ABCDEF"); - - EXPECT_EQ(Base16::rfc4648().decode("41"), "A"); - EXPECT_EQ(Base16::rfc4648().decode("4142"), "AB"); - EXPECT_EQ(Base16::rfc4648().decode("414243"), "ABC"); - EXPECT_EQ(Base16::rfc4648().decode("41424344"), "ABCD"); - EXPECT_EQ(Base16::rfc4648().decode("4142434445"), "ABCDE"); - EXPECT_EQ(Base16::rfc4648().decode("414243444546"), "ABCDEF"); -} - -TEST(base32, Encoder) { - // Note base64() encoding is NOT the same as the standard (rfc4648) - EXPECT_EQ(Base32::base32().encode("A"), "21"); - EXPECT_EQ(Base32::base32().encode("AB"), "ga2"); - EXPECT_EQ(Base32::base32().encode("ABC"), "42gi3"); - EXPECT_EQ(Base32::base32().encode("ABCD"), "10k4gq4"); - EXPECT_EQ(Base32::base32().encode("ABCDE"), "85146h25"); - EXPECT_EQ(Base32::base32().encode("ABCDEF"), "21891k8ha6"); - EXPECT_EQ(Base32::base32().encode("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), "21891k8ha68t44iiib9h6ksjqga5956l2lapblgmaq"); - - EXPECT_EQ(Base32::rfc4648().encode("A"), "IE======"); - EXPECT_EQ(Base32::rfc4648().encode("AB"), "IFBA===="); - EXPECT_EQ(Base32::rfc4648().encode("ABC"), "IFBEG==="); - EXPECT_EQ(Base32::rfc4648().encode("ABCD"), "IFBEGRA="); - EXPECT_EQ(Base32::rfc4648().encode("ABCDE"), "IFBEGRCF"); - EXPECT_EQ(Base32::rfc4648().encode("ABCDEF"), "IFBEGRCFIY======"); - EXPECT_EQ(Base32::rfc4648().encode("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), "IFBEGRCFIZDUQSKKJNGE2TSPKBIVEU2UKVLFOWCZLI======"); - - EXPECT_EQ(Base32::crockford().encode(519571), "FVCK"); - EXPECT_EQ(Base32::crockfordchk().encode(1234), "16JD"); - EXPECT_EQ(Base32::crockfordchk().encode("Hello World"), "28CNP6RVS0AXQQ4V348"); -} - -TEST(base32, Decoder) { - // Note base64() encoding is NOT the same as the standard (rfc4648) - EXPECT_EQ(Base32::base32().decode("21"), "A"); - EXPECT_EQ(Base32::base32().decode("ga2"), "AB"); - EXPECT_EQ(Base32::base32().decode("42gi3"), "ABC"); - EXPECT_EQ(Base32::base32().decode("10k4gq4"), "ABCD"); - EXPECT_EQ(Base32::base32().decode("85146h25"), "ABCDE"); - EXPECT_EQ(Base32::base32().decode("21891k8ha6"), "ABCDEF"); - EXPECT_EQ(Base32::base32().decode("21891k8ha68t44iiib9h6ksjqga5956l2lapblgmaq"), "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); - - EXPECT_EQ(Base32::rfc4648().decode("IE======"), "A"); - EXPECT_EQ(Base32::rfc4648().decode("IFBA===="), "AB"); - EXPECT_EQ(Base32::rfc4648().decode("IFBEG==="), "ABC"); - EXPECT_EQ(Base32::rfc4648().decode("IFBEGRA="), "ABCD"); - EXPECT_EQ(Base32::rfc4648().decode("IFBEGRCF"), "ABCDE"); - EXPECT_EQ(Base32::rfc4648().decode("IFBEGRCFIY======"), "ABCDEF"); - EXPECT_EQ(Base32::rfc4648().decode("IFBEGRCFIZDUQSKKJNGE2TSPKBIVEU2UKVLFOWCZLI======"), "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); - - EXPECT_EQ(Base32::crockford().decode("FVCK"), 519571); - EXPECT_EQ(Base32::crockfordchk().is_valid("16JD"), true); - EXPECT_EQ(Base32::crockfordchk().decode("16JD"), 1234); - - EXPECT_EQ(Base32::crockfordchk().decode("2-8cn-p6r-vso-axq-q4v-348"), "Hello World"); -} - -TEST(base58, Encoder) { - EXPECT_EQ(Base58::base58().decode("1TFvCj"), 987654321); - EXPECT_EQ(Base58::base58().encode(987654321), "1TFvCj"); - EXPECT_EQ(Base58::base58().encode("Hello world!"), "1LDlk6QWOejX6rPrJ"); - EXPECT_EQ(Base58::bitcoin().encode("Hello world!"), "2NEpo7TZRhna7vSvL"); -} - -TEST(base62, Encoder) { - EXPECT_EQ(Base62::base62().decode("14q60P"), 987654321); - EXPECT_EQ(Base62::base62().encode(987654321), "14q60P"); - EXPECT_EQ(Base62::base62().encode("Hello world!"), "T8dgcjRGuYUueWht"); - EXPECT_EQ(Base62::inverted().encode("Hello world!"), "t8DGCJrgUyuUEwHT"); -} - -TEST(base64, Encoder) { - // Note Base64 encoding is NOT the same as the standard (rfc4648) - EXPECT_EQ(Base64::base64().encode("A"), "BB"); - EXPECT_EQ(Base64::base64().encode("AB"), "EFC"); - EXPECT_EQ(Base64::base64().encode("ABC"), "QUJD"); - EXPECT_EQ(Base64::base64().encode("ABCD"), "BBQkNE"); - EXPECT_EQ(Base64::base64().encode("ABCDE"), "EFCQ0RF"); - EXPECT_EQ(Base64::base64().encode("ABCDEF"), "QUJDREVG"); - EXPECT_EQ(Base64::base64().encode("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), "EFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFla"); - - EXPECT_EQ(Base64::rfc4648().encode("A"), "QQ=="); - EXPECT_EQ(Base64::rfc4648().encode("AB"), "QUI="); - EXPECT_EQ(Base64::rfc4648().encode("ABC"), "QUJD"); - EXPECT_EQ(Base64::rfc4648().encode("ABCD"), "QUJDRA=="); - EXPECT_EQ(Base64::rfc4648().encode("ABCDE"), "QUJDREU="); - EXPECT_EQ(Base64::rfc4648().encode("ABCDEF"), "QUJDREVG"); - EXPECT_EQ(Base64::rfc4648().encode("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo="); -} - -TEST(base64, Decoder) { - // Note Base64 encoding is NOT the same as the standard (rfc4648) - EXPECT_EQ(Base64::base64().decode("BB"), "A"); - EXPECT_EQ(Base64::base64().decode("EFC"), "AB"); - EXPECT_EQ(Base64::base64().decode("QUJD"), "ABC"); - EXPECT_EQ(Base64::base64().decode("BBQkNE"), "ABCD"); - EXPECT_EQ(Base64::base64().decode("EFCQ0RF"), "ABCDE"); - EXPECT_EQ(Base64::base64().decode("QUJDREVG"), "ABCDEF"); - EXPECT_EQ(Base64::base64().decode("EFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFla"), "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); - - EXPECT_EQ(Base64::rfc4648().decode("QQ=="), "A"); - EXPECT_EQ(Base64::rfc4648().decode("QUI="), "AB"); - EXPECT_EQ(Base64::rfc4648().decode("QUJD"), "ABC"); - EXPECT_EQ(Base64::rfc4648().decode("QUJDRA=="), "ABCD"); - EXPECT_EQ(Base64::rfc4648().decode("QUJDREU="), "ABCDE"); - EXPECT_EQ(Base64::rfc4648().decode("QUJDREVG"), "ABCDEF"); - EXPECT_EQ(Base64::rfc4648().decode("QUJDREVG\nR0hJSktM\nTU5PUFFS\nU1RVVldY\nWVo="), "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); -} - -TEST(base58, ShouldEncodeAndDecodeIntegers) { - auto data = 987654321; - - auto gmpEncoded = Base58::base58().encode(data); - auto bitcoinEncoded = Base58::bitcoin().encode(data); - auto rippleEncoded = Base58::ripple().encode(data); - auto flickrEncoded = Base58::flickr().encode(data); - - EXPECT_EQ(gmpEncoded, "1TFvCj"); - EXPECT_EQ(bitcoinEncoded, "2WGzDn"); - EXPECT_EQ(rippleEncoded, "pWGzD8"); - EXPECT_EQ(flickrEncoded, "2vgZdM"); - - auto gmpDecoded = Base58::base58().decode(gmpEncoded); - auto bitcoinDecoded = Base58::bitcoin().decode(bitcoinEncoded); - auto rippleDecoded = Base58::ripple().decode(rippleEncoded); - auto flickrDecoded = Base58::flickr().decode(flickrEncoded); - - EXPECT_EQ(gmpDecoded, data); - EXPECT_EQ(bitcoinDecoded, data); - EXPECT_EQ(rippleDecoded, data); - EXPECT_EQ(flickrDecoded, data); - - auto encoded = Base58::base58().encode(data); - auto decoded = Base58::base58().decode(encoded); - - EXPECT_EQ(decoded, data); -} - -TEST(base58, LongText) { - auto data = "Lorem ipsum dolor consectetur."; - - auto gmpEncoded = Base58::base58().encode(data); - auto bitcoinEncoded = Base58::bitcoin().encode(data); - auto rippleEncoded = Base58::ripple().encode(data); - auto flickrEncoded = Base58::flickr().encode(data); - - EXPECT_EQ(gmpEncoded, "FIHZQEpJ739QdqChX1PkgTBqP1FaDgJWQiGvY92YA"); - EXPECT_EQ(bitcoinEncoded, "GKJcTFtL84ATguDka2SojWCuS2GdEjLZTmHzbA3bB"); - EXPECT_EQ(rippleEncoded, "GKJcTEtL3hwTguDk2pSojWUuSpGdNjLZTmHzbwsbB"); - EXPECT_EQ(flickrEncoded, "gjiBsfTk84asFUdKz2rNJvcUr2gCeJkysLhZAa3Ab"); - - auto gmpDecoded = Base58::base58().decode(gmpEncoded); - auto bitcoinDecoded = Base58::bitcoin().decode(bitcoinEncoded); - auto rippleDecoded = Base58::ripple().decode(rippleEncoded); - auto flickrDecoded = Base58::flickr().decode(flickrEncoded); - - EXPECT_EQ(gmpDecoded, data); - EXPECT_EQ(bitcoinDecoded, data); - EXPECT_EQ(rippleDecoded, data); - EXPECT_EQ(flickrDecoded, data); -} - -TEST(base58, Tests) { - EXPECT_EQ(test_base2.encode(uinteger_t("000f", 16)), "1111"); - // EXPECT_EQ(test_base2.encode(uinteger_t("00ff", 16)), "011111111"); // ->> - EXPECT_EQ(test_base2.encode(uinteger_t("00ff", 16)), "11111111"); - EXPECT_EQ(test_base2.encode(uinteger_t("0fff", 16)), "111111111111"); - EXPECT_EQ(test_base2.encode(uinteger_t("ff00ff00", 16)), "11111111000000001111111100000000"); - // EXPECT_EQ(test_base16.encode(uinteger_t("0000000f", 16)), "000f"); // ->> - EXPECT_EQ(test_base16.encode(uinteger_t("0000000f", 16)), "f"); - // EXPECT_EQ(test_base16.encode(uinteger_t("000fff", 16)), "0fff"); // ->> - EXPECT_EQ(test_base16.encode(uinteger_t("000fff", 16)), "fff"); - EXPECT_EQ(test_base16.encode(uinteger_t("ffff", 16)), "ffff"); - // EXPECT_EQ(test_base58.encode(uinteger_t("", 16)), ""); // ->> - EXPECT_EQ(test_base58.encode(uinteger_t("", 16)), "1"); - EXPECT_EQ(test_base58.encode(uinteger_t("61", 16)), "2g"); - EXPECT_EQ(test_base58.encode(uinteger_t("626262", 16)), "a3gV"); - EXPECT_EQ(test_base58.encode(uinteger_t("636363", 16)), "aPEr"); - EXPECT_EQ(test_base58.encode(uinteger_t("73696d706c792061206c6f6e6720737472696e67", 16)), "2cFupjhnEsSn59qHXstmK2ffpLv2"); - // EXPECT_EQ(test_base58.encode(uinteger_t("00eb15231dfceb60925886b67d065299925915aeb172c06647", 16)), "1NS17iag9jJgTHD1VXjvLCEnZuQ3rJDE9L"); // ->> - EXPECT_EQ(test_base58.encode(uinteger_t("00eb15231dfceb60925886b67d065299925915aeb172c06647", 16)), "NS17iag9jJgTHD1VXjvLCEnZuQ3rJDE9L"); - EXPECT_EQ(test_base58.encode(uinteger_t("516b6fcd0f", 16)), "ABnLTmg"); - EXPECT_EQ(test_base58.encode(uinteger_t("bf4f89001e670274dd", 16)), "3SEo3LWLoPntC"); - EXPECT_EQ(test_base58.encode(uinteger_t("572e4794", 16)), "3EFU7m"); - EXPECT_EQ(test_base58.encode(uinteger_t("ecac89cad93923c02321", 16)), "EJDM8drfXA6uyA"); - EXPECT_EQ(test_base58.encode(uinteger_t("10c8511e", 16)), "Rt5zm"); - // EXPECT_EQ(test_base58.encode(uinteger_t("00000000000000000000", 16)), "1111111111"); // ->> - EXPECT_EQ(test_base58.encode(uinteger_t("00000000000000000000", 16)), "1"); - EXPECT_EQ(test_base58.encode(uinteger_t("801184cd2cdd640ca42cfc3a091c51d549b2f016d454b2774019c2b2d2e08529fd206ec97e", 16)), "5Hx15HFGyep2CfPxsJKe2fXJsCVn5DEiyoeGGF6JZjGbTRnqfiD"); - // EXPECT_EQ(test_base58.encode(uinteger_t("003c176e659bea0f29a3e9bf7880c112b1b31b4dc826268187", 16)), "16UjcYNBG9GTK4uq2f7yYEbuifqCzoLMGS"); // ->> - EXPECT_EQ(test_base58.encode(uinteger_t("003c176e659bea0f29a3e9bf7880c112b1b31b4dc826268187", 16)), "6UjcYNBG9GTK4uq2f7yYEbuifqCzoLMGS"); - EXPECT_EQ(test_base58.encode(uinteger_t("ffffffffffffffffffff", 16)), "FPBt6CHo3fovdL"); - EXPECT_EQ(test_base58.encode(uinteger_t("ffffffffffffffffffffffffff", 16)), "NKioeUVktgzXLJ1B3t"); - EXPECT_EQ(test_base58.encode(uinteger_t("ffffffffffffffffffffffffffffffff", 16)), "YcVfxkQb6JRzqk5kF2tNLv"); - EXPECT_EQ(test_base2.encode(uinteger_t("fb6f9ac3", 16)), "11111011011011111001101011000011"); - EXPECT_EQ(test_base2.encode(uinteger_t("179eea7a", 16)), "10111100111101110101001111010"); - EXPECT_EQ(test_base2.encode(uinteger_t("6db825db", 16)), "1101101101110000010010111011011"); - EXPECT_EQ(test_base2.encode(uinteger_t("93976aa7", 16)), "10010011100101110110101010100111"); - EXPECT_EQ(test_base58.encode(uinteger_t("ef41b9ce7e830af7", 16)), "h26E62FyLQN"); - EXPECT_EQ(test_base58.encode(uinteger_t("606cbc791036d2e9", 16)), "H8Sa62HVULG"); - EXPECT_EQ(test_base58.encode(uinteger_t("bdcb0ea69c2c8ec8", 16)), "YkESUPpnfoD"); - EXPECT_EQ(test_base58.encode(uinteger_t("1a2358ba67fb71d5", 16)), "5NaBN89ajtQ"); - EXPECT_EQ(test_base58.encode(uinteger_t("e6173f0f4d5fb5d7", 16)), "fVAoezT1ZkS"); - EXPECT_EQ(test_base58.encode(uinteger_t("91c81cbfdd58bbd2", 16)), "RPGNSU3bqTX"); - EXPECT_EQ(test_base58.encode(uinteger_t("329e0bf0e388dbfe", 16)), "9U41ZkwwysT"); - EXPECT_EQ(test_base58.encode(uinteger_t("30b10393210fa65b", 16)), "99NMW3WHjjY"); - EXPECT_EQ(test_base58.encode(uinteger_t("ab3bdd18e3623654", 16)), "VeBbqBb4rCT"); - EXPECT_EQ(test_base58.encode(uinteger_t("fe29d1751ec4af8a", 16)), "jWhmYLN9dUm"); - EXPECT_EQ(test_base58.encode(uinteger_t("c1273ab5488769807d", 16)), "3Tbh4kL3WKW6g"); - EXPECT_EQ(test_base58.encode(uinteger_t("6c7907904de934f852", 16)), "2P5jNYhfpTJxy"); - EXPECT_EQ(test_base58.encode(uinteger_t("05f0be055db47a0dc9", 16)), "5PN768Kr5oEp"); - EXPECT_EQ(test_base58.encode(uinteger_t("3511e6206829b35b12", 16)), "gBREojGaJ6DF"); - EXPECT_EQ(test_base58.encode(uinteger_t("d1c7c2ddc4a459d503", 16)), "3fsekq5Esq2KC"); - EXPECT_EQ(test_base58.encode(uinteger_t("1f88efd17ab073e9a1", 16)), "QHJbmW9ZY7jn"); - EXPECT_EQ(test_base58.encode(uinteger_t("0f45dadf4e64c5d5c2", 16)), "CGyVUMmCKLRf"); - EXPECT_EQ(test_base58.encode(uinteger_t("de1e5c5f718bb7fafa", 16)), "3pyy8U7w3KUa5"); - EXPECT_EQ(test_base58.encode(uinteger_t("123190b93e9a49a46c", 16)), "ES3DeFrG1zbd"); - EXPECT_EQ(test_base58.encode(uinteger_t("8bee94a543e7242e5a", 16)), "2nJnuWyLpGf6y"); - EXPECT_EQ(test_base58.encode(uinteger_t("9fd5f2285362f5cfd834", 16)), "9yqFhqeewcW3pF"); - EXPECT_EQ(test_base58.encode(uinteger_t("6987bac63ad23828bb31", 16)), "6vskE5Y1LhS3U4"); - EXPECT_EQ(test_base58.encode(uinteger_t("19d4a0f9d459cc2a08b0", 16)), "2TAsHPuaLhh5Aw"); - EXPECT_EQ(test_base58.encode(uinteger_t("a1e47ffdbea5a807ab26", 16)), "A6XzPgSUJDf1W5"); - EXPECT_EQ(test_base58.encode(uinteger_t("35c231e5b3a86a9b83db", 16)), "42B8reRwPAAoAa"); - EXPECT_EQ(test_base58.encode(uinteger_t("b2351012a48b8347c351", 16)), "B1hPyomGx4Vhqa"); - EXPECT_EQ(test_base58.encode(uinteger_t("71d402694dd9517ea653", 16)), "7Pv2SyAQx2Upu8"); - EXPECT_EQ(test_base58.encode(uinteger_t("55227c0ec7955c2bd6e8", 16)), "5nR64BkskyjHMq"); - EXPECT_EQ(test_base58.encode(uinteger_t("17b3d8ee7907c1be34df", 16)), "2LEg7TxosoxTGS"); - EXPECT_EQ(test_base58.encode(uinteger_t("7e7bba7b68bb8e95827f", 16)), "879o2ATGnmYyAW"); - EXPECT_EQ(test_base58.encode(uinteger_t("db9c13f5ba7654b01407fb", 16)), "wTYfxjDVbiks874"); - EXPECT_EQ(test_base58.encode(uinteger_t("6186449d20f5fd1e6c4393", 16)), "RBeiWhzZNL6VtMG"); - EXPECT_EQ(test_base58.encode(uinteger_t("5248751cebf4ad1c1a83c3", 16)), "MQSVNnc8ehFCqtW"); - EXPECT_EQ(test_base58.encode(uinteger_t("32090ef18cd479fc376a74", 16)), "DQdu351ExDaeYeX"); - EXPECT_EQ(test_base58.encode(uinteger_t("7cfa5d6ed1e467d986c426", 16)), "XzW67T5qfEnFcaZ"); - EXPECT_EQ(test_base58.encode(uinteger_t("9d8707723c7ede51103b6d", 16)), "g4eTCg6QJnB1UU4"); - EXPECT_EQ(test_base58.encode(uinteger_t("6f4d1e392d6a9b4ed8b223", 16)), "Ubo7kZY5aDpAJp2"); - EXPECT_EQ(test_base58.encode(uinteger_t("38057d98797cd39f80a0c9", 16)), "EtjQ2feamJvuqse"); - EXPECT_EQ(test_base58.encode(uinteger_t("de7e59903177e20880e915", 16)), "xB2N7yRBnDYEoT2"); - EXPECT_EQ(test_base58.encode(uinteger_t("b2ea24a28bc4a60b5c4b8d", 16)), "mNFMpJ2P3TGYqhv"); - EXPECT_EQ(test_base58.encode(uinteger_t("cf84938958589b6ffba6114d", 16)), "4v8ZbsGh2ePz5sipt"); - EXPECT_EQ(test_base58.encode(uinteger_t("dee13be7b8d8a08c94a3c02a", 16)), "5CwmE9jQqwtHkTF45"); - EXPECT_EQ(test_base58.encode(uinteger_t("14cb9c6b3f8cd2e02710f569", 16)), "Pm85JHVAAdeUdxtp"); - EXPECT_EQ(test_base58.encode(uinteger_t("ca3f2d558266bdcc44c79cb5", 16)), "4pMwomBAQHuUnoLUC"); - EXPECT_EQ(test_base58.encode(uinteger_t("c031215be44cbad745f38982", 16)), "4dMeTrcxiVw9RWvj3"); - EXPECT_EQ(test_base58.encode(uinteger_t("1435ab1dbc403111946270a5", 16)), "P7wX3sCWNrbqhBEC"); - EXPECT_EQ(test_base58.encode(uinteger_t("d8c6e4d775e7a66a0d0f9f41", 16)), "56GLoRDGWGuGJJwPN"); - EXPECT_EQ(test_base58.encode(uinteger_t("dcee35e74f0fd74176fce2f4", 16)), "5Ap1zyuYiJJFwWcMR"); - EXPECT_EQ(test_base58.encode(uinteger_t("bfcc0ca4b4855d1cf8993fc0", 16)), "4cvafQW4PEhARKv9D"); - EXPECT_EQ(test_base58.encode(uinteger_t("e02a3ac25ece7b54584b670a", 16)), "5EMM28xkpxZ1kkVUM"); - EXPECT_EQ(test_base58.encode(uinteger_t("fe4d938fc3719f064cabb4bfff", 16)), "NBXKkbHwrAsiWTLAk6"); - EXPECT_EQ(test_base58.encode(uinteger_t("9289cb4f6b15c57e6086b87ea5", 16)), "DCvDpjEXEbHjZqskKv"); - EXPECT_EQ(test_base58.encode(uinteger_t("fc266f35626b3612bfe978537b", 16)), "N186PVoBWrNre35BGE"); - EXPECT_EQ(test_base58.encode(uinteger_t("33ff08c06d92502bf258c07166", 16)), "5LC4SoW6jmTtbkbePw"); - EXPECT_EQ(test_base58.encode(uinteger_t("6a81cac1f3666bc59dc67b1c3c", 16)), "9sXgUySUzwiqDU5WHy"); - EXPECT_EQ(test_base58.encode(uinteger_t("9dfb8e7e744c544c0f323ea729", 16)), "EACsmGmkgcwsrPFzLg"); - EXPECT_EQ(test_base58.encode(uinteger_t("1e7a1e284f70838b38442b682b", 16)), "3YEVk9bE7rw5qExMkv"); - EXPECT_EQ(test_base58.encode(uinteger_t("2a862ad57901a8235f5dc74eaf", 16)), "4YS259nuTLfeXa5Wuc"); - EXPECT_EQ(test_base58.encode(uinteger_t("74c82096baef21f9d3089e5462", 16)), "AjAcKEhUfrqm8smvM7"); - EXPECT_EQ(test_base58.encode(uinteger_t("7a3edbc23d7b600263920261cc", 16)), "BBZXyRgey5S5DDZkcK"); - EXPECT_EQ(test_base58.encode(uinteger_t("20435664c357d25a9c8df751cf4f", 16)), "CrwNL6Fbv4pbRx1zd9g"); - EXPECT_EQ(test_base58.encode(uinteger_t("51a7aa87cf5cb1c12d045ec3422d", 16)), "X27NHGgKXmGzzQvDtpC"); - EXPECT_EQ(test_base58.encode(uinteger_t("344d2e116aa26f1062a2cb6ebbef", 16)), "LEDLDvL1Hg4qt1efVXt"); - EXPECT_EQ(test_base58.encode(uinteger_t("6941add7be4c0b5c7163e4928f8e", 16)), "fhMyN6gwoxE3uYraVzV"); - EXPECT_EQ(test_base58.encode(uinteger_t("10938fcbb7c4ab991649734a14bf", 16)), "76TPrSDxzGQfSzMu974"); - EXPECT_EQ(test_base58.encode(uinteger_t("eafe04d944ba504e9af9117b07de", 16)), "2VPgov563ryfe4L2Bj6M"); - EXPECT_EQ(test_base58.encode(uinteger_t("58d0aeed4d35da20b6f052127edf", 16)), "ZenZhXF9YwP8nQvNtNz"); - EXPECT_EQ(test_base58.encode(uinteger_t("d734984e2f5aecf25f7a3e353f8a", 16)), "2N7n3jFsTdyN49Faoq6h"); - EXPECT_EQ(test_base58.encode(uinteger_t("57d873fdb405b7daf4bafa62068a", 16)), "ZJ7NwoP4wHvwyZg3Wjs"); - EXPECT_EQ(test_base58.encode(uinteger_t("bda4ec7b40d0d65ca95dec4c4d3b", 16)), "2CijxjsNyvqTwPCfDcpA"); - EXPECT_EQ(test_base58.encode(uinteger_t("826c4abdceb1b91f0d4ad665f86d2e", 16)), "4edfvuDQu9KzVxLuXHfMo"); - EXPECT_EQ(test_base58.encode(uinteger_t("e7ecb35d07e65b960cb10574a4f51a", 16)), "7VLRYdB4cToipp2J2p3v9"); - EXPECT_EQ(test_base58.encode(uinteger_t("4f2d72ead87b31d6869fba39eac6dc", 16)), "3DUjqJRcfdWhpsrLrGcQs"); - EXPECT_EQ(test_base58.encode(uinteger_t("8b4f5788d60030950d5dfbf94c585d", 16)), "4u44JSRH5jP5X39YhPsmE"); - EXPECT_EQ(test_base58.encode(uinteger_t("ee4c0a0025d1a74ace9fe349355cc5", 16)), "7fgACjABRQUGUEpN6VBBA"); - EXPECT_EQ(test_base58.encode(uinteger_t("58ac05b9a0b4b66083ff1d489b8d84", 16)), "3UtJPyTwGXapcxHx8Rom5"); - EXPECT_EQ(test_base58.encode(uinteger_t("1aa35c05e1132e8e049aafaef035d8", 16)), "kE2eSU7gM2619pT82iGP"); - EXPECT_EQ(test_base58.encode(uinteger_t("771b0c28608484562a292e5d5d2b30", 16)), "4LGYeWhyfrjUByibUqdVR"); - EXPECT_EQ(test_base58.encode(uinteger_t("78ff9a0e56f9e88dc1cd654b40d019", 16)), "4PLggs66qAdbmZgkaPihe"); - EXPECT_EQ(test_base58.encode(uinteger_t("6d691bdd736346aa5a0a95b373b2ab", 16)), "44Y6qTgSvRMkdqpQ5ufkN"); -} diff --git a/contrib/base-x/uinteger_t.hh b/contrib/base-x/uinteger_t.hh deleted file mode 100644 index 901460f75c4..00000000000 --- a/contrib/base-x/uinteger_t.hh +++ /dev/null @@ -1,2546 +0,0 @@ -/* -uinteger_t.hh -An arbitrary precision unsigned integer type for C++ - -Copyright (c) 2017 German Mendez Bravo (Kronuz) @ german dot mb at gmail.com -Copyright (c) 2013 - 2017 Jason Lee @ calccrypto at gmail.com - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -With much help from Auston Sterling - -Thanks to Stefan Deigmüller for finding -a bug in operator*. - -Thanks to François Dessenne for convincing me -to do a general rewrite of this class. - -Germán Mández Bravo (Kronuz) converted Jason Lee's uint128_t -to header-only and extended to arbitrary bit length. -*/ - -#ifndef __uint_t__ -#define __uint_t__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define ASSERT assert - -// Compatibility inlines -#ifndef __has_builtin // Optional of course -#define __has_builtin(x) 0 // Compatibility with non-clang compilers -#endif - -#if defined _MSC_VER -# define HAVE___ADDCARRY_U64 -# define HAVE___SUBBORROW_U64 -# define HAVE___ADDCARRY_U32 -# define HAVE___SUBBORROW_U32 -# define HAVE___ADDCARRY_U16 -# define HAVE___SUBBORROW_U16 -# define HAVE___UMUL128 -# define HAVE___UMUL64 -# define HAVE___UMUL32 -# include -#endif - -#if (defined(__clang__) && __has_builtin(__builtin_clzll)) || (defined(__GNUC__ ) && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 3))) -# define HAVE____BUILTIN_CLZLL -#endif -#if (defined(__clang__) && __has_builtin(__builtin_clzl)) || (defined(__GNUC__ ) && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 3))) -# define HAVE____BUILTIN_CLZL -#endif -#if (defined(__clang__) && __has_builtin(__builtin_clz)) || (defined(__GNUC__ ) && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 3))) -# define HAVE____BUILTIN_CLZ -#endif -#if (defined(__clang__) && __has_builtin(__builtin_addcll)) -# define HAVE____BUILTIN_ADDCLL -#endif -#if (defined(__clang__) && __has_builtin(__builtin_addcl)) -# define HAVE____BUILTIN_ADDCL -#endif -#if (defined(__clang__) && __has_builtin(__builtin_addc)) -# define HAVE____BUILTIN_ADDC -#endif -#if (defined(__clang__) && __has_builtin(__builtin_subcll)) -# define HAVE____BUILTIN_SUBCLL -#endif -#if (defined(__clang__) && __has_builtin(__builtin_subcl)) -# define HAVE____BUILTIN_SUBCL -#endif -#if (defined(__clang__) && __has_builtin(__builtin_subc)) -# define HAVE____BUILTIN_SUBC -#endif - -#if defined __SIZEOF_INT128__ -#define HAVE____INT128_T -#endif - - -#ifndef DIGIT_T -#define DIGIT_T std::uint64_t -#endif - -#ifndef HALF_DIGIT_T -#define HALF_DIGIT_T std::uint32_t -#endif - -class uinteger_t; - -namespace std { // This is probably not a good idea - // Give uinteger_t type traits - template <> struct is_arithmetic : std::true_type {}; - template <> struct is_integral : std::true_type {}; - template <> struct is_unsigned : std::true_type {}; -} - -class uinteger_t { -public: - using digit = DIGIT_T; - using half_digit = HALF_DIGIT_T; - - static constexpr std::size_t digit_octets = sizeof(digit); // number of octets per digit - static constexpr std::size_t digit_bits = digit_octets * 8; // number of bits per digit - static constexpr std::size_t half_digit_octets = sizeof(half_digit); // number of octets per half_digit - static constexpr std::size_t half_digit_bits = half_digit_octets * 8; // number of bits per half_digit - - using container = std::vector; - - template - struct is_result { - static const bool value = false; - }; - - template - struct is_result> { - static const bool value = true; - }; - - template - struct is_result> { - static const bool value = true; - }; - -private: - static_assert(digit_octets == half_digit_octets * 2, "half_digit must be exactly half the size of digit"); - - static constexpr std::size_t karatsuba_cutoff = 1024 / digit_bits; - static constexpr double growth_factor = 1.5; - - std::size_t _begin; - std::size_t _end; - container _value_instance; - container& _value; - bool _carry; - -public: - // Window to vector (uses _begin and _end) - - void reserve(std::size_t sz) { - _value.reserve(sz + _begin); - } - - std::size_t grow(std::size_t n) { - // expands the vector using a growth factor - // and returns the new capacity. - auto cc = _value.capacity(); - if (n >= cc) { - cc = n * growth_factor; - _value.reserve(cc); - } - return cc; - } - - void resize(std::size_t sz) { - grow(sz + _begin); - _value.resize(sz + _begin); - } - - void resize(std::size_t sz, const digit& c) { - grow(sz + _begin); - _value.resize(sz + _begin, c); - } - - void clear() { - _value.clear(); - _begin = 0; - _end = 0; - _carry = false; - } - - digit* data() noexcept { - return _value.data() + _begin; - } - - const digit* data() const noexcept { - return _value.data() + _begin; - } - - std::size_t size() const noexcept { - return _end ? _end - _begin : _value.size() - _begin; - } - - void prepend(std::size_t sz, const digit& c) { - // Efficiently prepend by growing backwards by growth factor - auto min = std::min(_begin, sz); - if (min) { - // If there is some space before `_begin`, we try using it first: - _begin -= min; - std::fill_n(_value.begin() + _begin, min, c); - sz -= min; - } - if (sz) { - ASSERT(_begin == 0); // _begin should be 0 in here - // If there's still more room needed, we grow the vector: - // Ex.: grow using prepend(3, y) - // sz = 3 - // _begin = 0 (B) - // _end = 1 (E) - // initially (capacity == 12): - // |xxxxxxxxxx- | - // B E - // after reclaiming space after `_end` (same capacity == 12): - // |xxxxxxxxxx | - // B - // _end = 0 - // csz = 10 - // grow returns the new capacity (22) - // isz = 12 (22 - 10) - // _begin = 9 (12 - 3) - // after (capacity == (12 + 3) * 1.5 == 22): - // |---------yyyxxxxxxxxxx| - // B - if (_end) { - // reclaim space after `_end` - _value.resize(_end); - _end = 0; - } - auto csz = _value.size(); - auto isz = grow(csz + sz) - csz; - _value.insert(_value.begin(), isz, c); - _begin = isz - sz; - } - } - - void prepend(const digit& c) { - prepend(1, c); - } - - void prepend(const uinteger_t& num) { - prepend(num.size(), 0); - std::copy(num.begin(), num.end(), begin()); - } - - void append(std::size_t sz, const digit& c) { - // Efficiently append by growing by growth factor - if (_end) { - // reclaim space after `_end` - _value.resize(_end); - _end = 0; - } - auto nsz = _value.size() + sz; - grow(nsz); - _value.resize(nsz, c); - } - - void append(const digit& c) { - append(1, c); - } - - void append(const uinteger_t& num) { - auto sz = num.size(); - append(sz, 0); - std::copy(num.begin(), num.end(), end() - sz); - } - - container::iterator begin() noexcept { - return _value.begin() + _begin; - } - - container::const_iterator begin() const noexcept { - return _value.cbegin() + _begin; - } - - container::iterator end() noexcept { - return _end ? _value.begin() + _end : _value.end(); - } - - container::const_iterator end() const noexcept { - return _end ? _value.cbegin() + _end : _value.cend(); - } - - container::reverse_iterator rbegin() noexcept { - return _end ? container::reverse_iterator(_value.begin() + _end) : _value.rbegin(); - } - - container::const_reverse_iterator rbegin() const noexcept { - return _end ? container::const_reverse_iterator(_value.cbegin() + _end) : _value.crbegin(); - } - - container::reverse_iterator rend() noexcept { - return container::reverse_iterator(_value.begin() + _begin); - } - - container::const_reverse_iterator rend() const noexcept { - return container::const_reverse_iterator(_value.cbegin() + _begin); - } - - container::reference front() { - return *begin(); - } - - container::const_reference front() const { - return *begin(); - } - - container::reference back() { - return *rbegin(); - } - - container::const_reference back() const { - return *rbegin(); - } - -private: - // Optimized primitives for operations - - static digit _bits(digit x) { - #if defined HAVE____BUILTIN_CLZLL - if (digit_octets == sizeof(unsigned long long)) { - return x ? digit_bits - __builtin_clzll(x) : 1; - } - #endif - #if defined HAVE____BUILTIN_CLZL - if (digit_octets == sizeof(unsigned long)) { - return x ? digit_bits - __builtin_clzl(x) : 1; - } - #endif - #if defined HAVE____BUILTIN_CLZ - if (digit_octets == sizeof(unsigned)) { - return x ? digit_bits - __builtin_clz(x) : 1; - } - #endif - { - digit c = x ? 0 : 1; - while (x) { - x >>= 1; - ++c; - } - return c; - } - } - - static digit _mult(digit x, digit y, digit* lo) { - #if defined HAVE___UMUL128 - if (digit_bits == 64) { - digit h; - digit l = _umul128(x, y, &h); // _umul128(x, y, *hi) -> lo - return h; - } - #endif - #if defined HAVE___UMUL64 - if (digit_bits == 32) { - digit h; - digit l = _umul64(x, y, &h); // _umul64(x, y, *hi) -> lo - return h; - } - #endif - #if defined HAVE___UMUL32 - if (digit_bits == 16) { - digit h; - digit l = _umul32(x, y, &h); // _umul32(x, y, *hi) -> lo - return h; - } - #endif - #if defined HAVE____INT128_T - if (digit_bits == 64) { - auto r = static_cast<__uint128_t>(x) * static_cast<__uint128_t>(y); - *lo = r; - return r >> digit_bits; - } - #endif - if (digit_bits == 64) { - digit x0 = x & 0xffffffffUL; - digit x1 = x >> 32; - digit y0 = y & 0xffffffffUL; - digit y1 = y >> 32; - - digit u = (x0 * y0); - digit v = (x1 * y0) + (u >> 32); - digit w = (x0 * y1) + (v & 0xffffffffUL); - - *lo = (w << 32) + (u & 0xffffffffUL); // low - return (x1 * y1) + (v >> 32) + (w >> 32); // high - } if (digit_bits == 32) { - auto r = static_cast(x) * static_cast(y); - *lo = r; - return r >> 32; - } if (digit_bits == 16) { - auto r = static_cast(x) * static_cast(y); - *lo = r; - return r >> 16; - } if (digit_bits == 8) { - auto r = static_cast(x) * static_cast(y); - *lo = r; - return r >> 8; - } - } - - static digit _multadd(digit x, digit y, digit a, digit c, digit* lo) { - #if defined HAVE___UMUL128 && defined HAVE___ADDCARRY_U64 - if (digit_bits == 64) { - digit h; - digit l = _umul128(x, y, &h); // _umul128(x, y, *hi) -> lo - return h + _addcarry_u64(c, l, a, lo); // _addcarry_u64(carryin, x, y, *sum) -> carryout - } - #endif - #if defined HAVE___UMUL64 && defined HAVE___ADDCARRY_U32 - if (digit_bits == 32) { - digit h; - digit l = _umul64(x, y, &h); // _umul64(x, y, *hi) -> lo - return h + _addcarry_u32(c, l, a, lo); // _addcarry_u32(carryin, x, y, *sum) -> carryout - } - #endif - #if defined HAVE___UMUL32 && defined HAVE___ADDCARRY_U16 - if (digit_bits == 16) { - digit h; - digit l = _umul32(x, y, &h); // _umul32(x, y, *hi) -> lo - return h + _addcarry_u16(c, l, a, lo); // _addcarry_u16(carryin, x, y, *sum) -> carryout - } - #endif - #if defined HAVE____INT128_T - if (digit_bits == 64) { - auto r = static_cast<__uint128_t>(x) * static_cast<__uint128_t>(y) + static_cast<__uint128_t>(a) + static_cast<__uint128_t>(c); - *lo = r; - return r >> digit_bits; - } - #endif - if (digit_bits == 64) { - digit x0 = x & 0xffffffffUL; - digit x1 = x >> 32; - digit y0 = y & 0xffffffffUL; - digit y1 = y >> 32; - - digit u = (x0 * y0) + (a & 0xffffffffUL) + (c & 0xffffffffUL); - digit v = (x1 * y0) + (u >> 32) + (a >> 32) + (c >> 32); - digit w = (x0 * y1) + (v & 0xffffffffUL); - - *lo = (w << 32) + (u & 0xffffffffUL); // low - return (x1 * y1) + (v >> 32) + (w >> 32); // high - } - if (digit_bits == 32) { - auto r = static_cast(x) * static_cast(y) + static_cast(a) + static_cast(c); - *lo = r; - return r >> 32; - } - if (digit_bits == 16) { - auto r = static_cast(x) * static_cast(y) + static_cast(a) + static_cast(c); - *lo = r; - return r >> 16; - } - if (digit_bits == 8) { - auto r = static_cast(x) * static_cast(y) + static_cast(a) + static_cast(c); - *lo = r; - return r >> 8; - } - } - - static digit _divmod(digit x_hi, digit x_lo, digit y, digit* result) { - #if defined HAVE____INT128_T - if (digit_bits == 64) { - auto x = static_cast<__uint128_t>(x_hi) << digit_bits | static_cast<__uint128_t>(x_lo); - digit q = x / y; - digit r = x % y; - - *result = q; - return r; - } - #endif - if (digit_bits == 64) { - // quotient - digit q = x_lo << 1; - - // remainder - digit r = x_hi; - - digit carry = x_lo >> 63; - int i; - - for (i = 0; i < 64; i++) { - auto tmp = r >> 63; - r <<= 1; - r |= carry; - carry = tmp; - - if (carry == 0) { - if (r >= y) { - carry = 1; - } else { - tmp = q >> 63; - q <<= 1; - q |= carry; - carry = tmp; - continue; - } - } - - r -= y; - r -= (1 - carry); - carry = 1; - tmp = q >> 63; - q <<= 1; - q |= carry; - carry = tmp; - } - - *result = q; - return r; - } - if (digit_bits == 32) { - auto x = static_cast(x_hi) << 32 | static_cast(x_lo); - digit q = x / y; - digit r = x % y; - - *result = q; - return r; - } - if (digit_bits == 16) { - auto x = static_cast(x_hi) << 16 | static_cast(x_lo); - digit q = x / y; - digit r = x % y; - - *result = q; - return r; - } - if (digit_bits == 8) { - auto x = static_cast(x_hi) << 8 | static_cast(x_lo); - digit q = x / y; - digit r = x % y; - - *result = q; - return r; - } - } - - static digit _addcarry(digit x, digit y, digit c, digit* result) { - #if defined HAVE___ADDCARRY_U64 - if (digit_bits == 64) { - return _addcarry_u64(c, x, y, result); // _addcarry_u64(carryin, x, y, *sum) -> carryout - } - #endif - #if defined HAVE___ADDCARRY_U32 - if (digit_bits == 32) { - return _addcarry_u32(c, x, y, result); // _addcarry_u32(carryin, x, y, *sum) -> carryout - } - #endif - #if defined HAVE___ADDCARRY_U16 - if (digit_bits == 16) { - return _addcarry_u16(c, x, y, result); // _addcarry_u16(carryin, x, y, *sum) -> carryout - } - #endif - #if defined HAVE____BUILTIN_ADDCLL - if (digit_octets == sizeof(unsigned long long)) { - unsigned long long carryout; - *result = __builtin_addcll(x, y, c, &carryout); // __builtin_addcll(x, y, carryin, *carryout) -> sum - return carryout; - } - #endif - #if defined HAVE____BUILTIN_ADDCL - if (digit_octets == sizeof(unsigned long)) { - unsigned long carryout; - *result = __builtin_addcl(x, y, c, &carryout); // __builtin_addcl(x, y, carryin, *carryout) -> sum - return carryout; - } - #endif - #if defined HAVE____BUILTIN_ADDC - if (digit_octets == sizeof(unsigned)) { - unsigned carryout; - *result = __builtin_addc(x, y, c, &carryout); // __builtin_addc(x, y, carryin, *carryout) -> sum - return carryout; - } - #endif - #if defined HAVE____INT128_T - if (digit_bits == 64) { - auto r = static_cast<__uint128_t>(x) + static_cast<__uint128_t>(y) + static_cast<__uint128_t>(c); - *result = r; - return static_cast(r >> digit_bits); - } - #endif - if (digit_bits == 64) { - digit x0 = x & 0xffffffffUL; - digit x1 = x >> 32; - digit y0 = y & 0xffffffffUL; - digit y1 = y >> 32; - - auto u = x0 + y0 + c; - auto v = x1 + y1 + static_cast(u >> 32); - *result = (v << 32) + (u & 0xffffffffUL); - return static_cast(v >> 32); - } - if (digit_bits == 32) { - auto r = static_cast(x) + static_cast(y) + static_cast(c); - *result = r; - return static_cast(r >> 32); - } - if (digit_bits == 16) { - auto r = static_cast(x) + static_cast(y) + static_cast(c); - *result = r; - return static_cast(r >> 16); - } - if (digit_bits == 8) { - auto r = static_cast(x) + static_cast(y) + static_cast(c); - *result = r; - return static_cast(r >> 8); - } - } - - static digit _subborrow(digit x, digit y, digit c, digit* result) { - #if defined HAVE___SUBBORROW_U64 - if (digit_bits == 64) { - return _subborrow_u64(c, x, y, result); // _subborrow_u64(carryin, x, y, *sum) -> carryout - } - #endif - #if defined HAVE___SUBBORROW_U32 - if (digit_bits == 64) { - return _subborrow_u32(c, x, y, result); // _subborrow_u32(carryin, x, y, *sum) -> carryout - } - #endif - #if defined HAVE___SUBBORROW_U16 - if (digit_bits == 64) { - return _subborrow_u16(c, x, y, result); // _subborrow_u16(carryin, x, y, *sum) -> carryout - } - #endif - #if defined HAVE____BUILTIN_SUBCLL - if (digit_octets == sizeof(unsigned long long)) { - unsigned long long carryout; - *result = __builtin_subcll(x, y, c, &carryout); // __builtin_subcll(x, y, carryin, *carryout) -> sum - return carryout; - } - #endif - #if defined HAVE____BUILTIN_SUBCL - if (digit_octets == sizeof(unsigned long)) { - unsigned long carryout; - *result = __builtin_subcl(x, y, c, &carryout); // __builtin_subcl(x, y, carryin, *carryout) -> sum - return carryout; - } - #endif - #if defined HAVE____BUILTIN_SUBC - if (digit_octets == sizeof(unsigned)) { - unsigned carryout; - *result = __builtin_subc(x, y, c, &carryout); // __builtin_subc(x, y, carryin, *carryout) -> sum - return carryout; - } - #endif - #if defined HAVE____INT128_T - if (digit_bits == 64) { - auto r = static_cast<__uint128_t>(x) - static_cast<__uint128_t>(y) - static_cast<__uint128_t>(c); - *result = r; - return static_cast(r >> 64); - } - #endif - if (digit_bits == 64) { - digit x0 = x & 0xffffffffUL; - digit x1 = x >> 32; - digit y0 = y & 0xffffffffUL; - digit y1 = y >> 32; - - auto u = x0 - y0 - c; - auto v = x1 - y1 - static_cast(u >> 32); - *result = (v << 32) + (u & 0xffffffffUL); - return static_cast(v >> 32); - } - if (digit_bits == 32) { - auto r = static_cast(x) - static_cast(y) - static_cast(c); - *result = r; - return static_cast(r >> 32); - } - if (digit_bits == 16) { - auto r = static_cast(x) - static_cast(y) - static_cast(c); - *result = r; - return static_cast(r >> 16); - } - if (digit_bits == 8) { - auto r = static_cast(x) - static_cast(y) - static_cast(c); - *result = r; - return static_cast(r >> 8); - } - } - - // Helper functions - - void trim(digit mask = 0) { - auto rit = rbegin(); - auto rit_e = rend(); - - // Masks the last value of internal vector - mask &= (digit_bits - 1); - if (mask && rit != rit_e) { - *rit &= (static_cast(1) << mask) - 1; - } - - // Removes all unused zeros from the internal vector - auto rit_f = std::find_if(rit, rit_e, [](const digit& c) { return c; }); - resize(rit_e - rit_f); // shrink - } - - static constexpr char chr(int ord) { - constexpr const char _[256] = { - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', - 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', - 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - }; - return _[ord]; - } - - static constexpr int ord(int chr) { - constexpr const int _[256] = { - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, - - -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, -1, - -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, -1, - - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - }; - return _[chr]; - } - -public: - static constexpr unsigned base_bits(int base) { - constexpr const unsigned _[256] = { - 0, 1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, - }; - return _[base - 1]; - } - - static constexpr unsigned base_size(int base) { - constexpr const unsigned _[256] = { - 0, 64, 41, 32, 28, 25, 23, 22, 21, 20, 19, 18, 18, 17, 17, 16, - 16, 16, 16, 15, 15, 15, 15, 14, 14, 14, 14, 14, 14, 14, 13, 13, - 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 12, 12, 12, 12, 12, - 12, 12, 12, 12, 12, 12, 12, 12, 11, 11, 11, 11, 11, 11, 11, 11, - - 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, - 11, 11, 11, 11, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, - 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, - 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, - - 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 8, - }; - return _[base - 1]; - } - - static const uinteger_t uint_0() { - static uinteger_t uint_0(0); - return uint_0; - } - - static const uinteger_t uint_1() { - static uinteger_t uint_1(1); - return uint_1; - } - -private: - // Public Implementation -#ifdef UINT_T_PUBLIC_IMPLEMENTATION -public: -#endif - static uinteger_t& bitwise_and(uinteger_t& lhs, const uinteger_t& rhs) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - if (lhs_sz > rhs_sz) { - lhs.resize(rhs_sz); // shrink - } - - auto lhs_it = lhs.begin(); - auto lhs_it_e = lhs.end(); - - auto rhs_it = rhs.begin(); - - for (; lhs_it != lhs_it_e; ++lhs_it, ++rhs_it) { - *lhs_it &= *rhs_it; - } - - // Finish up - lhs.trim(); - return lhs; - } - - static uinteger_t& bitwise_and(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - auto result_sz = std::max(lhs_sz, rhs_sz); - result.resize(result_sz); - - // not using `end()` because resize of `result.resize()` could have - // resized `lhs` or `rhs` if `result` is also either `rhs` or `lhs`. - auto lhs_it = lhs.begin(); - auto lhs_it_e = lhs_it + lhs_sz; - - auto rhs_it = rhs.begin(); - auto rhs_it_e = rhs_it + rhs_sz; - - auto it = result.begin(); - - if (lhs_sz < rhs_sz) { - for (; lhs_it != lhs_it_e; ++lhs_it, ++rhs_it, ++it) { - *it = *lhs_it & *rhs_it; - } - for (; rhs_it != rhs_it_e; ++rhs_it, ++it) { - *it = 0; - } - } else { - for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it, ++it) { - *it = *lhs_it & *rhs_it; - } - for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { - *it = 0; - } - } - - // Finish up - result.trim(); - return result; - } - - static uinteger_t bitwise_and(const uinteger_t& lhs, const uinteger_t& rhs) { - uinteger_t result; - bitwise_and(result, lhs, rhs); - return result; - } - - static uinteger_t& bitwise_or(uinteger_t& lhs, const uinteger_t& rhs) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - if (lhs_sz < rhs_sz) { - lhs.resize(rhs_sz, 0); // grow - } - - auto lhs_it = lhs.begin(); - - auto rhs_it = rhs.begin(); - auto rhs_it_e = rhs.end(); - - for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it) { - *lhs_it |= *rhs_it; - } - - // Finish up - lhs.trim(); - return lhs; - } - - static uinteger_t& bitwise_or(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - auto result_sz = std::max(lhs_sz, rhs_sz); - result.resize(result_sz); - - // not using `end()` because resize of `result.resize()` could have - // resized `lhs` or `rhs` if `result` is also either `rhs` or `lhs`. - auto lhs_it = lhs.begin(); - auto lhs_it_e = lhs_it + lhs_sz; - - auto rhs_it = rhs.begin(); - auto rhs_it_e = rhs_it + rhs_sz; - - auto it = result.begin(); - - if (lhs_sz < rhs_sz) { - for (; lhs_it != lhs_it_e; ++lhs_it, ++rhs_it, ++it) { - *it = *lhs_it | *rhs_it; - } - for (; rhs_it != rhs_it_e; ++rhs_it, ++it) { - *it = *rhs_it; - } - } else { - for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it, ++it) { - *it = *lhs_it | *rhs_it; - } - for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { - *it = *lhs_it; - } - } - - // Finish up - result.trim(); - return result; - } - static uinteger_t bitwise_or(const uinteger_t& lhs, const uinteger_t& rhs) { - uinteger_t result; - bitwise_or(result, lhs, rhs); - return result; - } - - static uinteger_t& bitwise_xor(uinteger_t& lhs, const uinteger_t& rhs) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - if (lhs_sz < rhs_sz) { - lhs.resize(rhs_sz, 0); // grow - } - - auto lhs_it = lhs.begin(); - - auto rhs_it = rhs.begin(); - auto rhs_it_e = rhs.end(); - - for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it) { - *lhs_it ^= *rhs_it; - } - - // Finish up - lhs.trim(); - return lhs; - } - - static uinteger_t& bitwise_xor(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - auto result_sz = std::max(lhs_sz, rhs_sz); - result.resize(result_sz); - - // not using `end()` because resize of `result.resize()` could have - // resized `lhs` or `rhs` if `result` is also either `rhs` or `lhs`. - auto lhs_it = lhs.begin(); - auto lhs_it_e = lhs_it + lhs_sz; - - auto rhs_it = rhs.begin(); - auto rhs_it_e = rhs_it + rhs_sz; - - auto it = result.begin(); - - if (lhs_sz < rhs_sz) { - for (; lhs_it != lhs_it_e; ++lhs_it, ++rhs_it, ++it) { - *it = *lhs_it ^ *rhs_it; - } - for (; rhs_it != rhs_it_e; ++rhs_it, ++it) { - *it = *rhs_it; - } - } else { - for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it, ++it) { - *it = *lhs_it ^ *rhs_it; - } - for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { - *it = *lhs_it; - } - } - - // Finish up - result.trim(); - return result; - } - - static uinteger_t bitwise_xor(const uinteger_t& lhs, const uinteger_t& rhs) { - uinteger_t result; - bitwise_xor(result, lhs, rhs); - return result; - } - - static uinteger_t& bitwise_inv(uinteger_t& lhs) { - auto lhs_sz = lhs.size(); - - auto b = lhs.bits(); - - if (!lhs_sz) { - lhs.append(0); - } - - // not using `end()` because resize of `result.resize()` could have - // resized `lhs` if `result` is also `lhs`. - auto lhs_it = lhs.begin(); - auto lhs_it_e = lhs_it + lhs_sz; - - for (; lhs_it != lhs_it_e; ++lhs_it) { - *lhs_it = ~*lhs_it; - } - - // Finish up - lhs.trim(b ? b : 1); - return lhs; - } - - static uinteger_t& bitwise_inv(uinteger_t& result, const uinteger_t& lhs) { - auto lhs_sz = lhs.size(); - - auto b = lhs.bits(); - - auto result_sz = lhs_sz ? lhs_sz : 1; - result.resize(result_sz); - - // not using `end()` because resize of `result.resize()` could have - // resized `lhs` if `result` is also `lhs`. - auto lhs_it = lhs.begin(); - auto lhs_it_e = lhs_it + lhs_sz; - - auto it = result.begin(); - auto it_e = it + result_sz; - - for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { - *it = ~*lhs_it; - } - for (; it != it_e; ++it) { - *it = ~static_cast(0); - } - - // Finish up - result.trim(b ? b : 1); - return result; - } - - static uinteger_t bitwise_inv(const uinteger_t& lhs) { - uinteger_t result; - bitwise_inv(result, lhs); - return result; - } - - static uinteger_t& bitwise_lshift(uinteger_t& lhs, const uinteger_t& rhs) { - if (!rhs) { - return lhs; - } - - uinteger_t shifts_q; - uinteger_t shifts_r; - auto _digit_bits = digit_bits; - auto uint_digit_bits = uinteger_t(_digit_bits); - divmod(shifts_q, shifts_r, rhs, uint_digit_bits); - std::size_t shifts = static_cast(shifts_q); - std::size_t shift = static_cast(shifts_r); - - if (shifts) { - lhs.prepend(shifts, 0); - } - if (shift) { - digit shifted = 0; - auto lhs_it = lhs.begin() + shifts; - auto lhs_it_e = lhs.end(); - for (; lhs_it != lhs_it_e; ++lhs_it) { - auto v = (*lhs_it << shift) | shifted; - shifted = *lhs_it >> (_digit_bits - shift); - *lhs_it = v; - } - if (shifted) { - lhs.append(shifted); - } - } - - // Finish up - lhs.trim(); - return lhs; - } - - static uinteger_t& bitwise_lshift(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { - if (&result._value == &lhs._value) { - bitwise_lshift(result, rhs); - return result; - } - if (!rhs) { - result = lhs; - return result; - } - - auto lhs_sz = lhs.size(); - - uinteger_t shifts_q; - uinteger_t shifts_r; - auto _digit_bits = digit_bits; - auto uint_digit_bits = uinteger_t(_digit_bits); - divmod(shifts_q, shifts_r, rhs, uint_digit_bits); - std::size_t shifts = static_cast(shifts_q); - std::size_t shift = static_cast(shifts_r); - - auto result_sz = lhs_sz + shifts; - result.grow(result_sz + 1); - result.resize(shifts, 0); - result.resize(result_sz); - - // not using `end()` because resize of `result.resize()` could have - // resized `lhs` if `result` is also `lhs`. - auto lhs_it = lhs.begin(); - auto lhs_it_e = lhs_it + lhs_sz; - - auto it = result.begin() + shifts; - - if (shift) { - digit shifted = 0; - for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { - auto v = (*lhs_it << shift) | shifted; - shifted = *lhs_it >> (_digit_bits - shift); - *it = v; - } - if (shifted) { - result.append(shifted); - } - } else { - for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { - *it = *lhs_it; - } - } - - // Finish up - result.trim(); - return result; - } - - static uinteger_t bitwise_lshift(const uinteger_t& lhs, const uinteger_t& rhs) { - uinteger_t result; - bitwise_lshift(result, lhs, rhs); - return result; - } - - static uinteger_t& bitwise_rshift(uinteger_t& lhs, const uinteger_t& rhs) { - if (!rhs) { - return lhs; - } - - auto lhs_sz = lhs.size(); - - auto _digit_bits = digit_bits; - if (compare(rhs, uinteger_t(lhs_sz * _digit_bits)) >= 0) { - lhs = uint_0(); - return lhs; - } - - uinteger_t shifts_q; - uinteger_t shifts_r; - auto uint_digit_bits = uinteger_t(_digit_bits); - divmod(shifts_q, shifts_r, rhs, uint_digit_bits); - std::size_t shifts = static_cast(shifts_q); - std::size_t shift = static_cast(shifts_r); - - if (shifts) { - lhs._begin += shifts; - } - if (shift) { - digit shifted = 0; - auto lhs_rit = lhs.rbegin(); - auto lhs_rit_e = lhs.rend(); - for (; lhs_rit != lhs_rit_e; ++lhs_rit) { - auto v = (*lhs_rit >> shift) | shifted; - shifted = *lhs_rit << (_digit_bits - shift); - *lhs_rit = v; - } - lhs.trim(); - } - - return lhs; - } - - static uinteger_t& bitwise_rshift(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { - if (&result._value == &lhs._value) { - bitwise_lshift(result, rhs); - return result; - } - if (!rhs) { - result = lhs; - return result; - } - - auto lhs_sz = lhs.size(); - - auto _digit_bits = digit_bits; - if (compare(rhs, uinteger_t(lhs_sz * _digit_bits)) >= 0) { - result = uint_0(); - return result; - } - - uinteger_t shifts_q; - uinteger_t shifts_r; - auto uint_digit_bits = uinteger_t(_digit_bits); - divmod(shifts_q, shifts_r, rhs, uint_digit_bits); - std::size_t shifts = static_cast(shifts_q); - std::size_t shift = static_cast(shifts_r); - - auto result_sz = lhs_sz - shifts; - result.resize(result_sz); - - // not using `end()` because resize of `result.resize()` could have - // resized `lhs` if `result` is also `lhs`. - auto lhs_rit = lhs.rbegin(); - auto lhs_rit_e = lhs_rit + lhs_sz - shifts; - - auto rit = result.rbegin(); - auto rit_e = rit + result_sz; - - if (shift) { - digit shifted = 0; - for (; lhs_rit != lhs_rit_e; ++lhs_rit, ++rit) { - ASSERT(rit != rit_e); (void)(rit_e); - auto v = (*lhs_rit >> shift) | shifted; - shifted = *lhs_rit << (_digit_bits - shift); - *rit = v; - } - } else { - for (; lhs_rit != lhs_rit_e; ++lhs_rit, ++rit) { - ASSERT(rit != rit_e); (void)(rit_e); - *rit = *lhs_rit; - } - } - - // Finish up - result.trim(); - return result; - } - - static uinteger_t bitwise_rshift(const uinteger_t& lhs, const uinteger_t& rhs) { - uinteger_t result; - bitwise_rshift(result, lhs, rhs); - return result; - } - - static int compare(const uinteger_t& lhs, const uinteger_t& rhs) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - if (lhs_sz > rhs_sz) return 1; - if (lhs_sz < rhs_sz) return -1; - - auto lhs_rit = lhs.rbegin(); - auto lhs_rit_e = lhs.rend(); - - auto rhs_rit = rhs.rbegin(); - - for (; lhs_rit != lhs_rit_e && *lhs_rit == *rhs_rit; ++lhs_rit, ++rhs_rit); - - if (lhs_rit != lhs_rit_e) { - if (*lhs_rit > *rhs_rit) return 1; - if (*lhs_rit < *rhs_rit) return -1; - } - - return 0; - } - - static uinteger_t& long_add(uinteger_t& lhs, const uinteger_t& rhs) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - if (lhs_sz < rhs_sz) { - lhs.reserve(rhs_sz + 1); - lhs.resize(rhs_sz, 0); // grow - } - - // not using `end()` because resize of `lhs.resize()` could have - // resized `lhs`. - auto lhs_it = lhs.begin(); - auto lhs_it_e = lhs_it + lhs_sz; - - auto rhs_it = rhs.begin(); - auto rhs_it_e = rhs_it + rhs_sz; - - digit carry = 0; - if (lhs_sz < rhs_sz) { - for (; lhs_it != lhs_it_e; ++rhs_it, ++lhs_it) { - carry = _addcarry(*lhs_it, *rhs_it, carry, &*lhs_it); - } - for (; carry && rhs_it != rhs_it_e; ++rhs_it, ++lhs_it) { - carry = _addcarry(0, *rhs_it, carry, &*lhs_it); - } - for (; rhs_it != rhs_it_e; ++rhs_it, ++lhs_it) { - *lhs_it = *rhs_it; - } - } else { - for (; rhs_it != rhs_it_e; ++rhs_it, ++lhs_it) { - carry = _addcarry(*lhs_it, *rhs_it, carry, &*lhs_it); - } - for (; carry && lhs_it != lhs_it_e; ++lhs_it) { - carry = _addcarry(*lhs_it, 0, carry, &*lhs_it); - } - } - - if (carry) { - lhs.append(1); - } - - lhs._carry = false; - - // Finish up - lhs.trim(); - return lhs; - } - - static uinteger_t& long_add(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - auto result_sz = std::max(lhs_sz, rhs_sz); - result.reserve(result_sz + 1); - result.resize(result_sz, 0); - - // not using `end()` because resize of `result.resize()` could have - // resized `lhs` or `rhs` if `result` is also either `rhs` or `lhs`. - auto lhs_it = lhs.begin(); - auto lhs_it_e = lhs_it + lhs_sz; - - auto rhs_it = rhs.begin(); - auto rhs_it_e = rhs_it + rhs_sz; - - auto it = result.begin(); - - digit carry = 0; - if (lhs_sz < rhs_sz) { - for (; lhs_it != lhs_it_e; ++lhs_it, ++rhs_it, ++it) { - carry = _addcarry(*lhs_it, *rhs_it, carry, &*it); - } - for (; carry && rhs_it != rhs_it_e; ++rhs_it, ++it) { - carry = _addcarry(0, *rhs_it, carry, &*it); - } - for (; rhs_it != rhs_it_e; ++rhs_it, ++it) { - *it = *rhs_it; - } - } else { - for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it, ++it) { - carry = _addcarry(*lhs_it, *rhs_it, carry, &*it); - } - for (; carry && lhs_it != lhs_it_e; ++lhs_it, ++it) { - carry = _addcarry(*lhs_it, 0, carry, &*it); - } - for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { - *it = *lhs_it; - } - } - - if (carry) { - result.append(1); - } - result._carry = false; - - // Finish up - result.trim(); - return result; - } - - static uinteger_t& add(uinteger_t& lhs, const uinteger_t& rhs) { - // First try saving some calculations: - if (!rhs) { - return lhs; - } - if (!lhs) { - lhs = rhs; - return lhs; - } - - return long_add(lhs, rhs); - } - - static uinteger_t& add(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { - // First try saving some calculations: - if (!rhs) { - result = lhs; - return result; - } - if (!lhs) { - result = rhs; - return result; - } - - return long_add(result, lhs, rhs); - } - - static uinteger_t add(const uinteger_t& lhs, const uinteger_t& rhs) { - uinteger_t result; - add(result, lhs, rhs); - return result; - } - - static uinteger_t& long_sub(uinteger_t& lhs, const uinteger_t& rhs) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - if (lhs_sz < rhs_sz) { - lhs.resize(rhs_sz, 0); // grow - } - - // not using `end()` because resize of `lhs.resize()` could have - // resized `lhs`. - auto lhs_it = lhs.begin(); - auto lhs_it_e = lhs_it + lhs_sz; - - auto rhs_it = rhs.begin(); - auto rhs_it_e = rhs_it + rhs_sz; - - digit borrow = 0; - if (lhs_sz < rhs_sz) { - for (; lhs_it != lhs_it_e; ++lhs_it, ++rhs_it) { - borrow = _subborrow(*lhs_it, *rhs_it, borrow, &*lhs_it); - } - for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it) { - borrow = _subborrow(0, *rhs_it, borrow, &*lhs_it); - } - } else { - for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it) { - borrow = _subborrow(*lhs_it, *rhs_it, borrow, &*lhs_it); - } - for (; borrow && lhs_it != lhs_it_e; ++lhs_it) { - borrow = _subborrow(*lhs_it, 0, borrow, &*lhs_it); - } - } - - lhs._carry = borrow; - - // Finish up - lhs.trim(); - return lhs; - } - - static uinteger_t& long_sub(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - auto result_sz = std::max(lhs_sz, rhs_sz); - result.resize(result_sz, 0); - - // not using `end()` because resize of `result.resize()` could have - // resized `lhs` or `rhs` if `result` is also either `rhs` or `lhs`. - auto lhs_it = lhs.begin(); - auto lhs_it_e = lhs_it + lhs_sz; - - auto rhs_it = rhs.begin(); - auto rhs_it_e = rhs_it + rhs_sz; - - auto it = result.begin(); - - digit borrow = 0; - if (lhs_sz < rhs_sz) { - for (; lhs_it != lhs_it_e; ++lhs_it, ++rhs_it, ++it) { - borrow = _subborrow(*lhs_it, *rhs_it, borrow, &*it); - } - for (; rhs_it != rhs_it_e; ++rhs_it, ++it) { - borrow = _subborrow(0, *rhs_it, borrow, &*it); - } - } else { - for (; rhs_it != rhs_it_e; ++lhs_it, ++rhs_it, ++it) { - borrow = _subborrow(*lhs_it, *rhs_it, borrow, &*it); - } - for (; borrow && lhs_it != lhs_it_e; ++lhs_it, ++it) { - borrow = _subborrow(*lhs_it, 0, borrow, &*it); - } - for (; lhs_it != lhs_it_e; ++lhs_it, ++it) { - *it = *lhs_it; - } - } - - result._carry = borrow; - - // Finish up - result.trim(); - return result; - } - - static uinteger_t& sub(uinteger_t& lhs, const uinteger_t& rhs) { - // First try saving some calculations: - if (!rhs) { - return lhs; - } - - return long_sub(lhs, rhs); - } - - static uinteger_t& sub(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { - // First try saving some calculations: - if (!rhs) { - result = lhs; - return result; - } - - return long_sub(result, lhs, rhs); - } - - static uinteger_t sub(const uinteger_t& lhs, const uinteger_t& rhs) { - uinteger_t result; - sub(result, lhs, rhs); - return result; - } - - // Single word long multiplication - // Fastests, but ONLY for single sized rhs - static uinteger_t& single_mult(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - ASSERT(rhs_sz == 1); (void)(rhs_sz); - auto n = rhs.front(); - - uinteger_t tmp; - tmp.resize(lhs_sz + 1, 0); - - auto it_lhs = lhs.begin(); - auto it_lhs_e = lhs.end(); - - auto it_result = tmp.begin(); - - digit carry = 0; - for (; it_lhs != it_lhs_e; ++it_lhs, ++it_result) { - carry = _multadd(*it_lhs, n, 0, carry, &*it_result); - } - if (carry) { - *it_result = carry; - } - - result = std::move(tmp); - - // Finish up - result.trim(); - return result; - } - - static uinteger_t& long_mult(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - if (lhs_sz > rhs_sz) { - // rhs should be the largest: - return long_mult(result, rhs, lhs); - } - - if (lhs_sz == 1) { - return single_mult(result, rhs, lhs); - } - - uinteger_t tmp; - tmp.resize(lhs_sz + rhs_sz, 0); - - auto it_lhs = lhs.begin(); - auto it_lhs_e = lhs.end(); - - auto it_rhs = rhs.begin(); - auto it_rhs_e = rhs.end(); - - auto it_result = tmp.begin(); - auto it_result_s = it_result; - auto it_result_l = it_result; - - for (; it_lhs != it_lhs_e; ++it_lhs, ++it_result) { - if (auto lhs_it_val = *it_lhs) { - auto _it_rhs = it_rhs; - auto _it_result = it_result; - digit carry = 0; - for (; _it_rhs != it_rhs_e; ++_it_rhs, ++_it_result) { - carry = _multadd(*_it_rhs, lhs_it_val, *_it_result, carry, &*_it_result); - } - if (carry) { - *_it_result++ = carry; - } - if (it_result_l < _it_result) { - it_result_l = _it_result; - } - } - } - - tmp.resize(it_result_l - it_result_s); // shrink - - result = std::move(tmp); - - // Finish up - result.trim(); - return result; - } - - // A helper for Karatsuba multiplication to split a number in two, at n. - static std::pair karatsuba_mult_split(const uinteger_t& num, std::size_t n) { - const uinteger_t a(num, num._begin, num._begin + n); - const uinteger_t b(num, num._begin + n, num._end); - return std::make_pair(std::move(a), std::move(b)); - } - - // If rhs has at least twice the digits of lhs, and lhs is big enough that - // Karatsuba would pay off *if* the inputs had balanced sizes. - // View rhs as a sequence of slices, each with lhs.size() digits, - // and multiply the slices by lhs, one at a time. - static uinteger_t& karatsuba_lopsided_mult(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs, std::size_t cutoff) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - ASSERT(lhs_sz > cutoff); - ASSERT(2 * lhs_sz <= rhs_sz); - - auto rhs_begin = rhs._begin; - std::size_t shift = 0; - - uinteger_t r; - while (rhs_sz > 0) { - // Multiply the next slice of rhs by lhs and add into result: - auto slice_size = std::min(lhs_sz, rhs_sz); - const uinteger_t rhs_slice(rhs, rhs_begin, rhs_begin + slice_size); - uinteger_t p; - karatsuba_mult(p, lhs, rhs_slice, cutoff); - uinteger_t rs(r, shift, 0); - add(rs, rs, p); - shift += slice_size; - rhs_sz -= slice_size; - rhs_begin += slice_size; - } - - result = std::move(r); - return result; - } - - // Karatsuba multiplication - static uinteger_t& karatsuba_mult(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs, std::size_t cutoff = 1) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - if (lhs_sz > rhs_sz) { - // rhs should be the largest: - return karatsuba_mult(result, rhs, lhs, cutoff); - } - - if (lhs_sz <= cutoff) { - return long_mult(result, lhs, rhs); - } - - // If a is too small compared to b, splitting on b gives a degenerate case - // in which Karatsuba may be (even much) less efficient than long multiplication. - if (2 * lhs_sz <= rhs_sz) { - return karatsuba_lopsided_mult(result, lhs, rhs, cutoff); - } - - // Karatsuba: - // - // A B - // x C D - // --------------------- - // AD BD - // AC BC - // --------------------- - // AC AD + BC BD - // - // AD + BC = - // AC + AD + BC + BD - AC - BD - // (A + B) (C + D) - AC - BD - - // Calculate the split point near the middle of the largest (rhs). - auto shift = rhs_sz >> 1; - - // Split to get A and B: - const auto lhs_pair = karatsuba_mult_split(lhs, shift); - const auto& A = lhs_pair.second; // hi - const auto& B = lhs_pair.first; // lo - - // Split to get C and D: - const auto rhs_pair = karatsuba_mult_split(rhs, shift); - const auto& C = rhs_pair.second; // hi - const auto& D = rhs_pair.first; // lo - - // Get the pieces: - uinteger_t AC; - karatsuba_mult(AC, A, C, cutoff); - - uinteger_t BD; - karatsuba_mult(BD, B, D, cutoff); - uinteger_t AD_BC, AB, CD; - karatsuba_mult(AD_BC, A + B, C + D, cutoff); - AD_BC -= AC; - AD_BC -= BD; - - // Join the pieces, AC and BD (can't overlap) into BD: - BD.reserve(shift * 2 + AC.size()); - BD.resize(shift * 2, 0); - BD.append(AC); - - // And add AD_BC to the middle: (AC BD) + ( AD + BC ): - uinteger_t BDs(BD, shift, 0); - add(BDs, BDs, AD_BC); - - result = std::move(BD); - - // Finish up - result.trim(); - return result; - } - - static uinteger_t& mult(uinteger_t& lhs, const uinteger_t& rhs) { - // Hard to see how this could have a further optimized implementation. - return mult(lhs, lhs, rhs); - } - - static uinteger_t& mult(uinteger_t& result, const uinteger_t& lhs, const uinteger_t& rhs) { - // First try saving some calculations: - if (!lhs || !rhs) { - result = uint_0(); - return result; - } - if (compare(lhs, uint_1()) == 0) { - result = rhs; - return result; - } - if (compare(rhs, uint_1()) == 0) { - result = lhs; - return result; - } - - return karatsuba_mult(result, lhs, rhs, karatsuba_cutoff); - } - - static uinteger_t mult(const uinteger_t& lhs, const uinteger_t& rhs) { - uinteger_t result; - mult(result, lhs, rhs); - return result; - } - - // Single word long division - // Fastests, but ONLY for single sized rhs - static std::pair, std::reference_wrapper> single_divmod(uinteger_t& quotient, uinteger_t& remainder, const uinteger_t& lhs, const uinteger_t& rhs) { - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - - ASSERT(rhs_sz == 1); (void)(rhs_sz); - auto n = rhs.front(); - - auto rit_lhs = lhs.rbegin(); - auto rit_lhs_e = lhs.rend(); - - auto q = uint_0(); - q.resize(lhs_sz, 0); - auto rit_q = q.rbegin(); - - digit r = 0; - for (; rit_lhs != rit_lhs_e; ++rit_lhs, ++rit_q) { - r = _divmod(r, *rit_lhs, n, &*rit_q); - } - - q.trim(); - - quotient = std::move(q); - remainder = r; - return std::make_pair(std::ref(quotient), std::ref(remainder)); - } - - // Implementation of Knuth's Algorithm D - static std::pair, std::reference_wrapper> knuth_divmod(uinteger_t& quotient, uinteger_t& remainder, const uinteger_t& lhs, const uinteger_t& rhs) { - uinteger_t v(lhs); - uinteger_t w(rhs); - - auto v_size = v.size(); - auto w_size = w.size(); - ASSERT(v_size >= w_size && w_size >= 2); - - // D1. normalize: shift rhs left so that its top digit is >= 63 bits. - // shift lhs left by the same amount. Results go into w and v. - auto d = uinteger_t(digit_bits - _bits(w.back())); - v <<= d; - w <<= d; - - if (*v.rbegin() >= *w.rbegin()) { - v.append(0); - } - v_size = v.size(); - v.append(0); - - // Now *v.rbegin() < *w.rbegin() so quotient has at most - // (and usually exactly) k = v.size() - w.size() digits. - auto k = v_size - w_size; - auto q = uint_0(); - q.resize(k + 1, 0); - - auto rit_q = q.rend() - (k + 1); - - auto it_v_b = v.begin(); - auto it_v_k = it_v_b + k; - - auto it_w = w.begin(); - auto it_w_e = w.end(); - - auto rit_w = w.rbegin(); - auto wm1 = *rit_w++; - auto wm2 = *rit_w; - - // D2. inner loop: divide v[k+0..k+n] by w[0..n] - for (; it_v_k >= it_v_b; --it_v_k, ++rit_q) { - // D3. Compute estimate quotient digit q; may overestimate by 1 (rare) - digit _q; - auto _r = _divmod(*(it_v_k + w_size), *(it_v_k + w_size - 1), wm1, &_q); - digit mullo = 0; - auto mulhi = _mult(_q, wm2, &mullo); - auto rlo = *(it_v_k + w_size - 2); - while (mulhi > _r || (mulhi == _r && mullo > rlo)) { - --_q; - if (_addcarry(_r, wm1, 0, &_r)) { - break; - } - mulhi = _mult(_q, wm2, &mullo); - } - - // D4. Multiply and subtract _q * w0[0:size_w] from vk[0:size_w+1] - auto _it_v = it_v_k; - auto _it_w = it_w; - mulhi = 0; - digit carry = 0; - for (; _it_w != it_w_e; ++_it_v, ++_it_w) { - mullo = 0; - mulhi = _multadd(*_it_w, _q, 0, mulhi, &mullo); - carry = _subborrow(*_it_v, mullo, carry, &*_it_v); - } - carry = _subborrow(*_it_v, 0, carry, &*_it_v); - - if (carry) { - // D6. Add w back if q was too large (this branch taken rarely) - --_q; - - _it_v = it_v_k; - _it_w = it_w; - carry = 0; - for (; _it_w != it_w_e; ++_it_v, ++_it_w) { - carry = _addcarry(*_it_v, *_it_w, carry, &*_it_v); - } - carry = _addcarry(*_it_v, 0, carry, &*_it_v); - } - - /* store quotient digit */ - *rit_q = _q; - } - - // D8. unnormalize: unshift remainder. - v.resize(w_size); - v >>= d; - - q.trim(); - v.trim(); - - quotient = std::move(q); - remainder = std::move(v); - return std::make_pair(std::ref(quotient), std::ref(remainder)); - } - - static std::pair, std::reference_wrapper> divmod(uinteger_t& quotient, uinteger_t& remainder, const uinteger_t& lhs, const uinteger_t& rhs) { - // First try saving some calculations: - if (!rhs) { - throw std::domain_error("Error: division or modulus by 0"); - } - auto lhs_sz = lhs.size(); - auto rhs_sz = rhs.size(); - if (lhs_sz == 1 && rhs_sz == 1) { - // Fast division and modulo for single value - auto a = *lhs.begin(); - auto b = *rhs.begin(); - quotient = a / b; - remainder = a % b; - return std::make_pair(std::ref(quotient), std::ref(remainder)); - } - if (compare(rhs, uint_1()) == 0) { - quotient = lhs; - remainder = uint_0(); - return std::make_pair(std::ref(quotient), std::ref(remainder)); - } - auto compared = compare(lhs, rhs); - if (compared == 0) { - quotient = uint_1(); - remainder = uint_0(); - return std::make_pair(std::ref(quotient), std::ref(remainder)); - } - if (!lhs || compared < 0) { - quotient = uint_0(); - remainder = lhs; - return std::make_pair(std::ref(quotient), std::ref(remainder)); - } - if (rhs_sz == 1) { - return single_divmod(quotient, remainder, lhs, rhs); - } - - return knuth_divmod(quotient, remainder, lhs, rhs); - } - - static std::pair divmod(const uinteger_t& lhs, const uinteger_t& rhs) { - uinteger_t quotient; - uinteger_t remainder; - divmod(quotient, remainder, lhs, rhs); - return std::make_pair(std::move(quotient), std::move(remainder)); - } - -private: - // Constructors - - template ::value and not std::is_same>::value>> - void _uint_t(const T& value) { - append(static_cast(value)); - } - - template ::value and not std::is_same>::value>> - void _uint_t(const T& value, Args... args) { - _uint_t(args...); - append(static_cast(value)); - } - - // This constructor creates a window view of the _value - uinteger_t(const uinteger_t& o, std::size_t begin, std::size_t end) : - _begin(begin), - _end(end), - _value(o._value), - _carry(o._carry) { } - -public: - uinteger_t() : - _begin(0), - _end(0), - _value(_value_instance), - _carry(false) { } - - uinteger_t(const uinteger_t& o) : - _begin(0), - _end(0), - _value_instance(o.begin(), o.end()), - _value(_value_instance), - _carry(o._carry) { } - - uinteger_t(uinteger_t&& o) : - _begin(std::move(o._begin)), - _end(std::move(o._end)), - _value_instance(std::move(o._value_instance)), - _value(_value_instance), - _carry(std::move(o._carry)) { } - - template ::value and not std::is_same>::value>> - uinteger_t(const T& value) : - _begin(0), - _end(0), - _value(_value_instance), - _carry(false) { - if (value) { - append(static_cast(value)); - } - } - - template ::value and not std::is_same>::value>> - uinteger_t(const T& value, Args... args) : - _begin(0), - _end(0), - _value(_value_instance), - _carry(false) { - _uint_t(args...); - append(static_cast(value)); - trim(); - } - - template ::value and not std::is_same>::value>> - uinteger_t(std::initializer_list list) : - _begin(0), - _end(0), - _value(_value_instance), - _carry(false) { - reserve(list.size()); - for (const auto& value : list) { - append(static_cast(value)); - } - trim(); - } - - template - explicit uinteger_t(T (&s)[N], int base=10) : - uinteger_t(s, N - 1, base) { } - - explicit uinteger_t(const unsigned char* bytes, std::size_t sz, int base) : - uinteger_t(strtouint(bytes, sz, base)) { } - - explicit uinteger_t(const char* bytes, std::size_t sz, int base) : - uinteger_t(strtouint(bytes, sz, base)) { } - - template - explicit uinteger_t(const std::vector& bytes, int base=10) : - uinteger_t(bytes.data(), bytes.size(), base) { } - - explicit uinteger_t(const std::string& bytes, int base=10) : - uinteger_t(bytes.data(), bytes.size(), base) { } - - // Assignment Operator - uinteger_t& operator=(const uinteger_t& o) { - _begin = 0; - _end = 0; - _value = container(o.begin(), o.end()); - _carry = o._carry; - return *this; - } - uinteger_t& operator=(uinteger_t&& o) { - _begin = std::move(o._begin); - _end = std::move(o._end); - _value_instance = std::move(o._value_instance); - _carry = std::move(o._carry); - return *this; - } - - // Typecast Operators - explicit operator bool() const { - return static_cast(size()); - } - explicit operator unsigned char() const { - return static_cast(size() ? front() : 0); - } - explicit operator unsigned short() const { - return static_cast(size() ? front() : 0); - } - explicit operator unsigned int() const { - return static_cast(size() ? front() : 0); - } - explicit operator unsigned long() const { - return static_cast(size() ? front() : 0); - } - explicit operator unsigned long long() const { - return static_cast(size() ? front() : 0); - } - explicit operator char() const { - return static_cast(size() ? front() : 0); - } - explicit operator short() const { - return static_cast(size() ? front() : 0); - } - explicit operator int() const { - return static_cast(size() ? front() : 0); - } - explicit operator long() const { - return static_cast(size() ? front() : 0); - } - explicit operator long long() const { - return static_cast(size() ? front() : 0); - } - - // Bitwise Operators - uinteger_t operator&(const uinteger_t& rhs) const { - return bitwise_and(*this, rhs); - } - - uinteger_t& operator&=(const uinteger_t& rhs) { - return bitwise_and(*this, rhs); - } - - uinteger_t operator|(const uinteger_t& rhs) const { - return bitwise_or(*this, rhs); - } - - uinteger_t& operator|=(const uinteger_t& rhs) { - return bitwise_or(*this, rhs); - } - - uinteger_t operator^(const uinteger_t& rhs) const { - return bitwise_xor(*this, rhs); - } - - uinteger_t& operator^=(const uinteger_t& rhs) { - return bitwise_xor(*this, rhs); - } - - uinteger_t operator~() const { - return bitwise_inv(*this); - } - - uinteger_t inv() { - return bitwise_inv(*this); - } - - // Bit Shift Operators - uinteger_t operator<<(const uinteger_t& rhs) const { - return bitwise_lshift(*this, rhs); - } - - uinteger_t& operator<<=(const uinteger_t& rhs) { - return bitwise_lshift(*this, rhs); - } - - uinteger_t operator>>(const uinteger_t& rhs) const { - return bitwise_rshift(*this, rhs); - } - - uinteger_t& operator>>=(const uinteger_t& rhs) { - return bitwise_rshift(*this, rhs); - } - - // Logical Operators - bool operator!() const { - return !static_cast(*this); - } - - bool operator&&(const uinteger_t& rhs) const { - return static_cast(*this) && rhs; - } - - bool operator||(const uinteger_t& rhs) const { - return static_cast(*this) || rhs; - } - - // Comparison Operators - bool operator==(const uinteger_t& rhs) const { - return compare(*this, rhs) == 0; - } - - bool operator!=(const uinteger_t& rhs) const { - return compare(*this, rhs) != 0; - } - - bool operator>(const uinteger_t& rhs) const { - return compare(*this, rhs) > 0; - } - - bool operator<(const uinteger_t& rhs) const { - return compare(*this, rhs) < 0; - } - - bool operator>=(const uinteger_t& rhs) const { - return compare(*this, rhs) >= 0; - } - - bool operator<=(const uinteger_t& rhs) const { - return compare(*this, rhs) <= 0; - } - - // Arithmetic Operators - uinteger_t operator+(const uinteger_t& rhs) const { - return add(*this, rhs); - } - - uinteger_t& operator+=(const uinteger_t& rhs) { - return add(*this, rhs); - } - - uinteger_t operator-(const uinteger_t& rhs) const { - return sub(*this, rhs); - } - - uinteger_t& operator-=(const uinteger_t& rhs) { - return sub(*this, rhs); - } - - uinteger_t operator*(const uinteger_t& rhs) const { - return mult(*this, rhs); - } - - uinteger_t& operator*=(const uinteger_t& rhs) { - return mult(*this, rhs); - } - - std::pair divmod(const uinteger_t& rhs) const { - return divmod(*this, rhs); - } - - uinteger_t operator/(const uinteger_t& rhs) const { - return divmod(*this, rhs).first; - } - - uinteger_t& operator/=(const uinteger_t& rhs) { - uinteger_t quotient; - uinteger_t remainder; - divmod(quotient, remainder, *this, rhs); - *this = std::move(quotient); - return *this; - } - - uinteger_t operator%(const uinteger_t& rhs) const { - return divmod(*this, rhs).second; - } - - uinteger_t& operator%=(const uinteger_t& rhs) { - uinteger_t quotient; - uinteger_t remainder; - divmod(quotient, remainder, *this, rhs); - *this = std::move(remainder); - return *this; - } - - // Increment Operator - uinteger_t& operator++() { - return *this += uint_1(); - } - uinteger_t operator++(int) { - uinteger_t temp(*this); - ++*this; - return temp; - } - - // Decrement Operator - uinteger_t& operator--() { - return *this -= uint_1(); - } - uinteger_t operator--(int) { - uinteger_t temp(*this); - --*this; - return temp; - } - - // Nothing done since promotion doesn't work here - uinteger_t operator+() const { - return *this; - } - - // two's complement - uinteger_t operator-() const { - return uint_0() - *this; - } - - // Get private value at index - const digit& value(std::size_t idx) const { - static const digit zero = 0; - return idx < size() ? *(begin() + idx) : zero; - } - - // Get value of bit N - bool operator[](std::size_t n) const { - auto nd = n / digit_bits; - auto nm = n % digit_bits; - return nd < size() ? (*(begin() + nd) >> nm) & 1 : 0; - } - - // Get bitsize of value - std::size_t bits() const { - auto sz = size(); - if (sz) { - return _bits(back()) + (sz - 1) * digit_bits; - } - return 0; - } - - // Get string representation of value - template ::value>> - Result str(int alphabet_base = 10) const { - auto num_sz = size(); - if (alphabet_base >= 2 && alphabet_base <= 36) { - Result result; - if (num_sz) { - auto alphabet_base_bits = base_bits(alphabet_base); - result.reserve(num_sz * base_size(alphabet_base)); - if (alphabet_base_bits) { - digit alphabet_base_mask = alphabet_base - 1; - std::size_t shift = 0; - auto ptr = reinterpret_cast(data()); - digit v = *ptr++; - v <<= half_digit_bits; - for (auto i = num_sz * 2 - 1; i; --i) { - v >>= half_digit_bits; - v |= (static_cast(*ptr++) << half_digit_bits); - do { - auto d = static_cast((v >> shift) & alphabet_base_mask); - result.push_back(chr(d)); - shift += alphabet_base_bits; - } while (shift <= half_digit_bits); - shift -= half_digit_bits; - } - v >>= (shift + half_digit_bits); - while (v) { - auto d = static_cast(v & alphabet_base_mask); - result.push_back(chr(d)); - v >>= alphabet_base_bits; - } - auto s = chr(0); - auto rit_f = std::find_if(result.rbegin(), result.rend(), [s](const char& c) { return c != s; }); - result.resize(result.rend() - rit_f); // shrink - } else { - uinteger_t uint_base = alphabet_base; - uinteger_t quotient = *this; - do { - auto r = quotient.divmod(uint_base); - auto d = static_cast(r.second); - result.push_back(chr(d)); - quotient = std::move(r.first); - } while (quotient); - } - std::reverse(result.begin(), result.end()); - } else { - result.push_back(chr(0)); - } - return result; - } else if (alphabet_base == 256) { - if (num_sz) { - auto ptr = reinterpret_cast(data()); - Result result(ptr, ptr + num_sz * digit_octets); - auto rit_f = std::find_if(result.rbegin(), result.rend(), [](const char& c) { return c; }); - result.resize(result.rend() - rit_f); // shrink - std::reverse(result.begin(), result.end()); - return result; - } else { - Result result; - result.push_back('\x00'); - return result; - } - } else { - throw std::invalid_argument("Base must be in the range [2, 36]"); - } - } - - static uinteger_t strtouint(const void* encoded, std::size_t encoded_size, int alphabet_base) { - const char* data = (const char *)encoded; - uinteger_t result; - - if (alphabet_base >= 2 && alphabet_base <= 36) { - uinteger_t alphabet_base_bits = base_bits(alphabet_base); - uinteger_t uint_base = alphabet_base; - if (alphabet_base_bits) { - for (; encoded_size; --encoded_size, ++data) { - auto d = ord(static_cast(*data)); - if (d < 0) { - throw std::invalid_argument("Error: Not a digit in base " + std::to_string(alphabet_base) + ": '" + std::string(1, *data) + "' at " + std::to_string(encoded_size)); - } - result = (result << alphabet_base_bits) | d; - } - } else { - for (; encoded_size; --encoded_size, ++data) { - auto d = ord(static_cast(*data)); - if (d < 0) { - throw std::invalid_argument("Error: Not a digit in base " + std::to_string(alphabet_base) + ": '" + std::string(1, *data) + "' at " + std::to_string(encoded_size)); - } - result = (result * uint_base) + d; - } - } - } else if (encoded_size && alphabet_base == 256) { - auto value_size = encoded_size / digit_octets; - auto value_padding = encoded_size % digit_octets; - if (value_padding) { - value_padding = digit_octets - value_padding; - ++value_size; - } - result.resize(value_size); // grow (no initialization) - *result.begin() = 0; // initialize value - auto ptr = reinterpret_cast(result.data()); - std::copy(data, data + encoded_size, ptr + value_padding); - std::reverse(ptr, ptr + value_size * digit_octets); - } else { - throw std::invalid_argument("Error: Cannot convert from base " + std::to_string(alphabet_base)); - } - - return result; - } - - template ::value>> - Result bin() const { - return str(2); - } - - template ::value>> - Result oct() const { - return str(8); - } - - template ::value>> - Result hex() const { - return str(16); - } - - template ::value>> - Result raw() const { - return str(256); - } -}; - -namespace std { // This is probably not a good idea - // Make it work with std::string() - inline std::string to_string(uinteger_t& num) { - return num.str(); - } - inline const std::string to_string(const uinteger_t& num) { - return num.str(); - } -} - -// lhs type T as first arguemnt -// If the output is not a bool, casts to type T - -// Bitwise Operators -template ::value and not std::is_same>::value>> -uinteger_t operator&(const T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) & rhs; -} - -template ::value and not std::is_same>::value>> -T& operator&=(T& lhs, const uinteger_t& rhs) { - return lhs = static_cast(rhs & lhs); -} - -template ::value and not std::is_same>::value>> -uinteger_t operator|(const T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) | rhs; -} - -template ::value and not std::is_same>::value>> -T& operator|=(T& lhs, const uinteger_t& rhs) { - return lhs = static_cast(rhs | lhs); -} - -template ::value and not std::is_same>::value>> -uinteger_t operator^(const T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) ^ rhs; -} - -template ::value and not std::is_same>::value>> -T& operator^=(T& lhs, const uinteger_t& rhs) { - return lhs = static_cast(rhs ^ lhs); -} - -// Bitshift operators -template ::value and not std::is_same>::value>> -inline uinteger_t operator<<(T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) << rhs; -} - -template ::value and not std::is_same>::value>> -T& operator<<=(T& lhs, const uinteger_t& rhs) { - return lhs = static_cast(lhs << rhs); -} - -template ::value and not std::is_same>::value>> -inline uinteger_t operator>>(T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) >> rhs; -} - -template ::value and not std::is_same>::value>> -T& operator>>=(T& lhs, const uinteger_t& rhs) { - return lhs = static_cast(lhs >> rhs); -} - -// Comparison Operators -template ::value and not std::is_same>::value>> -bool operator==(const T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) == rhs; -} - -template ::value and not std::is_same>::value>> -bool operator!=(const T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) != rhs; -} - -template ::value and not std::is_same>::value>> -bool operator>(const T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) > rhs; -} - -template ::value and not std::is_same>::value>> -bool operator<(const T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) < rhs; -} - -template ::value and not std::is_same>::value>> -bool operator>=(const T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) >= rhs; -} - -template ::value and not std::is_same>::value>> -bool operator<=(const T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) <= rhs; -} - -// Arithmetic Operators -template ::value and not std::is_same>::value>> -uinteger_t operator+(const T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) + rhs; -} - -template ::value and not std::is_same>::value>> -T& operator+=(T& lhs, const uinteger_t& rhs) { - return lhs = static_cast(rhs + lhs); -} - -template ::value and not std::is_same>::value>> -uinteger_t operator-(const T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) - rhs; -} - -template ::value and not std::is_same>::value>> -T& operator-=(T& lhs, const uinteger_t& rhs) { - return lhs = static_cast(lhs - rhs); -} - -template ::value and not std::is_same>::value>> -uinteger_t operator*(const T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) * rhs; -} - -template ::value and not std::is_same>::value>> -T& operator*=(T& lhs, const uinteger_t& rhs) { - return lhs = static_cast(rhs * lhs); -} - -template ::value and not std::is_same>::value>> -uinteger_t operator/(const T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) / rhs; -} - -template ::value and not std::is_same>::value>> -T& operator/=(T& lhs, const uinteger_t& rhs) { - return lhs = static_cast(lhs / rhs); -} - -template ::value and not std::is_same>::value>> -uinteger_t operator%(const T& lhs, const uinteger_t& rhs) { - return uinteger_t(lhs) % rhs; -} - -template ::value and not std::is_same>::value>> -T& operator%=(T& lhs, const uinteger_t& rhs) { - return lhs = static_cast(lhs % rhs); -} - -// IO Operator -inline std::ostream& operator<<(std::ostream& stream, const uinteger_t& rhs) { - if (stream.flags() & stream.oct) { - stream << rhs.str(8); - } else if (stream.flags() & stream.dec) { - stream << rhs.str(10); - } else if (stream.flags() & stream.hex) { - stream << rhs.str(16); - } - return stream; -} - -#endif From b89f01438ef987b42058862bae8aa7cc46277f44 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Jun 2022 03:05:55 +0200 Subject: [PATCH 080/123] Lower mutex scope --- programs/server/embedded.xml | 1 - src/Common/ThreadStatus.cpp | 11 +++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/programs/server/embedded.xml b/programs/server/embedded.xml index 2b6c4d9f770..0f11efab8a3 100644 --- a/programs/server/embedded.xml +++ b/programs/server/embedded.xml @@ -12,7 +12,6 @@ ./ - 8589934592 true diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index b5c2e7be11f..c7d0a42ee79 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -164,12 +164,15 @@ ThreadStatus::~ThreadStatus() if (thread_group) { - std::lock_guard guard(thread_group->mutex); - thread_group->finished_threads_counters_memory.emplace_back(ThreadGroupStatus::ProfileEventsCountersAndMemory{ + ThreadGroupStatus::ProfileEventsCountersAndMemory counters + { performance_counters.getPartiallyAtomicSnapshot(), memory_tracker.get(), - thread_id, - }); + thread_id + }; + + std::lock_guard guard(thread_group->mutex); + thread_group->finished_threads_counters_memory.emplace_back(std::move(counters)); thread_group->threads.erase(this); } From e33f236d502fe9f7906392f4ceffaee44e3c4ce4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Jun 2022 03:59:01 +0200 Subject: [PATCH 081/123] Slight improvement --- src/Common/ThreadStatus.cpp | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index c7d0a42ee79..37331b91d56 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -148,19 +148,10 @@ ThreadStatus::ThreadStatus() ThreadStatus::~ThreadStatus() { - try - { - if (untracked_memory > 0) - memory_tracker.alloc(untracked_memory); - else - memory_tracker.free(-untracked_memory); - } - catch (const DB::Exception &) - { - /// It's a minor tracked memory leak here (not the memory itself but it's counter). - /// We've already allocated a little bit more than the limit and cannot track it in the thread memory tracker or its parent. - tryLogCurrentException(log); - } + if (untracked_memory > 0) + memory_tracker.allocImpl(untracked_memory, false); + else + memory_tracker.free(-untracked_memory); if (thread_group) { From 0e5742fb8e386e5af544b4271b8639bb23d414ad Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Jun 2022 04:28:14 +0200 Subject: [PATCH 082/123] Better hardware benchmark --- benchmark/hardware.sh | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index a4cafd501e2..ce9b3e7d99d 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -TABLE="hits_100m_obfuscated" +TABLE="hits" QUERIES_FILE="queries.sql" TRIES=3 @@ -20,7 +20,7 @@ uptime echo "Starting clickhouse-server" -./clickhouse server > server.log 2>&1 & +./clickhouse server >/dev/null 2>&1 & PID=$! function finish { @@ -37,15 +37,23 @@ for i in {1..30}; do if [[ $i == 30 ]]; then exit 1; fi done -echo "Will download the dataset" -./clickhouse client --max_insert_threads $(nproc || 4) --progress --query " - CREATE OR REPLACE TABLE ${TABLE} ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime) - AS SELECT * FROM url('https://datasets.clickhouse.com/hits/native/hits_100m_obfuscated_{0..255}.native.zst')" +if [[ $(./clickhouse client --query "EXISTS hits") == '1' && $(./clickhouse client --query "SELECT count() FROM hits") == '100000000' ]]; then + echo "Dataset already downloaded" +else + echo "Will download the dataset" + ./clickhouse client --max_insert_threads $(nproc || 4) --progress --query " + CREATE OR REPLACE TABLE ${TABLE} ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime) + AS SELECT * FROM url('https://datasets.clickhouse.com/hits/native/hits_100m_obfuscated_{0..255}.native.zst')" -./clickhouse client --query "SELECT 'The dataset size is: ', count() FROM ${TABLE}" + ./clickhouse client --query "SELECT 'The dataset size is: ', count() FROM ${TABLE}" +fi -echo "Will prepare the dataset" -./clickhouse client --query "OPTIMIZE TABLE ${TABLE} FINAL" +if [[ $(./clickhouse client --query "SELECT count() FROM system.parts WHERE table = 'hits' AND database = 'default' AND active") == '1' ]]; then + echo "Dataset already prepared" +else + echo "Will prepare the dataset" + ./clickhouse client --query "OPTIMIZE TABLE ${TABLE} FINAL" +fi echo echo "Will perform benchmark. Results:" From 4c3f42f1c2ff39790437ed0d3500ee28cc9b8c60 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Jun 2022 04:29:43 +0200 Subject: [PATCH 083/123] Better hardware benchmark --- benchmark/hardware.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index ce9b3e7d99d..374f2c515de 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -1,6 +1,5 @@ #!/bin/bash -e -TABLE="hits" QUERIES_FILE="queries.sql" TRIES=3 @@ -42,24 +41,24 @@ if [[ $(./clickhouse client --query "EXISTS hits") == '1' && $(./clickhouse clie else echo "Will download the dataset" ./clickhouse client --max_insert_threads $(nproc || 4) --progress --query " - CREATE OR REPLACE TABLE ${TABLE} ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime) + CREATE OR REPLACE TABLE hits ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime) AS SELECT * FROM url('https://datasets.clickhouse.com/hits/native/hits_100m_obfuscated_{0..255}.native.zst')" - ./clickhouse client --query "SELECT 'The dataset size is: ', count() FROM ${TABLE}" + ./clickhouse client --query "SELECT 'The dataset size is: ', count() FROM hits" fi if [[ $(./clickhouse client --query "SELECT count() FROM system.parts WHERE table = 'hits' AND database = 'default' AND active") == '1' ]]; then echo "Dataset already prepared" else echo "Will prepare the dataset" - ./clickhouse client --query "OPTIMIZE TABLE ${TABLE} FINAL" + ./clickhouse client --query "OPTIMIZE TABLE hits FINAL" fi echo echo "Will perform benchmark. Results:" echo -cat "$QUERIES_FILE" | sed "s/{table}/${TABLE}/g" | while read query; do +cat "$QUERIES_FILE" | sed "s/{table}/hits/g" | while read query; do sync if [ "${OS}" = "Darwin" ] then From 5cf6586ed631416bc1fbaad53218b930de2d1e92 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Jun 2022 04:43:07 +0200 Subject: [PATCH 084/123] Add benchmark for c6a_metal --- .../hardware/results/aws_c6a_metal.json | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 website/benchmark/hardware/results/aws_c6a_metal.json diff --git a/website/benchmark/hardware/results/aws_c6a_metal.json b/website/benchmark/hardware/results/aws_c6a_metal.json new file mode 100644 index 00000000000..adaf3cce48d --- /dev/null +++ b/website/benchmark/hardware/results/aws_c6a_metal.json @@ -0,0 +1,54 @@ +[ + { + "system": "AWS c6a.metal", + "system_full": "AWS c6a.metal 192 vCPU (96 cores) 384GiB RAM, 200 GB EBS", + "time": "2022-06-25 00:00:00", + "kind": "cloud", + "result": + [ +[0.031, 0.044, 0.001], +[0.050, 0.013, 0.012], +[0.061, 0.017, 0.017], +[0.117, 0.019, 0.021], +[0.280, 0.099, 0.121], +[0.802, 0.113, 0.225], +[0.020, 0.001, 0.001], +[0.024, 0.014, 0.013], +[0.243, 0.116, 0.115], +[0.886, 0.133, 0.129], +[0.325, 0.102, 0.107], +[0.369, 0.093, 0.098], +[0.883, 0.186, 0.178], +[1.532, 0.220, 0.221], +[0.727, 0.208, 0.196], +[0.215, 0.181, 0.184], +[1.676, 0.419, 0.413], +[1.324, 0.306, 0.300], +[3.088, 0.880, 0.765], +[0.119, 0.038, 0.026], +[8.946, 0.218, 0.135], +[9.994, 0.162, 0.134], +[19.043, 0.603, 0.616], +[14.138, 0.257, 0.222], +[1.821, 0.061, 0.048], +[0.856, 0.044, 0.046], +[2.364, 0.064, 0.050], +[9.074, 0.303, 0.291], +[7.130, 0.315, 0.307], +[0.430, 0.400, 0.412], +[1.321, 0.119, 0.114], +[4.649, 0.207, 0.183], +[3.786, 0.861, 0.848], +[8.971, 0.681, 0.652], +[8.974, 0.661, 0.645], +[0.251, 0.230, 0.223], +[0.112, 0.066, 0.073], +[0.053, 0.029, 0.031], +[0.069, 0.031, 0.028], +[0.230, 0.168, 0.171], +[0.044, 0.018, 0.020], +[0.035, 0.015, 0.020], +[0.021, 0.008, 0.007] + ] + } +] From ff8c52b087c3b10e0e5ad31b65699022a8a38ee3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Jun 2022 06:11:37 +0300 Subject: [PATCH 085/123] Update partition.md --- docs/en/sql-reference/statements/alter/partition.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md index 27178c91de8..d960f057a00 100644 --- a/docs/en/sql-reference/statements/alter/partition.md +++ b/docs/en/sql-reference/statements/alter/partition.md @@ -7,9 +7,9 @@ sidebar_label: PARTITION The following operations with [partitions](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md) are available: -- [DETACH PARTITION\|Part](#detach-partitionpart) — Moves a partition or part to the `detached` directory and forget it. -- [DROP PARTITION\|Part](#drop-partitionpart) — Deletes a partition or part. -- [ATTACH PARTITION\|Part](#attach-partitionpart) — Adds a partition or part from the `detached` directory to the table. +- [DETACH PARTITION\|PART](#detach-partitionpart) — Moves a partition or part to the `detached` directory and forget it. +- [DROP PARTITION\|PART](#drop-partitionpart) — Deletes a partition or part. +- [ATTACH PARTITION\|PART](#attach-partitionpart) — Adds a partition or part from the `detached` directory to the table. - [ATTACH PARTITION FROM](#attach-partition-from) — Copies the data partition from one table to another and adds. - [REPLACE PARTITION](#replace-partition) — Copies the data partition from one table to another and replaces. - [MOVE PARTITION TO TABLE](#move_to_table-partition) — Moves the data partition from one table to another. From 0654684bd4eb0b63e58696783734765559318906 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Jun 2022 06:10:50 +0200 Subject: [PATCH 086/123] Fix wrong implementation of filesystem* functions --- src/Functions/filesystem.cpp | 2 +- .../queries/0_stateless/02345_filesystem_local.reference | 1 + tests/queries/0_stateless/02345_filesystem_local.sh | 8 ++++++++ 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02345_filesystem_local.reference create mode 100755 tests/queries/0_stateless/02345_filesystem_local.sh diff --git a/src/Functions/filesystem.cpp b/src/Functions/filesystem.cpp index bea7ffdf818..36db68617e9 100644 --- a/src/Functions/filesystem.cpp +++ b/src/Functions/filesystem.cpp @@ -36,7 +36,7 @@ public: static FunctionPtr create(ContextPtr context) { - return std::make_shared>(std::filesystem::space(context->getConfigRef().getString("path"))); + return std::make_shared>(std::filesystem::space(context->getPath())); } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override diff --git a/tests/queries/0_stateless/02345_filesystem_local.reference b/tests/queries/0_stateless/02345_filesystem_local.reference new file mode 100644 index 00000000000..9972842f982 --- /dev/null +++ b/tests/queries/0_stateless/02345_filesystem_local.reference @@ -0,0 +1 @@ +1 1 diff --git a/tests/queries/0_stateless/02345_filesystem_local.sh b/tests/queries/0_stateless/02345_filesystem_local.sh new file mode 100755 index 00000000000..6771df2ae2d --- /dev/null +++ b/tests/queries/0_stateless/02345_filesystem_local.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Checks that these functions are working inside clickhouse-local. Does not check specific values. +$CLICKHOUSE_LOCAL --query "SELECT filesystemAvailable() > 0, filesystemFree() <= filesystemCapacity()" From a45e3d47adacc59699de26b59e5966307b97b8fb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Jun 2022 07:16:25 +0200 Subject: [PATCH 087/123] Remove useless codec from system.asynchronous_metric_log --- src/Interpreters/AsynchronousMetricLog.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/AsynchronousMetricLog.h b/src/Interpreters/AsynchronousMetricLog.h index 836e967a7a6..900d84868bd 100644 --- a/src/Interpreters/AsynchronousMetricLog.h +++ b/src/Interpreters/AsynchronousMetricLog.h @@ -40,7 +40,7 @@ struct AsynchronousMetricLogElement return "event_date Date CODEC(Delta(2), ZSTD(1)), " "event_time DateTime CODEC(Delta(4), ZSTD(1)), " "metric LowCardinality(String) CODEC(ZSTD(1)), " - "value Float64 CODEC(Gorilla, ZSTD(3))"; + "value Float64 CODEC(ZSTD(3))"; } }; From ccb4802ab10218523d4249e705decb05edf23945 Mon Sep 17 00:00:00 2001 From: xinhuitian Date: Sat, 25 Jun 2022 14:01:41 +0800 Subject: [PATCH 088/123] fix some wrong links in alter docs --- docs/en/sql-reference/statements/alter/column.md | 2 +- docs/en/sql-reference/statements/alter/partition.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index 2a5e36eaa00..9387b442944 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -24,7 +24,7 @@ The following actions are supported: - [CLEAR COLUMN](#clear-column) — Resets column values. - [COMMENT COLUMN](#comment-column) — Adds a text comment to the column. - [MODIFY COLUMN](#modify-column) — Changes column’s type, default expression and TTL. -- [MODIFY COLUMN REMOVE](#modify-remove) — Removes one of the column properties. +- [MODIFY COLUMN REMOVE](#modify-column-remove) — Removes one of the column properties. - [MATERIALIZE COLUMN](#materialize-column) — Materializes the column in the parts where the column is missing. These actions are described in detail below. diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md index d960f057a00..921ffb71066 100644 --- a/docs/en/sql-reference/statements/alter/partition.md +++ b/docs/en/sql-reference/statements/alter/partition.md @@ -12,13 +12,13 @@ The following operations with [partitions](../../../engines/table-engines/merget - [ATTACH PARTITION\|PART](#attach-partitionpart) — Adds a partition or part from the `detached` directory to the table. - [ATTACH PARTITION FROM](#attach-partition-from) — Copies the data partition from one table to another and adds. - [REPLACE PARTITION](#replace-partition) — Copies the data partition from one table to another and replaces. -- [MOVE PARTITION TO TABLE](#move_to_table-partition) — Moves the data partition from one table to another. -- [CLEAR COLUMN IN PARTITION](#clear-column-partition) — Resets the value of a specified column in a partition. -- [CLEAR INDEX IN PARTITION](#clear-index-partition) — Resets the specified secondary index in a partition. +- [MOVE PARTITION TO TABLE](#move-partition-to-table) — Moves the data partition from one table to another. +- [CLEAR COLUMN IN PARTITION](#clear-column-in-partition) — Resets the value of a specified column in a partition. +- [CLEAR INDEX IN PARTITION](#clear-index-in-partition) — Resets the specified secondary index in a partition. - [FREEZE PARTITION](#freeze-partition) — Creates a backup of a partition. - [UNFREEZE PARTITION](#unfreeze-partition) — Removes a backup of a partition. -- [FETCH PARTITION\|PART](#fetch-partition) — Downloads a part or partition from another server. -- [MOVE PARTITION\|PART](#move-partition) — Move partition/data part to another disk or volume. +- [FETCH PARTITION\|PART](#fetch-partitionpart) — Downloads a part or partition from another server. +- [MOVE PARTITION\|PART](#move-partitionpart) — Move partition/data part to another disk or volume. - [UPDATE IN PARTITION](#update-in-partition) — Update data inside the partition by condition. - [DELETE IN PARTITION](#delete-in-partition) — Delete data inside the partition by condition. From 9bc23f579e6aa3b2b5317a8512fb3355d9704931 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Jun 2022 06:07:41 +0200 Subject: [PATCH 089/123] Automated hardware benchmark --- benchmark/hardware.sh | 61 +++++++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index 374f2c515de..c67ba2bb09c 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -58,6 +58,7 @@ echo echo "Will perform benchmark. Results:" echo +QUERY_NUM=1 cat "$QUERIES_FILE" | sed "s/{table}/hits/g" | while read query; do sync if [ "${OS}" = "Darwin" ] @@ -72,8 +73,12 @@ cat "$QUERIES_FILE" | sed "s/{table}/hits/g" | while read query; do RES=$(./clickhouse client --time --format=Null --query="$query" 2>&1 ||:) [[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null" [[ "$i" != $TRIES ]] && echo -n ", " + + echo "${QUERY_NUM},${i},${RES}" >> result.csv done echo "]," + + QUERY_NUM=$((QUERY_NUM + 1)) done @@ -81,22 +86,23 @@ echo echo "Benchmark complete. System info:" echo +touch {cpu_model,cpu,df,memory,memory_total,blk,mdstat,instance}.txt + if [ "${OS}" = "Darwin" ] then echo '----Version, build id-----------' ./clickhouse local --query "SELECT format('Version: {}', version())" - sw_vers | grep BuildVersion ./clickhouse local --query "SELECT format('The number of threads is: {}', value) FROM system.settings WHERE name = 'max_threads'" --output-format TSVRaw ./clickhouse local --query "SELECT format('Current time: {}', toString(now(), 'UTC'))" echo '----CPU-------------------------' - sysctl hw.model - sysctl -a | grep -E 'hw.activecpu|hw.memsize|hw.byteorder|cachesize' + sysctl hw.model | tee cpu_model.txt + sysctl -a | grep -E 'hw.activecpu|hw.memsize|hw.byteorder|cachesize' | tee cpu.txt echo '----Disk Free and Total--------' - df -h . + df -h . | tee df.txt echo '----Memory Free and Total-------' - vm_stat + vm_stat | tee memory.txt echo '----Physical Memory Amount------' - ls -l /var/vm + ls -l /var/vm | tee memory_total.txt echo '--------------------------------' else echo '----Version, build id-----------' @@ -104,22 +110,49 @@ else ./clickhouse local --query "SELECT format('The number of threads is: {}', value) FROM system.settings WHERE name = 'max_threads'" --output-format TSVRaw ./clickhouse local --query "SELECT format('Current time: {}', toString(now(), 'UTC'))" echo '----CPU-------------------------' - cat /proc/cpuinfo | grep -i -F 'model name' | uniq - lscpu + cat /proc/cpuinfo | grep -i -F 'model name' | uniq | tee cpu_model.txt + lscpu | tee cpu.txt echo '----Block Devices---------------' - lsblk + lsblk | tee blk.txt echo '----Disk Free and Total--------' - df -h . + df -h . | tee df.txt echo '----Memory Free and Total-------' - free -h + free -h | tee memory.txt echo '----Physical Memory Amount------' - cat /proc/meminfo | grep MemTotal + cat /proc/meminfo | grep MemTotal | tee memory_total.txt echo '----RAID Info-------------------' - cat /proc/mdstat + cat /proc/mdstat| tee mdstat.txt echo '--------------------------------' fi echo echo "Instance type from IMDS (if available):" -curl --connect-timeout 1 http://169.254.169.254/latest/meta-data/instance-type +curl -s --connect-timeout 1 'http://169.254.169.254/latest/meta-data/instance-type' | tee instance.txt echo + +echo "Uploading the results (if possible)" + +./clickhouse local --query " + SELECT + (SELECT generateUUIDv4()) AS test_id, + c1 AS query_num, + c2 AS try_num, + c3 AS time, + version() AS version, + now() AS test_time, + (SELECT value FROM system.settings WHERE name = 'max_threads') AS threads, + filesystemCapacity() AS fs_capacity, + filesystemAvailable() AS fs_available, + file('cpu_model.txt') AS cpu_model, + file('cpu.txt') AS cpu, + file('df.txt') AS df, + file('memory.txt') AS memory, + file('memory_total.txt') AS memory_total, + file('blk.txt') AS blk, + file('mdstat.txt') AS mdstat, + file('instance.txt') AS instance + FROM file('result.csv') +" | tee upload.tsv | ./clickhouse client --host play.clickhouse.com --secure --user benchmark --query " + INSERT INTO hardware_benchmark_results + (test_id, query_num, try_num, time, version, test_time, threads, fs_capacity, fs_available, cpu_model, cpu, df, memory, memory_total, blk, mdstat, instance) + FORMAT TSV" From d4b9d5f067cf388a1d4497ecf4689c38f3ff53d9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Jun 2022 07:05:56 +0200 Subject: [PATCH 090/123] Automatic upload from hardware benchmark --- benchmark/benchmark.sh | 180 --------------------------------------- benchmark/create_dump.sh | 3 - benchmark/hardware.sh | 72 ++++++++++++++-- 3 files changed, 64 insertions(+), 191 deletions(-) delete mode 100755 benchmark/benchmark.sh delete mode 100755 benchmark/create_dump.sh diff --git a/benchmark/benchmark.sh b/benchmark/benchmark.sh deleted file mode 100755 index 5fea1bfbb88..00000000000 --- a/benchmark/benchmark.sh +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env bash -# script to run query to databases - -function usage() -{ - cat < /proc/sys/vm/drop_caches" - - if [[ "$restart_server_each_query" == "1" && "$use_service" == "1" ]]; then - echo "restart server: $etc_init_d_service restart" - sudo $etc_init_d_service restart - fi - - for i in $(seq $TIMES) - do - if [[ -f $etc_init_d_service && "$use_service" == "1" ]]; then - sudo $etc_init_d_service status - server_status=$? - expect -f $expect_file "" - - if [[ "$?" != "0" || $server_status != "0" ]]; then - echo "restart server: $etc_init_d_service restart" - sudo $etc_init_d_service restart - fi - - #wait until can connect to server - restart_timer=0 - restart_limit=60 - expect -f $expect_file "" &> /dev/null - while [ "$?" != "0" ]; do - echo "waiting" - sleep 1 - let "restart_timer = $restart_timer + 1" - if (( $restart_limit < $restart_timer )); then - sudo $etc_init_d_service restart - restart_timer=0 - fi - expect -f $expect_file "" &> /dev/null - done - fi - - echo - echo "times: $i" - - echo "query:" "$query" - expect -f $expect_file "$query" - - done - fi - - let "index = $index + 1" - done -} - -temp_test_file=temp_queries_$table_name -cat $test_file | sed s/$table_name_pattern/$table_name/g > $temp_test_file -mapfile -t test_queries < $temp_test_file - -echo "start time: $(date)" -time execute "${test_queries[@]}" -echo "stop time: $(date)" diff --git a/benchmark/create_dump.sh b/benchmark/create_dump.sh deleted file mode 100755 index 3e26f8c1426..00000000000 --- a/benchmark/create_dump.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -table=hits_10m; time clickhouse-client --max_bytes_before_external_sort=30000000000 --query="SELECT toInt64(WatchID), JavaEnable, Title, GoodEvent, (EventTime < toDateTime('1971-01-01 00:00:00') ? toDateTime('1971-01-01 00:00:01') : EventTime), (EventDate < toDate('1971-01-01') ? toDate('1971-01-01') : EventDate), CounterID, ClientIP, RegionID, toInt64(UserID), CounterClass, OS, UserAgent, URL, Referer, Refresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, (ClientEventTime < toDateTime('1971-01-01 00:00:01') ? toDateTime('1971-01-01 00:00:01') : ClientEventTime), SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, toInt64(FUniqID), OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, (LocalEventTime < toDateTime('1971-01-01 00:00:01') ? toDateTime('1971-01-01 00:00:01') : LocalEventTime), Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, toInt64(RefererHash), toInt64(URLHash), CLID FROM $table ORDER BY rand()" | corrector_utf8 > /opt/dumps/${table}_corrected.tsv diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index c67ba2bb09c..5b8b822c25c 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -58,7 +58,9 @@ echo echo "Will perform benchmark. Results:" echo +>result.csv QUERY_NUM=1 + cat "$QUERIES_FILE" | sed "s/{table}/hits/g" | while read query; do sync if [ "${OS}" = "Darwin" ] @@ -132,12 +134,11 @@ echo echo "Uploading the results (if possible)" +UUID=$(clickhouse-local --query "SELECT generateUUIDv4()") + ./clickhouse local --query " SELECT - (SELECT generateUUIDv4()) AS test_id, - c1 AS query_num, - c2 AS try_num, - c3 AS time, + '${UUID}' AS run_id, version() AS version, now() AS test_time, (SELECT value FROM system.settings WHERE name = 'max_threads') AS threads, @@ -151,8 +152,63 @@ echo "Uploading the results (if possible)" file('blk.txt') AS blk, file('mdstat.txt') AS mdstat, file('instance.txt') AS instance +" | tee meta.tsv | ./clickhouse client --host play.clickhouse.com --secure --user benchmark --query " + INSERT INTO benchmark_runs + (run_id, version, test_time, threads, fs_capacity, fs_available, cpu_model, cpu, df, memory, memory_total, blk, mdstat, instance) + FORMAT TSV" || echo "Cannot upload results." + +./clickhouse local --query " + SELECT + '${UUID}' AS run_id, + c1 AS query_num, + c2 AS try_num, + c3 AS time FROM file('result.csv') -" | tee upload.tsv | ./clickhouse client --host play.clickhouse.com --secure --user benchmark --query " - INSERT INTO hardware_benchmark_results - (test_id, query_num, try_num, time, version, test_time, threads, fs_capacity, fs_available, cpu_model, cpu, df, memory, memory_total, blk, mdstat, instance) - FORMAT TSV" +" | tee results.tsv | ./clickhouse client --host play.clickhouse.com --secure --user benchmark --query " + INSERT INTO benchmark_results + (run_id, query_num, try_num, time) + FORMAT TSV" || echo "Cannot upload results. Please send the output to feedback@clickhouse.com" + +< Date: Sat, 25 Jun 2022 07:08:37 +0200 Subject: [PATCH 091/123] Slightly better --- benchmark/hardware.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index 5b8b822c25c..0e6e4db499f 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -40,7 +40,7 @@ if [[ $(./clickhouse client --query "EXISTS hits") == '1' && $(./clickhouse clie echo "Dataset already downloaded" else echo "Will download the dataset" - ./clickhouse client --max_insert_threads $(nproc || 4) --progress --query " + ./clickhouse client --receive_timeout 1000 --max_insert_threads $(nproc || 4) --progress --query " CREATE OR REPLACE TABLE hits ENGINE = MergeTree PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID), EventTime) AS SELECT * FROM url('https://datasets.clickhouse.com/hits/native/hits_100m_obfuscated_{0..255}.native.zst')" @@ -51,7 +51,7 @@ if [[ $(./clickhouse client --query "SELECT count() FROM system.parts WHERE tabl echo "Dataset already prepared" else echo "Will prepare the dataset" - ./clickhouse client --query "OPTIMIZE TABLE hits FINAL" + ./clickhouse client --receive_timeout 1000 --query "OPTIMIZE TABLE hits FINAL" fi echo From 756fd0dac89337aefa063fd863ad6b9ea2ce5994 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Jun 2022 18:27:26 +0200 Subject: [PATCH 092/123] Automated hardware benchmark --- benchmark/hardware.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index 0e6e4db499f..e1d5c56b5a8 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -203,7 +203,7 @@ CREATE USER benchmark IDENTIFIED WITH no_password SETTINGS max_rows_to_read = 1, CREATE QUOTA benchmark KEYED BY ip_address -FOR RANDOMIZED INTERVAL 1 MINUTE MAX query_inserts = 1, written_bytes = 100000, +FOR RANDOMIZED INTERVAL 1 MINUTE MAX query_inserts = 4, written_bytes = 100000, FOR RANDOMIZED INTERVAL 1 HOUR MAX query_inserts = 10, written_bytes = 500000, FOR RANDOMIZED INTERVAL 1 DAY MAX query_inserts = 50, written_bytes = 2000000 TO benchmark; From 37903bfee00e20dd1721d9ad22e60570c90be1d5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 25 Jun 2022 23:18:33 +0200 Subject: [PATCH 093/123] Fix after #38427 --- benchmark/hardware.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/hardware.sh b/benchmark/hardware.sh index e1d5c56b5a8..e8c9c58aca3 100755 --- a/benchmark/hardware.sh +++ b/benchmark/hardware.sh @@ -134,7 +134,7 @@ echo echo "Uploading the results (if possible)" -UUID=$(clickhouse-local --query "SELECT generateUUIDv4()") +UUID=$(./clickhouse local --query "SELECT generateUUIDv4()") ./clickhouse local --query " SELECT From 07f14c9bb295a054f7c3cfd07b66ee2c78eb8025 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 26 Jun 2022 00:03:01 +0200 Subject: [PATCH 094/123] Disable vectorscan-on-ARM for now --- contrib/vectorscan-cmake/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/vectorscan-cmake/CMakeLists.txt b/contrib/vectorscan-cmake/CMakeLists.txt index 140c174cd73..480ab3a384f 100644 --- a/contrib/vectorscan-cmake/CMakeLists.txt +++ b/contrib/vectorscan-cmake/CMakeLists.txt @@ -1,9 +1,11 @@ # We use vectorscan, a portable and API/ABI-compatible drop-in replacement for hyperscan. -if (ARCH_AMD64 OR ARCH_AARCH64) +if (ARCH_AMD64) option (ENABLE_VECTORSCAN "Enable vectorscan library" ${ENABLE_LIBRARIES}) endif() +# TODO: vectorscan supports ARM yet some tests involving cyrillic letters fail (PR #38171) ... needs further investigation + # TODO PPC should generally work but needs manual generation of ppc/config.h file on a PPC machine if (NOT ENABLE_VECTORSCAN) From 25cc406201f93b666dbb03771a0123a4820b7650 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 26 Jun 2022 01:29:06 +0300 Subject: [PATCH 095/123] Update src/Common/ThreadStatus.cpp Co-authored-by: Dmitry Novik --- src/Common/ThreadStatus.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index 37331b91d56..a4f99c1be1a 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -149,7 +149,7 @@ ThreadStatus::ThreadStatus() ThreadStatus::~ThreadStatus() { if (untracked_memory > 0) - memory_tracker.allocImpl(untracked_memory, false); + memory_tracker.allocNoThrow(untracked_memory); else memory_tracker.free(-untracked_memory); From 725d80d470398f81e67fa9b89f824ec8d862805f Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 25 Jun 2022 20:30:36 +0200 Subject: [PATCH 096/123] get rid of path separation --- src/Disks/DiskDecorator.h | 2 +- src/Disks/DiskRestartProxy.cpp | 4 +- src/Disks/DiskRestartProxy.h | 2 +- src/Disks/DiskWebServer.cpp | 4 +- src/Disks/DiskWebServer.h | 2 +- src/Disks/IDisk.h | 6 +-- src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 37 +++++++++++----- src/Disks/IO/ReadBufferFromRemoteFSGather.h | 32 +++++--------- src/Disks/IO/ReadBufferFromWebServer.h | 3 +- .../AzureBlobStorage/AzureObjectStorage.cpp | 20 +++++---- .../AzureBlobStorage/AzureObjectStorage.h | 22 +++++++--- .../ObjectStorages/DiskObjectStorage.cpp | 16 ++++--- src/Disks/ObjectStorages/DiskObjectStorage.h | 2 +- .../DiskObjectStorageMetadata.cpp | 18 ++++---- .../DiskObjectStorageMetadata.h | 8 ++-- ...jectStorageRemoteMetadataRestoreHelper.cpp | 14 +++--- .../DiskObjectStorageTransaction.cpp | 31 ++++++------- .../ObjectStorages/HDFS/HDFSObjectStorage.cpp | 31 +++++++------ .../ObjectStorages/HDFS/HDFSObjectStorage.h | 25 +++++++---- src/Disks/ObjectStorages/IMetadataStorage.h | 8 ++-- src/Disks/ObjectStorages/IObjectStorage.h | 43 +++++++++++-------- .../MetadataStorageFromDisk.cpp | 20 +++------ .../ObjectStorages/MetadataStorageFromDisk.h | 4 +- .../ObjectStorages/S3/S3ObjectStorage.cpp | 25 ++++++----- src/Disks/ObjectStorages/S3/S3ObjectStorage.h | 36 +++++++++++----- .../System/StorageSystemRemoteDataPaths.cpp | 4 +- 26 files changed, 233 insertions(+), 186 deletions(-) diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index 08a11cd3f03..e17a5aff3c7 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -75,7 +75,7 @@ public: void startup(ContextPtr context) override; void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map) override; String getCacheBasePath() const override { return delegate->getCacheBasePath(); } - std::vector getRemotePaths(const String & path) const override { return delegate->getRemotePaths(path); } + PathsWithSize getObjectStoragePaths(const String & path) const override { return delegate->getObjectStoragePaths(path); } void getRemotePathsRecursive(const String & path, std::vector & paths_map) override { return delegate->getRemotePathsRecursive(path, paths_map); } MetadataStoragePtr getMetadataStorage() override { return delegate->getMetadataStorage(); } diff --git a/src/Disks/DiskRestartProxy.cpp b/src/Disks/DiskRestartProxy.cpp index 674179fa4a0..99dfc8e96a0 100644 --- a/src/Disks/DiskRestartProxy.cpp +++ b/src/Disks/DiskRestartProxy.cpp @@ -318,10 +318,10 @@ String DiskRestartProxy::getCacheBasePath() const return DiskDecorator::getCacheBasePath(); } -std::vector DiskRestartProxy::getRemotePaths(const String & path) const +PathsWithSize DiskRestartProxy::getObjectStoragePaths(const String & path) const { ReadLock lock (mutex); - return DiskDecorator::getRemotePaths(path); + return DiskDecorator::getObjectStoragePaths(path); } void DiskRestartProxy::getRemotePathsRecursive(const String & path, std::vector & paths_map) diff --git a/src/Disks/DiskRestartProxy.h b/src/Disks/DiskRestartProxy.h index 52d68806ab0..e483936c817 100644 --- a/src/Disks/DiskRestartProxy.h +++ b/src/Disks/DiskRestartProxy.h @@ -65,7 +65,7 @@ public: String getUniqueId(const String & path) const override; bool checkUniqueId(const String & id) const override; String getCacheBasePath() const override; - std::vector getRemotePaths(const String & path) const override; + PathsWithSize getObjectStoragePaths(const String & path) const override; void getRemotePathsRecursive(const String & path, std::vector & paths_map) override; void restart(ContextPtr context); diff --git a/src/Disks/DiskWebServer.cpp b/src/Disks/DiskWebServer.cpp index 64c020511a4..83cfbbb0150 100644 --- a/src/Disks/DiskWebServer.cpp +++ b/src/Disks/DiskWebServer.cpp @@ -170,10 +170,10 @@ std::unique_ptr DiskWebServer::readFile(const String & p auto remote_path = fs_path.parent_path() / (escapeForFileName(fs_path.stem()) + fs_path.extension().string()); remote_path = remote_path.string().substr(url.size()); - std::vector blobs_to_read; + PathsWithSize blobs_to_read; blobs_to_read.emplace_back(remote_path, iter->second.size); - auto web_impl = std::make_unique(url, path, blobs_to_read, getContext(), read_settings); + auto web_impl = std::make_unique(url, blobs_to_read, getContext(), read_settings); if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) { diff --git a/src/Disks/DiskWebServer.h b/src/Disks/DiskWebServer.h index 6182c97f70b..40754e71fa0 100644 --- a/src/Disks/DiskWebServer.h +++ b/src/Disks/DiskWebServer.h @@ -169,7 +169,7 @@ public: throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Disk {} is read-only", getName()); } - std::vector getRemotePaths(const String &) const override { return {}; } + PathsWithSize getObjectStoragePaths(const String &) const override { return {}; } void getRemotePathsRecursive(const String &, std::vector &) override {} diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index f441f0827fb..27cac7a5456 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -219,13 +219,13 @@ public: /// Returns a list of paths because for Log family engines there might be /// multiple files in remote fs for single clickhouse file. - virtual std::vector getRemotePaths(const String &) const + virtual PathsWithSize getObjectStoragePaths(const String &) const { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getRemotePaths() not implemented for disk: {}`", getType()); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getObjectStoragePaths() not implemented for disk: {}`", getType()); } /// For one local path there might be multiple remote paths in case of Log family engines. - using LocalPathWithRemotePaths = std::pair>; + using LocalPathWithRemotePaths = std::pair; virtual void getRemotePathsRecursive(const String &, std::vector &) { diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 19f3a12b38d..804fb69a8f5 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -40,7 +40,7 @@ SeekableReadBufferPtr ReadBufferFromRemoteFSGather::createImplementationBuffer(c appendFilesystemCacheLog(); } - current_file_path = fs::path(common_path_prefix) / path; + current_file_path = path; current_file_size = file_size; total_bytes_read_from_current_file = 0; @@ -50,18 +50,30 @@ SeekableReadBufferPtr ReadBufferFromRemoteFSGather::createImplementationBuffer(c #if USE_AWS_S3 SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBufferImpl(const String & path, size_t file_size) { - auto remote_path = fs::path(common_path_prefix) / path; auto remote_file_reader_creator = [=, this]() { return std::make_unique( - client_ptr, bucket, remote_path, version_id, max_single_read_retries, - settings, /* use_external_buffer */true, /* offset */ 0, read_until_position, /* restricted_seek */true); + client_ptr, + bucket, + path, + version_id, + max_single_read_retries, + settings, + /* use_external_buffer */true, + /* offset */0, + read_until_position, + /* restricted_seek */true); }; if (with_cache) { return std::make_shared( - remote_path, settings.remote_fs_cache, remote_file_reader_creator, settings, query_id, read_until_position ? read_until_position : file_size); + path, + settings.remote_fs_cache, + remote_file_reader_creator, + settings, + query_id, + read_until_position ? read_until_position : file_size); } return remote_file_reader_creator(); @@ -82,24 +94,27 @@ SeekableReadBufferPtr ReadBufferFromAzureBlobStorageGather::createImplementation SeekableReadBufferPtr ReadBufferFromWebServerGather::createImplementationBufferImpl(const String & path, size_t /* file_size */) { current_file_path = path; - return std::make_unique(fs::path(uri) / path, context, settings, /* use_external_buffer */true, read_until_position); + return std::make_unique( + path, + context, + settings, + /* use_external_buffer */true, + read_until_position); } #if USE_HDFS SeekableReadBufferPtr ReadBufferFromHDFSGather::createImplementationBufferImpl(const String & path, size_t /* file_size */) { - return std::make_unique(hdfs_uri, fs::path(hdfs_directory) / path, config, settings.remote_fs_buffer_size); + return std::make_unique(hdfs_uri, path, config); } #endif ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather( - const std::string & common_path_prefix_, - const BlobsPathToSize & blobs_to_read_, + const PathsWithSize & blobs_to_read_, const ReadSettings & settings_) : ReadBuffer(nullptr, 0) - , common_path_prefix(common_path_prefix_) , blobs_to_read(blobs_to_read_) , settings(settings_) , query_id(CurrentThread::isInitialized() && CurrentThread::get().getQueryContext() != nullptr ? CurrentThread::getQueryId() : "") @@ -295,7 +310,7 @@ size_t ReadBufferFromRemoteFSGather::getFileSize() const { size_t size = 0; for (const auto & object : blobs_to_read) - size += object.bytes_size; + size += object.size; return size; } diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h index d6b1f9d9479..8edacfd36c9 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.h +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.h @@ -27,8 +27,7 @@ friend class ReadIndirectBufferFromRemoteFS; public: ReadBufferFromRemoteFSGather( - const std::string & common_path_prefix_, - const BlobsPathToSize & blobs_to_read_, + const PathsWithSize & blobs_to_read_, const ReadSettings & settings_); ~ReadBufferFromRemoteFSGather() override; @@ -54,9 +53,7 @@ public: protected: virtual SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) = 0; - std::string common_path_prefix; - - BlobsPathToSize blobs_to_read; + PathsWithSize blobs_to_read; ReadSettings settings; @@ -112,11 +109,10 @@ public: std::shared_ptr client_ptr_, const String & bucket_, const String & version_id_, - const std::string & common_path_prefix_, - const BlobsPathToSize & blobs_to_read_, + const PathsWithSize & blobs_to_read_, size_t max_single_read_retries_, const ReadSettings & settings_) - : ReadBufferFromRemoteFSGather(common_path_prefix_, blobs_to_read_, settings_) + : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_) , client_ptr(std::move(client_ptr_)) , bucket(bucket_) , version_id(version_id_) @@ -142,12 +138,11 @@ class ReadBufferFromAzureBlobStorageGather final : public ReadBufferFromRemoteFS public: ReadBufferFromAzureBlobStorageGather( std::shared_ptr blob_container_client_, - const std::string & common_path_prefix_, - const BlobsPathToSize & blobs_to_read_, + const PathsWithSize & blobs_to_read_, size_t max_single_read_retries_, size_t max_single_download_retries_, const ReadSettings & settings_) - : ReadBufferFromRemoteFSGather(common_path_prefix_, blobs_to_read_, settings_) + : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_) , blob_container_client(blob_container_client_) , max_single_read_retries(max_single_read_retries_) , max_single_download_retries(max_single_download_retries_) @@ -169,11 +164,10 @@ class ReadBufferFromWebServerGather final : public ReadBufferFromRemoteFSGather public: ReadBufferFromWebServerGather( const String & uri_, - const std::string & common_path_prefix_, - const BlobsPathToSize & blobs_to_read_, + const PathsWithSize & blobs_to_read_, ContextPtr context_, const ReadSettings & settings_) - : ReadBufferFromRemoteFSGather(common_path_prefix_, blobs_to_read_, settings_) + : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_) , uri(uri_) , context(context_) { @@ -195,15 +189,12 @@ public: ReadBufferFromHDFSGather( const Poco::Util::AbstractConfiguration & config_, const String & hdfs_uri_, - const std::string & common_path_prefix_, - const BlobsPathToSize & blobs_to_read_, + const PathsWithSize & blobs_to_read_, const ReadSettings & settings_) - : ReadBufferFromRemoteFSGather(common_path_prefix_, blobs_to_read_, settings_) + : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_) , config(config_) + , hdfs_uri(hdfs_uri_) { - const size_t begin_of_path = hdfs_uri_.find('/', hdfs_uri_.find("//") + 2); - hdfs_directory = hdfs_uri_.substr(begin_of_path); - hdfs_uri = hdfs_uri_.substr(0, begin_of_path); } SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) override; @@ -211,7 +202,6 @@ public: private: const Poco::Util::AbstractConfiguration & config; String hdfs_uri; - String hdfs_directory; }; #endif diff --git a/src/Disks/IO/ReadBufferFromWebServer.h b/src/Disks/IO/ReadBufferFromWebServer.h index ea746fb75a1..1e4219d53ee 100644 --- a/src/Disks/IO/ReadBufferFromWebServer.h +++ b/src/Disks/IO/ReadBufferFromWebServer.h @@ -19,7 +19,8 @@ class ReadBufferFromWebServer : public SeekableReadBuffer { public: explicit ReadBufferFromWebServer( - const String & url_, ContextPtr context_, + const String & url_, + ContextPtr context_, const ReadSettings & settings_ = {}, bool use_external_buffer_ = false, size_t read_until_position = 0); diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp index 4ea7c609a51..32fd285dcdb 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp @@ -67,16 +67,18 @@ std::unique_ptr AzureObjectStorage::readObject( /// NOLINT } std::unique_ptr AzureObjectStorage::readObjects( /// NOLINT - const std::string & common_path_prefix, - const BlobsPathToSize & blobs_to_read, + const PathsWithSize & paths_to_read, const ReadSettings & read_settings, std::optional, std::optional) const { auto settings_ptr = settings.get(); auto reader_impl = std::make_unique( - client.get(), common_path_prefix, blobs_to_read, - settings_ptr->max_single_read_retries, settings_ptr->max_single_download_retries, read_settings); + client.get(), + paths_to_read, + settings_ptr->max_single_read_retries, + settings_ptr->max_single_download_retries, + read_settings); if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) { @@ -111,7 +113,7 @@ std::unique_ptr AzureObjectStorage::writeObject( /// NO return std::make_unique(std::move(buffer), std::move(finalize_callback), path); } -void AzureObjectStorage::listPrefix(const std::string & path, BlobsPathToSize & children) const +void AzureObjectStorage::listPrefix(const std::string & path, PathsWithSize & children) const { auto client_ptr = client.get(); @@ -134,10 +136,10 @@ void AzureObjectStorage::removeObject(const std::string & path) throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Failed to delete file in AzureBlob Storage: {}", path); } -void AzureObjectStorage::removeObjects(const std::vector & paths) +void AzureObjectStorage::removeObjects(const PathsWithSize & paths) { auto client_ptr = client.get(); - for (const auto & path : paths) + for (const auto & [path, _] : paths) { auto delete_info = client_ptr->DeleteBlob(path); if (!delete_info.Value.Deleted) @@ -151,10 +153,10 @@ void AzureObjectStorage::removeObjectIfExists(const std::string & path) auto delete_info = client_ptr->DeleteBlob(path); } -void AzureObjectStorage::removeObjectsIfExist(const std::vector & paths) +void AzureObjectStorage::removeObjectsIfExist(const PathsWithSize & paths) { auto client_ptr = client.get(); - for (const auto & path : paths) + for (const auto & [path, _] : paths) auto delete_info = client_ptr->DeleteBlob(path); } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 37c3ba72ed9..559be0ad257 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -59,8 +59,7 @@ public: std::optional file_size = {}) const override; std::unique_ptr readObjects( /// NOLINT - const std::string & common_path_prefix, - const BlobsPathToSize & blobs_to_read, + const PathsWithSize & blobs_to_read, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const override; @@ -74,15 +73,16 @@ public: size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, const WriteSettings & write_settings = {}) override; - void listPrefix(const std::string & path, BlobsPathToSize & children) const override; + void listPrefix(const std::string & path, PathsWithSize & children) const override; + /// Remove file. Throws exception if file doesn't exists or it's a directory. void removeObject(const std::string & path) override; - void removeObjects(const std::vector & paths) override; + void removeObjects(const PathsWithSize & paths) override; void removeObjectIfExists(const std::string & path) override; - void removeObjectsIfExist(const std::vector & paths) override; + void removeObjectsIfExist(const PathsWithSize & paths) override; ObjectMetadata getObjectMetadata(const std::string & path) const override; @@ -95,11 +95,19 @@ public: void startup() override {} - void applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) override; + void applyNewSettings( + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) override; String getObjectsNamespace() const override { return ""; } - std::unique_ptr cloneObjectStorage(const std::string & new_namespace, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) override; + std::unique_ptr cloneObjectStorage( + const std::string & new_namespace, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) override; + private: const String name; diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index 0f2c320ed67..540672d9b0a 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -108,9 +108,9 @@ DiskObjectStorage::DiskObjectStorage( , metadata_helper(std::make_unique(this, ReadSettings{})) {} -std::vector DiskObjectStorage::getRemotePaths(const String & local_path) const +PathsWithSize DiskObjectStorage::getObjectStoragePaths(const String & local_path) const { - return metadata_storage->getRemotePaths(local_path); + return metadata_storage->getObjectStoragePaths(local_path); } void DiskObjectStorage::getRemotePathsRecursive(const String & local_path, std::vector & paths_map) @@ -120,7 +120,7 @@ void DiskObjectStorage::getRemotePathsRecursive(const String & local_path, std:: { try { - paths_map.emplace_back(local_path, getRemotePaths(local_path)); + paths_map.emplace_back(local_path, getObjectStoragePaths(local_path)); } catch (const Exception & e) { @@ -244,9 +244,9 @@ String DiskObjectStorage::getUniqueId(const String & path) const { LOG_TRACE(log, "Remote path: {}, Path: {}", remote_fs_root_path, path); String id; - auto blobs_paths = metadata_storage->getRemotePaths(path); + auto blobs_paths = metadata_storage->getObjectStoragePaths(path); if (!blobs_paths.empty()) - id = blobs_paths[0]; + id = blobs_paths[0].path; return id; } @@ -438,7 +438,11 @@ std::unique_ptr DiskObjectStorage::readFile( std::optional read_hint, std::optional file_size) const { - return object_storage->readObjects(remote_fs_root_path, metadata_storage->getBlobs(path), settings, read_hint, file_size); + return object_storage->readObjects( + metadata_storage->getObjectStoragePaths(path), + settings, + read_hint, + file_size); } std::unique_ptr DiskObjectStorage::writeFile( diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h index b1a1d263ede..ef29dc8f071 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.h +++ b/src/Disks/ObjectStorages/DiskObjectStorage.h @@ -49,7 +49,7 @@ public: const String & getPath() const override { return metadata_storage->getPath(); } - std::vector getRemotePaths(const String & local_path) const override; + PathsWithSize getObjectStoragePaths(const String & local_path) const override; void getRemotePathsRecursive(const String & local_path, std::vector & paths_map) override; diff --git a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp index 6ed049b865d..6763e37ed69 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp @@ -26,14 +26,14 @@ void DiskObjectStorageMetadata::deserialize(ReadBuffer & buf) assertChar('\n', buf); - UInt32 remote_fs_objects_count; - readIntText(remote_fs_objects_count, buf); + UInt32 storage_objects_count; + readIntText(storage_objects_count, buf); assertChar('\t', buf); readIntText(total_size, buf); assertChar('\n', buf); - remote_fs_objects.resize(remote_fs_objects_count); + storage_objects.resize(storage_objects_count); - for (size_t i = 0; i < remote_fs_objects_count; ++i) + for (size_t i = 0; i < storage_objects_count; ++i) { String remote_fs_object_path; size_t remote_fs_object_size; @@ -50,8 +50,8 @@ void DiskObjectStorageMetadata::deserialize(ReadBuffer & buf) remote_fs_object_path = remote_fs_object_path.substr(remote_fs_root_path.size()); } assertChar('\n', buf); - remote_fs_objects[i].relative_path = remote_fs_object_path; - remote_fs_objects[i].bytes_size = remote_fs_object_size; + storage_objects[i].path = remote_fs_object_path; + storage_objects[i].size = remote_fs_object_size; } readIntText(ref_count, buf); @@ -75,12 +75,12 @@ void DiskObjectStorageMetadata::serialize(WriteBuffer & buf, bool sync) const writeIntText(VERSION_READ_ONLY_FLAG, buf); writeChar('\n', buf); - writeIntText(remote_fs_objects.size(), buf); + writeIntText(storage_objects.size(), buf); writeChar('\t', buf); writeIntText(total_size, buf); writeChar('\n', buf); - for (const auto & [remote_fs_object_path, remote_fs_object_size] : remote_fs_objects) + for (const auto & [remote_fs_object_path, remote_fs_object_size] : storage_objects) { writeIntText(remote_fs_object_size, buf); writeChar('\t', buf); @@ -120,7 +120,7 @@ DiskObjectStorageMetadata::DiskObjectStorageMetadata( void DiskObjectStorageMetadata::addObject(const String & path, size_t size) { total_size += size; - remote_fs_objects.emplace_back(path, size); + storage_objects.emplace_back(path, size); } diff --git a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h index 27df9e57cf4..14dd819b4f8 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h +++ b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h @@ -19,8 +19,8 @@ private: const std::string & common_metadata_path; - /// Remote FS objects paths and their sizes. - std::vector remote_fs_objects; + /// Relative paths of blobs. + std::vector storage_objects; /// URI const std::string & remote_fs_root_path; @@ -60,9 +60,9 @@ public: return remote_fs_root_path; } - std::vector getBlobs() const + std::vector getBlobs() const { - return remote_fs_objects; + return storage_objects; } bool isReadOnly() const diff --git a/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp b/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp index 96667b8496a..f820d06646a 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp @@ -84,13 +84,13 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::migrateFileToRestorableSchema { LOG_TRACE(disk->log, "Migrate file {} to restorable schema", disk->metadata_storage->getPath() + path); - auto blobs = disk->metadata_storage->getBlobs(path); - for (const auto & [key, _] : blobs) + auto objects = disk->metadata_storage->getObjectStoragePaths(path); + for (const auto & [object_path, size] : objects) { ObjectAttributes metadata { {"path", path} }; - updateObjectMetadata(disk->remote_fs_root_path + key, metadata); + updateObjectMetadata(object_path, metadata); } } void DiskObjectStorageRemoteMetadataRestoreHelper::migrateToRestorableSchemaRecursive(const String & path, Futures & results) @@ -346,7 +346,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFiles(IObjectStorage * LOG_INFO(disk->log, "Starting restore files for disk {}", disk->name); std::vector> results; - auto restore_files = [this, &source_object_storage, &restore_information, &results](const BlobsPathToSize & keys) + auto restore_files = [this, &source_object_storage, &restore_information, &results](const PathsWithSize & keys) { std::vector keys_names; for (const auto & [key, size] : keys) @@ -379,7 +379,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFiles(IObjectStorage * return true; }; - BlobsPathToSize children; + PathsWithSize children; source_object_storage->listPrefix(restore_information.source_path, children); restore_files(children); @@ -456,7 +456,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFileOperations(IObject bool send_metadata = source_object_storage->getObjectsNamespace() != disk->object_storage->getObjectsNamespace() || disk->remote_fs_root_path != restore_information.source_path; std::set renames; - auto restore_file_operations = [this, &source_object_storage, &restore_information, &renames, &send_metadata](const BlobsPathToSize & keys) + auto restore_file_operations = [this, &source_object_storage, &restore_information, &renames, &send_metadata](const PathsWithSize & keys) { const String rename = "rename"; const String hardlink = "hardlink"; @@ -523,7 +523,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::restoreFileOperations(IObject return true; }; - BlobsPathToSize children; + PathsWithSize children; source_object_storage->listPrefix(restore_information.source_path + "operations/", children); restore_file_operations(children); diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index cd43ea81be0..1d16012437a 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -65,7 +65,7 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation std::string path; bool delete_metadata_only; bool remove_from_cache{false}; - std::vector paths_to_remove; + PathsWithSize paths_to_remove; bool if_exists; RemoveObjectStorageOperation( @@ -96,13 +96,13 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation try { uint32_t hardlink_count = metadata_storage.getHardlinkCount(path); - auto remote_objects = metadata_storage.getRemotePaths(path); + auto objects = metadata_storage.getObjectStoragePaths(path); tx->unlinkMetadata(path); if (hardlink_count == 0) { - paths_to_remove = remote_objects; + paths_to_remove = objects; remove_from_cache = true; } } @@ -134,7 +134,7 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation if (remove_from_cache) { for (const auto & path_to_remove : paths_to_remove) - object_storage.removeFromCache(path_to_remove); + object_storage.removeFromCache(path_to_remove.path); } } @@ -143,10 +143,10 @@ struct RemoveObjectStorageOperation final : public IDiskObjectStorageOperation struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOperation { std::string path; - std::unordered_map> paths_to_remove; + std::unordered_map paths_to_remove; bool keep_all_batch_data; NameSet file_names_remove_metadata_only; - std::vector path_to_remove_from_cache; + PathsWithSize path_to_remove_from_cache; RemoveRecursiveObjectStorageOperation( IObjectStorage & object_storage_, @@ -169,14 +169,14 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp try { uint32_t hardlink_count = metadata_storage.getHardlinkCount(path_to_remove); - auto remote_objects = metadata_storage.getRemotePaths(path_to_remove); + auto objects_paths = metadata_storage.getObjectStoragePaths(path_to_remove); tx->unlinkMetadata(path_to_remove); if (hardlink_count == 0) { - paths_to_remove[path_to_remove] = remote_objects; - path_to_remove_from_cache.insert(path_to_remove_from_cache.end(), remote_objects.begin(), remote_objects.end()); + paths_to_remove[path_to_remove] = objects_paths; + path_to_remove_from_cache.insert(path_to_remove_from_cache.end(), objects_paths.begin(), objects_paths.end()); } } @@ -217,7 +217,7 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp { if (!keep_all_batch_data) { - std::vector remove_from_remote; + PathsWithSize remove_from_remote; for (auto && [local_path, remote_paths] : paths_to_remove) { if (!file_names_remove_metadata_only.contains(fs::path(local_path).filename())) @@ -228,7 +228,7 @@ struct RemoveRecursiveObjectStorageOperation final : public IDiskObjectStorageOp object_storage.removeObjects(remove_from_remote); } - for (const auto & path_to_remove : path_to_remove_from_cache) + for (const auto & [path_to_remove, _] : path_to_remove_from_cache) object_storage.removeFromCache(path_to_remove); } }; @@ -238,7 +238,7 @@ struct ReplaceFileObjectStorageOperation final : public IDiskObjectStorageOperat { std::string path_from; std::string path_to; - std::vector blobs_to_remove; + PathsWithSize blobs_to_remove; ReplaceFileObjectStorageOperation( IObjectStorage & object_storage_, @@ -254,7 +254,7 @@ struct ReplaceFileObjectStorageOperation final : public IDiskObjectStorageOperat { if (metadata_storage.exists(path_to)) { - blobs_to_remove = metadata_storage.getRemotePaths(path_to); + blobs_to_remove = metadata_storage.getObjectStoragePaths(path_to); tx->replaceFile(path_from, path_to); } else @@ -328,14 +328,15 @@ struct CopyFileObjectStorageOperation final : public IDiskObjectStorageOperation void execute(MetadataTransactionPtr tx) override { tx->createEmptyMetadataFile(to_path); - auto source_blobs = metadata_storage.getBlobs(from_path); + auto source_blobs = metadata_storage.getObjectStoragePaths(from_path); /// Full paths + for (const auto & [blob_from, size] : source_blobs) { auto blob_name = getRandomASCIIString(); auto blob_to = fs::path(remote_fs_root_path) / blob_name; - object_storage.copyObject(fs::path(remote_fs_root_path) / blob_from, blob_to); + object_storage.copyObject(blob_from, blob_to); tx->addBlobToMetadata(to_path, blob_name, size); diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp index 4574b8cb52c..024d1e84ef0 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp @@ -49,13 +49,12 @@ std::unique_ptr HDFSObjectStorage::readObject( /// NOLINT } std::unique_ptr HDFSObjectStorage::readObjects( /// NOLINT - const std::string & common_path_prefix, - const BlobsPathToSize & blobs_to_read, + const PathsWithSize & paths_to_read, const ReadSettings & read_settings, std::optional, std::optional) const { - auto hdfs_impl = std::make_unique(config, common_path_prefix, common_path_prefix, blobs_to_read, read_settings); + auto hdfs_impl = std::make_unique(config, hdfs_root_path, paths_to_read, read_settings); auto buf = std::make_unique(std::move(hdfs_impl)); return std::make_unique(std::move(buf), settings->min_bytes_for_seek); } @@ -69,7 +68,9 @@ std::unique_ptr HDFSObjectStorage::writeObject( /// NOL const WriteSettings &) { if (attributes.has_value()) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "HDFS API doesn't support custom attributes/metadata for stored objects"); + throw Exception( + ErrorCodes::UNSUPPORTED_METHOD, + "HDFS API doesn't support custom attributes/metadata for stored objects"); /// Single O_WRONLY in libhdfs adds O_TRUNC auto hdfs_buffer = std::make_unique( @@ -80,7 +81,7 @@ std::unique_ptr HDFSObjectStorage::writeObject( /// NOL } -void HDFSObjectStorage::listPrefix(const std::string & path, BlobsPathToSize & children) const +void HDFSObjectStorage::listPrefix(const std::string & path, PathsWithSize & children) const { const size_t begin_of_path = path.find('/', path.find("//") + 2); int32_t num_entries; @@ -104,10 +105,10 @@ void HDFSObjectStorage::removeObject(const std::string & path) } -void HDFSObjectStorage::removeObjects(const std::vector & paths) +void HDFSObjectStorage::removeObjects(const PathsWithSize & paths) { - for (const auto & hdfs_path : paths) - removeObject(hdfs_path); + for (const auto & [path, _] : paths) + removeObject(path); } void HDFSObjectStorage::removeObjectIfExists(const std::string & path) @@ -116,15 +117,17 @@ void HDFSObjectStorage::removeObjectIfExists(const std::string & path) removeObject(path); } -void HDFSObjectStorage::removeObjectsIfExist(const std::vector & paths) +void HDFSObjectStorage::removeObjectsIfExist(const PathsWithSize & paths) { - for (const auto & hdfs_path : paths) - removeObjectIfExists(hdfs_path); + for (const auto & [path, _] : paths) + removeObjectIfExists(path); } ObjectMetadata HDFSObjectStorage::getObjectMetadata(const std::string &) const { - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "HDFS API doesn't support custom attributes/metadata for stored objects"); + throw Exception( + ErrorCodes::UNSUPPORTED_METHOD, + "HDFS API doesn't support custom attributes/metadata for stored objects"); } void HDFSObjectStorage::copyObject( /// NOLINT @@ -133,7 +136,9 @@ void HDFSObjectStorage::copyObject( /// NOLINT std::optional object_to_attributes) { if (object_to_attributes.has_value()) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "HDFS API doesn't support custom attributes/metadata for stored objects"); + throw Exception( + ErrorCodes::UNSUPPORTED_METHOD, + "HDFS API doesn't support custom attributes/metadata for stored objects"); auto in = readObject(object_from); auto out = writeObject(object_to, WriteMode::Rewrite); diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h index a9a223a3d7e..221a77ff08b 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h @@ -50,6 +50,7 @@ public: , hdfs_builder(createHDFSBuilder(hdfs_root_path_, config)) , hdfs_fs(createHDFSFS(hdfs_builder.get())) , settings(std::move(settings_)) + , hdfs_root_path(hdfs_root_path_) {} bool exists(const std::string & hdfs_uri) const override; @@ -61,8 +62,7 @@ public: std::optional file_size = {}) const override; std::unique_ptr readObjects( /// NOLINT - const std::string & common_path_prefix, - const BlobsPathToSize & blobs_to_read, + const PathsWithSize & paths_to_read, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const override; @@ -76,15 +76,16 @@ public: size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, const WriteSettings & write_settings = {}) override; - void listPrefix(const std::string & path, BlobsPathToSize & children) const override; + void listPrefix(const std::string & path, PathsWithSize & children) const override; + /// Remove file. Throws exception if file doesn't exists or it's a directory. void removeObject(const std::string & path) override; - void removeObjects(const std::vector & paths) override; + void removeObjects(const PathsWithSize & paths) override; void removeObjectIfExists(const std::string & path) override; - void removeObjectsIfExist(const std::vector & paths) override; + void removeObjectsIfExist(const PathsWithSize & paths) override; ObjectMetadata getObjectMetadata(const std::string & path) const override; @@ -97,11 +98,18 @@ public: void startup() override; - void applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) override; + void applyNewSettings( + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) override; String getObjectsNamespace() const override { return ""; } - std::unique_ptr cloneObjectStorage(const std::string & new_namespace, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) override; + std::unique_ptr cloneObjectStorage( + const std::string & new_namespace, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) override; private: const Poco::Util::AbstractConfiguration & config; @@ -110,8 +118,7 @@ private: HDFSFSPtr hdfs_fs; SettingsPtr settings; - - + String hdfs_root_path; }; } diff --git a/src/Disks/ObjectStorages/IMetadataStorage.h b/src/Disks/ObjectStorages/IMetadataStorage.h index fbf611df1ac..49f4d4bcc51 100644 --- a/src/Disks/ObjectStorages/IMetadataStorage.h +++ b/src/Disks/ObjectStorages/IMetadataStorage.h @@ -119,11 +119,9 @@ public: /// Read multiple metadata files into strings and return mapping from file_path -> metadata virtual std::unordered_map getSerializedMetadata(const std::vector & file_paths) const = 0; - /// Return list of paths corresponding to metadata stored in local path - virtual std::vector getRemotePaths(const std::string & path) const = 0; - - /// Return [(remote_path, size_in_bytes), ...] for metadata path - virtual BlobsPathToSize getBlobs(const std::string & path) const = 0; + /// Return [(object_storage_path, size_in_bytes), ...] for metadata path + /// object_storage_path is a full path to the blob. + virtual PathsWithSize getObjectStoragePaths(const std::string & path) const = 0; }; using MetadataStoragePtr = std::shared_ptr; diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h index 4921059c6b7..864b9ae8caa 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.h +++ b/src/Disks/ObjectStorages/IObjectStorage.h @@ -25,23 +25,23 @@ class WriteBufferFromFileBase; using ObjectAttributes = std::map; -/// Path to blob with it's size -struct BlobPathWithSize +/// Path to a file and its size. +/// Path can be either relative or absolute - according to the context of use. +struct PathWithSize { - std::string relative_path; - uint64_t bytes_size; + std::string path; + uint64_t size; /// Size in bytes - BlobPathWithSize() = default; - BlobPathWithSize(const BlobPathWithSize & other) = default; + PathWithSize() = default; - BlobPathWithSize(const std::string & relative_path_, uint64_t bytes_size_) - : relative_path(relative_path_) - , bytes_size(bytes_size_) + PathWithSize(const std::string & path_, uint64_t size_) + : path(path_) + , size(size_) {} }; -/// List of blobs with their sizes -using BlobsPathToSize = std::vector; +/// List of paths with their sizes +using PathsWithSize = std::vector; struct ObjectMetadata { @@ -65,8 +65,8 @@ public: /// Path exists or not virtual bool exists(const std::string & path) const = 0; - /// List on prefix, return children with their sizes. - virtual void listPrefix(const std::string & path, BlobsPathToSize & children) const = 0; + /// List on prefix, return children (relative paths) with their sizes. + virtual void listPrefix(const std::string & path, PathsWithSize & children) const = 0; /// Get object metadata if supported. It should be possible to receive /// at least size of object @@ -81,8 +81,7 @@ public: /// Read multiple objects with common prefix virtual std::unique_ptr readObjects( /// NOLINT - const std::string & common_path_prefix, - const BlobsPathToSize & blobs_to_read, + const PathsWithSize & paths_to_read, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const = 0; @@ -101,13 +100,13 @@ public: /// Remove multiple objects. Some object storages can do batch remove in a more /// optimal way. - virtual void removeObjects(const std::vector & paths) = 0; + virtual void removeObjects(const PathsWithSize & paths) = 0; /// Remove object on path if exists virtual void removeObjectIfExists(const std::string & path) = 0; /// Remove objects on path if exists - virtual void removeObjectsIfExist(const std::vector & paths) = 0; + virtual void removeObjectsIfExist(const PathsWithSize & paths) = 0; /// Copy object with different attributes if required virtual void copyObject( /// NOLINT @@ -140,7 +139,10 @@ public: void removeFromCache(const std::string & path); /// Apply new settings, in most cases reiniatilize client and some other staff - virtual void applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) = 0; + virtual void applyNewSettings( + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) = 0; /// Sometimes object storages have something similar to chroot or namespace, for example /// buckets in S3. If object storage doesn't have any namepaces return empty string. @@ -148,7 +150,10 @@ public: /// FIXME: confusing function required for a very specific case. Create new instance of object storage /// in different namespace. - virtual std::unique_ptr cloneObjectStorage(const std::string & new_namespace, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) = 0; + virtual std::unique_ptr cloneObjectStorage( + const std::string & new_namespace, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, ContextPtr context) = 0; protected: FileCachePtr cache; diff --git a/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp b/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp index ae87e8c61c0..393603ff11a 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp @@ -300,18 +300,18 @@ MetadataTransactionPtr MetadataStorageFromDisk::createTransaction() const return std::make_shared(*this); } -std::vector MetadataStorageFromDisk::getRemotePaths(const std::string & path) const +PathsWithSize MetadataStorageFromDisk::getObjectStoragePaths(const std::string & path) const { auto metadata = readMetadata(path); - std::vector remote_paths; - auto blobs = metadata->getBlobs(); + PathsWithSize object_storage_paths = metadata->getBlobs(); /// Relative paths. auto root_path = metadata->getBlobsCommonPrefix(); - remote_paths.reserve(blobs.size()); - for (const auto & [remote_path, _] : blobs) - remote_paths.push_back(fs::path(root_path) / remote_path); - return remote_paths; + /// Relative paths -> absolute. + for (auto & [object_path, _] : object_storage_paths) + object_path = fs::path(root_path) / object_path; + + return object_storage_paths; } uint32_t MetadataStorageFromDisk::getHardlinkCount(const std::string & path) const @@ -320,12 +320,6 @@ uint32_t MetadataStorageFromDisk::getHardlinkCount(const std::string & path) con return metadata->getRefCount(); } -BlobsPathToSize MetadataStorageFromDisk::getBlobs(const std::string & path) const -{ - auto metadata = readMetadata(path); - return metadata->getBlobs(); -} - void MetadataStorageFromDiskTransaction::unlinkMetadata(const std::string & path) { auto metadata = metadata_storage.readMetadata(path); diff --git a/src/Disks/ObjectStorages/MetadataStorageFromDisk.h b/src/Disks/ObjectStorages/MetadataStorageFromDisk.h index 1ac68e193f2..8dd47d18fdb 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromDisk.h +++ b/src/Disks/ObjectStorages/MetadataStorageFromDisk.h @@ -59,9 +59,7 @@ public: std::unordered_map getSerializedMetadata(const std::vector & file_paths) const override; - BlobsPathToSize getBlobs(const std::string & path) const override; - - std::vector getRemotePaths(const std::string & path) const override; + PathsWithSize getObjectStoragePaths(const std::string & path) const override; uint32_t getHardlinkCount(const std::string & path) const override; diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index ffe4d2dd942..62995e04e9b 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -109,8 +109,7 @@ bool S3ObjectStorage::exists(const std::string & path) const std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT - const std::string & common_path_prefix, - const BlobsPathToSize & blobs_to_read, + const PathsWithSize & paths_to_read, const ReadSettings & read_settings, std::optional, std::optional) const @@ -128,8 +127,12 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT auto settings_ptr = s3_settings.get(); auto s3_impl = std::make_unique( - client.get(), bucket, version_id, common_path_prefix, blobs_to_read, - settings_ptr->s3_settings.max_single_read_retries, disk_read_settings); + client.get(), + bucket, + version_id, + paths_to_read, + settings_ptr->s3_settings.max_single_read_retries, + read_settings); if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) { @@ -192,7 +195,7 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN return std::make_unique(std::move(s3_buffer), std::move(finalize_callback), path); } -void S3ObjectStorage::listPrefix(const std::string & path, BlobsPathToSize & children) const +void S3ObjectStorage::listPrefix(const std::string & path, PathsWithSize & children) const { auto settings_ptr = s3_settings.get(); auto client_ptr = client.get(); @@ -253,14 +256,14 @@ void S3ObjectStorage::removeObjectImpl(const std::string & path, bool if_exists) } } -void S3ObjectStorage::removeObjectsImpl(const std::vector & paths, bool if_exists) +void S3ObjectStorage::removeObjectsImpl(const PathsWithSize & paths, bool if_exists) { if (paths.empty()) return; if (!s3_capabilities.support_batch_delete) { - for (const auto & path : paths) + for (const auto & [path, _] : paths) removeObjectImpl(path, if_exists); } else @@ -278,12 +281,12 @@ void S3ObjectStorage::removeObjectsImpl(const std::vector & paths, for (; current_position < paths.size() && current_chunk.size() < chunk_size_limit; ++current_position) { Aws::S3::Model::ObjectIdentifier obj; - obj.SetKey(paths[current_position]); + obj.SetKey(paths[current_position].path); current_chunk.push_back(obj); if (!keys.empty()) keys += ", "; - keys += paths[current_position]; + keys += paths[current_position].path; } Aws::S3::Model::Delete delkeys; @@ -308,12 +311,12 @@ void S3ObjectStorage::removeObjectIfExists(const std::string & path) removeObjectImpl(path, true); } -void S3ObjectStorage::removeObjects(const std::vector & paths) +void S3ObjectStorage::removeObjects(const PathsWithSize & paths) { removeObjectsImpl(paths, false); } -void S3ObjectStorage::removeObjectsIfExist(const std::vector & paths) +void S3ObjectStorage::removeObjectsIfExist(const PathsWithSize & paths) { removeObjectsImpl(paths, true); } diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index 16e6c9c8cd5..5c53ea1f894 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -66,8 +66,7 @@ public: std::optional file_size = {}) const override; std::unique_ptr readObjects( /// NOLINT - const std::string & common_path_prefix, - const BlobsPathToSize & blobs_to_read, + const PathsWithSize & paths_to_read, const ReadSettings & read_settings = ReadSettings{}, std::optional read_hint = {}, std::optional file_size = {}) const override; @@ -81,15 +80,16 @@ public: size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, const WriteSettings & write_settings = {}) override; - void listPrefix(const std::string & path, BlobsPathToSize & children) const override; + void listPrefix(const std::string & path, PathsWithSize & children) const override; + /// Remove file. Throws exception if file doesn't exist or it's a directory. void removeObject(const std::string & path) override; - void removeObjects(const std::vector & paths) override; + void removeObjects(const PathsWithSize & paths) override; void removeObjectIfExists(const std::string & path) override; - void removeObjectsIfExist(const std::vector & paths) override; + void removeObjectsIfExist(const PathsWithSize & paths) override; ObjectMetadata getObjectMetadata(const std::string & path) const override; @@ -108,26 +108,42 @@ public: void startup() override; - void applyNewSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) override; + void applyNewSettings( + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) override; String getObjectsNamespace() const override { return bucket; } - std::unique_ptr cloneObjectStorage(const std::string & new_namespace, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, ContextPtr context) override; + std::unique_ptr cloneObjectStorage( + const std::string & new_namespace, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + ContextPtr context) override; + private: void setNewSettings(std::unique_ptr && s3_settings_); void setNewClient(std::unique_ptr && client_); - void copyObjectImpl(const String & src_bucket, const String & src_key, const String & dst_bucket, const String & dst_key, + void copyObjectImpl( + const String & src_bucket, + const String & src_key, + const String & dst_bucket, + const String & dst_key, std::optional head = std::nullopt, std::optional metadata = std::nullopt) const; - void copyObjectMultipartImpl(const String & src_bucket, const String & src_key, const String & dst_bucket, const String & dst_key, + void copyObjectMultipartImpl( + const String & src_bucket, + const String & src_key, + const String & dst_bucket, + const String & dst_key, std::optional head = std::nullopt, std::optional metadata = std::nullopt) const; void removeObjectImpl(const std::string & path, bool if_exists); - void removeObjectsImpl(const std::vector & paths, bool if_exists); + void removeObjectsImpl(const PathsWithSize & paths, bool if_exists); Aws::S3::Model::HeadObjectOutcome requestObjectHeadData(const std::string & bucket_from, const std::string & key) const; diff --git a/src/Storages/System/StorageSystemRemoteDataPaths.cpp b/src/Storages/System/StorageSystemRemoteDataPaths.cpp index a009f9d25c9..d39a0d2482d 100644 --- a/src/Storages/System/StorageSystemRemoteDataPaths.cpp +++ b/src/Storages/System/StorageSystemRemoteDataPaths.cpp @@ -68,11 +68,11 @@ Pipe StorageSystemRemoteDataPaths::read( col_base_path->insert(disk->getPath()); col_cache_base_path->insert(cache_base_path); col_local_path->insert(local_path); - col_remote_path->insert(remote_path); + col_remote_path->insert(remote_path.path); if (cache) { - auto cache_paths = cache->tryGetCachePaths(cache->hash(remote_path)); + auto cache_paths = cache->tryGetCachePaths(cache->hash(remote_path.path)); col_cache_paths->insert(Array(cache_paths.begin(), cache_paths.end())); } else From 8da6136f882866c3dafbde05bed6a2333b276870 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 26 Jun 2022 13:21:05 +0200 Subject: [PATCH 097/123] Fix --- src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 2 +- src/Disks/IO/ReadBufferFromRemoteFSGather.h | 3 +++ src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 804fb69a8f5..e8197eba008 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -95,7 +95,7 @@ SeekableReadBufferPtr ReadBufferFromWebServerGather::createImplementationBufferI { current_file_path = path; return std::make_unique( - path, + fs::path(uri) / path, context, settings, /* use_external_buffer */true, diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h index 8edacfd36c9..1c4cdc678f4 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.h +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.h @@ -195,6 +195,8 @@ public: , config(config_) , hdfs_uri(hdfs_uri_) { + const size_t begin_of_path = hdfs_uri_.find('/', hdfs_uri_.find("//") + 2); + hdfs_uri = hdfs_uri_.substr(0, begin_of_path); } SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) override; @@ -203,6 +205,7 @@ private: const Poco::Util::AbstractConfiguration & config; String hdfs_uri; }; + #endif } diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 62995e04e9b..58bd29d2d73 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -132,7 +132,7 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT version_id, paths_to_read, settings_ptr->s3_settings.max_single_read_retries, - read_settings); + disk_read_settings); if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool) { From a15238d147617d1697e0922e79bdc85b49951f69 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 26 Jun 2022 16:44:55 +0200 Subject: [PATCH 098/123] Review fixes, fix hdfs tests --- src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 9 +++++++-- src/Disks/IO/ReadBufferFromRemoteFSGather.h | 10 +++------- .../ObjectStorages/DiskObjectStorageMetadata.cpp | 8 ++++++-- .../ObjectStorages/DiskObjectStorageMetadata.h | 15 +++++++++++++-- ...skObjectStorageRemoteMetadataRestoreHelper.cpp | 2 +- src/Disks/ObjectStorages/IObjectStorage.h | 6 +++--- .../ObjectStorages/MetadataStorageFromDisk.cpp | 11 +++++++---- 7 files changed, 40 insertions(+), 21 deletions(-) diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index e8197eba008..b53ba0f8e29 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -106,7 +106,12 @@ SeekableReadBufferPtr ReadBufferFromWebServerGather::createImplementationBufferI #if USE_HDFS SeekableReadBufferPtr ReadBufferFromHDFSGather::createImplementationBufferImpl(const String & path, size_t /* file_size */) { - return std::make_unique(hdfs_uri, path, config); + size_t begin_of_path = path.find('/', path.find("//") + 2); + auto hdfs_path = path.substr(begin_of_path); + auto hdfs_uri = path.substr(0, begin_of_path); + LOG_TEST(log, "HDFS uri: {}, path: {}", hdfs_path, hdfs_uri); + + return std::make_unique(hdfs_uri, hdfs_path, config); } #endif @@ -310,7 +315,7 @@ size_t ReadBufferFromRemoteFSGather::getFileSize() const { size_t size = 0; for (const auto & object : blobs_to_read) - size += object.size; + size += object.bytes_size; return size; } diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h index 1c4cdc678f4..e282a2cfc6c 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.h +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.h @@ -66,6 +66,8 @@ protected: String query_id; + Poco::Logger * log; + private: SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size); @@ -92,8 +94,6 @@ private: */ size_t bytes_to_ignore = 0; - Poco::Logger * log; - size_t total_bytes_read_from_current_file = 0; bool enable_cache_log = false; @@ -188,22 +188,18 @@ class ReadBufferFromHDFSGather final : public ReadBufferFromRemoteFSGather public: ReadBufferFromHDFSGather( const Poco::Util::AbstractConfiguration & config_, - const String & hdfs_uri_, + const String &, const PathsWithSize & blobs_to_read_, const ReadSettings & settings_) : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_) , config(config_) - , hdfs_uri(hdfs_uri_) { - const size_t begin_of_path = hdfs_uri_.find('/', hdfs_uri_.find("//") + 2); - hdfs_uri = hdfs_uri_.substr(0, begin_of_path); } SeekableReadBufferPtr createImplementationBufferImpl(const String & path, size_t file_size) override; private: const Poco::Util::AbstractConfiguration & config; - String hdfs_uri; }; #endif diff --git a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp index 6763e37ed69..9ac04809b97 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp @@ -11,6 +11,7 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_FORMAT; + extern const int LOGICAL_ERROR; } void DiskObjectStorageMetadata::deserialize(ReadBuffer & buf) @@ -50,8 +51,8 @@ void DiskObjectStorageMetadata::deserialize(ReadBuffer & buf) remote_fs_object_path = remote_fs_object_path.substr(remote_fs_root_path.size()); } assertChar('\n', buf); - storage_objects[i].path = remote_fs_object_path; - storage_objects[i].size = remote_fs_object_size; + storage_objects[i].relative_path = remote_fs_object_path; + storage_objects[i].bytes_size = remote_fs_object_size; } readIntText(ref_count, buf); @@ -119,6 +120,9 @@ DiskObjectStorageMetadata::DiskObjectStorageMetadata( void DiskObjectStorageMetadata::addObject(const String & path, size_t size) { + if (path.starts_with(remote_fs_root_path)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected relative path"); + total_size += size; storage_objects.emplace_back(path, size); } diff --git a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h index 14dd819b4f8..adebbcb952d 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h +++ b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h @@ -12,6 +12,17 @@ namespace DB struct DiskObjectStorageMetadata { private: + struct RelativePathWithSize + { + String relative_path; + size_t bytes_size; + + RelativePathWithSize() = default; + + RelativePathWithSize(const String & relative_path_, size_t bytes_size_) + : relative_path(relative_path_), bytes_size(bytes_size_) {} + }; + /// Metadata file version. static constexpr uint32_t VERSION_ABSOLUTE_PATHS = 1; static constexpr uint32_t VERSION_RELATIVE_PATHS = 2; @@ -20,7 +31,7 @@ private: const std::string & common_metadata_path; /// Relative paths of blobs. - std::vector storage_objects; + std::vector storage_objects; /// URI const std::string & remote_fs_root_path; @@ -60,7 +71,7 @@ public: return remote_fs_root_path; } - std::vector getBlobs() const + std::vector getBlobsRelativePaths() const { return storage_objects; } diff --git a/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp b/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp index f820d06646a..a8140e8954e 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageRemoteMetadataRestoreHelper.cpp @@ -85,7 +85,7 @@ void DiskObjectStorageRemoteMetadataRestoreHelper::migrateFileToRestorableSchema LOG_TRACE(disk->log, "Migrate file {} to restorable schema", disk->metadata_storage->getPath() + path); auto objects = disk->metadata_storage->getObjectStoragePaths(path); - for (const auto & [object_path, size] : objects) + for (const auto & [object_path, _] : objects) { ObjectAttributes metadata { {"path", path} diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h index 864b9ae8caa..64022ec046d 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.h +++ b/src/Disks/ObjectStorages/IObjectStorage.h @@ -30,13 +30,13 @@ using ObjectAttributes = std::map; struct PathWithSize { std::string path; - uint64_t size; /// Size in bytes + uint64_t bytes_size; PathWithSize() = default; - PathWithSize(const std::string & path_, uint64_t size_) + PathWithSize(const std::string & path_, uint64_t bytes_size_) : path(path_) - , size(size_) + , bytes_size(bytes_size_) {} }; diff --git a/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp b/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp index 393603ff11a..32e6fe5834d 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFromDisk.cpp @@ -304,12 +304,15 @@ PathsWithSize MetadataStorageFromDisk::getObjectStoragePaths(const std::string & { auto metadata = readMetadata(path); - PathsWithSize object_storage_paths = metadata->getBlobs(); /// Relative paths. - auto root_path = metadata->getBlobsCommonPrefix(); + auto object_storage_relative_paths = metadata->getBlobsRelativePaths(); /// Relative paths. + fs::path root_path = metadata->getBlobsCommonPrefix(); + + PathsWithSize object_storage_paths; + object_storage_paths.reserve(object_storage_relative_paths.size()); /// Relative paths -> absolute. - for (auto & [object_path, _] : object_storage_paths) - object_path = fs::path(root_path) / object_path; + for (auto & [object_relative_path, size] : object_storage_relative_paths) + object_storage_paths.emplace_back(root_path / object_relative_path, size); return object_storage_paths; } From bb7c6279645f76d3bedd72514dfc2d520990fa92 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Jun 2022 15:16:57 +0200 Subject: [PATCH 099/123] Cosmetics: Pass patterns around as std::string_view instead of StringRef - The patterns are not used in hashing, there should not be a performance impact when we use stuff from the standard library instead. - added forgotten .reserve() in FunctionsMultiStringPosition.h --- src/Common/Volnitsky.h | 16 ++++++++-------- src/Functions/FunctionsMultiStringFuzzySearch.h | 3 +-- src/Functions/FunctionsMultiStringPosition.h | 3 ++- src/Functions/FunctionsMultiStringSearch.h | 3 +-- src/Functions/MultiMatchAllIndicesImpl.h | 4 ++-- src/Functions/MultiMatchAnyImpl.h | 6 +++--- src/Functions/MultiSearchAllPositionsImpl.h | 2 +- src/Functions/MultiSearchFirstIndexImpl.h | 2 +- src/Functions/MultiSearchFirstPositionImpl.h | 2 +- src/Functions/MultiSearchImpl.h | 2 +- src/Functions/PositionImpl.h | 8 ++++---- src/Functions/Regexps.h | 6 +++--- src/Functions/hyperscanRegexpChecker.cpp | 8 ++++---- src/Functions/hyperscanRegexpChecker.h | 2 +- 14 files changed, 33 insertions(+), 34 deletions(-) diff --git a/src/Common/Volnitsky.h b/src/Common/Volnitsky.h index 7eca0c0fe53..a6aef293ac1 100644 --- a/src/Common/Volnitsky.h +++ b/src/Common/Volnitsky.h @@ -476,7 +476,7 @@ class MultiVolnitskyBase { private: /// needles and their offsets - const std::vector & needles; + const std::vector & needles; /// fallback searchers @@ -502,7 +502,7 @@ private: static constexpr size_t small_limit = VolnitskyTraits::hash_size / 8; public: - explicit MultiVolnitskyBase(const std::vector & needles_) : needles{needles_}, step{0}, last{0} + explicit MultiVolnitskyBase(const std::vector & needles_) : needles{needles_}, step{0}, last{0} { fallback_searchers.reserve(needles.size()); hash = std::unique_ptr(new OffsetId[VolnitskyTraits::hash_size]); /// No zero initialization, it will be done later. @@ -535,8 +535,8 @@ public: for (; last < size; ++last) { - const char * cur_needle_data = needles[last].data; - const size_t cur_needle_size = needles[last].size; + const char * cur_needle_data = needles[last].data(); + const size_t cur_needle_size = needles[last].size(); /// save the indices of fallback searchers if (VolnitskyTraits::isFallbackNeedle(cur_needle_size)) @@ -593,7 +593,7 @@ public: { const auto res = pos - (hash[cell_num].off - 1); const size_t ind = hash[cell_num].id; - if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) + if (res + needles[ind].size() <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) return true; } } @@ -625,7 +625,7 @@ public: { const auto res = pos - (hash[cell_num].off - 1); const size_t ind = hash[cell_num].id; - if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) + if (res + needles[ind].size() <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) answer = std::min(answer, ind); } } @@ -663,7 +663,7 @@ public: { const auto res = pos - (hash[cell_num].off - 1); const size_t ind = hash[cell_num].id; - if (res + needles[ind].size <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) + if (res + needles[ind].size() <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) answer = std::min(answer, res - haystack); } } @@ -699,7 +699,7 @@ public: const auto * res = pos - (hash[cell_num].off - 1); const size_t ind = hash[cell_num].id; if (answer[ind] == 0 - && res + needles[ind].size <= haystack_end + && res + needles[ind].size() <= haystack_end && fallback_searchers[ind].compare(haystack, haystack_end, res)) answer[ind] = count_chars(haystack, res); } diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index e082cfbec2f..3c2c0e523ff 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -13,7 +13,6 @@ #include #include #include -#include #include @@ -115,7 +114,7 @@ public: + ", should be at most " + std::to_string(LimitArgs), ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - std::vector refs; + std::vector refs; refs.reserve(src_arr.size()); for (const auto & el : src_arr) diff --git a/src/Functions/FunctionsMultiStringPosition.h b/src/Functions/FunctionsMultiStringPosition.h index 357606f4042..855b5448b87 100644 --- a/src/Functions/FunctionsMultiStringPosition.h +++ b/src/Functions/FunctionsMultiStringPosition.h @@ -98,7 +98,8 @@ public: + ", should be at most 255", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - std::vector refs; + std::vector refs; + refs.reserve(src_arr.size()); for (const auto & el : src_arr) refs.emplace_back(el.get()); diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index 4bc8af3f214..6ab85e632e4 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -13,7 +13,6 @@ #include #include #include -#include namespace DB @@ -107,7 +106,7 @@ public: + ", should be at most " + std::to_string(LimitArgs), ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - std::vector refs; + std::vector refs; refs.reserve(src_arr.size()); for (const auto & el : src_arr) diff --git a/src/Functions/MultiMatchAllIndicesImpl.h b/src/Functions/MultiMatchAllIndicesImpl.h index 80a71548deb..a9479a9d7f5 100644 --- a/src/Functions/MultiMatchAllIndicesImpl.h +++ b/src/Functions/MultiMatchAllIndicesImpl.h @@ -44,7 +44,7 @@ struct MultiMatchAllIndicesImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const std::vector & needles, PaddedPODArray & res, PaddedPODArray & offsets) { @@ -54,7 +54,7 @@ struct MultiMatchAllIndicesImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const std::vector & needles, PaddedPODArray & res, PaddedPODArray & offsets, [[maybe_unused]] std::optional edit_distance) diff --git a/src/Functions/MultiMatchAnyImpl.h b/src/Functions/MultiMatchAnyImpl.h index fbbefe7be1d..f8d790c98cc 100644 --- a/src/Functions/MultiMatchAnyImpl.h +++ b/src/Functions/MultiMatchAnyImpl.h @@ -46,7 +46,7 @@ struct MultiMatchAnyImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const std::vector & needles, PaddedPODArray & res, PaddedPODArray & offsets) { @@ -56,7 +56,7 @@ struct MultiMatchAnyImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const std::vector & needles, PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets, [[maybe_unused]] std::optional edit_distance) @@ -120,7 +120,7 @@ struct MultiMatchAnyImpl memset(accum.data(), 0, accum.size()); for (size_t j = 0; j < needles.size(); ++j) { - MatchImpl::vectorConstant(haystack_data, haystack_offsets, needles[j].toString(), nullptr, accum); + MatchImpl::vectorConstant(haystack_data, haystack_offsets, std::string(needles[j].data(), needles[j].size()), nullptr, accum); for (size_t i = 0; i < res.size(); ++i) { if constexpr (FindAny) diff --git a/src/Functions/MultiSearchAllPositionsImpl.h b/src/Functions/MultiSearchAllPositionsImpl.h index f54fe41f20c..4356d6110f1 100644 --- a/src/Functions/MultiSearchAllPositionsImpl.h +++ b/src/Functions/MultiSearchAllPositionsImpl.h @@ -15,7 +15,7 @@ struct MultiSearchAllPositionsImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const std::vector & needles, PaddedPODArray & res) { auto res_callback = [](const UInt8 * start, const UInt8 * end) -> UInt64 diff --git a/src/Functions/MultiSearchFirstIndexImpl.h b/src/Functions/MultiSearchFirstIndexImpl.h index 26709119f6e..e452ac03902 100644 --- a/src/Functions/MultiSearchFirstIndexImpl.h +++ b/src/Functions/MultiSearchFirstIndexImpl.h @@ -22,7 +22,7 @@ struct MultiSearchFirstIndexImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const std::vector & needles, PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets) { diff --git a/src/Functions/MultiSearchFirstPositionImpl.h b/src/Functions/MultiSearchFirstPositionImpl.h index 1db8dcbde83..1aa89b54a1e 100644 --- a/src/Functions/MultiSearchFirstPositionImpl.h +++ b/src/Functions/MultiSearchFirstPositionImpl.h @@ -22,7 +22,7 @@ struct MultiSearchFirstPositionImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const std::vector & needles, PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets) { diff --git a/src/Functions/MultiSearchImpl.h b/src/Functions/MultiSearchImpl.h index 7cb0cefe580..91c3d71fdb9 100644 --- a/src/Functions/MultiSearchImpl.h +++ b/src/Functions/MultiSearchImpl.h @@ -22,7 +22,7 @@ struct MultiSearchImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const std::vector & needles, PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets) { diff --git a/src/Functions/PositionImpl.h b/src/Functions/PositionImpl.h index 82e58cdc643..5380fcc36d9 100644 --- a/src/Functions/PositionImpl.h +++ b/src/Functions/PositionImpl.h @@ -38,7 +38,7 @@ struct PositionCaseSensitiveASCII return SearcherInSmallHaystack(needle_data, needle_size); } - static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) + static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) { return MultiSearcherInBigHaystack(needles); } @@ -74,7 +74,7 @@ struct PositionCaseInsensitiveASCII return SearcherInSmallHaystack(needle_data, needle_size); } - static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) + static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) { return MultiSearcherInBigHaystack(needles); } @@ -106,7 +106,7 @@ struct PositionCaseSensitiveUTF8 return SearcherInSmallHaystack(needle_data, needle_size); } - static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) + static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) { return MultiSearcherInBigHaystack(needles); } @@ -154,7 +154,7 @@ struct PositionCaseInsensitiveUTF8 return SearcherInSmallHaystack(needle_data, needle_size); } - static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) + static MultiSearcherInBigHaystack createMultiSearcherInBigHaystack(const std::vector & needles) { return MultiSearcherInBigHaystack(needles); } diff --git a/src/Functions/Regexps.h b/src/Functions/Regexps.h index ac37875f91e..a85c89774a2 100644 --- a/src/Functions/Regexps.h +++ b/src/Functions/Regexps.h @@ -278,15 +278,15 @@ inline Regexps constructRegexps(const std::vector & str_patterns, std::o /// template has its own copy of local static variables which must not be the same /// for different hyperscan compilations. template -inline Regexps * get(const std::vector & patterns, std::optional edit_distance) +inline Regexps * get(const std::vector & patterns, std::optional edit_distance) { /// C++11 has thread-safe function-local static on most modern compilers. static Pool known_regexps; /// Different variables for different pattern parameters. std::vector str_patterns; str_patterns.reserve(patterns.size()); - for (const StringRef & ref : patterns) - str_patterns.push_back(ref.toString()); + for (const auto & pattern : patterns) + str_patterns.push_back(std::string(pattern.data(), pattern.size())); /// Get the lock for finding database. std::unique_lock lock(known_regexps.mutex); diff --git a/src/Functions/hyperscanRegexpChecker.cpp b/src/Functions/hyperscanRegexpChecker.cpp index b3c46e34daa..325817fa47e 100644 --- a/src/Functions/hyperscanRegexpChecker.cpp +++ b/src/Functions/hyperscanRegexpChecker.cpp @@ -9,16 +9,16 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -void checkRegexp(const std::vector & refs, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length) +void checkRegexp(const std::vector & regexps, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length) { if (max_hyperscan_regexp_length > 0 || max_hyperscan_regexp_total_length > 0) { size_t total_regexp_length = 0; - for (const auto & pattern : refs) + for (const auto & regexp : regexps) { - if (max_hyperscan_regexp_length > 0 && pattern.size > max_hyperscan_regexp_length) + if (max_hyperscan_regexp_length > 0 && regexp.size() > max_hyperscan_regexp_length) throw Exception("Regexp length too large", ErrorCodes::BAD_ARGUMENTS); - total_regexp_length += pattern.size; + total_regexp_length += regexp.size(); } if (max_hyperscan_regexp_total_length > 0 && total_regexp_length > max_hyperscan_regexp_total_length) diff --git a/src/Functions/hyperscanRegexpChecker.h b/src/Functions/hyperscanRegexpChecker.h index 1eea53722c1..ee8cc7d3954 100644 --- a/src/Functions/hyperscanRegexpChecker.h +++ b/src/Functions/hyperscanRegexpChecker.h @@ -5,6 +5,6 @@ namespace DB { -void checkRegexp(const std::vector & refs, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length); +void checkRegexp(const std::vector & refs, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length); } From 2ebfd01c2ee712ebe597daa345932b971587b9ca Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Jun 2022 15:20:03 +0200 Subject: [PATCH 100/123] Cosmetics: Pull out settings variable --- src/Functions/FunctionsMultiStringFuzzySearch.h | 10 +++++----- src/Functions/FunctionsMultiStringSearch.h | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index 3c2c0e523ff..6c016c9d6cb 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -36,12 +36,12 @@ public: static constexpr auto name = Impl::name; static FunctionPtr create(ContextPtr context) { - if (Impl::is_using_hyperscan && !context->getSettingsRef().allow_hyperscan) - throw Exception( - "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED); + const auto & settings = context->getSettingsRef(); - return std::make_shared( - context->getSettingsRef().max_hyperscan_regexp_length, context->getSettingsRef().max_hyperscan_regexp_total_length); + if (Impl::is_using_hyperscan && !settings.allow_hyperscan) + throw Exception("Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED); + + return std::make_shared(settings.max_hyperscan_regexp_length, settings.max_hyperscan_regexp_total_length); } FunctionsMultiStringFuzzySearch(size_t max_hyperscan_regexp_length_, size_t max_hyperscan_regexp_total_length_) diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index 6ab85e632e4..0286e119608 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -49,12 +49,12 @@ public: static constexpr auto name = Impl::name; static FunctionPtr create(ContextPtr context) { - if (Impl::is_using_hyperscan && !context->getSettingsRef().allow_hyperscan) - throw Exception( - "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED); + const auto & settings = context->getSettingsRef(); - return std::make_shared( - context->getSettingsRef().max_hyperscan_regexp_length, context->getSettingsRef().max_hyperscan_regexp_total_length); + if (Impl::is_using_hyperscan && !settings.allow_hyperscan) + throw Exception("Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED); + + return std::make_shared(settings.max_hyperscan_regexp_length, settings.max_hyperscan_regexp_total_length); } FunctionsMultiStringSearch(size_t max_hyperscan_regexp_length_, size_t max_hyperscan_regexp_total_length_) From 072e0855a82fc3f07ac77e7c89bc82eee6ea3d18 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Jun 2022 15:21:32 +0200 Subject: [PATCH 101/123] Cosmetics: Make member variables const --- src/Functions/FunctionsMultiStringFuzzySearch.h | 4 ++-- src/Functions/FunctionsMultiStringSearch.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index 6c016c9d6cb..fa09d54a75c 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -143,8 +143,8 @@ public: } private: - size_t max_hyperscan_regexp_length; - size_t max_hyperscan_regexp_total_length; + const size_t max_hyperscan_regexp_length; + const size_t max_hyperscan_regexp_total_length; }; } diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index 0286e119608..a5e27727545 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -134,8 +134,8 @@ public: } private: - size_t max_hyperscan_regexp_length; - size_t max_hyperscan_regexp_total_length; + const size_t max_hyperscan_regexp_length; + const size_t max_hyperscan_regexp_total_length; }; } From e5c74a14f7b5d21da3be38727166c5279a2032a6 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Jun 2022 15:24:17 +0200 Subject: [PATCH 102/123] Cosmetics: More consistent naming - rename utility function and file to "checkHyperscanRegexp" --- src/Functions/FunctionsMultiStringFuzzySearch.h | 4 ++-- src/Functions/FunctionsMultiStringSearch.h | 4 ++-- ...rscanRegexpChecker.cpp => checkHyperscanRegexp.cpp} | 4 ++-- src/Functions/checkHyperscanRegexp.h | 10 ++++++++++ src/Functions/hyperscanRegexpChecker.h | 10 ---------- 5 files changed, 16 insertions(+), 16 deletions(-) rename src/Functions/{hyperscanRegexpChecker.cpp => checkHyperscanRegexp.cpp} (79%) create mode 100644 src/Functions/checkHyperscanRegexp.h delete mode 100644 src/Functions/hyperscanRegexpChecker.h diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index fa09d54a75c..cc45a407099 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include @@ -121,7 +121,7 @@ public: refs.emplace_back(el.get()); if (Impl::is_using_hyperscan) - checkRegexp(refs, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + checkHyperscanRegexp(refs, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); auto col_res = ColumnVector::create(); auto col_offsets = ColumnArray::ColumnOffsets::create(); diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index a5e27727545..76c56fa0030 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include @@ -113,7 +113,7 @@ public: refs.emplace_back(el.get()); if (Impl::is_using_hyperscan) - checkRegexp(refs, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + checkHyperscanRegexp(refs, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); auto col_res = ColumnVector::create(); auto col_offsets = ColumnArray::ColumnOffsets::create(); diff --git a/src/Functions/hyperscanRegexpChecker.cpp b/src/Functions/checkHyperscanRegexp.cpp similarity index 79% rename from src/Functions/hyperscanRegexpChecker.cpp rename to src/Functions/checkHyperscanRegexp.cpp index 325817fa47e..ba9705208d5 100644 --- a/src/Functions/hyperscanRegexpChecker.cpp +++ b/src/Functions/checkHyperscanRegexp.cpp @@ -1,4 +1,4 @@ -#include +#include #include @@ -9,7 +9,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -void checkRegexp(const std::vector & regexps, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length) +void checkHyperscanRegexp(const std::vector & regexps, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length) { if (max_hyperscan_regexp_length > 0 || max_hyperscan_regexp_total_length > 0) { diff --git a/src/Functions/checkHyperscanRegexp.h b/src/Functions/checkHyperscanRegexp.h new file mode 100644 index 00000000000..2aac44115fc --- /dev/null +++ b/src/Functions/checkHyperscanRegexp.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +namespace DB +{ + +void checkHyperscanRegexp(const std::vector & regexps, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length); + +} diff --git a/src/Functions/hyperscanRegexpChecker.h b/src/Functions/hyperscanRegexpChecker.h deleted file mode 100644 index ee8cc7d3954..00000000000 --- a/src/Functions/hyperscanRegexpChecker.h +++ /dev/null @@ -1,10 +0,0 @@ -#pragma once - -#include - -namespace DB -{ - -void checkRegexp(const std::vector & refs, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length); - -} From 1273756911cd4d83253ca11a281d73aa2c83bb91 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Jun 2022 15:33:34 +0200 Subject: [PATCH 103/123] Cosmetics: fmt-based exceptions --- .../FunctionsMultiStringFuzzySearch.h | 34 +++++++------------ src/Functions/FunctionsMultiStringSearch.h | 24 ++++++------- 2 files changed, 22 insertions(+), 36 deletions(-) diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index cc45a407099..1d6b8cfeffd 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -39,15 +39,14 @@ public: const auto & settings = context->getSettingsRef(); if (Impl::is_using_hyperscan && !settings.allow_hyperscan) - throw Exception("Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED); + throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0"); return std::make_shared(settings.max_hyperscan_regexp_length, settings.max_hyperscan_regexp_total_length); } FunctionsMultiStringFuzzySearch(size_t max_hyperscan_regexp_length_, size_t max_hyperscan_regexp_total_length_) : max_hyperscan_regexp_length(max_hyperscan_regexp_length_), max_hyperscan_regexp_total_length(max_hyperscan_regexp_total_length_) - { - } + {} String getName() const override { return name; } @@ -59,17 +58,15 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (!isString(arguments[0])) - throw Exception( - "Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0]->getName(), getName()); if (!isUnsignedInteger(arguments[1])) - throw Exception( - "Illegal type " + arguments[1]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[1]->getName(), getName()); const DataTypeArray * array_type = checkAndGetDataType(arguments[2].get()); if (!array_type || !checkAndGetDataType(array_type->getNestedType().get())) - throw Exception( - "Illegal type " + arguments[2]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[2]->getName(), getName()); + return Impl::getReturnType(); } @@ -92,27 +89,20 @@ public: else if ((col_const_num = checkAndGetColumnConst(num_ptr.get()))) edit_distance = col_const_num->getValue(); else - throw Exception( - "Illegal column " + arguments[1].column->getName() - + ". The number is not const or does not fit in UInt32", - ErrorCodes::ILLEGAL_COLUMN); - + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}. The number is not const or does not fit in UInt32", arguments[1].column->getName()); const ColumnPtr & arr_ptr = arguments[2].column; const ColumnConst * col_const_arr = checkAndGetColumnConst(arr_ptr.get()); if (!col_const_arr) - throw Exception( - "Illegal column " + arguments[2].column->getName() + ". The array is not const", - ErrorCodes::ILLEGAL_COLUMN); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}. The array is not const", arguments[2].column->getName()); Array src_arr = col_const_arr->getValue(); if (src_arr.size() > LimitArgs) - throw Exception( - "Number of arguments for function " + getName() + " doesn't match: passed " + std::to_string(src_arr.size()) - + ", should be at most " + std::to_string(LimitArgs), - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be at most {}", + getName(), std::to_string(src_arr.size()), std::to_string(LimitArgs)); std::vector refs; refs.reserve(src_arr.size()); @@ -134,7 +124,7 @@ public: Impl::vectorConstant( col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res, edit_distance); else - throw Exception("Illegal column " + arguments[0].column->getName(), ErrorCodes::ILLEGAL_COLUMN); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}", arguments[0].column->getName()); if constexpr (Impl::is_column_array) return ColumnArray::create(std::move(col_res), std::move(col_offsets)); diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index 76c56fa0030..53eda533cb6 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -52,15 +52,14 @@ public: const auto & settings = context->getSettingsRef(); if (Impl::is_using_hyperscan && !settings.allow_hyperscan) - throw Exception("Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0", ErrorCodes::FUNCTION_NOT_ALLOWED); + throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0"); return std::make_shared(settings.max_hyperscan_regexp_length, settings.max_hyperscan_regexp_total_length); } FunctionsMultiStringSearch(size_t max_hyperscan_regexp_length_, size_t max_hyperscan_regexp_total_length_) : max_hyperscan_regexp_length(max_hyperscan_regexp_length_), max_hyperscan_regexp_total_length(max_hyperscan_regexp_total_length_) - { - } + {} String getName() const override { return name; } @@ -72,13 +71,12 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (!isString(arguments[0])) - throw Exception( - "Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0]->getName(), getName()); const DataTypeArray * array_type = checkAndGetDataType(arguments[1].get()); if (!array_type || !checkAndGetDataType(array_type->getNestedType().get())) - throw Exception( - "Illegal type " + arguments[1]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[1]->getName(), getName()); + return Impl::getReturnType(); } @@ -94,17 +92,15 @@ public: const ColumnConst * col_const_arr = checkAndGetColumnConst(arr_ptr.get()); if (!col_const_arr) - throw Exception( - "Illegal column " + arguments[1].column->getName() + ". The array is not const", - ErrorCodes::ILLEGAL_COLUMN); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}. The array is not const", arguments[1].column->getName()); Array src_arr = col_const_arr->getValue(); if (src_arr.size() > LimitArgs) throw Exception( - "Number of arguments for function " + getName() + " doesn't match: passed " + std::to_string(src_arr.size()) - + ", should be at most " + std::to_string(LimitArgs), - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be at most {}", + getName(), std::to_string(src_arr.size()), std::to_string(LimitArgs)); std::vector refs; refs.reserve(src_arr.size()); @@ -125,7 +121,7 @@ public: if (col_haystack_vector) Impl::vectorConstant(col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res); else - throw Exception("Illegal column " + arguments[0].column->getName(), ErrorCodes::ILLEGAL_COLUMN); + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}", arguments[0].column->getName()); if constexpr (Impl::is_column_array) return ColumnArray::create(std::move(col_res), std::move(col_offsets)); From 4bc59c18e38bb7a42df2ff9be1c79cf9d1867d57 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Jun 2022 15:34:40 +0200 Subject: [PATCH 104/123] Cosmetics: Move some code around + docs + whitespaces + minor stuff --- .../FunctionsMultiStringFuzzySearch.h | 9 ++++--- src/Functions/FunctionsMultiStringSearch.h | 11 +++++---- src/Functions/MultiMatchAllIndicesImpl.h | 24 ++++++++----------- src/Functions/MultiMatchAnyImpl.h | 18 +++++++------- src/Functions/Regexps.h | 8 +++---- 5 files changed, 34 insertions(+), 36 deletions(-) diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index 1d6b8cfeffd..b0f2de96622 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -72,13 +72,12 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { - using ResultType = typename Impl::ResultType; - const ColumnPtr & column_haystack = arguments[0].column; + const ColumnPtr & num_ptr = arguments[1].column; + const ColumnPtr & arr_ptr = arguments[2].column; const ColumnString * col_haystack_vector = checkAndGetColumn(&*column_haystack); - const ColumnPtr & num_ptr = arguments[1].column; const ColumnConst * col_const_num = nullptr; UInt32 edit_distance = 0; @@ -91,7 +90,6 @@ public: else throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}. The number is not const or does not fit in UInt32", arguments[1].column->getName()); - const ColumnPtr & arr_ptr = arguments[2].column; const ColumnConst * col_const_arr = checkAndGetColumnConst(arr_ptr.get()); if (!col_const_arr) @@ -113,13 +111,14 @@ public: if (Impl::is_using_hyperscan) checkHyperscanRegexp(refs, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + using ResultType = typename Impl::ResultType; auto col_res = ColumnVector::create(); auto col_offsets = ColumnArray::ColumnOffsets::create(); auto & vec_res = col_res->getData(); auto & offsets_res = col_offsets->getData(); - /// The blame for resizing output is for the callee. + // the implementations are responsible for resizing the output column if (col_haystack_vector) Impl::vectorConstant( col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res, edit_distance); diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index 53eda533cb6..e3e14458764 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -18,6 +18,10 @@ namespace DB { /** + * multiMatchAny(haystack, [pattern_1, pattern_2, ..., pattern_n]) + * multiMatchAnyIndex(haystack, [pattern_1, pattern_2, ..., pattern_n]) + * multiMatchAllIndices(haystack, [pattern_1, pattern_2, ..., pattern_n]) + * * multiSearchAny(haystack, [pattern_1, pattern_2, ..., pattern_n]) -- find any of the const patterns inside haystack and return 0 or 1 * multiSearchAnyUTF8(haystack, [pattern_1, pattern_2, ..., pattern_n]) * multiSearchAnyCaseInsensitive(haystack, [pattern_1, pattern_2, ..., pattern_n]) @@ -82,13 +86,11 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { - using ResultType = typename Impl::ResultType; - const ColumnPtr & column_haystack = arguments[0].column; + const ColumnPtr & arr_ptr = arguments[1].column; const ColumnString * col_haystack_vector = checkAndGetColumn(&*column_haystack); - const ColumnPtr & arr_ptr = arguments[1].column; const ColumnConst * col_const_arr = checkAndGetColumnConst(arr_ptr.get()); if (!col_const_arr) @@ -111,13 +113,14 @@ public: if (Impl::is_using_hyperscan) checkHyperscanRegexp(refs, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + using ResultType = typename Impl::ResultType; auto col_res = ColumnVector::create(); auto col_offsets = ColumnArray::ColumnOffsets::create(); auto & vec_res = col_res->getData(); auto & offsets_res = col_offsets->getData(); - /// The blame for resizing output is for the callee. + // the implementations are responsible for resizing the output column if (col_haystack_vector) Impl::vectorConstant(col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res); else diff --git a/src/Functions/MultiMatchAllIndicesImpl.h b/src/Functions/MultiMatchAllIndicesImpl.h index a9479a9d7f5..c77fd5fbbcc 100644 --- a/src/Functions/MultiMatchAllIndicesImpl.h +++ b/src/Functions/MultiMatchAllIndicesImpl.h @@ -26,10 +26,11 @@ namespace ErrorCodes } -template +template struct MultiMatchAllIndicesImpl { - using ResultType = Type; + using ResultType = ResultType_; + static constexpr bool is_using_hyperscan = true; /// Variable for understanding, if we used offsets for the output, most /// likely to determine whether the function returns ColumnVector of ColumnArray. @@ -45,18 +46,18 @@ struct MultiMatchAllIndicesImpl const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, const std::vector & needles, - PaddedPODArray & res, + PaddedPODArray & res, PaddedPODArray & offsets) { vectorConstant(haystack_data, haystack_offsets, needles, res, offsets, std::nullopt); } static void vectorConstant( - const ColumnString::Chars & haystack_data, - const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, - PaddedPODArray & res, - PaddedPODArray & offsets, + [[maybe_unused]] const ColumnString::Chars & haystack_data, + [[maybe_unused]] const ColumnString::Offsets & haystack_offsets, + [[maybe_unused]] const std::vector & needles, + [[maybe_unused]] PaddedPODArray & res, + [[maybe_unused]] PaddedPODArray & offsets, [[maybe_unused]] std::optional edit_distance) { offsets.resize(haystack_offsets.size()); @@ -76,7 +77,7 @@ struct MultiMatchAllIndicesImpl unsigned int /* flags */, void * context) -> int { - static_cast*>(context)->push_back(id); + static_cast*>(context)->push_back(id); return 0; }; const size_t haystack_offsets_size = haystack_offsets.size(); @@ -102,11 +103,6 @@ struct MultiMatchAllIndicesImpl offset = haystack_offsets[i]; } #else - (void)haystack_data; - (void)haystack_offsets; - (void)needles; - (void)res; - (void)offsets; throw Exception( "multi-search all indices is not implemented when vectorscan is off", ErrorCodes::NOT_IMPLEMENTED); diff --git a/src/Functions/MultiMatchAnyImpl.h b/src/Functions/MultiMatchAnyImpl.h index f8d790c98cc..b0a4d7d55e4 100644 --- a/src/Functions/MultiMatchAnyImpl.h +++ b/src/Functions/MultiMatchAnyImpl.h @@ -27,11 +27,13 @@ namespace ErrorCodes } -template +template struct MultiMatchAnyImpl { - static_assert(static_cast(FindAny) + static_cast(FindAnyIndex) == 1); - using ResultType = Type; + static_assert(FindAny ^ FindAnyIndex); + + using ResultType = ResultType_; + static constexpr bool is_using_hyperscan = true; /// Variable for understanding, if we used offsets for the output, most /// likely to determine whether the function returns ColumnVector of ColumnArray. @@ -47,7 +49,7 @@ struct MultiMatchAnyImpl const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, const std::vector & needles, - PaddedPODArray & res, + PaddedPODArray & res, PaddedPODArray & offsets) { vectorConstant(haystack_data, haystack_offsets, needles, res, offsets, std::nullopt); @@ -57,7 +59,7 @@ struct MultiMatchAnyImpl const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, const std::vector & needles, - PaddedPODArray & res, + PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets, [[maybe_unused]] std::optional edit_distance) { @@ -81,9 +83,9 @@ struct MultiMatchAnyImpl void * context) -> int { if constexpr (FindAnyIndex) - *reinterpret_cast(context) = id; + *reinterpret_cast(context) = id; else if constexpr (FindAny) - *reinterpret_cast(context) = 1; + *reinterpret_cast(context) = 1; /// Once we hit the callback, there is no need to search for others. return 1; }; @@ -110,7 +112,7 @@ struct MultiMatchAnyImpl offset = haystack_offsets[i]; } #else - /// Fallback if do not use vectorscan + // fallback if vectorscan is not compiled if constexpr (MultiSearchDistance) throw Exception( "Edit distance multi-search is not implemented when vectorscan is off", diff --git a/src/Functions/Regexps.h b/src/Functions/Regexps.h index a85c89774a2..87ad6577e15 100644 --- a/src/Functions/Regexps.h +++ b/src/Functions/Regexps.h @@ -167,9 +167,8 @@ struct Pool }; template -inline Regexps constructRegexps(const std::vector & str_patterns, std::optional edit_distance) +inline Regexps constructRegexps(const std::vector & str_patterns, [[maybe_unused]] std::optional edit_distance) { - (void)edit_distance; /// Common pointers std::vector patterns; std::vector flags; @@ -270,7 +269,7 @@ inline Regexps constructRegexps(const std::vector & str_patterns, std::o if (err != HS_SUCCESS) throw Exception("Could not allocate scratch space for hyperscan", ErrorCodes::CANNOT_ALLOCATE_MEMORY); - return Regexps{db, scratch}; + return {db, scratch}; } /// If CompileForEditDistance is False, edit_distance must be nullopt @@ -280,8 +279,7 @@ inline Regexps constructRegexps(const std::vector & str_patterns, std::o template inline Regexps * get(const std::vector & patterns, std::optional edit_distance) { - /// C++11 has thread-safe function-local static on most modern compilers. - static Pool known_regexps; /// Different variables for different pattern parameters. + static Pool known_regexps; /// Different variables for different pattern parameters, thread-safe in C++11 std::vector str_patterns; str_patterns.reserve(patterns.size()); From 580d89477f5b5d6f84c0ce259b24b418dcf66503 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Jun 2022 15:42:42 +0200 Subject: [PATCH 105/123] Minimally faster performance --- src/Functions/Regexps.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/Regexps.h b/src/Functions/Regexps.h index 87ad6577e15..3e12f7fa651 100644 --- a/src/Functions/Regexps.h +++ b/src/Functions/Regexps.h @@ -145,7 +145,7 @@ public: Regexps * operator()() { - std::unique_lock lock(mutex); + std::lock_guard lock(mutex); if (regexp) return &*regexp; regexp = constructor(); @@ -284,7 +284,7 @@ inline Regexps * get(const std::vector & patterns, std::option std::vector str_patterns; str_patterns.reserve(patterns.size()); for (const auto & pattern : patterns) - str_patterns.push_back(std::string(pattern.data(), pattern.size())); + str_patterns.emplace_back(std::string(pattern.data(), pattern.size())); /// Get the lock for finding database. std::unique_lock lock(known_regexps.mutex); From 89bfdd50bfa7b00304ed8805c08d4958fb231a89 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Jun 2022 15:44:42 +0200 Subject: [PATCH 106/123] Remove unnecessary check - getReturnTypeImpl() ensures that the haystack column has type "String" and we can simply assert that. --- src/Functions/FunctionsMultiStringFuzzySearch.h | 10 +++------- src/Functions/FunctionsMultiStringSearch.h | 9 +++------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index b0f2de96622..822c513b0d7 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -49,7 +49,6 @@ public: {} String getName() const override { return name; } - size_t getNumberOfArguments() const override { return 3; } bool useDefaultImplementationForConstants() const override { return true; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } @@ -77,6 +76,7 @@ public: const ColumnPtr & arr_ptr = arguments[2].column; const ColumnString * col_haystack_vector = checkAndGetColumn(&*column_haystack); + assert(col_haystack_vector); // getReturnTypeImpl() checks the data type const ColumnConst * col_const_num = nullptr; UInt32 edit_distance = 0; @@ -117,13 +117,9 @@ public: auto & vec_res = col_res->getData(); auto & offsets_res = col_offsets->getData(); - // the implementations are responsible for resizing the output column - if (col_haystack_vector) - Impl::vectorConstant( - col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res, edit_distance); - else - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}", arguments[0].column->getName()); + + Impl::vectorConstant(col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res, edit_distance); if constexpr (Impl::is_column_array) return ColumnArray::create(std::move(col_res), std::move(col_offsets)); diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index e3e14458764..0b2e68099d7 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -66,7 +66,6 @@ public: {} String getName() const override { return name; } - size_t getNumberOfArguments() const override { return 2; } bool useDefaultImplementationForConstants() const override { return true; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } @@ -90,6 +89,7 @@ public: const ColumnPtr & arr_ptr = arguments[1].column; const ColumnString * col_haystack_vector = checkAndGetColumn(&*column_haystack); + assert(col_haystack_vector); // getReturnTypeImpl() checks the data type const ColumnConst * col_const_arr = checkAndGetColumnConst(arr_ptr.get()); @@ -119,12 +119,9 @@ public: auto & vec_res = col_res->getData(); auto & offsets_res = col_offsets->getData(); - // the implementations are responsible for resizing the output column - if (col_haystack_vector) - Impl::vectorConstant(col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res); - else - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}", arguments[0].column->getName()); + + Impl::vectorConstant(col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res); if constexpr (Impl::is_column_array) return ColumnArray::create(std::move(col_res), std::move(col_offsets)); From 7913edc172abaeedb0d3267e732f0272ad95db85 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Jun 2022 16:12:38 +0200 Subject: [PATCH 107/123] Move check for hyperscan regexp constraints into implementations - This is preparation for non-const regexp arguments, where this check will run for each row. --- src/Functions/FunctionsMultiStringFuzzySearch.h | 8 +++----- src/Functions/FunctionsMultiStringSearch.h | 8 +++----- src/Functions/MultiMatchAllIndicesImpl.h | 16 ++++++++++++---- src/Functions/MultiMatchAnyImpl.h | 14 +++++++++++--- src/Functions/MultiSearchFirstIndexImpl.h | 4 +++- src/Functions/MultiSearchFirstPositionImpl.h | 4 +++- src/Functions/MultiSearchImpl.h | 4 +++- 7 files changed, 38 insertions(+), 20 deletions(-) diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index 822c513b0d7..514b0b0189a 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -108,9 +107,6 @@ public: for (const auto & el : src_arr) refs.emplace_back(el.get()); - if (Impl::is_using_hyperscan) - checkHyperscanRegexp(refs, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); - using ResultType = typename Impl::ResultType; auto col_res = ColumnVector::create(); auto col_offsets = ColumnArray::ColumnOffsets::create(); @@ -119,7 +115,9 @@ public: auto & offsets_res = col_offsets->getData(); // the implementations are responsible for resizing the output column - Impl::vectorConstant(col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res, edit_distance); + Impl::vectorConstant( + col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res, edit_distance, + max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); if constexpr (Impl::is_column_array) return ColumnArray::create(std::move(col_res), std::move(col_offsets)); diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index 0b2e68099d7..8af5689c4a6 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -110,9 +109,6 @@ public: for (const auto & el : src_arr) refs.emplace_back(el.get()); - if (Impl::is_using_hyperscan) - checkHyperscanRegexp(refs, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); - using ResultType = typename Impl::ResultType; auto col_res = ColumnVector::create(); auto col_offsets = ColumnArray::ColumnOffsets::create(); @@ -121,7 +117,9 @@ public: auto & offsets_res = col_offsets->getData(); // the implementations are responsible for resizing the output column - Impl::vectorConstant(col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res); + Impl::vectorConstant( + col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res, + max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); if constexpr (Impl::is_column_array) return ColumnArray::create(std::move(col_res), std::move(col_offsets)); diff --git a/src/Functions/MultiMatchAllIndicesImpl.h b/src/Functions/MultiMatchAllIndicesImpl.h index c77fd5fbbcc..cb8d29723a6 100644 --- a/src/Functions/MultiMatchAllIndicesImpl.h +++ b/src/Functions/MultiMatchAllIndicesImpl.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "Regexps.h" #include "config_functions.h" @@ -47,9 +48,11 @@ struct MultiMatchAllIndicesImpl const ColumnString::Offsets & haystack_offsets, const std::vector & needles, PaddedPODArray & res, - PaddedPODArray & offsets) + PaddedPODArray & offsets, + size_t max_hyperscan_regexp_length, + size_t max_hyperscan_regexp_total_length) { - vectorConstant(haystack_data, haystack_offsets, needles, res, offsets, std::nullopt); + vectorConstant(haystack_data, haystack_offsets, needles, res, offsets, std::nullopt, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); } static void vectorConstant( @@ -58,10 +61,15 @@ struct MultiMatchAllIndicesImpl [[maybe_unused]] const std::vector & needles, [[maybe_unused]] PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets, - [[maybe_unused]] std::optional edit_distance) + [[maybe_unused]] std::optional edit_distance, + [[maybe_unused]] size_t max_hyperscan_regexp_length, + [[maybe_unused]] size_t max_hyperscan_regexp_total_length) { - offsets.resize(haystack_offsets.size()); #if USE_VECTORSCAN + checkHyperscanRegexp(needles, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + + offsets.resize(haystack_offsets.size()); + const auto & hyperscan_regex = MultiRegexps::get(needles, edit_distance); hs_scratch_t * scratch = nullptr; hs_error_t err = hs_clone_scratch(hyperscan_regex->getScratch(), &scratch); diff --git a/src/Functions/MultiMatchAnyImpl.h b/src/Functions/MultiMatchAnyImpl.h index b0a4d7d55e4..349078b9d4a 100644 --- a/src/Functions/MultiMatchAnyImpl.h +++ b/src/Functions/MultiMatchAnyImpl.h @@ -3,6 +3,7 @@ #include #include #include +#include #include "Regexps.h" #include "config_functions.h" @@ -50,9 +51,11 @@ struct MultiMatchAnyImpl const ColumnString::Offsets & haystack_offsets, const std::vector & needles, PaddedPODArray & res, - PaddedPODArray & offsets) + PaddedPODArray & offsets, + size_t max_hyperscan_regexp_length, + size_t max_hyperscan_regexp_total_length) { - vectorConstant(haystack_data, haystack_offsets, needles, res, offsets, std::nullopt); + vectorConstant(haystack_data, haystack_offsets, needles, res, offsets, std::nullopt, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); } static void vectorConstant( @@ -61,10 +64,15 @@ struct MultiMatchAnyImpl const std::vector & needles, PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets, - [[maybe_unused]] std::optional edit_distance) + [[maybe_unused]] std::optional edit_distance, + size_t max_hyperscan_regexp_length, + size_t max_hyperscan_regexp_total_length) { (void)FindAny; (void)FindAnyIndex; + + checkHyperscanRegexp(needles, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + res.resize(haystack_offsets.size()); #if USE_VECTORSCAN const auto & hyperscan_regex = MultiRegexps::get(needles, edit_distance); diff --git a/src/Functions/MultiSearchFirstIndexImpl.h b/src/Functions/MultiSearchFirstIndexImpl.h index e452ac03902..b04ce157583 100644 --- a/src/Functions/MultiSearchFirstIndexImpl.h +++ b/src/Functions/MultiSearchFirstIndexImpl.h @@ -24,7 +24,9 @@ struct MultiSearchFirstIndexImpl const ColumnString::Offsets & haystack_offsets, const std::vector & needles, PaddedPODArray & res, - [[maybe_unused]] PaddedPODArray & offsets) + [[maybe_unused]] PaddedPODArray & offsets, + size_t /*max_hyperscan_regexp_length*/, + size_t /*max_hyperscan_regexp_total_length*/) { auto searcher = Impl::createMultiSearcherInBigHaystack(needles); const size_t haystack_string_size = haystack_offsets.size(); diff --git a/src/Functions/MultiSearchFirstPositionImpl.h b/src/Functions/MultiSearchFirstPositionImpl.h index 1aa89b54a1e..5bfc13be10b 100644 --- a/src/Functions/MultiSearchFirstPositionImpl.h +++ b/src/Functions/MultiSearchFirstPositionImpl.h @@ -24,7 +24,9 @@ struct MultiSearchFirstPositionImpl const ColumnString::Offsets & haystack_offsets, const std::vector & needles, PaddedPODArray & res, - [[maybe_unused]] PaddedPODArray & offsets) + [[maybe_unused]] PaddedPODArray & offsets, + size_t /*max_hyperscan_regexp_length*/, + size_t /*max_hyperscan_regexp_total_length*/) { auto res_callback = [](const UInt8 * start, const UInt8 * end) -> UInt64 { diff --git a/src/Functions/MultiSearchImpl.h b/src/Functions/MultiSearchImpl.h index 91c3d71fdb9..ac198b16765 100644 --- a/src/Functions/MultiSearchImpl.h +++ b/src/Functions/MultiSearchImpl.h @@ -24,7 +24,9 @@ struct MultiSearchImpl const ColumnString::Offsets & haystack_offsets, const std::vector & needles, PaddedPODArray & res, - [[maybe_unused]] PaddedPODArray & offsets) + [[maybe_unused]] PaddedPODArray & offsets, + size_t /*max_hyperscan_regexp_length*/, + size_t /*max_hyperscan_regexp_total_length*/) { auto searcher = Impl::createMultiSearcherInBigHaystack(needles); const size_t haystack_string_size = haystack_offsets.size(); From 3478db9fb6a81e4fcbd698cfca94d5508b709103 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 24 Jun 2022 16:42:39 +0200 Subject: [PATCH 108/123] Move check for regexp array size into implementations - This is not needed for non-const regexp array arguments (the cardinality of arrays is fixed per column) but it cleans up the code and runs the check only in functions which have restrictions on the number of patterns. - For functions using hyperscans, it was checked that the number of regexes is < 2^32. Removed the check because I don't think anyone will every specify 4 billion patterns. --- src/Functions/FunctionsMultiStringFuzzySearch.h | 10 +--------- src/Functions/FunctionsMultiStringSearch.h | 13 +------------ src/Functions/MultiSearchFirstIndexImpl.h | 11 +++++++++++ src/Functions/MultiSearchImpl.h | 11 +++++++++++ src/Functions/multiFuzzyMatchAllIndices.cpp | 4 +--- src/Functions/multiFuzzyMatchAny.cpp | 4 +--- src/Functions/multiFuzzyMatchAnyIndex.cpp | 4 +--- src/Functions/multiMatchAllIndices.cpp | 4 +--- src/Functions/multiMatchAny.cpp | 4 +--- src/Functions/multiMatchAnyIndex.cpp | 4 +--- src/Functions/multiSearchAnyCaseInsensitive.cpp | 3 +-- src/Functions/multiSearchFirstIndex.cpp | 3 +-- 12 files changed, 32 insertions(+), 43 deletions(-) diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index 514b0b0189a..b83b5c4192b 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -21,16 +21,13 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int FUNCTION_NOT_ALLOWED; } -template +template class FunctionsMultiStringFuzzySearch : public IFunction { - static_assert(LimitArgs > 0); - public: static constexpr auto name = Impl::name; static FunctionPtr create(ContextPtr context) @@ -96,11 +93,6 @@ public: Array src_arr = col_const_arr->getValue(); - if (src_arr.size() > LimitArgs) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Number of arguments for function {} doesn't match: passed {}, should be at most {}", - getName(), std::to_string(src_arr.size()), std::to_string(LimitArgs)); - std::vector refs; refs.reserve(src_arr.size()); diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index 8af5689c4a6..8b48f8f9eaa 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -36,18 +36,13 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int FUNCTION_NOT_ALLOWED; } -/// The argument limiting raises from Volnitsky searcher -- it is performance crucial to save only one byte for pattern number. -/// But some other searchers use this function, for example, multiMatchAny -- hyperscan does not have such restrictions -template ::max()> +template class FunctionsMultiStringSearch : public IFunction { - static_assert(LimitArgs > 0); - public: static constexpr auto name = Impl::name; static FunctionPtr create(ContextPtr context) @@ -97,12 +92,6 @@ public: Array src_arr = col_const_arr->getValue(); - if (src_arr.size() > LimitArgs) - throw Exception( - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Number of arguments for function {} doesn't match: passed {}, should be at most {}", - getName(), std::to_string(src_arr.size()), std::to_string(LimitArgs)); - std::vector refs; refs.reserve(src_arr.size()); diff --git a/src/Functions/MultiSearchFirstIndexImpl.h b/src/Functions/MultiSearchFirstIndexImpl.h index b04ce157583..4f01c45bdf4 100644 --- a/src/Functions/MultiSearchFirstIndexImpl.h +++ b/src/Functions/MultiSearchFirstIndexImpl.h @@ -7,6 +7,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + template struct MultiSearchFirstIndexImpl { @@ -28,6 +33,12 @@ struct MultiSearchFirstIndexImpl size_t /*max_hyperscan_regexp_length*/, size_t /*max_hyperscan_regexp_total_length*/) { + // For performance of Volnitsky search, it is crucial to save only one byte for pattern number. + if (needles.size() > std::numeric_limits::max()) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be at most {}", + name, std::to_string(needles.size()), std::to_string(std::numeric_limits::max())); + auto searcher = Impl::createMultiSearcherInBigHaystack(needles); const size_t haystack_string_size = haystack_offsets.size(); res.resize(haystack_string_size); diff --git a/src/Functions/MultiSearchImpl.h b/src/Functions/MultiSearchImpl.h index ac198b16765..0c951f3c5fc 100644 --- a/src/Functions/MultiSearchImpl.h +++ b/src/Functions/MultiSearchImpl.h @@ -7,6 +7,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + template struct MultiSearchImpl { @@ -28,6 +33,12 @@ struct MultiSearchImpl size_t /*max_hyperscan_regexp_length*/, size_t /*max_hyperscan_regexp_total_length*/) { + // For performance of Volnitsky search, it is crucial to save only one byte for pattern number. + if (needles.size() > std::numeric_limits::max()) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be at most {}", + name, std::to_string(needles.size()), std::to_string(std::numeric_limits::max())); + auto searcher = Impl::createMultiSearcherInBigHaystack(needles); const size_t haystack_string_size = haystack_offsets.size(); res.resize(haystack_string_size); diff --git a/src/Functions/multiFuzzyMatchAllIndices.cpp b/src/Functions/multiFuzzyMatchAllIndices.cpp index d0121ee3981..2c4ac05ddbe 100644 --- a/src/Functions/multiFuzzyMatchAllIndices.cpp +++ b/src/Functions/multiFuzzyMatchAllIndices.cpp @@ -13,9 +13,7 @@ struct NameMultiFuzzyMatchAllIndices static constexpr auto name = "multiFuzzyMatchAllIndices"; }; -using FunctionMultiFuzzyMatchAllIndices = FunctionsMultiStringFuzzySearch< - MultiMatchAllIndicesImpl, - std::numeric_limits::max()>; +using FunctionMultiFuzzyMatchAllIndices = FunctionsMultiStringFuzzySearch>; } diff --git a/src/Functions/multiFuzzyMatchAny.cpp b/src/Functions/multiFuzzyMatchAny.cpp index 640e93a23b0..fbd84ead31b 100644 --- a/src/Functions/multiFuzzyMatchAny.cpp +++ b/src/Functions/multiFuzzyMatchAny.cpp @@ -13,9 +13,7 @@ struct NameMultiFuzzyMatchAny static constexpr auto name = "multiFuzzyMatchAny"; }; -using FunctionMultiFuzzyMatchAny = FunctionsMultiStringFuzzySearch< - MultiMatchAnyImpl, - std::numeric_limits::max()>; +using FunctionMultiFuzzyMatchAny = FunctionsMultiStringFuzzySearch>; } diff --git a/src/Functions/multiFuzzyMatchAnyIndex.cpp b/src/Functions/multiFuzzyMatchAnyIndex.cpp index f8bad1bc461..255f710e2b3 100644 --- a/src/Functions/multiFuzzyMatchAnyIndex.cpp +++ b/src/Functions/multiFuzzyMatchAnyIndex.cpp @@ -13,9 +13,7 @@ struct NameMultiFuzzyMatchAnyIndex static constexpr auto name = "multiFuzzyMatchAnyIndex"; }; -using FunctionMultiFuzzyMatchAnyIndex = FunctionsMultiStringFuzzySearch< - MultiMatchAnyImpl, - std::numeric_limits::max()>; +using FunctionMultiFuzzyMatchAnyIndex = FunctionsMultiStringFuzzySearch>; } diff --git a/src/Functions/multiMatchAllIndices.cpp b/src/Functions/multiMatchAllIndices.cpp index 940c9e7e3bf..1c6577e5aa3 100644 --- a/src/Functions/multiMatchAllIndices.cpp +++ b/src/Functions/multiMatchAllIndices.cpp @@ -13,9 +13,7 @@ struct NameMultiMatchAllIndices static constexpr auto name = "multiMatchAllIndices"; }; -using FunctionMultiMatchAllIndices = FunctionsMultiStringSearch< - MultiMatchAllIndicesImpl, - std::numeric_limits::max()>; +using FunctionMultiMatchAllIndices = FunctionsMultiStringSearch>; } diff --git a/src/Functions/multiMatchAny.cpp b/src/Functions/multiMatchAny.cpp index 47510e0ecc2..4920e9c230f 100644 --- a/src/Functions/multiMatchAny.cpp +++ b/src/Functions/multiMatchAny.cpp @@ -13,9 +13,7 @@ struct NameMultiMatchAny static constexpr auto name = "multiMatchAny"; }; -using FunctionMultiMatchAny = FunctionsMultiStringSearch< - MultiMatchAnyImpl, - std::numeric_limits::max()>; +using FunctionMultiMatchAny = FunctionsMultiStringSearch>; } diff --git a/src/Functions/multiMatchAnyIndex.cpp b/src/Functions/multiMatchAnyIndex.cpp index a56d41dc95b..bf68e8576a5 100644 --- a/src/Functions/multiMatchAnyIndex.cpp +++ b/src/Functions/multiMatchAnyIndex.cpp @@ -13,9 +13,7 @@ struct NameMultiMatchAnyIndex static constexpr auto name = "multiMatchAnyIndex"; }; -using FunctionMultiMatchAnyIndex = FunctionsMultiStringSearch< - MultiMatchAnyImpl, - std::numeric_limits::max()>; +using FunctionMultiMatchAnyIndex = FunctionsMultiStringSearch>; } diff --git a/src/Functions/multiSearchAnyCaseInsensitive.cpp b/src/Functions/multiSearchAnyCaseInsensitive.cpp index 9bc950c0d3d..af463805ea5 100644 --- a/src/Functions/multiSearchAnyCaseInsensitive.cpp +++ b/src/Functions/multiSearchAnyCaseInsensitive.cpp @@ -13,8 +13,7 @@ struct NameMultiSearchAnyCaseInsensitive { static constexpr auto name = "multiSearchAnyCaseInsensitive"; }; -using FunctionMultiSearchCaseInsensitive - = FunctionsMultiStringSearch>; +using FunctionMultiSearchCaseInsensitive = FunctionsMultiStringSearch>; } diff --git a/src/Functions/multiSearchFirstIndex.cpp b/src/Functions/multiSearchFirstIndex.cpp index a96ebed029c..a5fbec2dc36 100644 --- a/src/Functions/multiSearchFirstIndex.cpp +++ b/src/Functions/multiSearchFirstIndex.cpp @@ -14,8 +14,7 @@ struct NameMultiSearchFirstIndex static constexpr auto name = "multiSearchFirstIndex"; }; -using FunctionMultiSearchFirstIndex - = FunctionsMultiStringSearch>; +using FunctionMultiSearchFirstIndex = FunctionsMultiStringSearch>; } From 2f15d45f27b8d9d8d32b557decc2ab9030032308 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 26 Jun 2022 16:25:43 +0000 Subject: [PATCH 109/123] Move check for regexp array size into implementations - This is not needed for non-const regexp array arguments (the cardinality of arrays is fixed per column) but it cleans up the code and runs the check only in functions which have restrictions on the number of patterns. - For functions using hyperscans, it was checked that the number of regexes is < 2^32. Removed the check because I don't think anyone will every specify 4 billion patterns. --- src/Functions/MultiSearchFirstPositionImpl.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Functions/MultiSearchFirstPositionImpl.h b/src/Functions/MultiSearchFirstPositionImpl.h index 5bfc13be10b..2260950c5d4 100644 --- a/src/Functions/MultiSearchFirstPositionImpl.h +++ b/src/Functions/MultiSearchFirstPositionImpl.h @@ -7,6 +7,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + template struct MultiSearchFirstPositionImpl { @@ -28,6 +33,12 @@ struct MultiSearchFirstPositionImpl size_t /*max_hyperscan_regexp_length*/, size_t /*max_hyperscan_regexp_total_length*/) { + // For performance of Volnitsky search, it is crucial to save only one byte for pattern number. + if (needles.size() > std::numeric_limits::max()) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be at most {}", + name, std::to_string(needles.size()), std::to_string(std::numeric_limits::max())); + auto res_callback = [](const UInt8 * start, const UInt8 * end) -> UInt64 { return 1 + Impl::countChars(reinterpret_cast(start), reinterpret_cast(end)); From c9ce0efa662b4af905450aa2a0db17cd7609ed89 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 25 Jun 2022 17:28:15 +0200 Subject: [PATCH 110/123] Instantiate MultiMatchAnyImpl template using enums - With this, invalid combinations of the FindAny/FindAnyIndex bools are no longer possible and we can remove the corresponding check - Also makes the instantiations more readable. --- src/Functions/MultiMatchAllIndicesImpl.h | 5 ++--- src/Functions/MultiMatchAnyImpl.h | 23 ++++++++++++++------- src/Functions/Regexps.h | 14 ++++++------- src/Functions/multiFuzzyMatchAllIndices.cpp | 2 +- src/Functions/multiFuzzyMatchAny.cpp | 2 +- src/Functions/multiFuzzyMatchAnyIndex.cpp | 2 +- src/Functions/multiMatchAllIndices.cpp | 2 +- src/Functions/multiMatchAny.cpp | 2 +- src/Functions/multiMatchAnyIndex.cpp | 2 +- 9 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/Functions/MultiMatchAllIndicesImpl.h b/src/Functions/MultiMatchAllIndicesImpl.h index cb8d29723a6..3234782d1c1 100644 --- a/src/Functions/MultiMatchAllIndicesImpl.h +++ b/src/Functions/MultiMatchAllIndicesImpl.h @@ -27,7 +27,7 @@ namespace ErrorCodes } -template +template struct MultiMatchAllIndicesImpl { using ResultType = ResultType_; @@ -69,8 +69,7 @@ struct MultiMatchAllIndicesImpl checkHyperscanRegexp(needles, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); offsets.resize(haystack_offsets.size()); - - const auto & hyperscan_regex = MultiRegexps::get(needles, edit_distance); + const auto & hyperscan_regex = MultiRegexps::get(needles, edit_distance); hs_scratch_t * scratch = nullptr; hs_error_t err = hs_clone_scratch(hyperscan_regex->getScratch(), &scratch); diff --git a/src/Functions/MultiMatchAnyImpl.h b/src/Functions/MultiMatchAnyImpl.h index 349078b9d4a..12b7f76d173 100644 --- a/src/Functions/MultiMatchAnyImpl.h +++ b/src/Functions/MultiMatchAnyImpl.h @@ -27,14 +27,24 @@ namespace ErrorCodes extern const int TOO_MANY_BYTES; } +// For more readable instantiations of MultiMatchAnyImpl<> +struct MultiMatchTraits +{ +enum class Find +{ + Any, + AnyIndex +}; +}; -template +template struct MultiMatchAnyImpl { - static_assert(FindAny ^ FindAnyIndex); - using ResultType = ResultType_; + static constexpr bool FindAny = (Find == MultiMatchTraits::Find::Any); + static constexpr bool FindAnyIndex = (Find == MultiMatchTraits::Find::AnyIndex); + static constexpr bool is_using_hyperscan = true; /// Variable for understanding, if we used offsets for the output, most /// likely to determine whether the function returns ColumnVector of ColumnArray. @@ -68,14 +78,11 @@ struct MultiMatchAnyImpl size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length) { - (void)FindAny; - (void)FindAnyIndex; - checkHyperscanRegexp(needles, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); res.resize(haystack_offsets.size()); #if USE_VECTORSCAN - const auto & hyperscan_regex = MultiRegexps::get(needles, edit_distance); + const auto & hyperscan_regex = MultiRegexps::get(needles, edit_distance); hs_scratch_t * scratch = nullptr; hs_error_t err = hs_clone_scratch(hyperscan_regex->getScratch(), &scratch); @@ -121,7 +128,7 @@ struct MultiMatchAnyImpl } #else // fallback if vectorscan is not compiled - if constexpr (MultiSearchDistance) + if constexpr (WithEditDistance) throw Exception( "Edit distance multi-search is not implemented when vectorscan is off", ErrorCodes::NOT_IMPLEMENTED); diff --git a/src/Functions/Regexps.h b/src/Functions/Regexps.h index 3e12f7fa651..b932b14a6a9 100644 --- a/src/Functions/Regexps.h +++ b/src/Functions/Regexps.h @@ -166,7 +166,7 @@ struct Pool std::map, std::optional>, RegexpsConstructor> storage; }; -template +template inline Regexps constructRegexps(const std::vector & str_patterns, [[maybe_unused]] std::optional edit_distance) { /// Common pointers @@ -180,7 +180,7 @@ inline Regexps constructRegexps(const std::vector & str_patterns, [[mayb patterns.reserve(str_patterns.size()); flags.reserve(str_patterns.size()); - if constexpr (CompileForEditDistance) + if constexpr (WithEditDistance) { ext_exprs.reserve(str_patterns.size()); ext_exprs_ptrs.reserve(str_patterns.size()); @@ -198,7 +198,7 @@ inline Regexps constructRegexps(const std::vector & str_patterns, [[mayb * as it is said in the Hyperscan documentation. https://intel.github.io/hyperscan/dev-reference/performance.html#single-match-flag */ flags.push_back(HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH | HS_FLAG_ALLOWEMPTY | HS_FLAG_UTF8); - if constexpr (CompileForEditDistance) + if constexpr (WithEditDistance) { /// Hyperscan currently does not support UTF8 matching with edit distance. flags.back() &= ~HS_FLAG_UTF8; @@ -223,7 +223,7 @@ inline Regexps constructRegexps(const std::vector & str_patterns, [[mayb } hs_error_t err; - if constexpr (!CompileForEditDistance) + if constexpr (!WithEditDistance) err = hs_compile_multi( patterns.data(), flags.data(), @@ -272,11 +272,11 @@ inline Regexps constructRegexps(const std::vector & str_patterns, [[mayb return {db, scratch}; } -/// If CompileForEditDistance is False, edit_distance must be nullopt +/// If WithEditDistance is False, edit_distance must be nullopt /// Also, we use templates here because each instantiation of function /// template has its own copy of local static variables which must not be the same /// for different hyperscan compilations. -template +template inline Regexps * get(const std::vector & patterns, std::optional edit_distance) { static Pool known_regexps; /// Different variables for different pattern parameters, thread-safe in C++11 @@ -299,7 +299,7 @@ inline Regexps * get(const std::vector & patterns, std::option .first; it->second.setConstructor([&str_patterns = it->first.first, edit_distance]() { - return constructRegexps(str_patterns, edit_distance); + return constructRegexps(str_patterns, edit_distance); }); } diff --git a/src/Functions/multiFuzzyMatchAllIndices.cpp b/src/Functions/multiFuzzyMatchAllIndices.cpp index 2c4ac05ddbe..93ffb936dc1 100644 --- a/src/Functions/multiFuzzyMatchAllIndices.cpp +++ b/src/Functions/multiFuzzyMatchAllIndices.cpp @@ -13,7 +13,7 @@ struct NameMultiFuzzyMatchAllIndices static constexpr auto name = "multiFuzzyMatchAllIndices"; }; -using FunctionMultiFuzzyMatchAllIndices = FunctionsMultiStringFuzzySearch>; +using FunctionMultiFuzzyMatchAllIndices = FunctionsMultiStringFuzzySearch>; } diff --git a/src/Functions/multiFuzzyMatchAny.cpp b/src/Functions/multiFuzzyMatchAny.cpp index fbd84ead31b..a627030d7af 100644 --- a/src/Functions/multiFuzzyMatchAny.cpp +++ b/src/Functions/multiFuzzyMatchAny.cpp @@ -13,7 +13,7 @@ struct NameMultiFuzzyMatchAny static constexpr auto name = "multiFuzzyMatchAny"; }; -using FunctionMultiFuzzyMatchAny = FunctionsMultiStringFuzzySearch>; +using FunctionMultiFuzzyMatchAny = FunctionsMultiStringFuzzySearch>; } diff --git a/src/Functions/multiFuzzyMatchAnyIndex.cpp b/src/Functions/multiFuzzyMatchAnyIndex.cpp index 255f710e2b3..4b24a06e171 100644 --- a/src/Functions/multiFuzzyMatchAnyIndex.cpp +++ b/src/Functions/multiFuzzyMatchAnyIndex.cpp @@ -13,7 +13,7 @@ struct NameMultiFuzzyMatchAnyIndex static constexpr auto name = "multiFuzzyMatchAnyIndex"; }; -using FunctionMultiFuzzyMatchAnyIndex = FunctionsMultiStringFuzzySearch>; +using FunctionMultiFuzzyMatchAnyIndex = FunctionsMultiStringFuzzySearch>; } diff --git a/src/Functions/multiMatchAllIndices.cpp b/src/Functions/multiMatchAllIndices.cpp index 1c6577e5aa3..47bd57029e2 100644 --- a/src/Functions/multiMatchAllIndices.cpp +++ b/src/Functions/multiMatchAllIndices.cpp @@ -13,7 +13,7 @@ struct NameMultiMatchAllIndices static constexpr auto name = "multiMatchAllIndices"; }; -using FunctionMultiMatchAllIndices = FunctionsMultiStringSearch>; +using FunctionMultiMatchAllIndices = FunctionsMultiStringSearch>; } diff --git a/src/Functions/multiMatchAny.cpp b/src/Functions/multiMatchAny.cpp index 4920e9c230f..324e435de26 100644 --- a/src/Functions/multiMatchAny.cpp +++ b/src/Functions/multiMatchAny.cpp @@ -13,7 +13,7 @@ struct NameMultiMatchAny static constexpr auto name = "multiMatchAny"; }; -using FunctionMultiMatchAny = FunctionsMultiStringSearch>; +using FunctionMultiMatchAny = FunctionsMultiStringSearch>; } diff --git a/src/Functions/multiMatchAnyIndex.cpp b/src/Functions/multiMatchAnyIndex.cpp index bf68e8576a5..6a11fa4eb35 100644 --- a/src/Functions/multiMatchAnyIndex.cpp +++ b/src/Functions/multiMatchAnyIndex.cpp @@ -13,7 +13,7 @@ struct NameMultiMatchAnyIndex static constexpr auto name = "multiMatchAnyIndex"; }; -using FunctionMultiMatchAnyIndex = FunctionsMultiStringSearch>; +using FunctionMultiMatchAnyIndex = FunctionsMultiStringSearch>; } From c2cea38b97e2282277d02e27c6ba249ed50179b5 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 25 Jun 2022 17:41:24 +0200 Subject: [PATCH 111/123] Move local variable into if statement --- src/Functions/FunctionsMultiStringFuzzySearch.h | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index b83b5c4192b..c161ffc50cd 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -74,15 +74,14 @@ public: const ColumnString * col_haystack_vector = checkAndGetColumn(&*column_haystack); assert(col_haystack_vector); // getReturnTypeImpl() checks the data type - const ColumnConst * col_const_num = nullptr; UInt32 edit_distance = 0; - if ((col_const_num = checkAndGetColumnConst(num_ptr.get()))) - edit_distance = col_const_num->getValue(); - else if ((col_const_num = checkAndGetColumnConst(num_ptr.get()))) - edit_distance = col_const_num->getValue(); - else if ((col_const_num = checkAndGetColumnConst(num_ptr.get()))) - edit_distance = col_const_num->getValue(); + if (const auto * col_const_uint8 = checkAndGetColumnConst(num_ptr.get())) + edit_distance = col_const_uint8->getValue(); + else if (const auto * col_const_uint16 = checkAndGetColumnConst(num_ptr.get())) + edit_distance = col_const_uint16->getValue(); + else if (const auto * col_const_uint32 = checkAndGetColumnConst(num_ptr.get())) + edit_distance = col_const_uint32->getValue(); else throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}. The number is not const or does not fit in UInt32", arguments[1].column->getName()); From e2b11899a1c6371c70f375a3ac119bd5b73a43d8 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 25 Jun 2022 17:53:11 +0200 Subject: [PATCH 112/123] Move check if cfg allows hyperscan into implementations - This is not needed for non-const regexp array arguments but cleans up the code and runs the check only in functions which actually use hyperscan. --- src/Functions/FunctionsMultiStringFuzzySearch.h | 16 ++++++++-------- src/Functions/FunctionsMultiStringSearch.h | 16 ++++++++-------- src/Functions/MultiMatchAllIndicesImpl.h | 10 +++++++--- src/Functions/MultiMatchAnyImpl.h | 11 ++++++++--- src/Functions/MultiSearchFirstIndexImpl.h | 2 +- src/Functions/MultiSearchFirstPositionImpl.h | 2 +- src/Functions/MultiSearchImpl.h | 2 +- 7 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index c161ffc50cd..33721dd19ef 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -30,18 +30,17 @@ class FunctionsMultiStringFuzzySearch : public IFunction { public: static constexpr auto name = Impl::name; + static FunctionPtr create(ContextPtr context) { const auto & settings = context->getSettingsRef(); - - if (Impl::is_using_hyperscan && !settings.allow_hyperscan) - throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0"); - - return std::make_shared(settings.max_hyperscan_regexp_length, settings.max_hyperscan_regexp_total_length); + return std::make_shared(settings.allow_hyperscan, settings.max_hyperscan_regexp_length, settings.max_hyperscan_regexp_total_length); } - FunctionsMultiStringFuzzySearch(size_t max_hyperscan_regexp_length_, size_t max_hyperscan_regexp_total_length_) - : max_hyperscan_regexp_length(max_hyperscan_regexp_length_), max_hyperscan_regexp_total_length(max_hyperscan_regexp_total_length_) + FunctionsMultiStringFuzzySearch(bool allow_hyperscan_, size_t max_hyperscan_regexp_length_, size_t max_hyperscan_regexp_total_length_) + : allow_hyperscan(allow_hyperscan_) + , max_hyperscan_regexp_length(max_hyperscan_regexp_length_) + , max_hyperscan_regexp_total_length(max_hyperscan_regexp_total_length_) {} String getName() const override { return name; } @@ -108,7 +107,7 @@ public: Impl::vectorConstant( col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res, edit_distance, - max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); if constexpr (Impl::is_column_array) return ColumnArray::create(std::move(col_res), std::move(col_offsets)); @@ -117,6 +116,7 @@ public: } private: + const bool allow_hyperscan; const size_t max_hyperscan_regexp_length; const size_t max_hyperscan_regexp_total_length; }; diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index 8b48f8f9eaa..b4ff354af41 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -45,18 +45,17 @@ class FunctionsMultiStringSearch : public IFunction { public: static constexpr auto name = Impl::name; + static FunctionPtr create(ContextPtr context) { const auto & settings = context->getSettingsRef(); - - if (Impl::is_using_hyperscan && !settings.allow_hyperscan) - throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0"); - - return std::make_shared(settings.max_hyperscan_regexp_length, settings.max_hyperscan_regexp_total_length); + return std::make_shared(settings.allow_hyperscan, settings.max_hyperscan_regexp_length, settings.max_hyperscan_regexp_total_length); } - FunctionsMultiStringSearch(size_t max_hyperscan_regexp_length_, size_t max_hyperscan_regexp_total_length_) - : max_hyperscan_regexp_length(max_hyperscan_regexp_length_), max_hyperscan_regexp_total_length(max_hyperscan_regexp_total_length_) + FunctionsMultiStringSearch(bool allow_hyperscan_, size_t max_hyperscan_regexp_length_, size_t max_hyperscan_regexp_total_length_) + : allow_hyperscan(allow_hyperscan_) + , max_hyperscan_regexp_length(max_hyperscan_regexp_length_) + , max_hyperscan_regexp_total_length(max_hyperscan_regexp_total_length_) {} String getName() const override { return name; } @@ -108,7 +107,7 @@ public: Impl::vectorConstant( col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res, - max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); if constexpr (Impl::is_column_array) return ColumnArray::create(std::move(col_res), std::move(col_offsets)); @@ -117,6 +116,7 @@ public: } private: + const bool allow_hyperscan; const size_t max_hyperscan_regexp_length; const size_t max_hyperscan_regexp_total_length; }; diff --git a/src/Functions/MultiMatchAllIndicesImpl.h b/src/Functions/MultiMatchAllIndicesImpl.h index 3234782d1c1..97149b48df1 100644 --- a/src/Functions/MultiMatchAllIndicesImpl.h +++ b/src/Functions/MultiMatchAllIndicesImpl.h @@ -20,8 +20,9 @@ namespace DB namespace ErrorCodes { - extern const int HYPERSCAN_CANNOT_SCAN_TEXT; extern const int CANNOT_ALLOCATE_MEMORY; + extern const int FUNCTION_NOT_ALLOWED; + extern const int HYPERSCAN_CANNOT_SCAN_TEXT; extern const int NOT_IMPLEMENTED; extern const int TOO_MANY_BYTES; } @@ -32,7 +33,6 @@ struct MultiMatchAllIndicesImpl { using ResultType = ResultType_; - static constexpr bool is_using_hyperscan = true; /// Variable for understanding, if we used offsets for the output, most /// likely to determine whether the function returns ColumnVector of ColumnArray. static constexpr bool is_column_array = true; @@ -49,10 +49,11 @@ struct MultiMatchAllIndicesImpl const std::vector & needles, PaddedPODArray & res, PaddedPODArray & offsets, + bool allow_hyperscan, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length) { - vectorConstant(haystack_data, haystack_offsets, needles, res, offsets, std::nullopt, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + vectorConstant(haystack_data, haystack_offsets, needles, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); } static void vectorConstant( @@ -62,9 +63,12 @@ struct MultiMatchAllIndicesImpl [[maybe_unused]] PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets, [[maybe_unused]] std::optional edit_distance, + bool allow_hyperscan, [[maybe_unused]] size_t max_hyperscan_regexp_length, [[maybe_unused]] size_t max_hyperscan_regexp_total_length) { + if (!allow_hyperscan) + throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0"); #if USE_VECTORSCAN checkHyperscanRegexp(needles, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); diff --git a/src/Functions/MultiMatchAnyImpl.h b/src/Functions/MultiMatchAnyImpl.h index 12b7f76d173..956458e2953 100644 --- a/src/Functions/MultiMatchAnyImpl.h +++ b/src/Functions/MultiMatchAnyImpl.h @@ -21,8 +21,9 @@ namespace DB namespace ErrorCodes { - extern const int HYPERSCAN_CANNOT_SCAN_TEXT; extern const int CANNOT_ALLOCATE_MEMORY; + extern const int FUNCTION_NOT_ALLOWED; + extern const int HYPERSCAN_CANNOT_SCAN_TEXT; extern const int NOT_IMPLEMENTED; extern const int TOO_MANY_BYTES; } @@ -45,7 +46,6 @@ struct MultiMatchAnyImpl static constexpr bool FindAny = (Find == MultiMatchTraits::Find::Any); static constexpr bool FindAnyIndex = (Find == MultiMatchTraits::Find::AnyIndex); - static constexpr bool is_using_hyperscan = true; /// Variable for understanding, if we used offsets for the output, most /// likely to determine whether the function returns ColumnVector of ColumnArray. static constexpr bool is_column_array = false; @@ -62,10 +62,11 @@ struct MultiMatchAnyImpl const std::vector & needles, PaddedPODArray & res, PaddedPODArray & offsets, + bool allow_hyperscan, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length) { - vectorConstant(haystack_data, haystack_offsets, needles, res, offsets, std::nullopt, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + vectorConstant(haystack_data, haystack_offsets, needles, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); } static void vectorConstant( @@ -75,9 +76,13 @@ struct MultiMatchAnyImpl PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets, [[maybe_unused]] std::optional edit_distance, + bool allow_hyperscan, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length) { + if (!allow_hyperscan) + throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0"); + checkHyperscanRegexp(needles, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); res.resize(haystack_offsets.size()); diff --git a/src/Functions/MultiSearchFirstIndexImpl.h b/src/Functions/MultiSearchFirstIndexImpl.h index 4f01c45bdf4..3a853625b37 100644 --- a/src/Functions/MultiSearchFirstIndexImpl.h +++ b/src/Functions/MultiSearchFirstIndexImpl.h @@ -16,7 +16,6 @@ template struct MultiSearchFirstIndexImpl { using ResultType = UInt64; - static constexpr bool is_using_hyperscan = false; /// Variable for understanding, if we used offsets for the output, most /// likely to determine whether the function returns ColumnVector of ColumnArray. static constexpr bool is_column_array = false; @@ -30,6 +29,7 @@ struct MultiSearchFirstIndexImpl const std::vector & needles, PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets, + bool /*allow_hyperscan*/, size_t /*max_hyperscan_regexp_length*/, size_t /*max_hyperscan_regexp_total_length*/) { diff --git a/src/Functions/MultiSearchFirstPositionImpl.h b/src/Functions/MultiSearchFirstPositionImpl.h index 2260950c5d4..87eb56c3b11 100644 --- a/src/Functions/MultiSearchFirstPositionImpl.h +++ b/src/Functions/MultiSearchFirstPositionImpl.h @@ -16,7 +16,6 @@ template struct MultiSearchFirstPositionImpl { using ResultType = UInt64; - static constexpr bool is_using_hyperscan = false; /// Variable for understanding, if we used offsets for the output, most /// likely to determine whether the function returns ColumnVector of ColumnArray. static constexpr bool is_column_array = false; @@ -30,6 +29,7 @@ struct MultiSearchFirstPositionImpl const std::vector & needles, PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets, + bool /*allow_hyperscan*/, size_t /*max_hyperscan_regexp_length*/, size_t /*max_hyperscan_regexp_total_length*/) { diff --git a/src/Functions/MultiSearchImpl.h b/src/Functions/MultiSearchImpl.h index 0c951f3c5fc..63ad41f1f7b 100644 --- a/src/Functions/MultiSearchImpl.h +++ b/src/Functions/MultiSearchImpl.h @@ -16,7 +16,6 @@ template struct MultiSearchImpl { using ResultType = UInt8; - static constexpr bool is_using_hyperscan = false; /// Variable for understanding, if we used offsets for the output, most /// likely to determine whether the function returns ColumnVector of ColumnArray. static constexpr bool is_column_array = false; @@ -30,6 +29,7 @@ struct MultiSearchImpl const std::vector & needles, PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets, + bool /*allow_hyperscan*/, size_t /*max_hyperscan_regexp_length*/, size_t /*max_hyperscan_regexp_total_length*/) { From b8f67185bf43e25005f697cdd0402479155bbeb3 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 25 Jun 2022 18:05:49 +0200 Subject: [PATCH 113/123] Cosmetics: Whitespaces --- src/Functions/FunctionsMultiStringFuzzySearch.h | 4 ---- src/Functions/FunctionsMultiStringSearch.h | 3 --- 2 files changed, 7 deletions(-) diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index 33721dd19ef..b42352d990e 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -74,7 +74,6 @@ public: assert(col_haystack_vector); // getReturnTypeImpl() checks the data type UInt32 edit_distance = 0; - if (const auto * col_const_uint8 = checkAndGetColumnConst(num_ptr.get())) edit_distance = col_const_uint8->getValue(); else if (const auto * col_const_uint16 = checkAndGetColumnConst(num_ptr.get())) @@ -85,15 +84,12 @@ public: throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}. The number is not const or does not fit in UInt32", arguments[1].column->getName()); const ColumnConst * col_const_arr = checkAndGetColumnConst(arr_ptr.get()); - if (!col_const_arr) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}. The array is not const", arguments[2].column->getName()); Array src_arr = col_const_arr->getValue(); - std::vector refs; refs.reserve(src_arr.size()); - for (const auto & el : src_arr) refs.emplace_back(el.get()); diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index b4ff354af41..1fe2e036a9a 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -85,15 +85,12 @@ public: assert(col_haystack_vector); // getReturnTypeImpl() checks the data type const ColumnConst * col_const_arr = checkAndGetColumnConst(arr_ptr.get()); - if (!col_const_arr) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}. The array is not const", arguments[1].column->getName()); Array src_arr = col_const_arr->getValue(); - std::vector refs; refs.reserve(src_arr.size()); - for (const auto & el : src_arr) refs.emplace_back(el.get()); From cb5d1a4a856e3362a276f801de4c456e1908429f Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 26 Jun 2022 15:45:52 +0000 Subject: [PATCH 114/123] Fix style check --- src/Functions/FunctionsMultiStringFuzzySearch.h | 1 - src/Functions/FunctionsMultiStringSearch.h | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index b42352d990e..b6428bb0f11 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -21,7 +21,6 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; - extern const int FUNCTION_NOT_ALLOWED; } diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index 1fe2e036a9a..be51f412f06 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -36,7 +36,6 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; - extern const int FUNCTION_NOT_ALLOWED; } From 959cbaab02b5c4cb961a1621cfbd6aa7cc932c31 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 26 Jun 2022 16:12:17 +0000 Subject: [PATCH 115/123] Move loop over patterns into implementations - This is preparation for non-const regexp arguments, where this loop will run for each row. --- src/Functions/FunctionsMultiStringFuzzySearch.h | 9 ++------- src/Functions/FunctionsMultiStringSearch.h | 9 ++------- src/Functions/MultiMatchAllIndicesImpl.h | 12 +++++++++--- src/Functions/MultiMatchAnyImpl.h | 12 +++++++++--- src/Functions/MultiSearchFirstIndexImpl.h | 12 +++++++++--- src/Functions/MultiSearchFirstPositionImpl.h | 12 +++++++++--- src/Functions/MultiSearchImpl.h | 12 +++++++++--- 7 files changed, 49 insertions(+), 29 deletions(-) diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index b6428bb0f11..865a5d182c8 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -86,12 +86,6 @@ public: if (!col_const_arr) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}. The array is not const", arguments[2].column->getName()); - Array src_arr = col_const_arr->getValue(); - std::vector refs; - refs.reserve(src_arr.size()); - for (const auto & el : src_arr) - refs.emplace_back(el.get()); - using ResultType = typename Impl::ResultType; auto col_res = ColumnVector::create(); auto col_offsets = ColumnArray::ColumnOffsets::create(); @@ -100,8 +94,9 @@ public: auto & offsets_res = col_offsets->getData(); // the implementations are responsible for resizing the output column + Array needles_arr = col_const_arr->getValue(); Impl::vectorConstant( - col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res, edit_distance, + col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), needles_arr, vec_res, offsets_res, edit_distance, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); if constexpr (Impl::is_column_array) diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index be51f412f06..04235e0a97a 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -87,12 +87,6 @@ public: if (!col_const_arr) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {}. The array is not const", arguments[1].column->getName()); - Array src_arr = col_const_arr->getValue(); - std::vector refs; - refs.reserve(src_arr.size()); - for (const auto & el : src_arr) - refs.emplace_back(el.get()); - using ResultType = typename Impl::ResultType; auto col_res = ColumnVector::create(); auto col_offsets = ColumnArray::ColumnOffsets::create(); @@ -101,8 +95,9 @@ public: auto & offsets_res = col_offsets->getData(); // the implementations are responsible for resizing the output column + Array needles_arr = col_const_arr->getValue(); Impl::vectorConstant( - col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), refs, vec_res, offsets_res, + col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), needles_arr, vec_res, offsets_res, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); if constexpr (Impl::is_column_array) diff --git a/src/Functions/MultiMatchAllIndicesImpl.h b/src/Functions/MultiMatchAllIndicesImpl.h index 97149b48df1..9c60dbffe91 100644 --- a/src/Functions/MultiMatchAllIndicesImpl.h +++ b/src/Functions/MultiMatchAllIndicesImpl.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -46,20 +47,20 @@ struct MultiMatchAllIndicesImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const Array & needles_arr, PaddedPODArray & res, PaddedPODArray & offsets, bool allow_hyperscan, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length) { - vectorConstant(haystack_data, haystack_offsets, needles, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); } static void vectorConstant( [[maybe_unused]] const ColumnString::Chars & haystack_data, [[maybe_unused]] const ColumnString::Offsets & haystack_offsets, - [[maybe_unused]] const std::vector & needles, + [[maybe_unused]] const Array & needles_arr, [[maybe_unused]] PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets, [[maybe_unused]] std::optional edit_distance, @@ -70,6 +71,11 @@ struct MultiMatchAllIndicesImpl if (!allow_hyperscan) throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0"); #if USE_VECTORSCAN + std::vector needles; + needles.reserve(needles_arr.size()); + for (const auto & needle : needles_arr) + needles.emplace_back(needle.get()); + checkHyperscanRegexp(needles, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); offsets.resize(haystack_offsets.size()); diff --git a/src/Functions/MultiMatchAnyImpl.h b/src/Functions/MultiMatchAnyImpl.h index 956458e2953..0752e87e8af 100644 --- a/src/Functions/MultiMatchAnyImpl.h +++ b/src/Functions/MultiMatchAnyImpl.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -59,20 +60,20 @@ struct MultiMatchAnyImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const Array & needles_arr, PaddedPODArray & res, PaddedPODArray & offsets, bool allow_hyperscan, size_t max_hyperscan_regexp_length, size_t max_hyperscan_regexp_total_length) { - vectorConstant(haystack_data, haystack_offsets, needles, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); + vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); } static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const Array & needles_arr, PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets, [[maybe_unused]] std::optional edit_distance, @@ -83,6 +84,11 @@ struct MultiMatchAnyImpl if (!allow_hyperscan) throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0"); + std::vector needles; + needles.reserve(needles_arr.size()); + for (const auto & needle : needles_arr) + needles.emplace_back(needle.get()); + checkHyperscanRegexp(needles, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length); res.resize(haystack_offsets.size()); diff --git a/src/Functions/MultiSearchFirstIndexImpl.h b/src/Functions/MultiSearchFirstIndexImpl.h index 3a853625b37..f69a3edbf8b 100644 --- a/src/Functions/MultiSearchFirstIndexImpl.h +++ b/src/Functions/MultiSearchFirstIndexImpl.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -26,7 +27,7 @@ struct MultiSearchFirstIndexImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const Array & needles_arr, PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets, bool /*allow_hyperscan*/, @@ -34,10 +35,15 @@ struct MultiSearchFirstIndexImpl size_t /*max_hyperscan_regexp_total_length*/) { // For performance of Volnitsky search, it is crucial to save only one byte for pattern number. - if (needles.size() > std::numeric_limits::max()) + if (needles_arr.size() > std::numeric_limits::max()) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Number of arguments for function {} doesn't match: passed {}, should be at most {}", - name, std::to_string(needles.size()), std::to_string(std::numeric_limits::max())); + name, std::to_string(needles_arr.size()), std::to_string(std::numeric_limits::max())); + + std::vector needles; + needles.reserve(needles_arr.size()); + for (const auto & needle : needles_arr) + needles.emplace_back(needle.get()); auto searcher = Impl::createMultiSearcherInBigHaystack(needles); const size_t haystack_string_size = haystack_offsets.size(); diff --git a/src/Functions/MultiSearchFirstPositionImpl.h b/src/Functions/MultiSearchFirstPositionImpl.h index 87eb56c3b11..21d558a6d58 100644 --- a/src/Functions/MultiSearchFirstPositionImpl.h +++ b/src/Functions/MultiSearchFirstPositionImpl.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -26,7 +27,7 @@ struct MultiSearchFirstPositionImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const Array & needles_arr, PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets, bool /*allow_hyperscan*/, @@ -34,10 +35,15 @@ struct MultiSearchFirstPositionImpl size_t /*max_hyperscan_regexp_total_length*/) { // For performance of Volnitsky search, it is crucial to save only one byte for pattern number. - if (needles.size() > std::numeric_limits::max()) + if (needles_arr.size() > std::numeric_limits::max()) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Number of arguments for function {} doesn't match: passed {}, should be at most {}", - name, std::to_string(needles.size()), std::to_string(std::numeric_limits::max())); + name, std::to_string(needles_arr.size()), std::to_string(std::numeric_limits::max())); + + std::vector needles; + needles.reserve(needles_arr.size()); + for (const auto & needle : needles_arr) + needles.emplace_back(needle.get()); auto res_callback = [](const UInt8 * start, const UInt8 * end) -> UInt64 { diff --git a/src/Functions/MultiSearchImpl.h b/src/Functions/MultiSearchImpl.h index 63ad41f1f7b..1124184f58c 100644 --- a/src/Functions/MultiSearchImpl.h +++ b/src/Functions/MultiSearchImpl.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -26,7 +27,7 @@ struct MultiSearchImpl static void vectorConstant( const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets, - const std::vector & needles, + const Array & needles_arr, PaddedPODArray & res, [[maybe_unused]] PaddedPODArray & offsets, bool /*allow_hyperscan*/, @@ -34,10 +35,15 @@ struct MultiSearchImpl size_t /*max_hyperscan_regexp_total_length*/) { // For performance of Volnitsky search, it is crucial to save only one byte for pattern number. - if (needles.size() > std::numeric_limits::max()) + if (needles_arr.size() > std::numeric_limits::max()) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Number of arguments for function {} doesn't match: passed {}, should be at most {}", - name, std::to_string(needles.size()), std::to_string(std::numeric_limits::max())); + name, std::to_string(needles_arr.size()), std::to_string(std::numeric_limits::max())); + + std::vector needles; + needles.reserve(needles_arr.size()); + for (const auto & needle : needles_arr) + needles.emplace_back(needle.get()); auto searcher = Impl::createMultiSearcherInBigHaystack(needles); const size_t haystack_string_size = haystack_offsets.size(); From 5727671cacd72fe00b93d965e18054c479289cd5 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 27 Jun 2022 00:34:57 +0200 Subject: [PATCH 116/123] Fix tests --- src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 12 ++++++++---- src/Disks/IO/ReadBufferFromRemoteFSGather.h | 1 - .../ObjectStorages/DiskObjectStorageMetadata.cpp | 2 +- src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp | 2 +- src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h | 2 -- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index b53ba0f8e29..1a4ae3f963f 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -84,16 +84,20 @@ SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBufferImpl(con #if USE_AZURE_BLOB_STORAGE SeekableReadBufferPtr ReadBufferFromAzureBlobStorageGather::createImplementationBufferImpl(const String & path, size_t /* file_size */) { - current_file_path = path; - return std::make_unique(blob_container_client, path, max_single_read_retries, - max_single_download_retries, settings.remote_fs_buffer_size, /* use_external_buffer */true, read_until_position); + return std::make_unique( + blob_container_client, + path, + max_single_read_retries, + max_single_download_retries, + settings.remote_fs_buffer_size, + /* use_external_buffer */true, + read_until_position); } #endif SeekableReadBufferPtr ReadBufferFromWebServerGather::createImplementationBufferImpl(const String & path, size_t /* file_size */) { - current_file_path = path; return std::make_unique( fs::path(uri) / path, context, diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h index e282a2cfc6c..6a39f4add46 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.h +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.h @@ -188,7 +188,6 @@ class ReadBufferFromHDFSGather final : public ReadBufferFromRemoteFSGather public: ReadBufferFromHDFSGather( const Poco::Util::AbstractConfiguration & config_, - const String &, const PathsWithSize & blobs_to_read_, const ReadSettings & settings_) : ReadBufferFromRemoteFSGather(blobs_to_read_, settings_) diff --git a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp index 9ac04809b97..4564e84316d 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageMetadata.cpp @@ -120,7 +120,7 @@ DiskObjectStorageMetadata::DiskObjectStorageMetadata( void DiskObjectStorageMetadata::addObject(const String & path, size_t size) { - if (path.starts_with(remote_fs_root_path)) + if (!remote_fs_root_path.empty() && path.starts_with(remote_fs_root_path)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected relative path"); total_size += size; diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp index 024d1e84ef0..82c700e1a63 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp @@ -54,7 +54,7 @@ std::unique_ptr HDFSObjectStorage::readObjects( /// NOLI std::optional, std::optional) const { - auto hdfs_impl = std::make_unique(config, hdfs_root_path, paths_to_read, read_settings); + auto hdfs_impl = std::make_unique(config, paths_to_read, read_settings); auto buf = std::make_unique(std::move(hdfs_impl)); return std::make_unique(std::move(buf), settings->min_bytes_for_seek); } diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h index 221a77ff08b..28f553906ea 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h @@ -50,7 +50,6 @@ public: , hdfs_builder(createHDFSBuilder(hdfs_root_path_, config)) , hdfs_fs(createHDFSFS(hdfs_builder.get())) , settings(std::move(settings_)) - , hdfs_root_path(hdfs_root_path_) {} bool exists(const std::string & hdfs_uri) const override; @@ -118,7 +117,6 @@ private: HDFSFSPtr hdfs_fs; SettingsPtr settings; - String hdfs_root_path; }; } From cbcf51530643f332a944902e91a71a6a19537de1 Mon Sep 17 00:00:00 2001 From: San Date: Mon, 27 Jun 2022 12:06:16 +1000 Subject: [PATCH 117/123] Update SECURITY.md.sh Update Security.md generation code to include bugcrowd program information. --- utils/security-generator/SECURITY.md.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/security-generator/SECURITY.md.sh b/utils/security-generator/SECURITY.md.sh index 97c696c1227..381f5b4eaa6 100755 --- a/utils/security-generator/SECURITY.md.sh +++ b/utils/security-generator/SECURITY.md.sh @@ -56,7 +56,7 @@ echo " We're extremely grateful for security researchers and users that report vulnerabilities to the ClickHouse Open Source Community. All reports are thoroughly investigated by developers. -To report a potential vulnerability in ClickHouse please send the details about it to [security@clickhouse.com](mailto:security@clickhouse.com). +To report a potential vulnerability in ClickHouse please send the details about it to [security@clickhouse.com](mailto:security@clickhouse.com). We do not offer any financial rewards for reporting issues to us using this method. Alternatively, you can also submit your findings through our public bug bounty program hosted by [Bugcrowd](https://bugcrowd.com/clickhouse) and be rewarded for it as per the program scope and rules of engagement. ### When Should I Report a Vulnerability? @@ -76,5 +76,5 @@ As the security issue moves from triage, to identified fix, to release planning ## Public Disclosure Timing -A public disclosure date is negotiated by the ClickHouse maintainers and the bug submitter. We prefer to fully disclose the bug as soon as possible once a user mitigation is available. It is reasonable to delay disclosure when the bug or the fix is not yet fully understood, the solution is not well-tested, or for vendor coordination. The timeframe for disclosure is from immediate (especially if it's already publicly known) to 90 days. For a vulnerability with a straightforward mitigation, we expect report date to disclosure date to be on the order of 7 days. +A public disclosure date is negotiated by the ClickHouse maintainers and the bug submitter. We prefer to fully disclose the bug as soon as possible once a user mitigation is available. It is reasonable to delay disclosure when the bug or the fix is not yet fully understood, the solution is not well-tested, or for vendor coordination. The timeframe for disclosure is from immediate (especially if it's already publicly known) to 90 days. For a vulnerability with a straightforward mitigation, we expect the report date to disclosure date to be on the order of 7 days. " From 674188bd7fe6acca2205c7171ba51725bf294c9c Mon Sep 17 00:00:00 2001 From: Miel Donkers Date: Mon, 27 Jun 2022 12:19:30 +0200 Subject: [PATCH 118/123] Docs: clarify processing of DateTime64 values --- .../en/sql-reference/data-types/datetime64.md | 25 +++++-- .../functions/type-conversion-functions.md | 74 +++++++++++++++++++ 2 files changed, 94 insertions(+), 5 deletions(-) diff --git a/docs/en/sql-reference/data-types/datetime64.md b/docs/en/sql-reference/data-types/datetime64.md index e67f86743a9..ddc71e75e44 100644 --- a/docs/en/sql-reference/data-types/datetime64.md +++ b/docs/en/sql-reference/data-types/datetime64.md @@ -7,7 +7,7 @@ sidebar_label: DateTime64 Allows to store an instant in time, that can be expressed as a calendar date and a time of a day, with defined sub-second precision -Tick size (precision): 10-precision seconds. Valid range: [ 0 : 9 ]. +Tick size (precision): 10-precision seconds. Valid range: [ 0 : 9 ]. Typically are used - 3 (milliseconds), 6 (microseconds), 9 (nanoseconds). **Syntax:** @@ -34,7 +34,7 @@ ENGINE = TinyLog; ``` ``` sql -INSERT INTO dt Values (1546300800000, 1), ('2019-01-01 00:00:00', 2); +INSERT INTO dt Values (1546300800123, 1), (1546300800.123, 2), ('2019-01-01 00:00:00', 3); ``` ``` sql @@ -43,12 +43,13 @@ SELECT * FROM dt; ``` text ┌───────────────timestamp─┬─event_id─┐ -│ 2019-01-01 03:00:00.000 │ 1 │ -│ 2019-01-01 00:00:00.000 │ 2 │ +│ 2019-01-01 03:00:00.123 │ 1 │ +│ 2019-01-01 03:00:00.123 │ 2 │ +│ 2019-01-01 00:00:00.000 │ 3 │ └─────────────────────────┴──────────┘ ``` -- When inserting datetime as an integer, it is treated as an appropriately scaled Unix Timestamp (UTC). `1546300800000` (with precision 3) represents `'2019-01-01 00:00:00'` UTC. However, as `timestamp` column has `Asia/Istanbul` (UTC+3) timezone specified, when outputting as a string the value will be shown as `'2019-01-01 03:00:00'`. +- When inserting datetime as an integer, it is treated as an appropriately scaled Unix Timestamp (UTC). `1546300800000` (with precision 3) represents `'2019-01-01 00:00:00'` UTC. However, as `timestamp` column has `Asia/Istanbul` (UTC+3) timezone specified, when outputting as a string the value will be shown as `'2019-01-01 03:00:00'`. Inserting datetime as a decimal will treat it similarly as an integer, except the value before the decimal point is the Unix Timestamp up to and including the seconds, and after the decimal point will be treated as the precision. - When inserting string value as datetime, it is treated as being in column timezone. `'2019-01-01 00:00:00'` will be treated as being in `Asia/Istanbul` timezone and stored as `1546290000000`. 2. Filtering on `DateTime64` values @@ -65,6 +66,20 @@ SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Asia/ Unlike `DateTime`, `DateTime64` values are not converted from `String` automatically. +``` sql +SELECT * FROM dt WHERE timestamp = toDateTime64(1546300800.123, 3); +``` + +``` text +┌───────────────timestamp─┬─event_id─┐ +│ 2019-01-01 00:00:00.123 │ 1 │ +│ 2019-01-01 00:00:00.123 │ 2 │ +└─────────────────────────┴──────────┘ +``` + +Contrary to inserting, the `toDateTime64` function will treat all values as the decimal variant, so precision needs to +be given after the decimal point. + 3. Getting a time zone for a `DateTime64`-type value: ``` sql diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index e2afddeaaf3..3f4db831e3d 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -301,6 +301,80 @@ Result: └─────────────────────────────────────────────────────────┴───────────────────────────────────────────────────────────┘ ``` +## toDateTime64 + +Converts the argument to the [DateTime64](../../sql-reference/data-types/datetime64.md) data type. + +**Syntax** + +``` sql +toDateTime64(expr, scale, [timezone]) +``` + +**Arguments** + +- `expr` — The value. [String](../../sql-reference/data-types/string.md), [UInt32](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [DateTime](../../sql-reference/data-types/datetime.md). +- `scale` - Tick size (precision): 10-precision seconds. Valid range: [ 0 : 9 ]. +- `timezone` - Time zone of the specified datetime64 object. + +**Returned value** + +- A calendar date and time of day, with sub-second precision. + +Type: [DateTime64](../../sql-reference/data-types/datetime64.md). + +**Example** + +1. The value is within the range: + +``` sql +SELECT toDateTime64('1955-01-01 00:00:00.000', 3) AS value, toTypeName(value); +``` + +``` text +┌───────────────────value─┬─toTypeName(toDateTime64('1955-01-01 00:00:00.000', 3))─┐ +│ 1955-01-01 00:00:00.000 │ DateTime64(3) │ +└─────────────────────────┴────────────────────────────────────────────────────────┘ +``` + +2. As decimal with precision: + +``` sql +SELECT toDateTime64(1546300800.000, 3) AS value, toTypeName(value); +``` + +``` text +┌───────────────────value─┬─toTypeName(toDateTime64(1546300800., 3))─┐ +│ 2019-01-01 00:00:00.000 │ DateTime64(3) │ +└─────────────────────────┴──────────────────────────────────────────┘ +``` + +Without the decimal point the value is still treated as Unix Timestamp in seconds: + +``` sql +SELECT toDateTime64(1546300800000, 3) AS value, toTypeName(value); +``` + +``` text +┌───────────────────value─┬─toTypeName(toDateTime64(1546300800000, 3))─┐ +│ 2282-12-31 00:00:00.000 │ DateTime64(3) │ +└─────────────────────────┴────────────────────────────────────────────┘ +``` + + +3. With `timezone`: + +``` sql +SELECT toDateTime64('2019-01-01 00:00:00', 3, 'Asia/Istanbul') AS value, toTypeName(value); +``` + +``` text +┌───────────────────value─┬─toTypeName(toDateTime64('2019-01-01 00:00:00', 3, 'Asia/Istanbul'))─┐ +│ 2019-01-01 00:00:00.000 │ DateTime64(3, 'Asia/Istanbul') │ +└─────────────────────────┴─────────────────────────────────────────────────────────────────────┘ +``` + + ## toDecimal(32\|64\|128\|256) Converts `value` to the [Decimal](../../sql-reference/data-types/decimal.md) data type with precision of `S`. The `value` can be a number or a string. The `S` (scale) parameter specifies the number of decimal places. From 2487ba7f0006f1f020f90424aff727de57397d65 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 27 Jun 2022 13:16:52 +0200 Subject: [PATCH 119/123] Move `updateInputStream` to `ITransformingStep` (#37393) --- src/Interpreters/ActionsDAG.cpp | 17 +-- src/Interpreters/AggregateDescription.cpp | 27 ----- src/Interpreters/AggregateDescription.h | 4 +- src/Interpreters/Aggregator.cpp | 80 ++++++------ src/Interpreters/Aggregator.h | 49 +++----- src/Interpreters/InterpreterSelectQuery.cpp | 114 +++++++----------- src/Processors/QueryPlan/AggregatingStep.cpp | 30 +++-- src/Processors/QueryPlan/AggregatingStep.h | 13 +- src/Processors/QueryPlan/ArrayJoinStep.cpp | 27 +---- src/Processors/QueryPlan/ArrayJoinStep.h | 7 +- src/Processors/QueryPlan/CreatingSetsStep.cpp | 5 + src/Processors/QueryPlan/CreatingSetsStep.h | 2 + src/Processors/QueryPlan/CubeStep.cpp | 25 ++-- src/Processors/QueryPlan/CubeStep.h | 7 +- src/Processors/QueryPlan/DistinctStep.cpp | 17 +++ src/Processors/QueryPlan/DistinctStep.h | 2 + src/Processors/QueryPlan/ExpressionStep.cpp | 19 +-- src/Processors/QueryPlan/ExpressionStep.h | 4 +- src/Processors/QueryPlan/ExtremesStep.h | 6 + src/Processors/QueryPlan/FillingStep.cpp | 8 ++ src/Processors/QueryPlan/FillingStep.h | 2 + src/Processors/QueryPlan/FilterStep.cpp | 28 ++--- src/Processors/QueryPlan/FilterStep.h | 4 +- src/Processors/QueryPlan/ITransformingStep.h | 15 +++ src/Processors/QueryPlan/JoinStep.cpp | 15 +++ src/Processors/QueryPlan/JoinStep.h | 4 + src/Processors/QueryPlan/LimitByStep.h | 5 + src/Processors/QueryPlan/LimitStep.cpp | 7 -- src/Processors/QueryPlan/LimitStep.h | 8 +- .../QueryPlan/MergingAggregatedStep.cpp | 33 +++-- .../QueryPlan/MergingAggregatedStep.h | 9 +- src/Processors/QueryPlan/OffsetStep.h | 5 + .../Optimizations/filterPushDown.cpp | 67 ++++++---- .../Optimizations/liftUpArrayJoin.cpp | 26 +--- .../Optimizations/liftUpFunctions.cpp | 2 - src/Processors/QueryPlan/RollupStep.cpp | 25 +++- src/Processors/QueryPlan/RollupStep.h | 8 +- src/Processors/QueryPlan/SortingStep.cpp | 11 +- src/Processors/QueryPlan/SortingStep.h | 4 +- src/Processors/QueryPlan/TotalsHavingStep.cpp | 39 ++++-- src/Processors/QueryPlan/TotalsHavingStep.h | 23 ++-- src/Processors/QueryPlan/WindowStep.cpp | 33 ++--- src/Processors/QueryPlan/WindowStep.h | 3 +- .../TTL/TTLAggregationAlgorithm.cpp | 31 ++--- .../AggregatingInOrderTransform.cpp | 13 +- .../Transforms/AggregatingTransform.cpp | 4 +- .../Transforms/AggregatingTransform.h | 13 +- src/Processors/Transforms/CubeTransform.cpp | 5 +- src/Processors/Transforms/CubeTransform.h | 2 +- src/Processors/Transforms/RollupTransform.cpp | 4 +- src/Processors/Transforms/RollupTransform.h | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 49 +++----- .../01763_filter_push_down_bugs.reference | 1 + .../01763_filter_push_down_bugs.sql | 4 + .../0_stateless/01823_explain_json.reference | 6 +- .../queries/0_stateless/01823_explain_json.sh | 2 +- 56 files changed, 503 insertions(+), 472 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index eb073ee8752..2703773f464 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1786,7 +1786,9 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(NodeRawConstPtrs conjunctio actions->inputs.push_back(input); } - actions->index.push_back(input); + /// We should not add result_predicate into the index for the second time. + if (input->result_name != result_predicate->result_name) + actions->index.push_back(input); } return actions; @@ -1840,13 +1842,14 @@ ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown( if (can_remove_filter) { /// If filter column is not needed, remove it from index. - for (auto i = index.begin(); i != index.end(); ++i) + std::erase_if(index, [&](const Node * node) { return node == predicate; }); + + /// At the very end of this method we'll call removeUnusedActions() with allow_remove_inputs=false, + /// so we need to manually remove predicate if it is an input node. + if (predicate->type == ActionType::INPUT) { - if (*i == predicate) - { - index.erase(i); - break; - } + std::erase_if(inputs, [&](const Node * node) { return node == predicate; }); + nodes.remove_if([&](const Node & node) { return &node == predicate; }); } } else diff --git a/src/Interpreters/AggregateDescription.cpp b/src/Interpreters/AggregateDescription.cpp index 1a0748b5f97..b0f51ea7c90 100644 --- a/src/Interpreters/AggregateDescription.cpp +++ b/src/Interpreters/AggregateDescription.cpp @@ -82,24 +82,6 @@ void AggregateDescription::explain(WriteBuffer & out, size_t indent) const } out << "\n"; } - - out << prefix << " Argument positions: "; - - if (arguments.empty()) - out << "none\n"; - else - { - bool first = true; - for (auto arg : arguments) - { - if (!first) - out << ", "; - first = false; - - out << arg; - } - out << '\n'; - } } void AggregateDescription::explain(JSONBuilder::JSONMap & map) const @@ -137,15 +119,6 @@ void AggregateDescription::explain(JSONBuilder::JSONMap & map) const args_array->add(name); map.add("Arguments", std::move(args_array)); - - if (!arguments.empty()) - { - auto args_pos_array = std::make_unique(); - for (auto pos : arguments) - args_pos_array->add(pos); - - map.add("Argument Positions", std::move(args_pos_array)); - } } } diff --git a/src/Interpreters/AggregateDescription.h b/src/Interpreters/AggregateDescription.h index 12c14f7a57c..8c3302a8b0b 100644 --- a/src/Interpreters/AggregateDescription.h +++ b/src/Interpreters/AggregateDescription.h @@ -14,8 +14,7 @@ struct AggregateDescription { AggregateFunctionPtr function; Array parameters; /// Parameters of the (parametric) aggregate function. - ColumnNumbers arguments; - Names argument_names; /// used if no `arguments` are specified. + Names argument_names; String column_name; /// What name to use for a column with aggregate function values void explain(WriteBuffer & out, size_t indent) const; /// Get description for EXPLAIN query. @@ -23,5 +22,4 @@ struct AggregateDescription }; using AggregateDescriptions = std::vector; - } diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 8a93dc5fd77..511e5c9e031 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -260,6 +260,14 @@ auto constructWithReserveIfPossible(size_t size_hint) else return std::make_unique(); } + +DB::ColumnNumbers calculateKeysPositions(const DB::Block & header, const DB::Aggregator::Params & params) +{ + DB::ColumnNumbers keys_positions(params.keys_size); + for (size_t i = 0; i < params.keys_size; ++i) + keys_positions[i] = header.getPositionByName(params.keys[i]); + return keys_positions; +} } namespace DB @@ -356,21 +364,17 @@ Aggregator::Params::StatsCollectingParams::StatsCollectingParams( Block Aggregator::getHeader(bool final) const { - return params.getHeader(final); + return params.getHeader(header, final); } Block Aggregator::Params::getHeader( - const Block & src_header, - const Block & intermediate_header, - const ColumnNumbers & keys, - const AggregateDescriptions & aggregates, - bool final) + const Block & header, bool only_merge, const Names & keys, const AggregateDescriptions & aggregates, bool final) { Block res; - if (intermediate_header) + if (only_merge) { - res = intermediate_header.cloneEmpty(); + res = header.cloneEmpty(); if (final) { @@ -386,14 +390,14 @@ Block Aggregator::Params::getHeader( else { for (const auto & key : keys) - res.insert(src_header.safeGetByPosition(key).cloneEmpty()); + res.insert(header.getByName(key).cloneEmpty()); for (const auto & aggregate : aggregates) { - size_t arguments_size = aggregate.arguments.size(); + size_t arguments_size = aggregate.argument_names.size(); DataTypes argument_types(arguments_size); for (size_t j = 0; j < arguments_size; ++j) - argument_types[j] = src_header.safeGetByPosition(aggregate.arguments[j]).type; + argument_types[j] = header.getByName(aggregate.argument_names[j]).type; DataTypePtr type; if (final) @@ -434,9 +438,6 @@ Aggregator::AggregateColumnsConstData Aggregator::Params::makeAggregateColumnsDa void Aggregator::Params::explain(WriteBuffer & out, size_t indent) const { Strings res; - const auto & header = src_header ? src_header - : intermediate_header; - String prefix(indent, ' '); { @@ -444,16 +445,13 @@ void Aggregator::Params::explain(WriteBuffer & out, size_t indent) const out << prefix << "Keys: "; bool first = true; - for (auto key : keys) + for (const auto & key : keys) { if (!first) out << ", "; first = false; - if (key >= header.columns()) - out << "unknown position " << key; - else - out << header.getByPosition(key).name; + out << key; } out << '\n'; @@ -470,18 +468,10 @@ void Aggregator::Params::explain(WriteBuffer & out, size_t indent) const void Aggregator::Params::explain(JSONBuilder::JSONMap & map) const { - const auto & header = src_header ? src_header - : intermediate_header; - auto keys_array = std::make_unique(); - for (auto key : keys) - { - if (key >= header.columns()) - keys_array->add(""); - else - keys_array->add(header.getByPosition(key).name); - } + for (const auto & key : keys) + keys_array->add(key); map.add("Keys", std::move(keys_array)); @@ -526,7 +516,8 @@ public: #endif -Aggregator::Aggregator(const Params & params_) : params(params_) +Aggregator::Aggregator(const Block & header_, const Params & params_) + : header(header_), keys_positions(calculateKeysPositions(header, params_)), params(params_) { /// Use query-level memory tracker if (auto * memory_tracker_child = CurrentThread::getMemoryTracker()) @@ -672,9 +663,9 @@ AggregatedDataVariants::Type Aggregator::chooseAggregationMethod() bool has_nullable_key = false; bool has_low_cardinality = false; - for (const auto & pos : params.keys) + for (const auto & key : params.keys) { - DataTypePtr type = (params.src_header ? params.src_header : params.intermediate_header).safeGetByPosition(pos).type; + DataTypePtr type = header.getByName(key).type; if (type->lowCardinality()) { @@ -1277,11 +1268,15 @@ void NO_INLINE Aggregator::mergeOnIntervalWithoutKeyImpl( } -void Aggregator::prepareAggregateInstructions(Columns columns, AggregateColumns & aggregate_columns, Columns & materialized_columns, - AggregateFunctionInstructions & aggregate_functions_instructions, NestedColumnsHolder & nested_columns_holder) const +void Aggregator::prepareAggregateInstructions( + Columns columns, + AggregateColumns & aggregate_columns, + Columns & materialized_columns, + AggregateFunctionInstructions & aggregate_functions_instructions, + NestedColumnsHolder & nested_columns_holder) const { for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_columns[i].resize(params.aggregates[i].arguments.size()); + aggregate_columns[i].resize(params.aggregates[i].argument_names.size()); aggregate_functions_instructions.resize(params.aggregates_size + 1); aggregate_functions_instructions[params.aggregates_size].that = nullptr; @@ -1293,7 +1288,8 @@ void Aggregator::prepareAggregateInstructions(Columns columns, AggregateColumns for (size_t j = 0; j < aggregate_columns[i].size(); ++j) { - materialized_columns.push_back(columns.at(params.aggregates[i].arguments[j])->convertToFullColumnIfConst()); + const auto pos = header.getPositionByName(params.aggregates[i].argument_names[j]); + materialized_columns.push_back(columns.at(pos)->convertToFullColumnIfConst()); aggregate_columns[i][j] = materialized_columns.back().get(); auto full_column = allow_sparse_arguments @@ -1382,7 +1378,7 @@ bool Aggregator::executeOnBlock(Columns columns, /// Remember the columns we will work with for (size_t i = 0; i < params.keys_size; ++i) { - materialized_columns.push_back(recursiveRemoveSparse(columns.at(params.keys[i]))->convertToFullColumnIfConst()); + materialized_columns.push_back(recursiveRemoveSparse(columns.at(keys_positions[i]))->convertToFullColumnIfConst()); key_columns[i] = materialized_columns.back().get(); if (!result.isLowCardinality()) @@ -1954,11 +1950,11 @@ Block Aggregator::prepareBlockAndFill( MutableColumns final_aggregate_columns(params.aggregates_size); AggregateColumnsData aggregate_columns_data(params.aggregates_size); - Block header = getHeader(final); + Block res_header = getHeader(final); for (size_t i = 0; i < params.keys_size; ++i) { - key_columns[i] = header.safeGetByPosition(i).type->createColumn(); + key_columns[i] = res_header.safeGetByPosition(i).type->createColumn(); key_columns[i]->reserve(rows); } @@ -1967,7 +1963,7 @@ Block Aggregator::prepareBlockAndFill( if (!final) { const auto & aggregate_column_name = params.aggregates[i].column_name; - aggregate_columns[i] = header.getByName(aggregate_column_name).type->createColumn(); + aggregate_columns[i] = res_header.getByName(aggregate_column_name).type->createColumn(); /// The ColumnAggregateFunction column captures the shared ownership of the arena with the aggregate function states. ColumnAggregateFunction & column_aggregate_func = assert_cast(*aggregate_columns[i]); @@ -2003,7 +1999,7 @@ Block Aggregator::prepareBlockAndFill( filler(key_columns, aggregate_columns_data, final_aggregate_columns, final); - Block res = header.cloneEmpty(); + Block res = res_header.cloneEmpty(); for (size_t i = 0; i < params.keys_size; ++i) res.getByPosition(i).column = std::move(key_columns[i]); @@ -2018,7 +2014,7 @@ Block Aggregator::prepareBlockAndFill( } /// Change the size of the columns-constants in the block. - size_t columns = header.columns(); + size_t columns = res_header.columns(); for (size_t i = 0; i < columns; ++i) if (isColumnConst(*res.getByPosition(i).column)) res.getByPosition(i).column = res.getByPosition(i).column->cut(0, rows); diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 475fcd9e249..feb07727725 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -897,13 +897,8 @@ public: struct Params { - /// Data structure of source blocks. - Block src_header; - /// Data structure of intermediate blocks before merge. - Block intermediate_header; - /// What to count. - const ColumnNumbers keys; + const Names keys; const AggregateDescriptions aggregates; const size_t keys_size; const size_t aggregates_size; @@ -937,6 +932,8 @@ public: bool compile_aggregate_expressions; size_t min_count_to_compile_aggregate_expression; + bool only_merge; + struct StatsCollectingParams { StatsCollectingParams(); @@ -957,8 +954,7 @@ public: StatsCollectingParams stats_collecting_params; Params( - const Block & src_header_, - const ColumnNumbers & keys_, + const Names & keys_, const AggregateDescriptions & aggregates_, bool overflow_row_, size_t max_rows_to_group_by_, @@ -972,11 +968,9 @@ public: size_t min_free_disk_space_, bool compile_aggregate_expressions_, size_t min_count_to_compile_aggregate_expression_, - const Block & intermediate_header_ = {}, + bool only_merge_ = false, // true for projections const StatsCollectingParams & stats_collecting_params_ = {}) - : src_header(src_header_) - , intermediate_header(intermediate_header_) - , keys(keys_) + : keys(keys_) , aggregates(aggregates_) , keys_size(keys.size()) , aggregates_size(aggregates.size()) @@ -992,33 +986,22 @@ public: , min_free_disk_space(min_free_disk_space_) , compile_aggregate_expressions(compile_aggregate_expressions_) , min_count_to_compile_aggregate_expression(min_count_to_compile_aggregate_expression_) + , only_merge(only_merge_) , stats_collecting_params(stats_collecting_params_) { } /// Only parameters that matter during merge. - Params( - const Block & intermediate_header_, - const ColumnNumbers & keys_, - const AggregateDescriptions & aggregates_, - bool overflow_row_, - size_t max_threads_) - : Params(Block(), keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, 0, 0, 0, false, nullptr, max_threads_, 0, false, 0, {}, {}) + Params(const Names & keys_, const AggregateDescriptions & aggregates_, bool overflow_row_, size_t max_threads_) + : Params( + keys_, aggregates_, overflow_row_, 0, OverflowMode::THROW, 0, 0, 0, false, nullptr, max_threads_, 0, false, 0, true, {}) { - intermediate_header = intermediate_header_; } - static Block getHeader( - const Block & src_header, - const Block & intermediate_header, - const ColumnNumbers & keys, - const AggregateDescriptions & aggregates, - bool final); + static Block + getHeader(const Block & header, bool only_merge, const Names & keys, const AggregateDescriptions & aggregates, bool final); - Block getHeader(bool final) const - { - return getHeader(src_header, intermediate_header, keys, aggregates, final); - } + Block getHeader(const Block & header_, bool final) const { return getHeader(header_, only_merge, keys, aggregates, final); } /// Remember the columns we will work with ColumnRawPtrs makeRawKeyColumns(const Block & block) const; @@ -1029,7 +1012,7 @@ public: void explain(JSONBuilder::JSONMap & map) const; }; - explicit Aggregator(const Params & params_); + explicit Aggregator(const Block & header_, const Params & params_); /// Process one block. Return false if the processing should be aborted (with group_by_overflow_mode = 'break'). bool executeOnBlock(const Block & block, @@ -1106,6 +1089,10 @@ private: friend class ConvertingAggregatedToChunksSource; friend class AggregatingInOrderTransform; + /// Data structure of source blocks. + Block header; + /// Positions of aggregation key columns in the header. + const ColumnNumbers keys_positions; Params params; AggregatedDataVariants::Type method_chosen; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 77ed83446b1..047e50272fe 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1592,13 +1592,9 @@ static void executeMergeAggregatedImpl( const NamesAndTypesList & aggregation_keys, const AggregateDescriptions & aggregates) { - const auto & header_before_merge = query_plan.getCurrentDataStream().header; - - ColumnNumbers keys; + auto keys = aggregation_keys.getNames(); if (has_grouping_sets) - keys.push_back(header_before_merge.getPositionByName("__grouping_set")); - for (const auto & key : aggregation_keys) - keys.push_back(header_before_merge.getPositionByName(key.name)); + keys.insert(keys.begin(), "__grouping_set"); /** There are two modes of distributed aggregation. * @@ -1615,16 +1611,12 @@ static void executeMergeAggregatedImpl( * but it can work more slowly. */ - Aggregator::Params params(header_before_merge, keys, aggregates, overflow_row, settings.max_threads); - - auto transform_params = std::make_shared( - params, - final, - /* only_merge_= */ false); + Aggregator::Params params(keys, aggregates, overflow_row, settings.max_threads); auto merging_aggregated = std::make_unique( query_plan.getCurrentDataStream(), - std::move(transform_params), + params, + final, settings.distributed_aggregation_memory_efficient && is_remote_storage, settings.max_threads, settings.aggregation_memory_efficient_merge_threads); @@ -2174,11 +2166,12 @@ static Aggregator::Params getAggregatorParams( const ASTPtr & query_ptr, const SelectQueryExpressionAnalyzer & query_analyzer, const Context & context, - const Block & current_data_stream_header, - const ColumnNumbers & keys, + const Names & keys, const AggregateDescriptions & aggregates, - bool overflow_row, const Settings & settings, - size_t group_by_two_level_threshold, size_t group_by_two_level_threshold_bytes) + bool overflow_row, + const Settings & settings, + size_t group_by_two_level_threshold, + size_t group_by_two_level_threshold_bytes) { const auto stats_collecting_params = Aggregator::Params::StatsCollectingParams( query_ptr, @@ -2186,8 +2179,8 @@ static Aggregator::Params getAggregatorParams( settings.max_entries_for_hash_table_stats, settings.max_size_to_preallocate_for_aggregation); - return Aggregator::Params{ - current_data_stream_header, + return Aggregator::Params + { keys, aggregates, overflow_row, @@ -2204,42 +2197,30 @@ static Aggregator::Params getAggregatorParams( settings.min_free_disk_space_for_temporary_data, settings.compile_aggregate_expressions, settings.min_count_to_compile_aggregate_expression, - Block{}, + /* only_merge */ false, stats_collecting_params }; } -static GroupingSetsParamsList getAggregatorGroupingSetsParams( - const SelectQueryExpressionAnalyzer & query_analyzer, - const Block & header_before_aggregation, - const ColumnNumbers & all_keys -) +static GroupingSetsParamsList getAggregatorGroupingSetsParams(const SelectQueryExpressionAnalyzer & query_analyzer, const Names & all_keys) { GroupingSetsParamsList result; if (query_analyzer.useGroupingSetKey()) { auto const & aggregation_keys_list = query_analyzer.aggregationKeysList(); - ColumnNumbersList grouping_sets_with_keys; - ColumnNumbersList missing_columns_per_set; - for (const auto & aggregation_keys : aggregation_keys_list) { - ColumnNumbers keys; - std::unordered_set keys_set; + NameSet keys; for (const auto & key : aggregation_keys) - { - keys.push_back(header_before_aggregation.getPositionByName(key.name)); - keys_set.insert(keys.back()); - } + keys.insert(key.name); - ColumnNumbers missing_indexes; - for (size_t i = 0; i < all_keys.size(); ++i) - { - if (!keys_set.contains(all_keys[i])) - missing_indexes.push_back(i); - } - result.emplace_back(std::move(keys), std::move(missing_indexes)); + Names missing_keys; + for (const auto & key : all_keys) + if (!keys.contains(key)) + missing_keys.push_back(key); + + result.emplace_back(aggregation_keys.getNames(), std::move(missing_keys)); } } return result; @@ -2254,24 +2235,24 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac if (options.is_projection_query) return; - const auto & header_before_aggregation = query_plan.getCurrentDataStream().header; - AggregateDescriptions aggregates = query_analyzer->aggregates(); - for (auto & descr : aggregates) - if (descr.arguments.empty()) - for (const auto & name : descr.argument_names) - descr.arguments.push_back(header_before_aggregation.getPositionByName(name)); const Settings & settings = context->getSettingsRef(); - ColumnNumbers keys; - for (const auto & key : query_analyzer->aggregationKeys()) - keys.push_back(header_before_aggregation.getPositionByName(key.name)); + const auto & keys = query_analyzer->aggregationKeys().getNames(); - auto aggregator_params = getAggregatorParams(query_ptr, *query_analyzer, *context, header_before_aggregation, keys, aggregates, overflow_row, settings, - settings.group_by_two_level_threshold, settings.group_by_two_level_threshold_bytes); + auto aggregator_params = getAggregatorParams( + query_ptr, + *query_analyzer, + *context, + keys, + aggregates, + overflow_row, + settings, + settings.group_by_two_level_threshold, + settings.group_by_two_level_threshold_bytes); - auto grouping_sets_params = getAggregatorGroupingSetsParams(*query_analyzer, header_before_aggregation, keys); + auto grouping_sets_params = getAggregatorGroupingSetsParams(*query_analyzer, keys); SortDescription group_by_sort_description; @@ -2292,7 +2273,6 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac std::move(aggregator_params), std::move(grouping_sets_params), final, - /* only_merge_= */ false, settings.max_block_size, settings.aggregation_in_order_max_block_bytes, merge_threads, @@ -2339,11 +2319,9 @@ void InterpreterSelectQuery::executeTotalsAndHaving( { const Settings & settings = context->getSettingsRef(); - const auto & header_before = query_plan.getCurrentDataStream().header; - auto totals_having_step = std::make_unique( query_plan.getCurrentDataStream(), - getAggregatesMask(header_before, query_analyzer->aggregates()), + query_analyzer->aggregates(), overflow_row, expression, has_having ? getSelectQuery().having()->getColumnName() : "", @@ -2357,25 +2335,23 @@ void InterpreterSelectQuery::executeTotalsAndHaving( void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modificator modificator) { - const auto & header_before_transform = query_plan.getCurrentDataStream().header; - const Settings & settings = context->getSettingsRef(); - ColumnNumbers keys; - for (const auto & key : query_analyzer->aggregationKeys()) - keys.push_back(header_before_transform.getPositionByName(key.name)); + const auto & keys = query_analyzer->aggregationKeys().getNames(); - auto params = getAggregatorParams(query_ptr, *query_analyzer, *context, header_before_transform, keys, query_analyzer->aggregates(), false, settings, 0, 0); - auto transform_params = std::make_shared( - std::move(params), - /* final_= */ true, - /* only_merge_= */ false); + // Arguments will not be present in Rollup / Cube input header and they don't actually needed 'cause these steps will work with AggregateFunctionState-s anyway. + auto aggregates = query_analyzer->aggregates(); + for (auto & aggregate : aggregates) + aggregate.argument_names.clear(); + + auto params = getAggregatorParams(query_ptr, *query_analyzer, *context, keys, aggregates, false, settings, 0, 0); + const bool final = true; QueryPlanStepPtr step; if (modificator == Modificator::ROLLUP) - step = std::make_unique(query_plan.getCurrentDataStream(), std::move(transform_params)); + step = std::make_unique(query_plan.getCurrentDataStream(), std::move(params), final); else if (modificator == Modificator::CUBE) - step = std::make_unique(query_plan.getCurrentDataStream(), std::move(transform_params)); + step = std::make_unique(query_plan.getCurrentDataStream(), std::move(params), final); query_plan.addStep(std::move(step)); } diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 28f821d6f3f..1e62673bc26 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -69,7 +69,6 @@ AggregatingStep::AggregatingStep( Aggregator::Params params_, GroupingSetsParamsList grouping_sets_params_, bool final_, - bool only_merge_, size_t max_block_size_, size_t aggregation_in_order_max_block_bytes_, size_t merge_threads_, @@ -77,11 +76,11 @@ AggregatingStep::AggregatingStep( bool storage_has_evenly_distributed_read_, InputOrderInfoPtr group_by_info_, SortDescription group_by_sort_description_) - : ITransformingStep(input_stream_, appendGroupingColumn(params_.getHeader(final_), grouping_sets_params_), getTraits(), false) + : ITransformingStep( + input_stream_, appendGroupingColumn(params_.getHeader(input_stream_.header, final_), grouping_sets_params_), getTraits(), false) , params(std::move(params_)) , grouping_sets_params(std::move(grouping_sets_params_)) , final(final_) - , only_merge(only_merge_) , max_block_size(max_block_size_) , aggregation_in_order_max_block_bytes(aggregation_in_order_max_block_bytes_) , merge_threads(merge_threads_) @@ -121,7 +120,8 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B * 1. Parallel aggregation is done, and the results should be merged in parallel. * 2. An aggregation is done with store of temporary data on the disk, and they need to be merged in a memory efficient way. */ - auto transform_params = std::make_shared(std::move(params), final, only_merge); + const auto src_header = pipeline.getHeader(); + auto transform_params = std::make_shared(src_header, std::move(params), final); if (!grouping_sets_params.empty()) { @@ -153,7 +153,6 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B { Aggregator::Params params_for_set { - transform_params->params.src_header, grouping_sets_params[i].used_keys, transform_params->params.aggregates, transform_params->params.overflow_row, @@ -168,10 +167,9 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B transform_params->params.min_free_disk_space, transform_params->params.compile_aggregate_expressions, transform_params->params.min_count_to_compile_aggregate_expression, - transform_params->params.intermediate_header, - transform_params->params.stats_collecting_params - }; - auto transform_params_for_set = std::make_shared(std::move(params_for_set), final, only_merge); + /* only_merge */ false, + transform_params->params.stats_collecting_params}; + auto transform_params_for_set = std::make_shared(src_header, std::move(params_for_set), final); if (streams > 1) { @@ -234,15 +232,15 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B grouping_node = &dag->materializeNode(*grouping_node); index.push_back(grouping_node); - size_t missign_column_index = 0; const auto & missing_columns = grouping_sets_params[set_counter].missing_keys; for (size_t i = 0; i < output_header.columns(); ++i) { auto & col = output_header.getByPosition(i); - if (missign_column_index < missing_columns.size() && missing_columns[missign_column_index] == i) + const auto it = std::find_if( + missing_columns.begin(), missing_columns.end(), [&](const auto & missing_col) { return missing_col == col.name; }); + if (it != missing_columns.end()) { - ++missign_column_index; auto column_with_default = col.column->cloneEmpty(); col.type->insertDefaultInto(*column_with_default); auto column = ColumnConst::create(std::move(column_with_default), 0); @@ -391,4 +389,12 @@ void AggregatingStep::describePipeline(FormatSettings & settings) const } } +void AggregatingStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), + appendGroupingColumn(params.getHeader(input_streams.front().header, final), grouping_sets_params), + getDataStreamTraits()); +} + } diff --git a/src/Processors/QueryPlan/AggregatingStep.h b/src/Processors/QueryPlan/AggregatingStep.h index 1be74da583a..2879cd1e0e9 100644 --- a/src/Processors/QueryPlan/AggregatingStep.h +++ b/src/Processors/QueryPlan/AggregatingStep.h @@ -11,13 +11,10 @@ struct GroupingSetsParams { GroupingSetsParams() = default; - GroupingSetsParams(ColumnNumbers used_keys_, ColumnNumbers missing_keys_) - : used_keys(std::move(used_keys_)) - , missing_keys(std::move(missing_keys_)) - {} + GroupingSetsParams(Names used_keys_, Names missing_keys_) : used_keys(std::move(used_keys_)), missing_keys(std::move(missing_keys_)) { } - ColumnNumbers used_keys; - ColumnNumbers missing_keys; + Names used_keys; + Names missing_keys; }; using GroupingSetsParamsList = std::vector; @@ -33,7 +30,6 @@ public: Aggregator::Params params_, GroupingSetsParamsList grouping_sets_params_, bool final_, - bool only_merge_, size_t max_block_size_, size_t aggregation_in_order_max_block_bytes_, size_t merge_threads_, @@ -54,10 +50,11 @@ public: const Aggregator::Params & getParams() const { return params; } private: + void updateOutputStream() override; + Aggregator::Params params; GroupingSetsParamsList grouping_sets_params; bool final; - bool only_merge; size_t max_block_size; size_t aggregation_in_order_max_block_bytes; size_t merge_threads; diff --git a/src/Processors/QueryPlan/ArrayJoinStep.cpp b/src/Processors/QueryPlan/ArrayJoinStep.cpp index 3ca5b9109e6..bd1908a4a6d 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.cpp +++ b/src/Processors/QueryPlan/ArrayJoinStep.cpp @@ -34,40 +34,19 @@ ArrayJoinStep::ArrayJoinStep(const DataStream & input_stream_, ArrayJoinActionPt { } -void ArrayJoinStep::updateInputStream(DataStream input_stream, Block result_header) +void ArrayJoinStep::updateOutputStream() { output_stream = createOutputStream( - input_stream, - ArrayJoinTransform::transformHeader(input_stream.header, array_join), - getDataStreamTraits()); - - input_streams.clear(); - input_streams.emplace_back(std::move(input_stream)); - res_header = std::move(result_header); + input_streams.front(), ArrayJoinTransform::transformHeader(input_streams.front().header, array_join), getDataStreamTraits()); } -void ArrayJoinStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) +void ArrayJoinStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) { bool on_totals = stream_type == QueryPipelineBuilder::StreamType::Totals; return std::make_shared(header, array_join, on_totals); }); - - if (res_header && !blocksHaveEqualStructure(res_header, output_stream->header)) - { - auto actions_dag = ActionsDAG::makeConvertingActions( - pipeline.getHeader().getColumnsWithTypeAndName(), - res_header.getColumnsWithTypeAndName(), - ActionsDAG::MatchColumnsMode::Name); - - auto actions = std::make_shared(actions_dag, settings.getActionsSettings()); - - pipeline.addSimpleTransform([&](const Block & header) - { - return std::make_shared(header, actions); - }); - } } void ArrayJoinStep::describeActions(FormatSettings & settings) const diff --git a/src/Processors/QueryPlan/ArrayJoinStep.h b/src/Processors/QueryPlan/ArrayJoinStep.h index 83df4d021e8..2d9b2ebd0c8 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.h +++ b/src/Processors/QueryPlan/ArrayJoinStep.h @@ -13,18 +13,17 @@ public: explicit ArrayJoinStep(const DataStream & input_stream_, ArrayJoinActionPtr array_join_); String getName() const override { return "ArrayJoin"; } - void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) override; + void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; - void updateInputStream(DataStream input_stream, Block result_header); - const ArrayJoinActionPtr & arrayJoin() const { return array_join; } private: + void updateOutputStream() override; + ArrayJoinActionPtr array_join; - Block res_header; }; } diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 6b6f9d361ef..94d841ff095 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -49,6 +49,11 @@ void CreatingSetStep::transformPipeline(QueryPipelineBuilder & pipeline, const B pipeline.addCreatingSetsTransform(getOutputStream().header, std::move(subquery_for_set), network_transfer_limits, getContext()); } +void CreatingSetStep::updateOutputStream() +{ + output_stream = createOutputStream(input_streams.front(), Block{}, getDataStreamTraits()); +} + void CreatingSetStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, ' '); diff --git a/src/Processors/QueryPlan/CreatingSetsStep.h b/src/Processors/QueryPlan/CreatingSetsStep.h index e20c28e10f4..20cdd24c8a9 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.h +++ b/src/Processors/QueryPlan/CreatingSetsStep.h @@ -27,6 +27,8 @@ public: void describeActions(FormatSettings & settings) const override; private: + void updateOutputStream() override; + String description; SubqueryForSet subquery_for_set; SizeLimits network_transfer_limits; diff --git a/src/Processors/QueryPlan/CubeStep.cpp b/src/Processors/QueryPlan/CubeStep.cpp index 91c85a08412..b0c57491085 100644 --- a/src/Processors/QueryPlan/CubeStep.cpp +++ b/src/Processors/QueryPlan/CubeStep.cpp @@ -24,14 +24,15 @@ static ITransformingStep::Traits getTraits() }; } -CubeStep::CubeStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_) - : ITransformingStep(input_stream_, appendGroupingSetColumn(params_->getHeader()), getTraits()) - , keys_size(params_->params.keys_size) +CubeStep::CubeStep(const DataStream & input_stream_, Aggregator::Params params_, bool final_) + : ITransformingStep(input_stream_, appendGroupingSetColumn(params_.getHeader(input_stream_.header, final_)), getTraits()) + , keys_size(params_.keys_size) , params(std::move(params_)) + , final(final_) { /// Aggregation keys are distinct - for (auto key : params->params.keys) - output_stream->distinct_columns.insert(params->params.src_header.getByPosition(key).name); + for (const auto & key : params.keys) + output_stream->distinct_columns.insert(key); } ProcessorPtr addGroupingSetForTotals(const Block & header, const BuildQueryPipelineSettings & settings, UInt64 grouping_set_number) @@ -59,13 +60,23 @@ void CubeStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQue if (stream_type == QueryPipelineBuilder::StreamType::Totals) return addGroupingSetForTotals(header, settings, (UInt64(1) << keys_size) - 1); - return std::make_shared(header, std::move(params)); + auto transform_params = std::make_shared(header, std::move(params), final); + return std::make_shared(header, std::move(transform_params)); }); } const Aggregator::Params & CubeStep::getParams() const { - return params->params; + return params; } +void CubeStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), appendGroupingSetColumn(params.getHeader(input_streams.front().header, final)), getDataStreamTraits()); + + /// Aggregation keys are distinct + for (const auto & key : params.keys) + output_stream->distinct_columns.insert(key); +} } diff --git a/src/Processors/QueryPlan/CubeStep.h b/src/Processors/QueryPlan/CubeStep.h index d3e26f9379f..87f22de7fc6 100644 --- a/src/Processors/QueryPlan/CubeStep.h +++ b/src/Processors/QueryPlan/CubeStep.h @@ -13,7 +13,7 @@ using AggregatingTransformParamsPtr = std::shared_ptrdistinct_columns.empty() /// Columns already distinct, do nothing + && (!pre_distinct /// Main distinct + || input_streams.front().has_single_port)) /// pre_distinct for single port works as usual one + { + /// Build distinct set. + for (const auto & name : columns) + output_stream->distinct_columns.insert(name); + } +} + } diff --git a/src/Processors/QueryPlan/DistinctStep.h b/src/Processors/QueryPlan/DistinctStep.h index a48a779425d..a4424e01d72 100644 --- a/src/Processors/QueryPlan/DistinctStep.h +++ b/src/Processors/QueryPlan/DistinctStep.h @@ -24,6 +24,8 @@ public: void describeActions(FormatSettings & settings) const override; private: + void updateOutputStream() override; + SizeLimits set_size_limits; UInt64 limit_hint; Names columns; diff --git a/src/Processors/QueryPlan/ExpressionStep.cpp b/src/Processors/QueryPlan/ExpressionStep.cpp index 33d2ad6e1cf..c9f1e1ce242 100644 --- a/src/Processors/QueryPlan/ExpressionStep.cpp +++ b/src/Processors/QueryPlan/ExpressionStep.cpp @@ -38,19 +38,6 @@ ExpressionStep::ExpressionStep(const DataStream & input_stream_, ActionsDAGPtr a updateDistinctColumns(output_stream->header, output_stream->distinct_columns); } -void ExpressionStep::updateInputStream(DataStream input_stream, bool keep_header) -{ - Block out_header = keep_header ? std::move(output_stream->header) - : ExpressionTransform::transformHeader(input_stream.header, *actions_dag); - output_stream = createOutputStream( - input_stream, - std::move(out_header), - getDataStreamTraits()); - - input_streams.clear(); - input_streams.emplace_back(std::move(input_stream)); -} - void ExpressionStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) { auto expression = std::make_shared(actions_dag, settings.getActionsSettings()); @@ -101,4 +88,10 @@ void ExpressionStep::describeActions(JSONBuilder::JSONMap & map) const map.add("Expression", expression->toTree()); } +void ExpressionStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), ExpressionTransform::transformHeader(input_streams.front().header, *actions_dag), getDataStreamTraits()); +} + } diff --git a/src/Processors/QueryPlan/ExpressionStep.h b/src/Processors/QueryPlan/ExpressionStep.h index 94c2ba21bc1..96869a9e9ca 100644 --- a/src/Processors/QueryPlan/ExpressionStep.h +++ b/src/Processors/QueryPlan/ExpressionStep.h @@ -20,8 +20,6 @@ public: void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) override; - void updateInputStream(DataStream input_stream, bool keep_header); - void describeActions(FormatSettings & settings) const override; const ActionsDAGPtr & getExpression() const { return actions_dag; } @@ -29,6 +27,8 @@ public: void describeActions(JSONBuilder::JSONMap & map) const override; private: + void updateOutputStream() override; + ActionsDAGPtr actions_dag; }; diff --git a/src/Processors/QueryPlan/ExtremesStep.h b/src/Processors/QueryPlan/ExtremesStep.h index 7898796306c..57ccef077aa 100644 --- a/src/Processors/QueryPlan/ExtremesStep.h +++ b/src/Processors/QueryPlan/ExtremesStep.h @@ -12,6 +12,12 @@ public: String getName() const override { return "Extremes"; } void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; + +private: + void updateOutputStream() override + { + output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); + } }; } diff --git a/src/Processors/QueryPlan/FillingStep.cpp b/src/Processors/QueryPlan/FillingStep.cpp index 41bc2f44dba..8a370786820 100644 --- a/src/Processors/QueryPlan/FillingStep.cpp +++ b/src/Processors/QueryPlan/FillingStep.cpp @@ -57,4 +57,12 @@ void FillingStep::describeActions(JSONBuilder::JSONMap & map) const map.add("Sort Description", explainSortDescription(sort_description)); } +void FillingStep::updateOutputStream() +{ + if (!input_streams.front().has_single_port) + throw Exception("FillingStep expects single input", ErrorCodes::LOGICAL_ERROR); + + output_stream = createOutputStream( + input_streams.front(), FillingTransform::transformHeader(input_streams.front().header, sort_description), getDataStreamTraits()); +} } diff --git a/src/Processors/QueryPlan/FillingStep.h b/src/Processors/QueryPlan/FillingStep.h index 0393b2c525b..4e1b5b3654d 100644 --- a/src/Processors/QueryPlan/FillingStep.h +++ b/src/Processors/QueryPlan/FillingStep.h @@ -22,6 +22,8 @@ public: const SortDescription & getSortDescription() const { return sort_description; } private: + void updateOutputStream() override; + SortDescription sort_description; InterpolateDescriptionPtr interpolate_description; }; diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp index df75c37dc97..ff58abf8874 100644 --- a/src/Processors/QueryPlan/FilterStep.cpp +++ b/src/Processors/QueryPlan/FilterStep.cpp @@ -46,25 +46,6 @@ FilterStep::FilterStep( updateDistinctColumns(output_stream->header, output_stream->distinct_columns); } -void FilterStep::updateInputStream(DataStream input_stream, bool keep_header) -{ - Block out_header = std::move(output_stream->header); - if (keep_header) - out_header = FilterTransform::transformHeader( - input_stream.header, - *actions_dag, - filter_column_name, - remove_filter_column); - - output_stream = createOutputStream( - input_stream, - std::move(out_header), - getDataStreamTraits()); - - input_streams.clear(); - input_streams.emplace_back(std::move(input_stream)); -} - void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) { auto expression = std::make_shared(actions_dag, settings.getActionsSettings()); @@ -124,4 +105,13 @@ void FilterStep::describeActions(JSONBuilder::JSONMap & map) const map.add("Expression", expression->toTree()); } +void FilterStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), + FilterTransform::transformHeader(input_streams.front().header, *actions_dag, filter_column_name, remove_filter_column), + getDataStreamTraits()); +} + + } diff --git a/src/Processors/QueryPlan/FilterStep.h b/src/Processors/QueryPlan/FilterStep.h index 7ac5bc036e0..daea1e13c58 100644 --- a/src/Processors/QueryPlan/FilterStep.h +++ b/src/Processors/QueryPlan/FilterStep.h @@ -20,8 +20,6 @@ public: String getName() const override { return "Filter"; } void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) override; - void updateInputStream(DataStream input_stream, bool keep_header); - void describeActions(JSONBuilder::JSONMap & map) const override; void describeActions(FormatSettings & settings) const override; @@ -30,6 +28,8 @@ public: bool removesFilterColumn() const { return remove_filter_column; } private: + void updateOutputStream() override; + ActionsDAGPtr actions_dag; String filter_column_name; bool remove_filter_column; diff --git a/src/Processors/QueryPlan/ITransformingStep.h b/src/Processors/QueryPlan/ITransformingStep.h index 8f3641dd5bd..008642c71ee 100644 --- a/src/Processors/QueryPlan/ITransformingStep.h +++ b/src/Processors/QueryPlan/ITransformingStep.h @@ -55,6 +55,19 @@ public: const TransformTraits & getTransformTraits() const { return transform_traits; } const DataStreamTraits & getDataStreamTraits() const { return data_stream_traits; } + /// Updates the input stream of the given step. Used during query plan optimizations. + /// It won't do any validation of a new stream, so it is your responsibility to ensure that this update doesn't break anything + /// (e.g. you update data stream traits or correctly remove / add columns). + void updateInputStream(DataStream input_stream) + { + input_streams.clear(); + input_streams.emplace_back(std::move(input_stream)); + + updateOutputStream(); + + updateDistinctColumns(output_stream->header, output_stream->distinct_columns); + } + void describePipeline(FormatSettings & settings) const override; /// Append extra processors for this step. @@ -73,6 +86,8 @@ protected: TransformTraits transform_traits; private: + virtual void updateOutputStream() = 0; + /// We collect processors got after pipeline transformation. Processors processors; bool collect_processors; diff --git a/src/Processors/QueryPlan/JoinStep.cpp b/src/Processors/QueryPlan/JoinStep.cpp index 983be9d45fb..209d91af4d8 100644 --- a/src/Processors/QueryPlan/JoinStep.cpp +++ b/src/Processors/QueryPlan/JoinStep.cpp @@ -40,6 +40,14 @@ void JoinStep::describePipeline(FormatSettings & settings) const IQueryPlanStep::describePipeline(processors, settings); } +void JoinStep::updateLeftStream(const DataStream & left_stream_) +{ + input_streams = {left_stream_, input_streams.at(1)}; + output_stream = DataStream{ + .header = JoiningTransform::transformHeader(left_stream_.header, join), + }; +} + static ITransformingStep::Traits getStorageJoinTraits() { return ITransformingStep::Traits @@ -87,4 +95,11 @@ void FilledJoinStep::transformPipeline(QueryPipelineBuilder & pipeline, const Bu }); } +void FilledJoinStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), JoiningTransform::transformHeader(input_streams.front().header, join), getDataStreamTraits()); +} + + } diff --git a/src/Processors/QueryPlan/JoinStep.h b/src/Processors/QueryPlan/JoinStep.h index b9d3dff1b65..17a0cc2ae63 100644 --- a/src/Processors/QueryPlan/JoinStep.h +++ b/src/Processors/QueryPlan/JoinStep.h @@ -28,6 +28,8 @@ public: const JoinPtr & getJoin() const { return join; } + void updateLeftStream(const DataStream & left_stream_); + private: JoinPtr join; size_t max_block_size; @@ -47,6 +49,8 @@ public: void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; private: + void updateOutputStream() override; + JoinPtr join; size_t max_block_size; }; diff --git a/src/Processors/QueryPlan/LimitByStep.h b/src/Processors/QueryPlan/LimitByStep.h index eb91be8a814..0edda3247d6 100644 --- a/src/Processors/QueryPlan/LimitByStep.h +++ b/src/Processors/QueryPlan/LimitByStep.h @@ -20,6 +20,11 @@ public: void describeActions(FormatSettings & settings) const override; private: + void updateOutputStream() override + { + output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); + } + size_t group_length; size_t group_offset; Names columns; diff --git a/src/Processors/QueryPlan/LimitStep.cpp b/src/Processors/QueryPlan/LimitStep.cpp index aff7472e4aa..144ac16f0d5 100644 --- a/src/Processors/QueryPlan/LimitStep.cpp +++ b/src/Processors/QueryPlan/LimitStep.cpp @@ -36,13 +36,6 @@ LimitStep::LimitStep( { } -void LimitStep::updateInputStream(DataStream input_stream) -{ - input_streams.clear(); - input_streams.emplace_back(std::move(input_stream)); - output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); -} - void LimitStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { auto transform = std::make_shared( diff --git a/src/Processors/QueryPlan/LimitStep.h b/src/Processors/QueryPlan/LimitStep.h index f5bceeb29c7..1ae6b73cc3d 100644 --- a/src/Processors/QueryPlan/LimitStep.h +++ b/src/Processors/QueryPlan/LimitStep.h @@ -31,12 +31,14 @@ public: return limit + offset; } - /// Change input stream when limit is pushed up. TODO: add clone() for steps. - void updateInputStream(DataStream input_stream); - bool withTies() const { return with_ties; } private: + void updateOutputStream() override + { + output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); + } + size_t limit; size_t offset; bool always_read_till_end; diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.cpp b/src/Processors/QueryPlan/MergingAggregatedStep.cpp index 8dfb9f9c923..c898b901a6a 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.cpp +++ b/src/Processors/QueryPlan/MergingAggregatedStep.cpp @@ -25,23 +25,26 @@ static ITransformingStep::Traits getTraits() MergingAggregatedStep::MergingAggregatedStep( const DataStream & input_stream_, - AggregatingTransformParamsPtr params_, + Aggregator::Params params_, + bool final_, bool memory_efficient_aggregation_, size_t max_threads_, size_t memory_efficient_merge_threads_) - : ITransformingStep(input_stream_, params_->getHeader(), getTraits()) - , params(params_) + : ITransformingStep(input_stream_, params_.getHeader(input_stream_.header, final_), getTraits()) + , params(std::move(params_)) + , final(final_) , memory_efficient_aggregation(memory_efficient_aggregation_) , max_threads(max_threads_) , memory_efficient_merge_threads(memory_efficient_merge_threads_) { /// Aggregation keys are distinct - for (auto key : params->params.keys) - output_stream->distinct_columns.insert(params->params.intermediate_header.getByPosition(key).name); + for (const auto & key : params.keys) + output_stream->distinct_columns.insert(key); } void MergingAggregatedStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { + auto transform_params = std::make_shared(pipeline.getHeader(), std::move(params), final); if (!memory_efficient_aggregation) { /// We union several sources into one, paralleling the work. @@ -49,9 +52,7 @@ void MergingAggregatedStep::transformPipeline(QueryPipelineBuilder & pipeline, c /// Now merge the aggregated blocks pipeline.addSimpleTransform([&](const Block & header) - { - return std::make_shared(header, params, max_threads); - }); + { return std::make_shared(header, transform_params, max_threads); }); } else { @@ -59,18 +60,28 @@ void MergingAggregatedStep::transformPipeline(QueryPipelineBuilder & pipeline, c ? static_cast(memory_efficient_merge_threads) : static_cast(max_threads); - pipeline.addMergingAggregatedMemoryEfficientTransform(params, num_merge_threads); + pipeline.addMergingAggregatedMemoryEfficientTransform(transform_params, num_merge_threads); } } void MergingAggregatedStep::describeActions(FormatSettings & settings) const { - return params->params.explain(settings.out, settings.offset); + return params.explain(settings.out, settings.offset); } void MergingAggregatedStep::describeActions(JSONBuilder::JSONMap & map) const { - params->params.explain(map); + params.explain(map); } +void MergingAggregatedStep::updateOutputStream() +{ + output_stream = createOutputStream(input_streams.front(), params.getHeader(input_streams.front().header, final), getDataStreamTraits()); + + /// Aggregation keys are distinct + for (const auto & key : params.keys) + output_stream->distinct_columns.insert(key); +} + + } diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.h b/src/Processors/QueryPlan/MergingAggregatedStep.h index eeead41b5f9..136422c8c27 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.h +++ b/src/Processors/QueryPlan/MergingAggregatedStep.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include @@ -14,7 +15,8 @@ class MergingAggregatedStep : public ITransformingStep public: MergingAggregatedStep( const DataStream & input_stream_, - AggregatingTransformParamsPtr params_, + Aggregator::Params params_, + bool final_, bool memory_efficient_aggregation_, size_t max_threads_, size_t memory_efficient_merge_threads_); @@ -27,7 +29,10 @@ public: void describeActions(FormatSettings & settings) const override; private: - AggregatingTransformParamsPtr params; + void updateOutputStream() override; + + Aggregator::Params params; + bool final; bool memory_efficient_aggregation; size_t max_threads; size_t memory_efficient_merge_threads; diff --git a/src/Processors/QueryPlan/OffsetStep.h b/src/Processors/QueryPlan/OffsetStep.h index f16559bcfad..a32835b62a6 100644 --- a/src/Processors/QueryPlan/OffsetStep.h +++ b/src/Processors/QueryPlan/OffsetStep.h @@ -19,6 +19,11 @@ public: void describeActions(FormatSettings & settings) const override; private: + void updateOutputStream() override + { + output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); + } + size_t offset; }; diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index 2625bf38bf7..0c17c27e7aa 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -27,10 +27,19 @@ namespace DB::ErrorCodes namespace DB::QueryPlanOptimizations { -static size_t tryAddNewFilterStep( - QueryPlan::Node * parent_node, - QueryPlan::Nodes & nodes, - const Names & allowed_inputs) +static bool filterColumnIsNotAmongAggregatesArguments(const AggregateDescriptions & aggregates, const std::string & filter_column_name) +{ + for (const auto & aggregate : aggregates) + { + const auto & argument_names = aggregate.argument_names; + if (std::find(argument_names.begin(), argument_names.end(), filter_column_name) != argument_names.end()) + return false; + } + return true; +} + +static size_t +tryAddNewFilterStep(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, const Names & allowed_inputs, bool can_remove_filter = true) { QueryPlan::Node * child_node = parent_node->children.front(); @@ -62,11 +71,6 @@ static size_t tryAddNewFilterStep( /// Filter column was replaced to constant. const bool filter_is_constant = filter_node && filter_node->column && isColumnConst(*filter_node->column); - if (!filter_node || filter_is_constant) - /// This means that all predicates of filter were pushed down. - /// Replace current actions to expression, as we don't need to filter anything. - parent = std::make_unique(child->getOutputStream(), expression); - /// Add new Filter step before Aggregating. /// Expression/Filter -> Aggregating -> Something auto & node = nodes.emplace_back(); @@ -77,21 +81,31 @@ static size_t tryAddNewFilterStep( /// New filter column is the first one. auto split_filter_column_name = (*split_filter->getIndex().begin())->result_name; node.step = std::make_unique( - node.children.at(0)->step->getOutputStream(), std::move(split_filter), std::move(split_filter_column_name), true); + node.children.at(0)->step->getOutputStream(), std::move(split_filter), std::move(split_filter_column_name), can_remove_filter); + + if (auto * transforming_step = dynamic_cast(child.get())) + { + transforming_step->updateInputStream(node.step->getOutputStream()); + } + else + { + if (auto * join = typeid_cast(child.get())) + join->updateLeftStream(node.step->getOutputStream()); + else + throw Exception( + ErrorCodes::LOGICAL_ERROR, "We are trying to push down a filter through a step for which we cannot update input stream"); + } + + if (!filter_node || filter_is_constant) + /// This means that all predicates of filter were pushed down. + /// Replace current actions to expression, as we don't need to filter anything. + parent = std::make_unique(child->getOutputStream(), expression); + else + filter->updateInputStream(child->getOutputStream()); return 3; } -static Names getAggregatingKeys(const Aggregator::Params & params) -{ - Names keys; - keys.reserve(params.keys.size()); - for (auto pos : params.keys) - keys.push_back(params.src_header.getByPosition(pos).name); - - return keys; -} - size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes) { if (parent_node->children.size() != 1) @@ -112,9 +126,14 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes if (auto * aggregating = typeid_cast(child.get())) { const auto & params = aggregating->getParams(); - Names keys = getAggregatingKeys(params); + const auto & keys = params.keys; - if (auto updated_steps = tryAddNewFilterStep(parent_node, nodes, keys)) + const bool filter_column_is_not_among_aggregation_keys + = std::find(keys.begin(), keys.end(), filter->getFilterColumnName()) == keys.end(); + const bool can_remove_filter = filter_column_is_not_among_aggregation_keys + && filterColumnIsNotAmongAggregatesArguments(params.aggregates, filter->getFilterColumnName()); + + if (auto updated_steps = tryAddNewFilterStep(parent_node, nodes, keys, can_remove_filter)) return updated_steps; } @@ -213,7 +232,9 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes allowed_keys.push_back(name); } - if (auto updated_steps = tryAddNewFilterStep(parent_node, nodes, allowed_keys)) + const bool can_remove_filter + = std::find(source_columns.begin(), source_columns.end(), filter->getFilterColumnName()) == source_columns.end(); + if (auto updated_steps = tryAddNewFilterStep(parent_node, nodes, allowed_keys, can_remove_filter)) return updated_steps; } } diff --git a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp index e20c5f93d6e..36aab41df49 100644 --- a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp @@ -36,30 +36,6 @@ size_t tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & node auto description = parent->getStepDescription(); - /// All actions was moved before ARRAY JOIN. Swap Expression and ArrayJoin. - if (split_actions.second->trivial()) - { - auto expected_header = parent->getOutputStream().header; - - /// Expression/Filter -> ArrayJoin - std::swap(parent, child); - /// ArrayJoin -> Expression/Filter - - if (expression_step) - child = std::make_unique(child_node->children.at(0)->step->getOutputStream(), - std::move(split_actions.first)); - else - child = std::make_unique(child_node->children.at(0)->step->getOutputStream(), - std::move(split_actions.first), - filter_step->getFilterColumnName(), - filter_step->removesFilterColumn()); - - child->setStepDescription(std::move(description)); - - array_join_step->updateInputStream(child->getOutputStream(), expected_header); - return 2; - } - /// Add new expression step before ARRAY JOIN. /// Expression/Filter -> ArrayJoin -> Something auto & node = nodes.emplace_back(); @@ -70,7 +46,7 @@ size_t tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & node node.step = std::make_unique(node.children.at(0)->step->getOutputStream(), std::move(split_actions.first)); node.step->setStepDescription(description); - array_join_step->updateInputStream(node.step->getOutputStream(), {}); + array_join_step->updateInputStream(node.step->getOutputStream()); if (expression_step) parent = std::make_unique(array_join_step->getOutputStream(), split_actions.second); diff --git a/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp b/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp index 872adfeccc1..c3b03a5385f 100644 --- a/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp +++ b/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp @@ -70,8 +70,6 @@ size_t tryExecuteFunctionsAfterSorting(QueryPlan::Node * parent_node, QueryPlan: // so far the origin Expression (parent_node) -> Sorting (child_node) -> NeededCalculations (node_with_needed) sorting_step->updateInputStream(getChildOutputStream(*child_node)); - auto input_header = sorting_step->getInputStreams().at(0).header; - sorting_step->updateOutputStream(std::move(input_header)); auto description = parent_step->getStepDescription(); parent_step = std::make_unique(child_step->getOutputStream(), std::move(unneeded_for_sorting)); diff --git a/src/Processors/QueryPlan/RollupStep.cpp b/src/Processors/QueryPlan/RollupStep.cpp index 3b061f9c246..169976195ea 100644 --- a/src/Processors/QueryPlan/RollupStep.cpp +++ b/src/Processors/QueryPlan/RollupStep.cpp @@ -22,14 +22,15 @@ static ITransformingStep::Traits getTraits() }; } -RollupStep::RollupStep(const DataStream & input_stream_, AggregatingTransformParamsPtr params_) - : ITransformingStep(input_stream_, appendGroupingSetColumn(params_->getHeader()), getTraits()) +RollupStep::RollupStep(const DataStream & input_stream_, Aggregator::Params params_, bool final_) + : ITransformingStep(input_stream_, appendGroupingSetColumn(params_.getHeader(input_stream_.header, final_)), getTraits()) , params(std::move(params_)) - , keys_size(params->params.keys_size) + , keys_size(params.keys_size) + , final(final_) { /// Aggregation keys are distinct - for (auto key : params->params.keys) - output_stream->distinct_columns.insert(params->params.src_header.getByPosition(key).name); + for (const auto & key : params.keys) + output_stream->distinct_columns.insert(key); } ProcessorPtr addGroupingSetForTotals(const Block & header, const BuildQueryPipelineSettings & settings, UInt64 grouping_set_number); @@ -43,8 +44,20 @@ void RollupStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQ if (stream_type == QueryPipelineBuilder::StreamType::Totals) return addGroupingSetForTotals(header, settings, keys_size); - return std::make_shared(header, std::move(params)); + auto transform_params = std::make_shared(header, std::move(params), true); + return std::make_shared(header, std::move(transform_params)); }); } +void RollupStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), appendGroupingSetColumn(params.getHeader(input_streams.front().header, final)), getDataStreamTraits()); + + /// Aggregation keys are distinct + for (const auto & key : params.keys) + output_stream->distinct_columns.insert(key); +} + + } diff --git a/src/Processors/QueryPlan/RollupStep.h b/src/Processors/QueryPlan/RollupStep.h index 3dce6f74d9f..c59bf9f3ee9 100644 --- a/src/Processors/QueryPlan/RollupStep.h +++ b/src/Processors/QueryPlan/RollupStep.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include @@ -12,15 +13,18 @@ using AggregatingTransformParamsPtr = std::shared_ptrsort_mode = DataStream::SortMode::Stream; } -void SortingStep::updateInputStream(DataStream input_stream) +void SortingStep::updateOutputStream() { - input_streams.clear(); - input_streams.emplace_back(std::move(input_stream)); -} - -void SortingStep::updateOutputStream(Block result_header) -{ - output_stream = createOutputStream(input_streams.at(0), std::move(result_header), getDataStreamTraits()); + output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); output_stream->sort_description = result_description; output_stream->sort_mode = DataStream::SortMode::Stream; - updateDistinctColumns(output_stream->header, output_stream->distinct_columns); } void SortingStep::updateLimit(size_t limit_) diff --git a/src/Processors/QueryPlan/SortingStep.h b/src/Processors/QueryPlan/SortingStep.h index 23a31b14093..ce78bb863bf 100644 --- a/src/Processors/QueryPlan/SortingStep.h +++ b/src/Processors/QueryPlan/SortingStep.h @@ -49,14 +49,12 @@ public: /// Add limit or change it to lower value. void updateLimit(size_t limit_); - void updateInputStream(DataStream input_stream); - void updateOutputStream(Block result_header); - SortDescription getSortDescription() const { return result_description; } void convertToFinishSorting(SortDescription prefix_description); private: + void updateOutputStream() override; enum class Type { diff --git a/src/Processors/QueryPlan/TotalsHavingStep.cpp b/src/Processors/QueryPlan/TotalsHavingStep.cpp index 5e7e7011e0e..bb918a1a02d 100644 --- a/src/Processors/QueryPlan/TotalsHavingStep.cpp +++ b/src/Processors/QueryPlan/TotalsHavingStep.cpp @@ -27,7 +27,7 @@ static ITransformingStep::Traits getTraits(bool has_filter) TotalsHavingStep::TotalsHavingStep( const DataStream & input_stream_, - const ColumnsMask & aggregates_mask_, + const AggregateDescriptions & aggregates_, bool overflow_row_, const ActionsDAGPtr & actions_dag_, const std::string & filter_column_, @@ -36,16 +36,16 @@ TotalsHavingStep::TotalsHavingStep( double auto_include_threshold_, bool final_) : ITransformingStep( - input_stream_, - TotalsHavingTransform::transformHeader( - input_stream_.header, - actions_dag_.get(), - filter_column_, - remove_filter_, - final_, - aggregates_mask_), - getTraits(!filter_column_.empty())) - , aggregates_mask(aggregates_mask_) + input_stream_, + TotalsHavingTransform::transformHeader( + input_stream_.header, + actions_dag_.get(), + filter_column_, + remove_filter_, + final_, + getAggregatesMask(input_stream_.header, aggregates_)), + getTraits(!filter_column_.empty())) + , aggregates(aggregates_) , overflow_row(overflow_row_) , actions_dag(actions_dag_) , filter_column_name(filter_column_) @@ -62,7 +62,7 @@ void TotalsHavingStep::transformPipeline(QueryPipelineBuilder & pipeline, const auto totals_having = std::make_shared( pipeline.getHeader(), - aggregates_mask, + getAggregatesMask(pipeline.getHeader(), aggregates), overflow_row, expression_actions, filter_column_name, @@ -125,4 +125,19 @@ void TotalsHavingStep::describeActions(JSONBuilder::JSONMap & map) const } } +void TotalsHavingStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), + TotalsHavingTransform::transformHeader( + input_streams.front().header, + actions_dag.get(), + filter_column_name, + remove_filter, + final, + getAggregatesMask(input_streams.front().header, aggregates)), + getDataStreamTraits()); +} + + } diff --git a/src/Processors/QueryPlan/TotalsHavingStep.h b/src/Processors/QueryPlan/TotalsHavingStep.h index 4ad741a1b44..b60eab78b53 100644 --- a/src/Processors/QueryPlan/TotalsHavingStep.h +++ b/src/Processors/QueryPlan/TotalsHavingStep.h @@ -15,15 +15,15 @@ class TotalsHavingStep : public ITransformingStep { public: TotalsHavingStep( - const DataStream & input_stream_, - const ColumnsMask & aggregates_mask_, - bool overflow_row_, - const ActionsDAGPtr & actions_dag_, - const std::string & filter_column_, - bool remove_filter_, - TotalsMode totals_mode_, - double auto_include_threshold_, - bool final_); + const DataStream & input_stream_, + const AggregateDescriptions & aggregates_, + bool overflow_row_, + const ActionsDAGPtr & actions_dag_, + const std::string & filter_column_, + bool remove_filter_, + TotalsMode totals_mode_, + double auto_include_threshold_, + bool final_); String getName() const override { return "TotalsHaving"; } @@ -35,7 +35,10 @@ public: const ActionsDAGPtr & getActions() const { return actions_dag; } private: - const ColumnsMask aggregates_mask; + void updateOutputStream() override; + + const AggregateDescriptions aggregates; + bool overflow_row; ActionsDAGPtr actions_dag; String filter_column_name; diff --git a/src/Processors/QueryPlan/WindowStep.cpp b/src/Processors/QueryPlan/WindowStep.cpp index 48d16ed321f..b67b394b57b 100644 --- a/src/Processors/QueryPlan/WindowStep.cpp +++ b/src/Processors/QueryPlan/WindowStep.cpp @@ -44,17 +44,13 @@ static Block addWindowFunctionResultColumns(const Block & block, return result; } -WindowStep::WindowStep(const DataStream & input_stream_, - const WindowDescription & window_description_, - const std::vector & window_functions_) - : ITransformingStep( - input_stream_, - addWindowFunctionResultColumns(input_stream_.header, - window_functions_), - getTraits()) +WindowStep::WindowStep( + const DataStream & input_stream_, + const WindowDescription & window_description_, + const std::vector & window_functions_) + : ITransformingStep(input_stream_, addWindowFunctionResultColumns(input_stream_.header, window_functions_), getTraits()) , window_description(window_description_) , window_functions(window_functions_) - , input_header(input_stream_.header) { // We don't remove any columns, only add, so probably we don't have to update // the output DataStream::distinct_columns. @@ -70,11 +66,12 @@ void WindowStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQ // have resized it. pipeline.resize(1); - pipeline.addSimpleTransform([&](const Block & /*header*/) - { - return std::make_shared(input_header, - output_stream->header, window_description, window_functions); - }); + pipeline.addSimpleTransform( + [&](const Block & /*header*/) + { + return std::make_shared( + input_streams.front().header, output_stream->header, window_description, window_functions); + }); assertBlocksHaveEqualStructure(pipeline.getHeader(), output_stream->header, "WindowStep transform for '" + window_description.window_name + "'"); @@ -138,6 +135,14 @@ void WindowStep::describeActions(JSONBuilder::JSONMap & map) const map.add("Functions", std::move(functions_array)); } +void WindowStep::updateOutputStream() +{ + output_stream = createOutputStream( + input_streams.front(), addWindowFunctionResultColumns(input_streams.front().header, window_functions), getDataStreamTraits()); + + window_description.checkValid(); +} + const WindowDescription & WindowStep::getWindowDescription() const { return window_description; diff --git a/src/Processors/QueryPlan/WindowStep.h b/src/Processors/QueryPlan/WindowStep.h index 9b58cceb972..0bafdcbe414 100644 --- a/src/Processors/QueryPlan/WindowStep.h +++ b/src/Processors/QueryPlan/WindowStep.h @@ -28,9 +28,10 @@ public: const WindowDescription & getWindowDescription() const; private: + void updateOutputStream() override; + WindowDescription window_description; std::vector window_functions; - Block input_header; }; } diff --git a/src/Processors/TTL/TTLAggregationAlgorithm.cpp b/src/Processors/TTL/TTLAggregationAlgorithm.cpp index bce54ace054..d8b022f0acb 100644 --- a/src/Processors/TTL/TTLAggregationAlgorithm.cpp +++ b/src/Processors/TTL/TTLAggregationAlgorithm.cpp @@ -15,28 +15,31 @@ TTLAggregationAlgorithm::TTLAggregationAlgorithm( { current_key_value.resize(description.group_by_keys.size()); - ColumnNumbers keys; - for (const auto & key : description.group_by_keys) - keys.push_back(header.getPositionByName(key)); + const auto & keys = description.group_by_keys; key_columns.resize(description.group_by_keys.size()); AggregateDescriptions aggregates = description.aggregate_descriptions; - for (auto & descr : aggregates) - if (descr.arguments.empty()) - for (const auto & name : descr.argument_names) - descr.arguments.push_back(header.getPositionByName(name)); - columns_for_aggregator.resize(description.aggregate_descriptions.size()); const Settings & settings = storage_.getContext()->getSettingsRef(); - Aggregator::Params params(header, keys, aggregates, - false, settings.max_rows_to_group_by, settings.group_by_overflow_mode, 0, 0, - settings.max_bytes_before_external_group_by, settings.empty_result_for_aggregation_by_empty_set, - storage_.getContext()->getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data, - settings.compile_aggregate_expressions, settings.min_count_to_compile_aggregate_expression); + Aggregator::Params params( + keys, + aggregates, + false, + settings.max_rows_to_group_by, + settings.group_by_overflow_mode, + 0, + 0, + settings.max_bytes_before_external_group_by, + settings.empty_result_for_aggregation_by_empty_set, + storage_.getContext()->getTemporaryVolume(), + settings.max_threads, + settings.min_free_disk_space_for_temporary_data, + settings.compile_aggregate_expressions, + settings.min_count_to_compile_aggregate_expression); - aggregator = std::make_unique(params); + aggregator = std::make_unique(header, params); if (isMaxTTLExpired()) new_ttl_info.ttl_finished = true; diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.cpp b/src/Processors/Transforms/AggregatingInOrderTransform.cpp index f435d46a066..ce50ae5eeee 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.cpp +++ b/src/Processors/Transforms/AggregatingInOrderTransform.cpp @@ -87,7 +87,8 @@ void AggregatingInOrderTransform::consume(Chunk chunk) Columns key_columns(params->params.keys_size); for (size_t i = 0; i < params->params.keys_size; ++i) { - materialized_columns.push_back(columns.at(params->params.keys[i])->convertToFullColumnIfConst()); + const auto pos = inputs.front().getHeader().getPositionByName(params->params.keys[i]); + materialized_columns.push_back(chunk.getColumns().at(pos)->convertToFullColumnIfConst()); key_columns[i] = materialized_columns.back(); if (group_by_key) key_columns_raw[i] = materialized_columns.back().get(); @@ -95,7 +96,11 @@ void AggregatingInOrderTransform::consume(Chunk chunk) Aggregator::NestedColumnsHolder nested_columns_holder; Aggregator::AggregateFunctionInstructions aggregate_function_instructions; - params->aggregator.prepareAggregateInstructions(columns, aggregate_columns, materialized_columns, aggregate_function_instructions, nested_columns_holder); + if (!params->params.only_merge) + { + params->aggregator.prepareAggregateInstructions( + columns, aggregate_columns, materialized_columns, aggregate_function_instructions, nested_columns_holder); + } size_t key_end = 0; size_t key_begin = 0; @@ -123,7 +128,7 @@ void AggregatingInOrderTransform::consume(Chunk chunk) Int64 current_memory_usage = 0; Aggregator::AggregateColumnsConstData aggregate_columns_data(params->params.aggregates_size); - if (params->only_merge) + if (params->params.only_merge) { for (size_t i = 0, j = 0; i < columns.size(); ++i) { @@ -149,7 +154,7 @@ void AggregatingInOrderTransform::consume(Chunk chunk) /// Add data to aggr. state if interval is not empty. Empty when haven't found current key in new block. if (key_begin != key_end) { - if (params->only_merge) + if (params->params.only_merge) { if (group_by_key) params->aggregator.mergeOnBlockSmall(variants, key_begin, key_end, aggregate_columns_data, key_columns_raw); diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index b5b254c3e3c..f8332742978 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -524,7 +524,7 @@ void AggregatingTransform::consume(Chunk chunk) src_rows += num_rows; src_bytes += chunk.bytes(); - if (params->only_merge) + if (params->params.only_merge) { auto block = getInputs().front().getHeader().cloneWithColumns(chunk.detachColumns()); block = materializeBlock(block); @@ -549,7 +549,7 @@ void AggregatingTransform::initGenerate() /// To do this, we pass a block with zero rows to aggregate. if (variants.empty() && params->params.keys_size == 0 && !params->params.empty_result_for_aggregation_by_empty_set) { - if (params->only_merge) + if (params->params.only_merge) params->aggregator.mergeOnBlock(getInputs().front().getHeader(), variants, no_more_keys); else params->aggregator.executeOnBlock(getInputs().front().getHeader(), variants, key_columns, aggregate_columns, no_more_keys); diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 8d62664da59..789fa970ebd 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -34,24 +34,21 @@ struct AggregatingTransformParams AggregatorListPtr aggregator_list_ptr; Aggregator & aggregator; bool final; - /// Merge data for aggregate projections. - bool only_merge = false; - AggregatingTransformParams(const Aggregator::Params & params_, bool final_, bool only_merge_) + AggregatingTransformParams(const Block & header, const Aggregator::Params & params_, bool final_) : params(params_) , aggregator_list_ptr(std::make_shared()) - , aggregator(*aggregator_list_ptr->emplace(aggregator_list_ptr->end(), params)) + , aggregator(*aggregator_list_ptr->emplace(aggregator_list_ptr->end(), header, params)) , final(final_) - , only_merge(only_merge_) { } - AggregatingTransformParams(const Aggregator::Params & params_, const AggregatorListPtr & aggregator_list_ptr_, bool final_, bool only_merge_) + AggregatingTransformParams( + const Block & header, const Aggregator::Params & params_, const AggregatorListPtr & aggregator_list_ptr_, bool final_) : params(params_) , aggregator_list_ptr(aggregator_list_ptr_) - , aggregator(*aggregator_list_ptr->emplace(aggregator_list_ptr->end(), params)) + , aggregator(*aggregator_list_ptr->emplace(aggregator_list_ptr->end(), header, params)) , final(final_) - , only_merge(only_merge_) { } diff --git a/src/Processors/Transforms/CubeTransform.cpp b/src/Processors/Transforms/CubeTransform.cpp index 83ed346dabe..b80ca29327f 100644 --- a/src/Processors/Transforms/CubeTransform.cpp +++ b/src/Processors/Transforms/CubeTransform.cpp @@ -12,9 +12,12 @@ namespace ErrorCodes CubeTransform::CubeTransform(Block header, AggregatingTransformParamsPtr params_) : IAccumulatingTransform(std::move(header), appendGroupingSetColumn(params_->getHeader())) , params(std::move(params_)) - , keys(params->params.keys) , aggregates_mask(getAggregatesMask(params->getHeader(), params->params.aggregates)) { + keys.reserve(params->params.keys_size); + for (const auto & key : params->params.keys) + keys.emplace_back(input.getHeader().getPositionByName(key)); + if (keys.size() >= 8 * sizeof(mask)) throw Exception("Too many keys are used for CubeTransform.", ErrorCodes::LOGICAL_ERROR); } diff --git a/src/Processors/Transforms/CubeTransform.h b/src/Processors/Transforms/CubeTransform.h index 4575a01935d..bd33eabd750 100644 --- a/src/Processors/Transforms/CubeTransform.h +++ b/src/Processors/Transforms/CubeTransform.h @@ -21,7 +21,7 @@ protected: private: AggregatingTransformParamsPtr params; - const ColumnNumbers keys; + ColumnNumbers keys; const ColumnsMask aggregates_mask; Chunks consumed_chunks; diff --git a/src/Processors/Transforms/RollupTransform.cpp b/src/Processors/Transforms/RollupTransform.cpp index b69a691323c..e5351d1d5e2 100644 --- a/src/Processors/Transforms/RollupTransform.cpp +++ b/src/Processors/Transforms/RollupTransform.cpp @@ -8,9 +8,11 @@ namespace DB RollupTransform::RollupTransform(Block header, AggregatingTransformParamsPtr params_) : IAccumulatingTransform(std::move(header), appendGroupingSetColumn(params_->getHeader())) , params(std::move(params_)) - , keys(params->params.keys) , aggregates_mask(getAggregatesMask(params->getHeader(), params->params.aggregates)) { + keys.reserve(params->params.keys_size); + for (const auto & key : params->params.keys) + keys.emplace_back(input.getHeader().getPositionByName(key)); } void RollupTransform::consume(Chunk chunk) diff --git a/src/Processors/Transforms/RollupTransform.h b/src/Processors/Transforms/RollupTransform.h index 8fd27e3e6a2..1630df23579 100644 --- a/src/Processors/Transforms/RollupTransform.h +++ b/src/Processors/Transforms/RollupTransform.h @@ -20,7 +20,7 @@ protected: private: AggregatingTransformParamsPtr params; - const ColumnNumbers keys; + ColumnNumbers keys; const ColumnsMask aggregates_mask; Chunks consumed_chunks; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 30fb3efcf0e..6590445572d 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -263,23 +263,22 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( Pipe ordinary_pipe; if (query_info.projection->desc->type == ProjectionDescription::Type::Aggregate) { - auto make_aggregator_params = [&](const Block & header_before_aggregation, bool projection) + auto make_aggregator_params = [&](bool projection) { - ColumnNumbers keys; - for (const auto & key : query_info.projection->aggregation_keys) - keys.push_back(header_before_aggregation.getPositionByName(key.name)); + const auto & keys = query_info.projection->aggregation_keys.getNames(); AggregateDescriptions aggregates = query_info.projection->aggregate_descriptions; - if (!projection) - { - for (auto & descr : aggregates) - if (descr.arguments.empty()) - for (const auto & name : descr.argument_names) - descr.arguments.push_back(header_before_aggregation.getPositionByName(name)); - } + + /// This part is hacky. + /// We want AggregatingTransform to work with aggregate states instead of normal columns. + /// It is almost the same, just instead of adding new data to aggregation state we merge it with existing. + /// + /// It is needed because data in projection: + /// * is not merged completely (we may have states with the same key in different parts) + /// * is not split into buckets (so if we just use MergingAggregated, it will use single thread) + const bool only_merge = projection; Aggregator::Params params( - header_before_aggregation, keys, aggregates, query_info.projection->aggregate_overflow_row, @@ -293,23 +292,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( settings.max_threads, settings.min_free_disk_space_for_temporary_data, settings.compile_aggregate_expressions, - settings.min_count_to_compile_aggregate_expression); - - bool only_merge = false; - if (projection) - { - /// The source header is also an intermediate header - params.intermediate_header = header_before_aggregation; - - /// This part is hacky. - /// We want AggregatingTransform to work with aggregate states instead of normal columns. - /// It is almost the same, just instead of adding new data to aggregation state we merge it with existing. - /// - /// It is needed because data in projection: - /// * is not merged completely (we may have states with the same key in different parts) - /// * is not split into buckets (so if we just use MergingAggregated, it will use single thread) - only_merge = true; - } + settings.min_count_to_compile_aggregate_expression, + only_merge); return std::make_pair(params, only_merge); }; @@ -343,10 +327,10 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( /// TODO apply optimize_aggregation_in_order here too (like below) auto build_aggregate_pipe = [&](Pipe & pipe, bool projection) { - auto [params, only_merge] = make_aggregator_params(pipe.getHeader(), projection); + auto [params, only_merge] = make_aggregator_params(projection); AggregatingTransformParamsPtr transform_params = std::make_shared( - std::move(params), aggregator_list_ptr, query_info.projection->aggregate_final, only_merge); + pipe.getHeader(), std::move(params), aggregator_list_ptr, query_info.projection->aggregate_final); pipe.resize(pipe.numOutputPorts(), true, true); @@ -371,7 +355,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( { auto add_aggregating_step = [&](QueryPlanPtr & query_plan, bool projection) { - auto [params, only_merge] = make_aggregator_params(query_plan->getCurrentDataStream().header, projection); + auto [params, only_merge] = make_aggregator_params(projection); auto merge_threads = num_streams; auto temporary_data_merge_threads = settings.aggregation_memory_efficient_merge_threads @@ -390,7 +374,6 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( std::move(params), /* grouping_sets_params_= */ GroupingSetsParamsList{}, query_info.projection->aggregate_final, - only_merge, settings.max_block_size, settings.aggregation_in_order_max_block_bytes, merge_threads, diff --git a/tests/queries/0_stateless/01763_filter_push_down_bugs.reference b/tests/queries/0_stateless/01763_filter_push_down_bugs.reference index 6917117b3e2..5aa2e645509 100644 --- a/tests/queries/0_stateless/01763_filter_push_down_bugs.reference +++ b/tests/queries/0_stateless/01763_filter_push_down_bugs.reference @@ -5,3 +5,4 @@ String1_0 String2_0 String3_0 String4_0 1 String1_0 String2_0 String3_0 String4_0 1 1 [0,1,2] +1 diff --git a/tests/queries/0_stateless/01763_filter_push_down_bugs.sql b/tests/queries/0_stateless/01763_filter_push_down_bugs.sql index b13282e6dca..1058bf75144 100644 --- a/tests/queries/0_stateless/01763_filter_push_down_bugs.sql +++ b/tests/queries/0_stateless/01763_filter_push_down_bugs.sql @@ -37,3 +37,7 @@ WHERE String4 ='String4_0'; DROP TABLE IF EXISTS Test; select x, y from (select [0, 1, 2] as y, 1 as a, 2 as b) array join y as x where a = 1 and b = 2 and (x = 1 or x != 1) and x = 1; + +create table t(a UInt8) engine=MergeTree order by a; +insert into t select * from numbers(2); +select a from t t1 join t t2 on t1.a = t2.a where t1.a; diff --git a/tests/queries/0_stateless/01823_explain_json.reference b/tests/queries/0_stateless/01823_explain_json.reference index 9e36660204b..9df7c16e4f4 100644 --- a/tests/queries/0_stateless/01823_explain_json.reference +++ b/tests/queries/0_stateless/01823_explain_json.reference @@ -63,8 +63,7 @@ "Argument Types": ["UInt64"], "Result Type": "Float64" }, - "Arguments": ["number"], - "Argument Positions": [0] + "Arguments": ["number"] }, { "Name": "sumIf(number, greater(number, 0))", @@ -73,8 +72,7 @@ "Argument Types": ["UInt64", "UInt8"], "Result Type": "UInt64" }, - "Arguments": ["number", "greater(number, 0)"], - "Argument Positions": [0, 2] + "Arguments": ["number", "greater(number, 0)"] } ], -------- diff --git a/tests/queries/0_stateless/01823_explain_json.sh b/tests/queries/0_stateless/01823_explain_json.sh index 3db2dcb6dc4..4d7aa5f88d6 100755 --- a/tests/queries/0_stateless/01823_explain_json.sh +++ b/tests/queries/0_stateless/01823_explain_json.sh @@ -12,7 +12,7 @@ $CLICKHOUSE_CLIENT -q "explain json = 1, description = 0, header = 1 select 1, 2 echo "--------" $CLICKHOUSE_CLIENT -q "EXPLAIN json = 1, actions = 1, header = 1, description = 0 SELECT quantile(0.2)(number), sumIf(number, number > 0) from numbers(2) group by number, number + 1 FORMAT TSVRaw - " | grep Aggregating -A 42 + " | grep Aggregating -A 40 echo "--------" $CLICKHOUSE_CLIENT -q "EXPLAIN json = 1, actions = 1, description = 0 From 5f81bcd84f52efd7a7f99acd9f7e6191e6163478 Mon Sep 17 00:00:00 2001 From: Anton Kozlov Date: Mon, 27 Jun 2022 13:13:21 +0100 Subject: [PATCH 120/123] CLICKHOUSE-1331 Rewrite tuple functions as literals in backwards-compatibility mode (#38096) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- src/Interpreters/TreeRewriter.cpp | 34 ++++++++++++++++--- ...f_tuple_literal_over_distributed.reference | 1 + ...name_of_tuple_literal_over_distributed.sql | 4 +++ 3 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/02287_legacy_column_name_of_tuple_literal_over_distributed.reference create mode 100644 tests/queries/0_stateless/02287_legacy_column_name_of_tuple_literal_over_distributed.sql diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 3d14955c16a..7fa1fe2f012 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -834,17 +834,43 @@ std::vector getWindowFunctions(ASTPtr & query, const ASTSel class MarkTupleLiteralsAsLegacyData { public: - using TypeToVisit = ASTLiteral; + struct Data + { + }; - static void visit(ASTLiteral & literal, ASTPtr &) + static void visitLiteral(ASTLiteral & literal, ASTPtr &) { if (literal.value.getType() == Field::Types::Tuple) literal.use_legacy_column_name_of_tuple = true; } + static void visitFunction(ASTFunction & func, ASTPtr &ast) + { + if (func.name == "tuple" && func.arguments && !func.arguments->children.empty()) + { + // re-write tuple() function as literal + if (auto literal = func.toLiteral()) + { + ast = literal; + visitLiteral(*typeid_cast(ast.get()), ast); + } + } + } + + static void visit(ASTPtr & ast, Data &) + { + if (auto * identifier = typeid_cast(ast.get())) + visitFunction(*identifier, ast); + if (auto * identifier = typeid_cast(ast.get())) + visitLiteral(*identifier, ast); + } + + static bool needChildVisit(const ASTPtr & /*parent*/, const ASTPtr & /*child*/) + { + return true; + } }; -using MarkTupleLiteralsAsLegacyMatcher = OneTypeMatcher; -using MarkTupleLiteralsAsLegacyVisitor = InDepthNodeVisitor; +using MarkTupleLiteralsAsLegacyVisitor = InDepthNodeVisitor; void markTupleLiteralsAsLegacy(ASTPtr & query) { diff --git a/tests/queries/0_stateless/02287_legacy_column_name_of_tuple_literal_over_distributed.reference b/tests/queries/0_stateless/02287_legacy_column_name_of_tuple_literal_over_distributed.reference new file mode 100644 index 00000000000..9766475a418 --- /dev/null +++ b/tests/queries/0_stateless/02287_legacy_column_name_of_tuple_literal_over_distributed.reference @@ -0,0 +1 @@ +ok diff --git a/tests/queries/0_stateless/02287_legacy_column_name_of_tuple_literal_over_distributed.sql b/tests/queries/0_stateless/02287_legacy_column_name_of_tuple_literal_over_distributed.sql new file mode 100644 index 00000000000..a2a8e7c80f8 --- /dev/null +++ b/tests/queries/0_stateless/02287_legacy_column_name_of_tuple_literal_over_distributed.sql @@ -0,0 +1,4 @@ +SET legacy_column_name_of_tuple_literal=1; +SET prefer_localhost_replica=0; + +select if(in(dummy, tuple(0, 1)), 'ok', 'ok') from remote('localhost', system.one); \ No newline at end of file From c4d815c60b2a40491159de4b77e4ecae561f54f7 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 27 Jun 2022 15:44:20 +0300 Subject: [PATCH 121/123] Update test.py --- tests/integration/test_broken_detached_part_clean_up/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_broken_detached_part_clean_up/test.py b/tests/integration/test_broken_detached_part_clean_up/test.py index dd6744b4228..167d10ec7d1 100644 --- a/tests/integration/test_broken_detached_part_clean_up/test.py +++ b/tests/integration/test_broken_detached_part_clean_up/test.py @@ -278,7 +278,7 @@ def test_store_cleanup(started_cluster): assert "300" in store assert "456" in store assert "kek" not in store # changed - assert "12" not in store # changed + assert "\n12\n" not in store # changed assert "d---------" not in node1.exec_in_container( ["ls", "-l", f"{path_to_data}/store"] ) # changed From e515579cbe070583a8b650f8db272982842ebc71 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 27 Jun 2022 16:26:36 +0200 Subject: [PATCH 122/123] Update IObjectStorage.h --- src/Disks/ObjectStorages/IObjectStorage.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h index 64022ec046d..ad8d5d2530e 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.h +++ b/src/Disks/ObjectStorages/IObjectStorage.h @@ -25,8 +25,7 @@ class WriteBufferFromFileBase; using ObjectAttributes = std::map; -/// Path to a file and its size. -/// Path can be either relative or absolute - according to the context of use. +/// Path to a file (always absolute) and its size. struct PathWithSize { std::string path; From 83b87938e8b47f162fe02cc7a0fd98c954089639 Mon Sep 17 00:00:00 2001 From: Yuko Takagi <70714860+yukotakagi@users.noreply.github.com> Date: Mon, 27 Jun 2022 13:23:26 -0600 Subject: [PATCH 123/123] Add 22.7 release webinar. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e07a701d7c7..153a0d5ce11 100644 --- a/README.md +++ b/README.md @@ -16,3 +16,4 @@ ClickHouse® is an open-source column-oriented database management system that a ## Upcoming events * [Paris Meetup](https://www.meetup.com/clickhouse-france-user-group/events/286304312/) Please join us for an evening of talks (in English), food and discussion. Featuring talks of ClickHouse in production and at least one on the deep internals of ClickHouse itself. +* [v22.7 Release Webinar](https://clickhouse.com/company/events/v22-7-release-webinar/) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.